feat(aio): Add prompt config editing UI

frontend/src/lib/constants.tsx

@@ -350,6 +350,7 @@ export const FEATURE_FLAGS = {
     LLM_ANALYTICS_TAGS: 'llm-analytics-tags', // owner: #team-ai-observability
     LLM_ANALYTICS_TRACE_NAVIGATION: 'llm-analytics-trace-navigation', // owner: #team-ai-observability
     LLM_OBSERVABILITY_TRACE_SEARCH: 'llm-observability-trace-search', // owner: #team-ai-observability
+    LLM_PROMPT_CONFIG: 'llm-prompt-config', // owner: @jurajmajerik #team-ai-observability
     LLM_PROMPT_LABELS: 'prompt-labels', // owner: @jurajmajerik #team-ai-observability
     LOGS: 'logs', // owner: #team-logs
     LOGS_ALERTING: 'logs-alerting', // owner: #team-logs

posthog/api/llm_prompt_serializers.py

@@ -55,7 +55,7 @@ def validate_prompt_config_value(value: Any) -> Any:
         return None
     if not isinstance(value, dict):
         raise serializers.ValidationError(
-            'Config must be a JSON object, e.g. {"model": "gpt-4o", "temperature": 0}.',
+            'Config must be a JSON object, e.g. {"model": "your-model-name", "temperature": 0}.',
             code="invalid_config",
         )
     return validate_prompt_payload_size(value, field_label="Config")

products/ai_observability/frontend/prompts/llmPromptLogic.test.ts

@@ -19,6 +19,7 @@ import {
 } from '../generated/api'
 import type { LLMPromptApi, LLMPromptResolveResponseApi } from '../generated/api.schemas'
 import { PromptAnalyticsScope, PromptMode, llmPromptLogic } from './llmPromptLogic'
+import type { ResolvedLLMPrompt } from './llmPromptLogic'
 import { validatePromptLabelName } from './utils'
 
 jest.mock('../generated/api', () => ({
@@ -63,7 +64,7 @@ const mockPrompt = {
         },
     ],
     has_more: false,
-}
+} as unknown as ResolvedLLMPrompt
 
 const productionLabelV1 = {
     id: 'label-1',
@@ -344,6 +345,98 @@ describe('llmPromptLogic', () => {
         logic.unmount()
     })
 
+    it('sends the parsed config on publish and null when the editor is cleared', async () => {
+        const { versions, has_more, ...promptFields } = mockPrompt
+        const promptWithConfig = { ...promptFields, config: { model: 'gpt-4o' } }
+        mockResolve.mockResolvedValue({
+            prompt: promptWithConfig,
+            versions,
+            has_more,
+        } as unknown as LLMPromptResolveResponseApi)
+        mockPartialUpdate.mockResolvedValue({
+            ...promptWithConfig,
+            id: 'prompt-version-3',
+            version: 3,
+            latest_version: 3,
+        } as unknown as LLMPromptApi)
+
+        const logic = llmPromptLogic({ promptName: 'my-test-prompt' })
+        logic.mount()
+        await expectLogic(logic).toDispatchActions(['loadPromptSuccess'])
+
+        // The form is seeded with the stored config so an untouched publish round-trips it.
+        expect(JSON.parse(logic.values.promptForm.config)).toEqual({ model: 'gpt-4o' })
+
+        logic.actions.setPromptFormValues({ config: '{"temperature": 0.9}' })
+        logic.actions.submitPromptForm()
+        await expectLogic(logic).toDispatchActions(['submitPromptFormSuccess'])
+        expect(mockPartialUpdate).toHaveBeenLastCalledWith(
+            expect.anything(),
+            'my-test-prompt',
+            expect.objectContaining({ config: { temperature: 0.9 } })
+        )
+
+        // Clearing the editor publishes an explicit null; omitting the key would carry the old config forward.
+        logic.actions.setMode(PromptMode.Edit)
+        logic.actions.setPromptFormValues({ config: '' })
+        logic.actions.submitPromptForm()
+        await expectLogic(logic).toDispatchActions(['submitPromptFormSuccess'])
+        expect(mockPartialUpdate).toHaveBeenLastCalledWith(
+            expect.anything(),
+            'my-test-prompt',
+            expect.objectContaining({ config: null })
+        )
+
+        logic.unmount()
+    })
+
+    it('does not count a key reorder as a config change, matching jsonb storage', async () => {
+        const { versions, has_more, ...promptFields } = mockPrompt
+        mockResolve.mockResolvedValue({
+            prompt: { ...promptFields, config: { model: 'gpt-4o', temperature: 0.2 } },
+            versions,
+            has_more,
+        } as unknown as LLMPromptResolveResponseApi)
+
+        const logic = llmPromptLogic({ promptName: 'my-test-prompt' })
+        logic.mount()
+        await expectLogic(logic).toDispatchActions(['loadPromptSuccess'])
+
+        logic.actions.setPromptFormValues({ config: '{"temperature": 0.2, "model": "gpt-4o"}' })
+        expect(logic.values.isConfigChanged).toBe(false)
+
+        logic.actions.setPromptFormValues({ config: '{"temperature": 0.9, "model": "gpt-4o"}' })
+        expect(logic.values.isConfigChanged).toBe(true)
+
+        logic.unmount()
+    })
+
+    it('routes publish with invalid config through form errors instead of the review modal', async () => {
+        const { versions, has_more, ...promptFields } = mockPrompt
+        mockResolve.mockResolvedValue({
+            prompt: promptFields,
+            versions,
+            has_more,
+        } as unknown as LLMPromptResolveResponseApi)
+
+        const logic = llmPromptLogic({ promptName: 'my-test-prompt' })
+        logic.mount()
+        await expectLogic(logic).toDispatchActions(['loadPromptSuccess'])
+
+        logic.actions.setMode(PromptMode.Edit)
+        logic.actions.setPromptFormValues({ config: '["not", "an", "object"]' })
+        logic.actions.requestPublish()
+
+        await expectLogic(logic).toDispatchActions(['submitPromptForm', 'submitPromptFormFailure'])
+        expect(logic.values.isPublishReviewOpen).toBe(false)
+        expect(logic.values.promptFormErrors.config).toBe(
+            'Configuration must be a JSON object, e.g. {"model": "your-model-name"}'
+        )
+        expect(mockPartialUpdate).not.toHaveBeenCalled()
+
+        logic.unmount()
+    })
+
     it('guards cancel behind a confirmation only when the form is dirty', async () => {
         const { versions, has_more, ...promptFields } = mockPrompt
         mockResolve.mockResolvedValue({
@@ -436,6 +529,7 @@ describe('llmPromptLogic', () => {
         expect(updateSpy).toHaveBeenCalledTimes(1)
         expect(updateSpy).toHaveBeenCalledWith(String(MOCK_DEFAULT_TEAM.id), 'my-test-prompt', {
             prompt: 'My edited prompt.',
+            config: null,
             base_version: 2,
             version_description: 'Tightened the refusal criteria',
         })

products/ai_observability/frontend/prompts/llmPromptLogic.ts

@@ -76,6 +76,46 @@ export interface PromptLogicProps {
 export interface PromptFormValues {
     name: string
     prompt: string
+    // The config JSON as editor text; '' means no config and publishes null.
+    config: string
+}
+
+export function formatPromptConfig(config: LLMPrompt['config'] | undefined): string {
+    return config == null ? '' : JSON.stringify(config, null, 2)
+}
+
+// Sorted keys so comparisons match Postgres jsonb, which doesn't preserve key order:
+// a reordered-but-equal config must not be presented as a change the server won't store.
+function canonicalizeJson(value: unknown): unknown {
+    if (Array.isArray(value)) {
+        return value.map(canonicalizeJson)
+    }
+    if (typeof value === 'object' && value !== null) {
+        const record = value as Record<string, unknown>
+        return Object.fromEntries(
+            Object.keys(record)
+                .sort()
+                .map((key) => [key, canonicalizeJson(record[key])])
+        )
+    }
+    return value
+}
+
+export function parsePromptConfig(text: string): { config: Record<string, unknown> | null; error?: string } {
+    const trimmed = text.trim()
+    if (!trimmed) {
+        return { config: null }
+    }
+    let parsed: unknown
+    try {
+        parsed = JSON.parse(trimmed)
+    } catch {
+        return { config: null, error: 'Configuration must be valid JSON' }
+    }
+    if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {
+        return { config: null, error: 'Configuration must be a JSON object, e.g. {"model": "your-model-name"}' }
+    }
+    return { config: parsed as Record<string, unknown> }
 }
 
 export interface ResolvedLLMPrompt extends LLMPrompt {
@@ -91,8 +131,13 @@ export function isPrompt(prompt: LLMPrompt | ResolvedLLMPrompt | PromptFormValue
 const DEFAULT_PROMPT_FORM_VALUES: PromptFormValues = {
     name: '',
     prompt: '',
+    config: '',
 }
 
+// Seeded into the empty editor when "Add configuration" is clicked, so users see the
+// expected shape instead of a blank JSON editor.
+const STARTER_PROMPT_CONFIG = '{\n  "model": "your-model-name",\n  "temperature": 0.7\n}'
+
 const PROMPT_FETCHED_EVENT = '$llm_prompt_fetched'
 const PROMPT_VERSIONS_LIMIT = 50
 const DEFAULT_PROMPT_ANALYTICS_DATE_FROM = '-1d'
@@ -174,6 +219,8 @@ export interface llmPromptLogicValues {
         value: number
     }>
     defaultRelatedTracesQuery: DataTableNode | null
+    isConfigChanged: boolean
+    isConfigEditorVisible: boolean
     isDiffVisible: boolean
     isEditMode: boolean
     isHistoricalVersion: boolean
@@ -280,6 +327,9 @@ export interface llmPromptLogicActions {
     openPublishReview: () => {
         value: true
     }
+    removeConfig: () => {
+        value: true
+    }
     removeLabel: (labelName: string) => {
         labelName: string
     }
@@ -346,6 +396,9 @@ export interface llmPromptLogicActions {
     setVersionsLoading: (versionsLoading: boolean) => {
         versionsLoading: boolean
     }
+    showConfigEditor: () => {
+        value: true
+    }
     submitPromptForm: () => {
         value: boolean
     }
@@ -386,6 +439,7 @@ export interface llmPromptLogicMeta {
             prompt: PromptFormValues | ResolvedLLMPrompt | null,
             isNewPrompt: boolean
         ) => boolean
+        isConfigChanged: (promptForm: PromptFormValues, prompt: PromptFormValues | ResolvedLLMPrompt | null) => boolean
         nextVersion: (prompt: PromptFormValues | ResolvedLLMPrompt | null) => number | null
         promptVariables: (promptForm: PromptFormValues) => string[]
         breadcrumbs: (
@@ -468,6 +522,8 @@ export const llmPromptLogic = kea<llmPromptLogicType>([
         toggleMarkdownRendering: true,
         setCompareVersion: (compareVersion: number | null) => ({ compareVersion }),
         toggleOutlineExpanded: true,
+        showConfigEditor: true,
+        removeConfig: true,
         cancelEditing: true,
         setPublishConflict: (publishConflict: PublishConflict | null) => ({ publishConflict }),
         requestPublish: true,
@@ -544,6 +600,18 @@ export const llmPromptLogic = kea<llmPromptLogicType>([
                 toggleOutlineExpanded: (state) => !state,
             },
         ],
+        // Once shown, the config editor stays visible for the editing session even if the
+        // text is emptied (clearing the text is how a config gets removed). Resets on
+        // mode changes and reloads so view mode starts collapsed again.
+        isConfigEditorVisible: [
+            false,
+            {
+                showConfigEditor: () => true,
+                removeConfig: () => false,
+                setMode: () => false,
+                loadPromptSuccess: () => false,
+            },
+        ],
         publishConflict: [
             null as PublishConflict | null,
             {
@@ -611,9 +679,10 @@ export const llmPromptLogic = kea<llmPromptLogicType>([
             defaults: DEFAULT_PROMPT_FORM_VALUES,
             options: { showErrorsOnTouch: true },
 
-            errors: ({ name, prompt }) => ({
+            errors: ({ name, prompt, config }) => ({
                 name: validatePromptName(name),
                 prompt: !prompt?.trim() ? 'Prompt content is required' : undefined,
+                config: parsePromptConfig(config ?? '').error,
             }),
 
             submit: async (formValues) => {
@@ -622,10 +691,13 @@ export const llmPromptLogic = kea<llmPromptLogicType>([
                 try {
                     let savedPrompt: LLMPrompt
 
+                    const parsedConfig = parsePromptConfig(formValues.config).config
+
                     if (isNew) {
                         savedPrompt = (await llmPromptsCreate(String(ApiConfig.getCurrentTeamId()), {
                             name: formValues.name,
                             prompt: formValues.prompt,
+                            ...(parsedConfig ? { config: parsedConfig } : {}),
                         })) as unknown as LLMPrompt
                         llmPromptsLogic.findMounted()?.actions.loadPrompts(false)
                         lemonToast.success('Prompt created successfully')
@@ -648,6 +720,9 @@ export const llmPromptLogic = kea<llmPromptLogicType>([
                             props.promptName,
                             {
                                 prompt: formValues.prompt,
+                                // Always sent: the form is the source of truth, and null clears a
+                                // previously set config (omitting the key would carry it forward).
+                                config: parsedConfig,
                                 base_version: currentPrompt.latest_version,
                                 ...(versionDescription ? { version_description: versionDescription } : {}),
                             }
@@ -743,9 +818,32 @@ export const llmPromptLogic = kea<llmPromptLogicType>([
                 isNewPrompt: boolean
             ): boolean => {
                 if (isNewPrompt) {
-                    return !!promptForm.name.trim() || !!promptForm.prompt.trim()
+                    return !!promptForm.name.trim() || !!promptForm.prompt.trim() || !!promptForm.config.trim()
+                }
+                if (!isPrompt(prompt)) {
+                    return false
                 }
-                return isPrompt(prompt) ? promptForm.prompt !== prompt.prompt : false
+                return (
+                    promptForm.prompt !== prompt.prompt ||
+                    promptForm.config.trim() !== formatPromptConfig(prompt.config).trim()
+                )
+            },
+        ],
+
+        isConfigChanged: [
+            (s) => [s.promptForm, s.prompt],
+            (promptForm: PromptFormValues, prompt: PromptFormValues | ResolvedLLMPrompt | null): boolean => {
+                if (!isPrompt(prompt)) {
+                    return false
+                }
+                const parsed = parsePromptConfig(promptForm.config)
+                if (parsed.error) {
+                    return true
+                }
+                return (
+                    JSON.stringify(canonicalizeJson(parsed.config)) !==
+                    JSON.stringify(canonicalizeJson(prompt.config ?? null))
+                )
             },
         ],
 
@@ -1171,10 +1269,26 @@ export const llmPromptLogic = kea<llmPromptLogicType>([
             llmPromptsLogic.findMounted()?.actions.loadPrompts(false)
         },
 
+        showConfigEditor: () => {
+            if (!values.promptForm.config.trim()) {
+                actions.setPromptFormValue('config', STARTER_PROMPT_CONFIG)
+            }
+        },
+
+        // Only clears the form: the stored config goes away when the version is published,
+        // and the review modal shows that as a config change first.
+        removeConfig: () => {
+            actions.setPromptFormValue('config', '')
+        },
+
         requestPublish: () => {
-            // New prompts publish directly (v1, nothing to diff against); an empty form
-            // goes through submit so kea-forms surfaces the validation errors.
-            if (values.isNewPrompt || !values.promptForm.prompt?.trim()) {
+            // New prompts publish directly (v1, nothing to diff against); an empty form or
+            // invalid config goes through submit so kea-forms surfaces the validation errors.
+            if (
+                values.isNewPrompt ||
+                !values.promptForm.prompt?.trim() ||
+                parsePromptConfig(values.promptForm.config).error
+            ) {
                 actions.submitPromptForm()
                 return
             }
@@ -1378,6 +1492,7 @@ function getPromptFormDefaults(prompt: LLMPrompt): PromptFormValues {
     return {
         name: prompt.name,
         prompt: prompt.prompt,
+        config: formatPromptConfig(prompt.config),
     }
 }
 

products/ai_observability/frontend/prompts/promptActivityDescriber.tsx

@@ -40,11 +40,14 @@ export function promptActivityDescriber(logItem: ActivityLogItem, asNotification
     if (logItem.activity === 'published') {
         const version = changeAfter(logItem, 'version')
         const versionDescription = changeAfter(logItem, 'version_description')
+        // Config contents are never logged; the change entry only records that it changed.
+        const configChanged = logItem.detail?.changes?.some((change) => change.field === 'config')
         return {
             description: (
                 <>
                     <strong className="ph-no-capture">{user}</strong> published <b>v{version ?? '?'}</b> of prompt{' '}
                     <b>{promptName}</b>
+                    {configChanged ? <> (configuration changed)</> : null}
                     {versionDescription ? <>: "{versionDescription}"</> : null}
                 </>
             ),