diff --git a/frontend/src/lib/constants.tsx b/frontend/src/lib/constants.tsx index 40973bc7a286..bb15ef7dd8aa 100644 --- a/frontend/src/lib/constants.tsx +++ b/frontend/src/lib/constants.tsx @@ -350,6 +350,7 @@ export const FEATURE_FLAGS = { LLM_ANALYTICS_TAGS: 'llm-analytics-tags', // owner: #team-ai-observability LLM_ANALYTICS_TRACE_NAVIGATION: 'llm-analytics-trace-navigation', // owner: #team-ai-observability LLM_OBSERVABILITY_TRACE_SEARCH: 'llm-observability-trace-search', // owner: #team-ai-observability + LLM_PROMPT_CONFIG: 'llm-prompt-config', // owner: @jurajmajerik #team-ai-observability LLM_PROMPT_LABELS: 'prompt-labels', // owner: @jurajmajerik #team-ai-observability LOGS: 'logs', // owner: #team-logs LOGS_ALERTING: 'logs-alerting', // owner: #team-logs diff --git a/posthog/api/llm_prompt_serializers.py b/posthog/api/llm_prompt_serializers.py index cc9d98be5ac2..b876da7f1d05 100644 --- a/posthog/api/llm_prompt_serializers.py +++ b/posthog/api/llm_prompt_serializers.py @@ -55,7 +55,7 @@ def validate_prompt_config_value(value: Any) -> Any: return None if not isinstance(value, dict): raise serializers.ValidationError( - 'Config must be a JSON object, e.g. {"model": "gpt-4o", "temperature": 0}.', + 'Config must be a JSON object, e.g. {"model": "your-model-name", "temperature": 0}.', code="invalid_config", ) return validate_prompt_payload_size(value, field_label="Config") diff --git a/products/ai_observability/frontend/prompts/llmPromptLogic.test.ts b/products/ai_observability/frontend/prompts/llmPromptLogic.test.ts index a4f593e883b5..7cbe0c5c03c8 100644 --- a/products/ai_observability/frontend/prompts/llmPromptLogic.test.ts +++ b/products/ai_observability/frontend/prompts/llmPromptLogic.test.ts @@ -19,6 +19,7 @@ import { } from '../generated/api' import type { LLMPromptApi, LLMPromptResolveResponseApi } from '../generated/api.schemas' import { PromptAnalyticsScope, PromptMode, llmPromptLogic } from './llmPromptLogic' +import type { ResolvedLLMPrompt } from './llmPromptLogic' import { validatePromptLabelName } from './utils' jest.mock('../generated/api', () => ({ @@ -63,7 +64,7 @@ const mockPrompt = { }, ], has_more: false, -} +} as unknown as ResolvedLLMPrompt const productionLabelV1 = { id: 'label-1', @@ -344,6 +345,98 @@ describe('llmPromptLogic', () => { logic.unmount() }) + it('sends the parsed config on publish and null when the editor is cleared', async () => { + const { versions, has_more, ...promptFields } = mockPrompt + const promptWithConfig = { ...promptFields, config: { model: 'gpt-4o' } } + mockResolve.mockResolvedValue({ + prompt: promptWithConfig, + versions, + has_more, + } as unknown as LLMPromptResolveResponseApi) + mockPartialUpdate.mockResolvedValue({ + ...promptWithConfig, + id: 'prompt-version-3', + version: 3, + latest_version: 3, + } as unknown as LLMPromptApi) + + const logic = llmPromptLogic({ promptName: 'my-test-prompt' }) + logic.mount() + await expectLogic(logic).toDispatchActions(['loadPromptSuccess']) + + // The form is seeded with the stored config so an untouched publish round-trips it. + expect(JSON.parse(logic.values.promptForm.config)).toEqual({ model: 'gpt-4o' }) + + logic.actions.setPromptFormValues({ config: '{"temperature": 0.9}' }) + logic.actions.submitPromptForm() + await expectLogic(logic).toDispatchActions(['submitPromptFormSuccess']) + expect(mockPartialUpdate).toHaveBeenLastCalledWith( + expect.anything(), + 'my-test-prompt', + expect.objectContaining({ config: { temperature: 0.9 } }) + ) + + // Clearing the editor publishes an explicit null; omitting the key would carry the old config forward. + logic.actions.setMode(PromptMode.Edit) + logic.actions.setPromptFormValues({ config: '' }) + logic.actions.submitPromptForm() + await expectLogic(logic).toDispatchActions(['submitPromptFormSuccess']) + expect(mockPartialUpdate).toHaveBeenLastCalledWith( + expect.anything(), + 'my-test-prompt', + expect.objectContaining({ config: null }) + ) + + logic.unmount() + }) + + it('does not count a key reorder as a config change, matching jsonb storage', async () => { + const { versions, has_more, ...promptFields } = mockPrompt + mockResolve.mockResolvedValue({ + prompt: { ...promptFields, config: { model: 'gpt-4o', temperature: 0.2 } }, + versions, + has_more, + } as unknown as LLMPromptResolveResponseApi) + + const logic = llmPromptLogic({ promptName: 'my-test-prompt' }) + logic.mount() + await expectLogic(logic).toDispatchActions(['loadPromptSuccess']) + + logic.actions.setPromptFormValues({ config: '{"temperature": 0.2, "model": "gpt-4o"}' }) + expect(logic.values.isConfigChanged).toBe(false) + + logic.actions.setPromptFormValues({ config: '{"temperature": 0.9, "model": "gpt-4o"}' }) + expect(logic.values.isConfigChanged).toBe(true) + + logic.unmount() + }) + + it('routes publish with invalid config through form errors instead of the review modal', async () => { + const { versions, has_more, ...promptFields } = mockPrompt + mockResolve.mockResolvedValue({ + prompt: promptFields, + versions, + has_more, + } as unknown as LLMPromptResolveResponseApi) + + const logic = llmPromptLogic({ promptName: 'my-test-prompt' }) + logic.mount() + await expectLogic(logic).toDispatchActions(['loadPromptSuccess']) + + logic.actions.setMode(PromptMode.Edit) + logic.actions.setPromptFormValues({ config: '["not", "an", "object"]' }) + logic.actions.requestPublish() + + await expectLogic(logic).toDispatchActions(['submitPromptForm', 'submitPromptFormFailure']) + expect(logic.values.isPublishReviewOpen).toBe(false) + expect(logic.values.promptFormErrors.config).toBe( + 'Configuration must be a JSON object, e.g. {"model": "your-model-name"}' + ) + expect(mockPartialUpdate).not.toHaveBeenCalled() + + logic.unmount() + }) + it('guards cancel behind a confirmation only when the form is dirty', async () => { const { versions, has_more, ...promptFields } = mockPrompt mockResolve.mockResolvedValue({ @@ -436,6 +529,7 @@ describe('llmPromptLogic', () => { expect(updateSpy).toHaveBeenCalledTimes(1) expect(updateSpy).toHaveBeenCalledWith(String(MOCK_DEFAULT_TEAM.id), 'my-test-prompt', { prompt: 'My edited prompt.', + config: null, base_version: 2, version_description: 'Tightened the refusal criteria', }) diff --git a/products/ai_observability/frontend/prompts/llmPromptLogic.ts b/products/ai_observability/frontend/prompts/llmPromptLogic.ts index 5d498d2bc2f5..415495deac0d 100644 --- a/products/ai_observability/frontend/prompts/llmPromptLogic.ts +++ b/products/ai_observability/frontend/prompts/llmPromptLogic.ts @@ -76,6 +76,46 @@ export interface PromptLogicProps { export interface PromptFormValues { name: string prompt: string + // The config JSON as editor text; '' means no config and publishes null. + config: string +} + +export function formatPromptConfig(config: LLMPrompt['config'] | undefined): string { + return config == null ? '' : JSON.stringify(config, null, 2) +} + +// Sorted keys so comparisons match Postgres jsonb, which doesn't preserve key order: +// a reordered-but-equal config must not be presented as a change the server won't store. +function canonicalizeJson(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map(canonicalizeJson) + } + if (typeof value === 'object' && value !== null) { + const record = value as Record + return Object.fromEntries( + Object.keys(record) + .sort() + .map((key) => [key, canonicalizeJson(record[key])]) + ) + } + return value +} + +export function parsePromptConfig(text: string): { config: Record | null; error?: string } { + const trimmed = text.trim() + if (!trimmed) { + return { config: null } + } + let parsed: unknown + try { + parsed = JSON.parse(trimmed) + } catch { + return { config: null, error: 'Configuration must be valid JSON' } + } + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { + return { config: null, error: 'Configuration must be a JSON object, e.g. {"model": "your-model-name"}' } + } + return { config: parsed as Record } } export interface ResolvedLLMPrompt extends LLMPrompt { @@ -91,8 +131,13 @@ export function isPrompt(prompt: LLMPrompt | ResolvedLLMPrompt | PromptFormValue const DEFAULT_PROMPT_FORM_VALUES: PromptFormValues = { name: '', prompt: '', + config: '', } +// Seeded into the empty editor when "Add configuration" is clicked, so users see the +// expected shape instead of a blank JSON editor. +const STARTER_PROMPT_CONFIG = '{\n "model": "your-model-name",\n "temperature": 0.7\n}' + const PROMPT_FETCHED_EVENT = '$llm_prompt_fetched' const PROMPT_VERSIONS_LIMIT = 50 const DEFAULT_PROMPT_ANALYTICS_DATE_FROM = '-1d' @@ -174,6 +219,8 @@ export interface llmPromptLogicValues { value: number }> defaultRelatedTracesQuery: DataTableNode | null + isConfigChanged: boolean + isConfigEditorVisible: boolean isDiffVisible: boolean isEditMode: boolean isHistoricalVersion: boolean @@ -280,6 +327,9 @@ export interface llmPromptLogicActions { openPublishReview: () => { value: true } + removeConfig: () => { + value: true + } removeLabel: (labelName: string) => { labelName: string } @@ -346,6 +396,9 @@ export interface llmPromptLogicActions { setVersionsLoading: (versionsLoading: boolean) => { versionsLoading: boolean } + showConfigEditor: () => { + value: true + } submitPromptForm: () => { value: boolean } @@ -386,6 +439,7 @@ export interface llmPromptLogicMeta { prompt: PromptFormValues | ResolvedLLMPrompt | null, isNewPrompt: boolean ) => boolean + isConfigChanged: (promptForm: PromptFormValues, prompt: PromptFormValues | ResolvedLLMPrompt | null) => boolean nextVersion: (prompt: PromptFormValues | ResolvedLLMPrompt | null) => number | null promptVariables: (promptForm: PromptFormValues) => string[] breadcrumbs: ( @@ -468,6 +522,8 @@ export const llmPromptLogic = kea([ toggleMarkdownRendering: true, setCompareVersion: (compareVersion: number | null) => ({ compareVersion }), toggleOutlineExpanded: true, + showConfigEditor: true, + removeConfig: true, cancelEditing: true, setPublishConflict: (publishConflict: PublishConflict | null) => ({ publishConflict }), requestPublish: true, @@ -544,6 +600,18 @@ export const llmPromptLogic = kea([ toggleOutlineExpanded: (state) => !state, }, ], + // Once shown, the config editor stays visible for the editing session even if the + // text is emptied (clearing the text is how a config gets removed). Resets on + // mode changes and reloads so view mode starts collapsed again. + isConfigEditorVisible: [ + false, + { + showConfigEditor: () => true, + removeConfig: () => false, + setMode: () => false, + loadPromptSuccess: () => false, + }, + ], publishConflict: [ null as PublishConflict | null, { @@ -611,9 +679,10 @@ export const llmPromptLogic = kea([ defaults: DEFAULT_PROMPT_FORM_VALUES, options: { showErrorsOnTouch: true }, - errors: ({ name, prompt }) => ({ + errors: ({ name, prompt, config }) => ({ name: validatePromptName(name), prompt: !prompt?.trim() ? 'Prompt content is required' : undefined, + config: parsePromptConfig(config ?? '').error, }), submit: async (formValues) => { @@ -622,10 +691,13 @@ export const llmPromptLogic = kea([ try { let savedPrompt: LLMPrompt + const parsedConfig = parsePromptConfig(formValues.config).config + if (isNew) { savedPrompt = (await llmPromptsCreate(String(ApiConfig.getCurrentTeamId()), { name: formValues.name, prompt: formValues.prompt, + ...(parsedConfig ? { config: parsedConfig } : {}), })) as unknown as LLMPrompt llmPromptsLogic.findMounted()?.actions.loadPrompts(false) lemonToast.success('Prompt created successfully') @@ -648,6 +720,9 @@ export const llmPromptLogic = kea([ props.promptName, { prompt: formValues.prompt, + // Always sent: the form is the source of truth, and null clears a + // previously set config (omitting the key would carry it forward). + config: parsedConfig, base_version: currentPrompt.latest_version, ...(versionDescription ? { version_description: versionDescription } : {}), } @@ -743,9 +818,32 @@ export const llmPromptLogic = kea([ isNewPrompt: boolean ): boolean => { if (isNewPrompt) { - return !!promptForm.name.trim() || !!promptForm.prompt.trim() + return !!promptForm.name.trim() || !!promptForm.prompt.trim() || !!promptForm.config.trim() + } + if (!isPrompt(prompt)) { + return false } - return isPrompt(prompt) ? promptForm.prompt !== prompt.prompt : false + return ( + promptForm.prompt !== prompt.prompt || + promptForm.config.trim() !== formatPromptConfig(prompt.config).trim() + ) + }, + ], + + isConfigChanged: [ + (s) => [s.promptForm, s.prompt], + (promptForm: PromptFormValues, prompt: PromptFormValues | ResolvedLLMPrompt | null): boolean => { + if (!isPrompt(prompt)) { + return false + } + const parsed = parsePromptConfig(promptForm.config) + if (parsed.error) { + return true + } + return ( + JSON.stringify(canonicalizeJson(parsed.config)) !== + JSON.stringify(canonicalizeJson(prompt.config ?? null)) + ) }, ], @@ -1171,10 +1269,26 @@ export const llmPromptLogic = kea([ llmPromptsLogic.findMounted()?.actions.loadPrompts(false) }, + showConfigEditor: () => { + if (!values.promptForm.config.trim()) { + actions.setPromptFormValue('config', STARTER_PROMPT_CONFIG) + } + }, + + // Only clears the form: the stored config goes away when the version is published, + // and the review modal shows that as a config change first. + removeConfig: () => { + actions.setPromptFormValue('config', '') + }, + requestPublish: () => { - // New prompts publish directly (v1, nothing to diff against); an empty form - // goes through submit so kea-forms surfaces the validation errors. - if (values.isNewPrompt || !values.promptForm.prompt?.trim()) { + // New prompts publish directly (v1, nothing to diff against); an empty form or + // invalid config goes through submit so kea-forms surfaces the validation errors. + if ( + values.isNewPrompt || + !values.promptForm.prompt?.trim() || + parsePromptConfig(values.promptForm.config).error + ) { actions.submitPromptForm() return } @@ -1378,6 +1492,7 @@ function getPromptFormDefaults(prompt: LLMPrompt): PromptFormValues { return { name: prompt.name, prompt: prompt.prompt, + config: formatPromptConfig(prompt.config), } } diff --git a/products/ai_observability/frontend/prompts/promptActivityDescriber.tsx b/products/ai_observability/frontend/prompts/promptActivityDescriber.tsx index fd62a3719135..be090505ae45 100644 --- a/products/ai_observability/frontend/prompts/promptActivityDescriber.tsx +++ b/products/ai_observability/frontend/prompts/promptActivityDescriber.tsx @@ -40,11 +40,14 @@ export function promptActivityDescriber(logItem: ActivityLogItem, asNotification if (logItem.activity === 'published') { const version = changeAfter(logItem, 'version') const versionDescription = changeAfter(logItem, 'version_description') + // Config contents are never logged; the change entry only records that it changed. + const configChanged = logItem.detail?.changes?.some((change) => change.field === 'config') return { description: ( <> {user} published v{version ?? '?'} of prompt{' '} {promptName} + {configChanged ? <> (configuration changed) : null} {versionDescription ? <>: "{versionDescription}" : null} ), diff --git a/products/ai_observability/frontend/prompts/promptSceneComponents.tsx b/products/ai_observability/frontend/prompts/promptSceneComponents.tsx index 30f394dd5799..d43e32ac5251 100644 --- a/products/ai_observability/frontend/prompts/promptSceneComponents.tsx +++ b/products/ai_observability/frontend/prompts/promptSceneComponents.tsx @@ -26,6 +26,7 @@ import { LemonTable, LemonTableColumns } from 'lib/lemon-ui/LemonTable' import { LemonTableLink } from 'lib/lemon-ui/LemonTable/LemonTableLink' import { ProfilePicture } from 'lib/lemon-ui/ProfilePicture' import { featureFlagLogic } from 'lib/logic/featureFlagLogic' +import { CodeEditor } from 'lib/monaco/CodeEditor' import { lazyWithRetry } from 'lib/utils/retryImport' import { teamLogic } from 'scenes/teamLogic' import { urls } from 'scenes/urls' @@ -39,7 +40,7 @@ import { useTracesQueryContext } from '../AIObservabilityTracesScene' import { MarkdownOutline } from '../components/MarkdownOutline' import { CreatePromptExperimentModal } from './CreatePromptExperimentModal' import { createPromptExperimentModalLogic } from './createPromptExperimentModalLogic' -import { PromptAnalyticsScope, isPrompt, llmPromptLogic } from './llmPromptLogic' +import { PromptAnalyticsScope, formatPromptConfig, isPrompt, llmPromptLogic } from './llmPromptLogic' import { promptExperimentsLogic } from './promptExperimentsLogic' import { PromptLabelChip } from './PromptLabelChip' import { PromptLabelPicker } from './PromptLabelPicker' @@ -78,12 +79,16 @@ export function PromptViewDetails(): JSX.Element { const { prompt, isRenderingMarkdown, isDiffVisible, canCompareVersions, compareVersionOptions } = useValues(llmPromptLogic) const { toggleMarkdownRendering, setCompareVersion } = useActions(llmPromptLogic) + const { featureFlags } = useValues(featureFlagLogic) const markdownContainerRef = useRef(null) if (!prompt || !isPrompt(prompt)) { return <> } + const configEnabled = !!featureFlags[FEATURE_FLAGS.LLM_PROMPT_CONFIG] + const configJson = configEnabled && prompt.config != null ? formatPromptConfig(prompt.config) : null + const promptText = prompt.prompt const variableMatches = promptText.match(/\{\{([^}]+)\}\}/g) const variables = variableMatches @@ -156,6 +161,13 @@ export function PromptViewDetails(): JSX.Element { ))} )} + + {configJson !== null && !isDiffVisible ? ( +
+

Configuration

+ {configJson} +
+ ) : null} ) } @@ -215,12 +227,20 @@ export function PromptHeaderMeta(): JSX.Element | null { } export function PublishReviewModal(): JSX.Element | null { - const { isPublishReviewOpen, prompt, promptForm, nextVersion, isPromptFormSubmitting, versionDescription } = - useValues(llmPromptLogic) + const { + isPublishReviewOpen, + prompt, + promptForm, + nextVersion, + isPromptFormSubmitting, + versionDescription, + isConfigChanged, + } = useValues(llmPromptLogic) const { promptLabels } = useValues(llmPromptLogic) const { closePublishReview, submitPromptForm, setVersionDescription } = useActions(llmPromptLogic) const { featureFlags } = useValues(featureFlagLogic) const labelsEnabled = !!featureFlags[FEATURE_FLAGS.LLM_PROMPT_LABELS] + const showConfigChange = !!featureFlags[FEATURE_FLAGS.LLM_PROMPT_CONFIG] && isConfigChanged if (!isPrompt(prompt)) { return null @@ -293,6 +313,42 @@ export function PublishReviewModal(): JSX.Element | null { /> + {showConfigChange ? ( +
+
+ Configuration + + Changed + +
+
+ + +
+ } + > + + +
+ + ) : null} @@ -319,6 +376,9 @@ function PromptDiffView(): JSX.Element { const currentVersion = prompt.version const original = comparePrompt?.prompt ?? '' const modified = prompt.prompt + const originalConfig = formatPromptConfig(comparePrompt?.config) + const modifiedConfig = formatPromptConfig(prompt.config) + const showConfigDiff = !!featureFlags[FEATURE_FLAGS.LLM_PROMPT_CONFIG] && !!(originalConfig || modifiedConfig) return (
@@ -344,33 +404,66 @@ function PromptDiffView(): JSX.Element { Failed to load version for comparison. Try selecting a different version. ) : ( -
- - - + <> +
+ + + +
+ } + > + + +
+ {showConfigDiff ? ( +
+
Configuration
+
+ + +
+ } + > + +
- } - > - - -
+ + ) : null} + )} ) @@ -729,6 +822,81 @@ export function PromptExperiments({ prompt }: { prompt: LLMPrompt }): JSX.Elemen ) } +function PromptConfigEditField(): JSX.Element | null { + const { promptForm, isConfigEditorVisible } = useValues(llmPromptLogic) + const { showConfigEditor, removeConfig } = useActions(llmPromptLogic) + const { featureFlags } = useValues(featureFlagLogic) + + if (!featureFlags[FEATURE_FLAGS.LLM_PROMPT_CONFIG]) { + return null + } + + if (!isConfigEditorVisible && !promptForm.config.trim()) { + return ( +
+ } + size="small" + type="secondary" + onClick={showConfigEditor} + tooltip="Store model parameters or other settings with this prompt" + data-attr="llma-prompt-add-config-button" + > + Add configuration + +
+ ) + } + + return ( +
+ + Configuration + { + e.preventDefault() + removeConfig() + }} + tooltip="Removed from the prompt when you publish" + data-attr="llma-prompt-remove-config-button" + > + Remove + +
+ } + help={ + 'Optional JSON object with model parameters or other settings for your app, for example {"model": "your-model-name", "temperature": 0}. ' + + "Stored with this version and returned when you fetch the prompt. Don't store secrets here." + } + > + {({ value, onChange }) => ( +
+ onChange(newValue ?? '')} + height={200} + options={{ + minimap: { enabled: false }, + lineNumbers: 'off', + scrollBeyondLastLine: false, + wordWrap: 'on', + folding: false, + }} + /> +
+ )} + + + ) +} + export function PromptEditForm({ isHistoricalVersion, selectedVersion, @@ -816,6 +984,8 @@ export function PromptEditForm({ ))} )} + + ) }