From 5d8084cc8e3d6ac98be8d2d51fc1c4767edf072a Mon Sep 17 00:00:00 2001 From: MYRhouma Date: Tue, 8 Sep 2026 04:00:00 +0200 Subject: [PATCH 1/4] feat(vla): support Gemini and Anthropic assistants --- test-env/README.md | 10 +- test-env/compose.yml | 1 + vla-manager-api/README.md | 25 +++- .../src/vla_manager_api/assistant.py | 139 +++++++++++++++--- vla-manager-api/src/vla_manager_api/config.py | 8 +- .../tests/test_assistant_routes.py | 102 +++++++++++++ 6 files changed, 257 insertions(+), 28 deletions(-) diff --git a/test-env/README.md b/test-env/README.md index 72b4b466..97ba29d9 100644 --- a/test-env/README.md +++ b/test-env/README.md @@ -35,12 +35,18 @@ RabbitMQ. The template assistant is optional. To enable it, provide these variables before starting the stack: ```console -export VLA_MANAGER_AI_URL=https://your-provider.example/v1 +export VLA_MANAGER_AI_PROVIDER=gemini export VLA_MANAGER_AI_API_KEY=your-key -export VLA_MANAGER_AI_MODEL=your-model docker compose up -d --build ``` +Gemini uses Google's OpenAI-compatible endpoint and defaults to the low-cost +`gemini-3.1-flash-lite` model. For OpenAI-compatible services, set +`VLA_MANAGER_AI_PROVIDER=openai`, `VLA_MANAGER_AI_URL`, and `VLA_MANAGER_AI_MODEL`. +For Anthropic, set `VLA_MANAGER_AI_PROVIDER=anthropic`, `VLA_MANAGER_AI_URL`, and +`VLA_MANAGER_AI_MODEL`; its native Messages API is used automatically. `VLA_MANAGER_AI_URL` +may be a provider base URL or the complete endpoint URL. + The key is passed only to the VLA Manager API container. It is never included in the frontend bundle. Without these variables, the VLA Manager remains available and the assistant reports that it is not configured. To inspect a service while reviewing a failure: diff --git a/test-env/compose.yml b/test-env/compose.yml index 708cf674..510edaac 100755 --- a/test-env/compose.yml +++ b/test-env/compose.yml @@ -85,6 +85,7 @@ services: VLA_MANAGER_DB_URL: postgresql://postgres:postgres@postgres-vla-manager:5432/dva # Rendered requirements are validated by asking processing to compile them. VLA_MANAGER_API_PROCESSING_URL: http://dva-processing-vla-manager:5000 + VLA_MANAGER_AI_PROVIDER: ${VLA_MANAGER_AI_PROVIDER:-openai} VLA_MANAGER_AI_URL: ${VLA_MANAGER_AI_URL:-} VLA_MANAGER_AI_API_KEY: ${VLA_MANAGER_AI_API_KEY:-} VLA_MANAGER_AI_MODEL: ${VLA_MANAGER_AI_MODEL:-} diff --git a/vla-manager-api/README.md b/vla-manager-api/README.md index 8d578477..59e8d216 100644 --- a/vla-manager-api/README.md +++ b/vla-manager-api/README.md @@ -77,11 +77,32 @@ spec is still served and `/swagger/components.yaml` answers `404`. | `VLA_MANAGER_API_PORT` | `8000` | Listen port | | `VLA_MANAGER_API_LOG_LEVEL` | `info` | One of `critical`, `error`, `warning`, `info`, `debug` | | `VLA_MANAGER_API_PROCESSING_URL` | `http://localhost:5000` | URL to a DVA processing instance | -| `VLA_MANAGER_AI_URL` | *(empty)* | Base URL or full `/chat/completions` URL of an OpenAI-compatible assistant service. Empty disables the assistant. | +| `VLA_MANAGER_AI_PROVIDER` | `openai` | `openai` for OpenAI-compatible services, `gemini` for Gemini's compatibility endpoint, or `anthropic` for the native Messages API | +| `VLA_MANAGER_AI_URL` | *(empty)* | Provider base URL or complete endpoint URL. Defaults to the provider's public endpoint. Empty credentials disable the assistant. | | `VLA_MANAGER_AI_API_KEY` | *(empty)* | API key used only by the VLA Manager API. | -| `VLA_MANAGER_AI_MODEL` | *(empty)* | Model name sent to the assistant service. | +| `VLA_MANAGER_AI_MODEL` | *(empty)* | Model name sent to the assistant service. Gemini defaults to `gemini-3.1-flash-lite`; other providers require an explicit model. | | `VLA_MANAGER_AI_TIMEOUT_SECONDS` | `30` | Maximum assistant request duration. | The assistant returns a structured draft and never saves a template. The UI must show the draft for review and use the normal template validation and save actions afterward. Do not place the API key in frontend environment variables or browser code. + +Example provider settings: + +```console +# Gemini +VLA_MANAGER_AI_PROVIDER=gemini +VLA_MANAGER_AI_API_KEY=your-gemini-key + +# OpenAI or another OpenAI-compatible gateway +VLA_MANAGER_AI_PROVIDER=openai +VLA_MANAGER_AI_URL=https://api.openai.com/v1 +VLA_MANAGER_AI_API_KEY=your-openai-key +VLA_MANAGER_AI_MODEL=your-model + +# Anthropic +VLA_MANAGER_AI_PROVIDER=anthropic +VLA_MANAGER_AI_URL=https://api.anthropic.com/v1 +VLA_MANAGER_AI_API_KEY=your-anthropic-key +VLA_MANAGER_AI_MODEL=your-model +``` diff --git a/vla-manager-api/src/vla_manager_api/assistant.py b/vla-manager-api/src/vla_manager_api/assistant.py index 48c0c8c5..8e8fb269 100644 --- a/vla-manager-api/src/vla_manager_api/assistant.py +++ b/vla-manager-api/src/vla_manager_api/assistant.py @@ -11,6 +11,13 @@ from .config import cfg from .models import TemplateNew +_DEFAULT_URLS = { + "openai": "https://api.openai.com/v1", + "gemini": "https://generativelanguage.googleapis.com/v1beta/openai/", + "anthropic": "https://api.anthropic.com/v1", +} +_DEFAULT_MODELS = {"gemini": "gemini-3.1-flash-lite"} + class AssistantUnavailable(RuntimeError): """The configured assistant service cannot be reached or is disabled.""" @@ -41,9 +48,13 @@ def build_assistant_messages( "VLA Manager template fields name, description, criterionType, " "targetAspect, and evaluationMethod. evaluationMethod must contain " "engine, variableSchema, and implementationTemplate. Supported " - "engines are SCHEMA, JQ, and GREAT_EXPECTATIONS. Never save data, " - "invent unsupported engines, or claim that a generated proposal is " - "validated. examples may contain passing and failing JSON samples. " + "engines are SCHEMA, JQ, and GREAT_EXPECTATIONS. criterionType must " + "be one of VALID_INVALID, IN_RANGE, GREATER_THAN, LESS_THAN. " + "targetAspect must be one of SYNTAX, TIMELINESS, ACCURACY, " + "COMPLETENESS, CONSISTENCY. Never save data, invent unsupported " + "engines, or claim that a generated proposal is validated. examples " + 'must be an object with "passing" and "failing" values, never an ' + "array. Return plain JSON without markdown fences. " f"Available templates: {json.dumps(template_context)}. " f"Current draft: {json.dumps(current_template or {})}." ) @@ -53,32 +64,45 @@ def build_assistant_messages( return messages -def _completion_url() -> str: - base = cfg.ai_url.rstrip("/") +def _provider_config() -> tuple[str, str, str, str]: + provider = cfg.ai_provider + if provider not in _DEFAULT_URLS: + raise AssistantUnavailable( + "The configured assistant provider is not supported." + ) + + api_key = cfg.ai_api_key.strip() + base_url = cfg.ai_url.strip() or _DEFAULT_URLS[provider] + model = cfg.ai_model.strip() or _DEFAULT_MODELS.get(provider, "") + if not api_key or not model: + raise AssistantUnavailable("The template assistant is not configured.") + return provider, base_url, model, api_key + + +def _completion_url(base_url: str) -> str: + base = base_url.rstrip("/") if base.endswith("/chat/completions"): return base return f"{base}/chat/completions" -def _complete_sync(messages: list[dict[str, str]]) -> str: - if not cfg.ai_url or not cfg.ai_api_key or not cfg.ai_model: - raise AssistantUnavailable("The template assistant is not configured.") +def _messages_url(base_url: str) -> str: + base = base_url.rstrip("/") + if base.endswith("/messages"): + return base + return f"{base}/messages" - body = json.dumps( - { - "model": cfg.ai_model, - "messages": messages, - "temperature": 0.2, - "response_format": {"type": "json_object"}, - } - ).encode() + +def _post_json( + url: str, + body: dict[str, Any], + headers: dict[str, str], +) -> dict[str, Any]: + headers = {**headers, "Content-Type": "application/json"} request = Request( - _completion_url(), - data=body, - headers={ - "Authorization": f"Bearer {cfg.ai_api_key}", - "Content-Type": "application/json", - }, + url, + data=json.dumps(body).encode(), + headers=headers, method="POST", ) try: @@ -88,6 +112,34 @@ def _complete_sync(messages: list[dict[str, str]]) -> str: raise AssistantUnavailable( "The template assistant could not be reached." ) from exc + except (json.JSONDecodeError, TypeError, UnicodeDecodeError) as exc: + raise AssistantResponseError("The assistant returned invalid JSON.") from exc + + if not isinstance(payload, dict): + raise AssistantResponseError("The assistant response had an unexpected shape.") + return payload + + +def _complete_openai_compatible( + messages: list[dict[str, str]], + base_url: str, + model: str, + api_key: str, + provider: str, +) -> str: + headers = {"Authorization": f"Bearer {api_key}"} + if provider == "gemini": + headers["x-goog-api-client"] = "prometheus-x-data-veracity/0.1" + payload = _post_json( + _completion_url(base_url), + { + "model": model, + "messages": messages, + "temperature": 0.2, + "response_format": {"type": "json_object"}, + }, + headers, + ) try: content = payload["choices"][0]["message"]["content"] @@ -100,6 +152,49 @@ def _complete_sync(messages: list[dict[str, str]]) -> str: return content +def _complete_anthropic( + messages: list[dict[str, str]], + base_url: str, + model: str, + api_key: str, +) -> str: + system_messages = [item["content"] for item in messages if item["role"] == "system"] + chat_messages = [item for item in messages if item["role"] in {"user", "assistant"}] + body: dict[str, Any] = { + "model": model, + "max_tokens": 2048, + "temperature": 0.2, + "messages": chat_messages, + } + if system_messages: + body["system"] = "\n\n".join(system_messages) + payload = _post_json( + _messages_url(base_url), + body, + {"x-api-key": api_key, "anthropic-version": "2023-06-01"}, + ) + blocks = payload.get("content") + if not isinstance(blocks, list): + raise AssistantResponseError("The assistant response had an unexpected shape.") + content = "".join( + block["text"] + for block in blocks + if isinstance(block, dict) + and block.get("type") == "text" + and isinstance(block.get("text"), str) + ) + if not content: + raise AssistantResponseError("The assistant response was not text.") + return content + + +def _complete_sync(messages: list[dict[str, str]]) -> str: + provider, base_url, model, api_key = _provider_config() + if provider == "anthropic": + return _complete_anthropic(messages, base_url, model, api_key) + return _complete_openai_compatible(messages, base_url, model, api_key, provider) + + async def complete_assistant(messages: list[dict[str, str]]) -> str: """Call the configured service without blocking the API event loop.""" return await asyncio.to_thread(_complete_sync, messages) diff --git a/vla-manager-api/src/vla_manager_api/config.py b/vla-manager-api/src/vla_manager_api/config.py index 6ff91c4d..5af53248 100644 --- a/vla-manager-api/src/vla_manager_api/config.py +++ b/vla-manager-api/src/vla_manager_api/config.py @@ -8,6 +8,7 @@ from pydantic_settings import BaseSettings, SettingsConfigDict LogLevel = Literal["critical", "error", "warning", "info", "debug"] +AIProvider = Literal["openai", "gemini", "anthropic"] class Config(BaseSettings): @@ -41,8 +42,11 @@ class Config(BaseSettings): "/app/openapi.yaml", validation_alias="VLA_MANAGER_OPENAPI_FILE" ) - # Optional OpenAI-compatible chat completion service used by the - # template assistant. Empty URL disables the assistant cleanly. + # Optional chat completion service used by the template assistant. + # Empty credentials disable the assistant cleanly. + ai_provider: AIProvider = Field( + "openai", validation_alias="VLA_MANAGER_AI_PROVIDER" + ) ai_url: str = Field("", validation_alias="VLA_MANAGER_AI_URL") ai_api_key: str = Field("", validation_alias="VLA_MANAGER_AI_API_KEY") ai_model: str = Field("", validation_alias="VLA_MANAGER_AI_MODEL") diff --git a/vla-manager-api/tests/test_assistant_routes.py b/vla-manager-api/tests/test_assistant_routes.py index 388adcbb..82392c1e 100644 --- a/vla-manager-api/tests/test_assistant_routes.py +++ b/vla-manager-api/tests/test_assistant_routes.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json from collections.abc import Iterator import pytest @@ -11,6 +12,20 @@ from vla_manager_api.vla_repo import FakeVLARepo +class FakeResponse: + def __init__(self, payload: dict[str, object]) -> None: + self._payload = json.dumps(payload).encode() + + def __enter__(self) -> FakeResponse: + return self + + def __exit__(self, *_args: object) -> None: + return None + + def read(self) -> bytes: + return self._payload + + @pytest.fixture def fake_template_repo() -> FakeTemplateRepo: return FakeTemplateRepo() @@ -99,3 +114,90 @@ def test_assistant_reports_missing_model_configuration( assert response.status_code == 503 assert response.json()["type"] == "ASSISTANT_UNAVAILABLE" + + +def test_assistant_prompt_describes_the_template_enums_and_examples_shape() -> None: + from vla_manager_api.assistant import build_assistant_messages + + system_prompt = build_assistant_messages("draft", [], [], None)[0]["content"] + + assert ( + "criterionType must be one of VALID_INVALID, IN_RANGE, GREATER_THAN, LESS_THAN" + in system_prompt + ) + assert ( + "targetAspect must be one of SYNTAX, TIMELINESS, ACCURACY, COMPLETENESS, CONSISTENCY" + in system_prompt + ) + assert ( + 'examples must be an object with "passing" and "failing" values' + in system_prompt + ) + + +def test_gemini_uses_its_openai_compatible_defaults( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from vla_manager_api import assistant + + monkeypatch.setattr(assistant.cfg, "ai_provider", "gemini") + monkeypatch.setattr(assistant.cfg, "ai_url", "") + monkeypatch.setattr(assistant.cfg, "ai_model", "") + monkeypatch.setattr(assistant.cfg, "ai_api_key", "gemini-test-key") + captured: dict[str, object] = {} + + def fake_urlopen(request: object, timeout: float) -> FakeResponse: + captured["request"] = request + captured["timeout"] = timeout + return FakeResponse({"choices": [{"message": {"content": '{"message":"ok"}'}}]}) + + monkeypatch.setattr(assistant, "urlopen", fake_urlopen) + + result = assistant._complete_sync([{"role": "user", "content": "hello"}]) + + request = captured["request"] + assert result == '{"message":"ok"}' + assert request.full_url == ( + "https://generativelanguage.googleapis.com/v1beta/openai/chat/completions" + ) + assert request.headers["Authorization"] == "Bearer gemini-test-key" + assert request.headers["X-goog-api-client"] == "prometheus-x-data-veracity/0.1" + body = json.loads(request.data) + assert body["model"] == "gemini-3.1-flash-lite" + + +def test_anthropic_uses_messages_api_headers_and_system_prompt( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from vla_manager_api import assistant + + monkeypatch.setattr(assistant.cfg, "ai_provider", "anthropic") + monkeypatch.setattr(assistant.cfg, "ai_url", "https://api.anthropic.com/v1") + monkeypatch.setattr(assistant.cfg, "ai_model", "claude-test") + monkeypatch.setattr(assistant.cfg, "ai_api_key", "anthropic-test-key") + captured: dict[str, object] = {} + + def fake_urlopen(request: object, timeout: float) -> FakeResponse: + captured["request"] = request + captured["timeout"] = timeout + return FakeResponse({"content": [{"type": "text", "text": '{"message":"ok"}'}]}) + + monkeypatch.setattr(assistant, "urlopen", fake_urlopen) + + result = assistant._complete_sync( + [ + {"role": "system", "content": "Return JSON only."}, + {"role": "user", "content": "hello"}, + ] + ) + + request = captured["request"] + assert result == '{"message":"ok"}' + assert request.full_url == "https://api.anthropic.com/v1/messages" + assert request.headers["X-api-key"] == "anthropic-test-key" + assert request.headers["Anthropic-version"] == "2023-06-01" + assert "Authorization" not in request.headers + body = json.loads(request.data) + assert body["model"] == "claude-test" + assert body["system"] == "Return JSON only." + assert body["messages"] == [{"role": "user", "content": "hello"}] From 720cec2591295f4c9bfb1559d7f1f2fb30813f8e Mon Sep 17 00:00:00 2001 From: MYRhouma Date: Tue, 8 Sep 2026 04:42:00 +0200 Subject: [PATCH 2/4] feat(vla): polish template assistant workspace --- .../src/vla_manager_api/assistant.py | 6 +- .../tests/test_assistant_routes.py | 1 + vla-manager/src/api/assistantPresentation.js | 82 +++++ vla-manager/src/components/AssistantJson.vue | 15 + .../src/components/TemplateAssistant.vue | 283 +++++++++++++++--- vla-manager/test/template-assistant.test.mjs | 38 +++ 6 files changed, 385 insertions(+), 40 deletions(-) create mode 100644 vla-manager/src/api/assistantPresentation.js create mode 100644 vla-manager/src/components/AssistantJson.vue diff --git a/vla-manager-api/src/vla_manager_api/assistant.py b/vla-manager-api/src/vla_manager_api/assistant.py index 8e8fb269..5749a0e7 100644 --- a/vla-manager-api/src/vla_manager_api/assistant.py +++ b/vla-manager-api/src/vla_manager_api/assistant.py @@ -53,8 +53,10 @@ def build_assistant_messages( "targetAspect must be one of SYNTAX, TIMELINESS, ACCURACY, " "COMPLETENESS, CONSISTENCY. Never save data, invent unsupported " "engines, or claim that a generated proposal is validated. examples " - 'must be an object with "passing" and "failing" values, never an ' - "array. Return plain JSON without markdown fences. " + 'must be an object with "passing" and "failing" values. Each value ' + "should contain at least two representative examples when possible; " + "use an array for multiple examples, never a top-level array. Return " + "plain JSON without markdown fences. " f"Available templates: {json.dumps(template_context)}. " f"Current draft: {json.dumps(current_template or {})}." ) diff --git a/vla-manager-api/tests/test_assistant_routes.py b/vla-manager-api/tests/test_assistant_routes.py index 82392c1e..f9dfd7b5 100644 --- a/vla-manager-api/tests/test_assistant_routes.py +++ b/vla-manager-api/tests/test_assistant_routes.py @@ -133,6 +133,7 @@ def test_assistant_prompt_describes_the_template_enums_and_examples_shape() -> N 'examples must be an object with "passing" and "failing" values' in system_prompt ) + assert "at least two representative examples" in system_prompt def test_gemini_uses_its_openai_compatible_defaults( diff --git a/vla-manager/src/api/assistantPresentation.js b/vla-manager/src/api/assistantPresentation.js new file mode 100644 index 00000000..bacbbfbe --- /dev/null +++ b/vla-manager/src/api/assistantPresentation.js @@ -0,0 +1,82 @@ +export function normaliseAssistantExamples (examples) { + const source = examples && typeof examples === 'object' && !Array.isArray(examples) + ? examples + : {} + + return { + passing: toList(source.passing), + failing: toList(source.failing) + } +} + +export function formatAssistantJson (value) { + if (typeof value === 'string') return value + + try { + return JSON.stringify(value, null, 2) + } catch { + return String(value) + } +} + +export function tokeniseAssistantJson (value) { + const text = String(formatAssistantJson(value) ?? '') + const tokens = [] + let index = 0 + + while (index < text.length) { + const character = text[index] + + if (character === '"') { + const start = index + index += 1 + while (index < text.length) { + if (text[index] === '\\') { + index += 2 + } else if (text[index] === '"') { + index += 1 + break + } else { + index += 1 + } + } + const isKey = /^\s*:/.test(text.slice(index)) + tokens.push({ text: text.slice(start, index), type: isKey ? 'key' : 'string' }) + continue + } + + if (/[-0-9]/.test(character)) { + const match = text.slice(index).match(/^-?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?/) + if (match) { + tokens.push({ text: match[0], type: 'number' }) + index += match[0].length + continue + } + } + + const literal = text.slice(index).match(/^(true|false|null)\b/) + if (literal) { + tokens.push({ text: literal[0], type: literal[1] === 'null' ? 'null' : 'boolean' }) + index += literal[0].length + continue + } + + if ('{}[],:'.includes(character)) { + tokens.push({ text: character, type: 'punctuation' }) + index += 1 + continue + } + + const start = index + while (index < text.length && !/["{}\[\],:\s]/.test(text[index])) index += 1 + if (start === index) index += 1 + tokens.push({ text: text.slice(start, index), type: 'plain' }) + } + + return tokens +} + +function toList (value) { + if (value === undefined || value === null) return [] + return Array.isArray(value) ? value : [value] +} diff --git a/vla-manager/src/components/AssistantJson.vue b/vla-manager/src/components/AssistantJson.vue new file mode 100644 index 00000000..8cbd2a8d --- /dev/null +++ b/vla-manager/src/components/AssistantJson.vue @@ -0,0 +1,15 @@ + + + + + diff --git a/vla-manager/src/components/TemplateAssistant.vue b/vla-manager/src/components/TemplateAssistant.vue index 783563ad..72342b24 100644 --- a/vla-manager/src/components/TemplateAssistant.vue +++ b/vla-manager/src/components/TemplateAssistant.vue @@ -1,59 +1,191 @@ diff --git a/vla-manager/test/template-assistant.test.mjs b/vla-manager/test/template-assistant.test.mjs index 9ab8c85a..c48adb59 100644 --- a/vla-manager/test/template-assistant.test.mjs +++ b/vla-manager/test/template-assistant.test.mjs @@ -2,6 +2,44 @@ import test from 'node:test' import assert from 'node:assert/strict' import { applyTemplateProposal, assistantErrorMessage } from '../src/api/assistant.js' +import { + formatAssistantJson, + normaliseAssistantExamples, + tokeniseAssistantJson +} from '../src/api/assistantPresentation.js' + +test('normalises assistant examples into passing and failing lists', () => { + assert.deepEqual( + normaliseAssistantExamples({ + passing: { timestamp: '2026-09-08T08:00:00Z', production_kwh: 420 }, + failing: [ + { timestamp: 'not-a-date' }, + { timestamp: '2026-09-08T08:00:00Z', production_kwh: -2 } + ] + }), + { + passing: [{ timestamp: '2026-09-08T08:00:00Z', production_kwh: 420 }], + failing: [ + { timestamp: 'not-a-date' }, + { timestamp: '2026-09-08T08:00:00Z', production_kwh: -2 } + ] + } + ) +}) + +test('formats assistant JSON without losing primitive values', () => { + assert.equal(formatAssistantJson({ valid: true, count: 2 }), '{\n "valid": true,\n "count": 2\n}') + assert.equal(formatAssistantJson('not available'), 'not available') +}) + +test('tokenises JSON for readable syntax colouring', () => { + const tokens = tokeniseAssistantJson({ valid: true, count: 2, label: 'ok' }) + + assert.ok(tokens.some(token => token.type === 'key' && token.text === '"valid"')) + assert.ok(tokens.some(token => token.type === 'boolean' && token.text === 'true')) + assert.ok(tokens.some(token => token.type === 'number' && token.text === '2')) + assert.ok(tokens.some(token => token.type === 'string' && token.text === '"ok"')) +}) test('applies only template fields from an assistant proposal', () => { const current = { From b917cf5aa4ee76a879b3d79828862a402a33c4f1 Mon Sep 17 00:00:00 2001 From: MYRhouma Date: Tue, 8 Sep 2026 04:53:00 +0200 Subject: [PATCH 3/4] fix(vla): preserve variable editor focus --- vla-manager/src/api/templateVariables.js | 55 +++++++++++++++++++ .../src/components/TemplateAssistant.vue | 51 +++++------------ vla-manager/src/components/TemplateEditor.vue | 27 ++++----- vla-manager/test/template-assistant.test.mjs | 30 ++++++++++ 4 files changed, 110 insertions(+), 53 deletions(-) create mode 100644 vla-manager/src/api/templateVariables.js diff --git a/vla-manager/src/api/templateVariables.js b/vla-manager/src/api/templateVariables.js new file mode 100644 index 00000000..496520f4 --- /dev/null +++ b/vla-manager/src/api/templateVariables.js @@ -0,0 +1,55 @@ +export function renameTemplateVariable (schema, oldName, nextName) { + const name = nextName.trim() + const properties = schema.properties || {} + if (!name || name === oldName || properties[oldName] === undefined || properties[name] !== undefined) return schema + + const nextProperties = { ...properties, [name]: properties[oldName] } + delete nextProperties[oldName] + return { + ...schema, + properties: nextProperties, + required: (schema.required || []).map(item => item === oldName ? name : item) + } +} + +export function updateTemplateVariable (schema, name, patch) { + return { + ...schema, + properties: { + ...(schema.properties || {}), + [name]: { ...((schema.properties || {})[name] || {}), ...patch } + } + } +} + +export function removeTemplateVariable (schema, name) { + const properties = { ...(schema.properties || {}) } + delete properties[name] + return { + ...schema, + properties, + required: (schema.required || []).filter(item => item !== name) + } +} + +export function createVariableKeyStore () { + const keys = new Map() + let sequence = 0 + + return { + keyFor (name) { + if (!keys.has(name)) keys.set(name, `variable-${++sequence}`) + return keys.get(name) + }, + rename (oldName, nextName) { + const key = keys.get(oldName) + if (key) { + keys.delete(oldName) + keys.set(nextName, key) + } + }, + remove (name) { + keys.delete(name) + } + } +} diff --git a/vla-manager/src/components/TemplateAssistant.vue b/vla-manager/src/components/TemplateAssistant.vue index 72342b24..f2922d7e 100644 --- a/vla-manager/src/components/TemplateAssistant.vue +++ b/vla-manager/src/components/TemplateAssistant.vue @@ -2,8 +2,7 @@
- Design with AI - + Design with AI
@@ -13,34 +12,20 @@
- AI template designer - Build a requirement + Template assistant + Design a requirement
-
- -
-
-
Ready to design
-

Turn an idea into a VLA check

-

Describe the data rule in your own words. You will review every field before it reaches the editor.

-
-
+

Describe the quality rule in plain language. The assistant will prepare a draft for your review. Nothing is saved automatically.

- What should this data guarantee? -

Start with a field, a value range, a freshness rule, or a schema requirement.

-
- -
+ Start with the rule you need +

Mention the fields, values, freshness, or schema that your data should satisfy.

@@ -62,8 +47,9 @@
VLA assistant Generating
-
Drafting your requirement
- + + Preparing a draft +
@@ -81,9 +67,8 @@ class="assistant-input" type="textarea" maxlength="1000" - show-count :autosize="{ minRows: 3, maxRows: 7 }" - placeholder="For example: require every xAPI statement to contain an actor, verb, and object." + placeholder="For example: every xAPI statement must contain an actor, verb, and object." :disabled="loading" aria-label="Describe your requirement" @keydown.enter.exact.prevent="send" @@ -111,7 +96,7 @@
Not saved -

The assistant prepared this draft from your description. Check the wording, fields, and implementation before applying it.

+

Check the wording, fields, and implementation before applying this draft to the editor.

@@ -195,12 +180,6 @@ const proposal = ref(null) const examples = ref(null) const conversationEl = ref(null) -const suggestions = [ - 'Require an xAPI actor, verb, and object', - 'Keep hourly readings fresh and within range', - 'Validate a customer event JSON schema' -] - const exampleGroups = computed(() => { const normalised = normaliseAssistantExamples(examples.value) return [ @@ -219,10 +198,6 @@ function labelFor (value) { return String(value).toLowerCase().split('_').map(word => word.charAt(0).toUpperCase() + word.slice(1)).join(' ') } -function useSuggestion (suggestion) { - input.value = suggestion -} - function scrollConversation () { nextTick(() => { if (conversationEl.value) conversationEl.value.scrollTop = conversationEl.value.scrollHeight @@ -287,7 +262,7 @@ const UserIcon = defineComponent({ const ArrowUpIcon = defineComponent({ setup () { return () => h('svg', { viewBox: '0 0 24 24', fill: 'none', xmlns: 'http://www.w3.org/2000/svg' }, [ - h('path', { d: 'M12 19V5M6.5 10.5L12 5L17.5 10.5', stroke: 'currentColor', 'stroke-width': '2', 'stroke-linecap': 'round', 'stroke-linejoin': 'round' }) + h('path', { d: 'M12 19V5M6.5 10.5L12 5M12 5L17.5 10.5', stroke: 'currentColor', 'stroke-width': '2', 'stroke-linecap': 'round', 'stroke-linejoin': 'round' }) ]) } }) @@ -310,5 +285,5 @@ const CrossIcon = defineComponent({ diff --git a/vla-manager/src/components/TemplateEditor.vue b/vla-manager/src/components/TemplateEditor.vue index ba0d3409..4c8315f0 100644 --- a/vla-manager/src/components/TemplateEditor.vue +++ b/vla-manager/src/components/TemplateEditor.vue @@ -30,7 +30,7 @@
Template variablesAdd only the values a user must provide when this template is used.
Add variable
-
+
@@ -64,6 +64,7 @@ import TemplatePreview from './TemplatePreview.vue' import TemplateAssistant from './TemplateAssistant.vue' import { applyTemplateProposal } from '../api/assistant.js' import { createTemplate, renderTemplate, updateTemplate } from '../api/templates.js' +import { createVariableKeyStore, removeTemplateVariable, renameTemplateVariable, updateTemplateVariable } from '../api/templateVariables.js' const props = defineProps({ template: { type: Object, default: null } }) const emit = defineEmits(['saved', 'cancel']) @@ -86,7 +87,8 @@ function blankTemplate () { function clone (value) { return JSON.parse(JSON.stringify(value)) } const form = ref(blankTemplate()) const isEditing = computed(() => Boolean(props.template?.id)) -const variableRows = computed(() => Object.entries(form.value.evaluationMethod.variableSchema.properties || {}).map(([name, definition]) => ({ name, definition: definition || {} }))) +const variableKeys = createVariableKeyStore() +const variableRows = computed(() => Object.entries(form.value.evaluationMethod.variableSchema.properties || {}).map(([name, definition]) => ({ name, key: variableKeys.keyFor(name), definition: definition || {} }))) const canRender = computed(() => Boolean(form.value.id && Object.keys(errors.value).length === 0)) watch(() => props.template, value => { form.value = value ? clone(value) : blankTemplate(); previewValue.value = '' }, { immediate: true }) @@ -104,25 +106,20 @@ function addVariable () { updateVariable(name, { type: 'string', description: '' }, true) } function updateVariable (name, patch, create = false) { - const properties = { ...form.value.evaluationMethod.variableSchema.properties } - properties[name] = { ...(properties[name] || {}), ...patch } - form.value.evaluationMethod.variableSchema.properties = properties + form.value.evaluationMethod.variableSchema = updateTemplateVariable(form.value.evaluationMethod.variableSchema, name, patch) if (create) toggleRequired(name, true) } function renameVariable (oldName, nextName) { const name = nextName.trim() - if (!name || name === oldName || form.value.evaluationMethod.variableSchema.properties[name]) return - const properties = { ...form.value.evaluationMethod.variableSchema.properties } - properties[name] = properties[oldName] - delete properties[oldName] - form.value.evaluationMethod.variableSchema.properties = properties - form.value.evaluationMethod.variableSchema.required = (form.value.evaluationMethod.variableSchema.required || []).map(item => item === oldName ? name : item) + const schema = form.value.evaluationMethod.variableSchema + const nextSchema = renameTemplateVariable(schema, oldName, name) + if (nextSchema === schema) return + variableKeys.rename(oldName, name) + form.value.evaluationMethod.variableSchema = nextSchema } function removeVariable (name) { - const properties = { ...form.value.evaluationMethod.variableSchema.properties } - delete properties[name] - form.value.evaluationMethod.variableSchema.properties = properties - toggleRequired(name, false) + variableKeys.remove(name) + form.value.evaluationMethod.variableSchema = removeTemplateVariable(form.value.evaluationMethod.variableSchema, name) } function toggleRequired (name, checked) { const current = new Set(form.value.evaluationMethod.variableSchema.required || []) diff --git a/vla-manager/test/template-assistant.test.mjs b/vla-manager/test/template-assistant.test.mjs index c48adb59..d448c61c 100644 --- a/vla-manager/test/template-assistant.test.mjs +++ b/vla-manager/test/template-assistant.test.mjs @@ -7,6 +7,36 @@ import { normaliseAssistantExamples, tokeniseAssistantJson } from '../src/api/assistantPresentation.js' +import { createVariableKeyStore, renameTemplateVariable } from '../src/api/templateVariables.js' + +test('renames one template variable without disturbing siblings or focus identity', () => { + const schema = { + type: 'object', + properties: { + first: { type: 'string', description: 'First value' }, + second: { type: 'number', description: 'Second value' } + }, + required: ['first', 'second'] + } + const keys = createVariableKeyStore() + const firstKey = keys.keyFor('first') + const secondKey = keys.keyFor('second') + + const next = renameTemplateVariable(schema, 'first', 'firs') + keys.rename('first', 'firs') + + assert.deepEqual(next.properties, { + firs: { type: 'string', description: 'First value' }, + second: { type: 'number', description: 'Second value' } + }) + assert.deepEqual(next.required, ['firs', 'second']) + assert.deepEqual(schema.properties, { + first: { type: 'string', description: 'First value' }, + second: { type: 'number', description: 'Second value' } + }) + assert.equal(keys.keyFor('firs'), firstKey) + assert.equal(keys.keyFor('second'), secondKey) +}) test('normalises assistant examples into passing and failing lists', () => { assert.deepEqual( From cc3f5e320dedf9b65141f9d5b9173754062e7d3d Mon Sep 17 00:00:00 2001 From: MYRhouma Date: Tue, 8 Sep 2026 05:06:46 +0200 Subject: [PATCH 4/4] fix(vla): enforce executable drafts and reveal responses --- .../src/vla_manager_api/assistant.py | 44 +++++++++- .../tests/test_assistant_routes.py | 45 ++++++++++ vla-manager/src/api/assistantPresentation.js | 4 + .../src/components/TemplateAssistant.vue | 84 +++++++++++++++---- vla-manager/test/template-assistant.test.mjs | 8 ++ 5 files changed, 169 insertions(+), 16 deletions(-) diff --git a/vla-manager-api/src/vla_manager_api/assistant.py b/vla-manager-api/src/vla_manager_api/assistant.py index 5749a0e7..540732a3 100644 --- a/vla-manager-api/src/vla_manager_api/assistant.py +++ b/vla-manager-api/src/vla_manager_api/assistant.py @@ -56,7 +56,17 @@ def build_assistant_messages( 'must be an object with "passing" and "failing" values. Each value ' "should contain at least two representative examples when possible; " "use an array for multiple examples, never a top-level array. Return " - "plain JSON without markdown fences. " + "plain JSON without markdown fences. The implementationTemplate is " + "executed directly by the selected evaluator, so it must be code, " + "not an explanation. SCHEMA implementationTemplate must be a string " + "containing valid JSON Schema JSON, for example " + '\'{"type":"object","properties":{}}\'. ' + "JQ implementationTemplate must be the executable jq expression " + "that returns an object with a boolean success field and optional " + "details string. GREAT_EXPECTATIONS implementationTemplate must be " + "a string containing valid YAML expectation configuration with type, " + "kwargs, and optional meta fields. Never put prose such as 'check " + "that...' in implementationTemplate. " f"Available templates: {json.dumps(template_context)}. " f"Current draft: {json.dumps(current_template or {})}." ) @@ -214,6 +224,7 @@ def parse_assistant_response(content: str) -> dict[str, Any]: if proposal is not None and not isinstance(proposal, dict): raise AssistantResponseError("The assistant proposal is not an object.") if proposal is not None: + _validate_implementation_template(proposal) try: proposal = TemplateNew.model_validate(proposal).model_dump( by_alias=True, exclude_none=True @@ -226,3 +237,34 @@ def parse_assistant_response(content: str) -> dict[str, Any]: if examples is not None and not isinstance(examples, dict): raise AssistantResponseError("The assistant examples are not an object.") return {"message": response["message"], "proposal": proposal, "examples": examples} + + +def _validate_implementation_template(proposal: dict[str, Any]) -> None: + """Reject assistant prose before it reaches the template editor. + + Templates may contain Mustache placeholders, so static validation is only + applied when the implementation is self-contained. The selected evaluator + will validate rendered templates again when they are used. + """ + evaluation_method = proposal.get("evaluationMethod") + if not isinstance(evaluation_method, dict): + return + + engine = evaluation_method.get("engine") + implementation = evaluation_method.get("implementationTemplate") + if not isinstance(implementation, str) or not implementation.strip(): + return + if "{{" in implementation or "}}" in implementation: + return + + if engine == "SCHEMA": + try: + parsed = json.loads(implementation) + except json.JSONDecodeError as exc: + raise AssistantResponseError( + "The assistant returned a SCHEMA implementation that is not valid JSON Schema JSON." + ) from exc + if not isinstance(parsed, (dict, bool)): + raise AssistantResponseError( + "The assistant returned a SCHEMA implementation that is not a JSON Schema object." + ) diff --git a/vla-manager-api/tests/test_assistant_routes.py b/vla-manager-api/tests/test_assistant_routes.py index f9dfd7b5..1e74aeae 100644 --- a/vla-manager-api/tests/test_assistant_routes.py +++ b/vla-manager-api/tests/test_assistant_routes.py @@ -103,6 +103,39 @@ async def fake_complete(_messages: list[dict[str, str]]) -> str: assert response.json()["type"] == "ASSISTANT_INVALID_RESPONSE" +def test_assistant_rejects_prose_where_a_schema_implementation_is_required( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + async def fake_complete(_messages: list[dict[str, str]]) -> str: + return json.dumps( + { + "message": "Prepared a draft.", + "proposal": { + "name": "Production schema", + "description": "Checks production records.", + "criterionType": "VALID_INVALID", + "targetAspect": "SYNTAX", + "evaluationMethod": { + "engine": "SCHEMA", + "variableSchema": {"type": "object"}, + "implementationTemplate": "Check that production_kwh is present.", + }, + }, + } + ) + + monkeypatch.setattr( + "vla_manager_api.assistant_routes.complete_assistant", fake_complete + ) + + response = client.post( + "/assistant/template", json={"message": "Create a production schema."} + ) + + assert response.status_code == 502 + assert response.json()["type"] == "ASSISTANT_INVALID_RESPONSE" + + def test_assistant_reports_missing_model_configuration( client: TestClient, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -134,6 +167,18 @@ def test_assistant_prompt_describes_the_template_enums_and_examples_shape() -> N in system_prompt ) assert "at least two representative examples" in system_prompt + assert ( + "SCHEMA implementationTemplate must be a string containing valid JSON Schema" + in system_prompt + ) + assert ( + "JQ implementationTemplate must be the executable jq expression" + in system_prompt + ) + assert ( + "GREAT_EXPECTATIONS implementationTemplate must be a string containing valid YAML" + in system_prompt + ) def test_gemini_uses_its_openai_compatible_defaults( diff --git a/vla-manager/src/api/assistantPresentation.js b/vla-manager/src/api/assistantPresentation.js index bacbbfbe..ca32728e 100644 --- a/vla-manager/src/api/assistantPresentation.js +++ b/vla-manager/src/api/assistantPresentation.js @@ -76,6 +76,10 @@ export function tokeniseAssistantJson (value) { return tokens } +export function assistantTextChunks (value) { + return String(value ?? '').match(/\s+|\S+\s*/g) || [] +} + function toList (value) { if (value === undefined || value === null) return [] return Array.isArray(value) ? value : [value] diff --git a/vla-manager/src/components/TemplateAssistant.vue b/vla-manager/src/components/TemplateAssistant.vue index f2922d7e..27429223 100644 --- a/vla-manager/src/components/TemplateAssistant.vue +++ b/vla-manager/src/components/TemplateAssistant.vue @@ -28,7 +28,7 @@

Mention the fields, values, freshness, or schema that your data should satisfy.

-
+
-

{{ item.content }}

+

{{ item.content }}

@@ -46,10 +46,12 @@
VLA assistant Generating
-
- - Preparing a draft - +
+
+ + +
+
@@ -69,15 +71,15 @@ maxlength="1000" :autosize="{ minRows: 3, maxRows: 7 }" placeholder="For example: every xAPI statement must contain an actor, verb, and object." - :disabled="loading" + :disabled="busy" aria-label="Describe your requirement" @keydown.enter.exact.prevent="send" />
Enter to send · Shift + Enter for a new line
- Clear - + Clear + Generate draft @@ -162,10 +164,10 @@