From 38e45ce9c3ebca230ace53f170bb4564845c41d3 Mon Sep 17 00:00:00 2001 From: MYRhouma Date: Wed, 9 Sep 2026 03:07:00 +0200 Subject: [PATCH 01/14] feat(vla): add the catalog-aware VLA builder assistant Co-Authored-By: Claude Opus 5.5 Co-Authored-By: bzp99 --- docs/spec/vla-manager-api.yaml | 112 +++++++++++ test-env/README.md | 2 +- vla-manager-api/README.md | 2 +- .../src/vla_manager_api/assistant.py | 151 +++++++++++++- .../src/vla_manager_api/assistant_routes.py | 92 +++++++++ .../tests/test_assistant_routes.py | 147 +++++++++++++- vla-manager/src/api/assistant.js | 9 + vla-manager/src/api/vlaBuilderAssistant.js | 118 +++++++++++ vla-manager/src/components/CreateView.vue | 50 ++++- .../src/components/VlaBuilderAssistant.vue | 189 ++++++++++++++++++ vla-manager/src/mocks/handlers.js | 19 ++ .../test/vla-builder-assistant.test.mjs | 88 ++++++++ 12 files changed, 973 insertions(+), 6 deletions(-) create mode 100644 vla-manager/src/api/vlaBuilderAssistant.js create mode 100644 vla-manager/src/components/VlaBuilderAssistant.vue create mode 100644 vla-manager/test/vla-builder-assistant.test.mjs diff --git a/docs/spec/vla-manager-api.yaml b/docs/spec/vla-manager-api.yaml index 08e592af..74071259 100644 --- a/docs/spec/vla-manager-api.yaml +++ b/docs/spec/vla-manager-api.yaml @@ -70,6 +70,46 @@ paths: application/json: schema: $ref: '#/components/schemas/Error' + /assistant/vla: + post: + tags: [Assistant] + summary: Prepare a VLA draft from the template catalog + description: >- + Uses the supplied builder context and available templates to prepare + an unsaved VLA draft. Requirements contain catalog template IDs and + variable models only; the endpoint never persists a VLA or template. + operationId: assistVLA + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/AssistantVLARequest' + responses: + '200': + description: Catalog-backed VLA draft. + content: + application/json: + schema: + $ref: '#/components/schemas/AssistantVLAResponse' + '422': + description: Empty or invalid assistant request. + content: + application/json: + schema: + $ref: '#/components/schemas/ValidationError' + '502': + description: Assistant returned an invalid response. + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '503': + description: Assistant service is not configured or unavailable. + content: + application/json: + schema: + $ref: '#/components/schemas/Error' /template: get: tags: [Templates] @@ -767,6 +807,78 @@ components: type: object nullable: true additionalProperties: true + AssistantVLARequest: + type: object + required: [message] + additionalProperties: false + properties: + message: + type: string + minLength: 1 + maxLength: 4000 + conversation: + type: array + maxItems: 20 + items: + type: object + required: [role, content] + additionalProperties: false + properties: + role: + type: string + enum: [user, assistant] + content: + type: string + builderContext: + $ref: '#/components/schemas/BuilderContext' + BuilderContext: + type: object + additionalProperties: false + properties: + metadata: + type: object + additionalProperties: true + sampleData: + nullable: true + selectedPath: + type: string + nullable: true + fragments: + type: array + items: + type: object + additionalProperties: true + AssistantVLAResponse: + type: object + required: [message, requirements, missingTemplates] + additionalProperties: false + properties: + message: + type: string + metadata: + type: object + nullable: true + additionalProperties: true + requirements: + type: array + items: + type: object + required: [templateId, model, reason] + additionalProperties: false + properties: + templateId: + type: string + format: uuid + model: + type: object + additionalProperties: true + reason: + type: string + missingTemplates: + type: array + items: + type: object + additionalProperties: true QualityEngine: $ref: ./components.yaml#/schemas/QualityEngine Health: diff --git a/test-env/README.md b/test-env/README.md index 94d88fdd..ffd7d40f 100644 --- a/test-env/README.md +++ b/test-env/README.md @@ -41,7 +41,7 @@ docker compose up -d --build ``` Gemini uses Google's OpenAI-compatible endpoint and defaults to the low-cost -`gemini-3.1-flash-lite` model. For OpenAI-compatible services, set +`gemini-3.5-flash-lite` model. For OpenAI-compatible services, set `VLA_MANAGER_AI_PROVIDER=openai`, `VLA_MANAGER_AI_URL`, and `VLA_MANAGER_AI_MODEL`. For OpenRouter, set `VLA_MANAGER_AI_PROVIDER=openrouter` and `VLA_MANAGER_AI_MODEL`. For Anthropic, set `VLA_MANAGER_AI_PROVIDER=anthropic`, `VLA_MANAGER_AI_URL`, and diff --git a/vla-manager-api/README.md b/vla-manager-api/README.md index 49d1a9b6..9b8f16cd 100644 --- a/vla-manager-api/README.md +++ b/vla-manager-api/README.md @@ -86,7 +86,7 @@ spec is still served and `/swagger/components.yaml` answers `404`. | `VLA_MANAGER_AI_PROVIDER` | `openai` | `openai` for OpenAI-compatible services, `gemini` for Gemini's compatibility endpoint, `openrouter` for OpenRouter, or `anthropic` for the native Messages API | | `VLA_MANAGER_AI_URL` | *(empty)* | Provider base URL or complete endpoint URL. Defaults to the provider's public endpoint. Empty credentials disable the assistant. | | `VLA_MANAGER_AI_API_KEY` | *(empty)* | API key used only by the VLA Manager API. | -| `VLA_MANAGER_AI_MODEL` | *(empty)* | Model name sent to the assistant service. Gemini defaults to `gemini-3.1-flash-lite`; other providers require an explicit model. | +| `VLA_MANAGER_AI_MODEL` | *(empty)* | Model name sent to the assistant service. Gemini defaults to `gemini-3.5-flash-lite`; other providers require an explicit model. | | `VLA_MANAGER_AI_TIMEOUT_SECONDS` | `30` | Maximum assistant request duration. | The assistant returns a structured draft and never saves a template. The UI must show the diff --git a/vla-manager-api/src/vla_manager_api/assistant.py b/vla-manager-api/src/vla_manager_api/assistant.py index 5c240978..c9646969 100644 --- a/vla-manager-api/src/vla_manager_api/assistant.py +++ b/vla-manager-api/src/vla_manager_api/assistant.py @@ -9,6 +9,7 @@ from typing import Any from urllib.error import HTTPError, URLError from urllib.request import Request, urlopen +from uuid import UUID from structlog.contextvars import bound_contextvars @@ -24,7 +25,8 @@ "anthropic": "https://api.anthropic.com/v1", "openrouter": "https://openrouter.ai/api/v1", } -_DEFAULT_MODELS = {"gemini": "gemini-3.1-flash-lite"} +_DEFAULT_MODELS = {"gemini": "gemini-3.5-flash-lite"} +_MAX_SAMPLE_BYTES = 32 * 1024 # Request headers that carry credentials; logged as present, never by value. _SECRET_HEADERS = frozenset({"authorization", "x-api-key", "x-goog-api-key"}) @@ -87,6 +89,135 @@ def build_assistant_messages( return messages +def build_vla_assistant_messages( + message: str, + builder_context: dict[str, Any], + templates: list[dict[str, Any]], + conversation: list[dict[str, str]], +) -> list[dict[str, str]]: + """Build the bounded catalog context for VLA assembly requests.""" + catalog = [] + for item in templates: + evaluation = item.get("evaluationMethod") or {} + catalog.append( + { + "id": str(item.get("id", "")), + "name": item.get("name"), + "description": item.get("description"), + "engine": evaluation.get("engine"), + "criterionType": item.get("criterionType"), + "targetAspect": item.get("targetAspect"), + "variableSchema": evaluation.get("variableSchema", {}), + } + ) + + context = dict(builder_context or {}) + sample = context.get("sampleData") + sample_json = json.dumps(sample, ensure_ascii=False, separators=(",", ":")) + sample_truncated = len(sample_json.encode("utf-8")) > _MAX_SAMPLE_BYTES + if sample_truncated: + encoded = sample_json.encode("utf-8")[:_MAX_SAMPLE_BYTES] + sample_json = encoded.decode("utf-8", errors="ignore") + context["sampleData"] = sample_json + context["sampleDataTruncated"] = sample_truncated + + system = ( + "You are the VLA builder assistant. Return JSON only with the shape " + "{message, metadata, requirements, missingTemplates}; return template IDs " + "from the catalog. Use only template IDs " + "from the supplied catalog. Return requirements as templateId plus model " + "values and a short reason. Every required variable listed in a template's " + "variableSchema must be present and non-empty in model; if you cannot fill " + "one, omit that requirement and explain it in missingTemplates. Do not " + "return raw engine code or prose as an " + "implementation. If no catalog template can satisfy a requested rule, " + "leave requirements empty and describe it in missingTemplates. Metadata is " + "a suggestion for review, not an instruction to save anything. Match the " + "sample data when filling variables, and do not invent template IDs. " + f"Available templates: {json.dumps(catalog, ensure_ascii=False)}. " + f"Builder context: {json.dumps(context, ensure_ascii=False)}." + ) + messages = [{"role": "system", "content": system}] + messages.extend(conversation[-10:]) + messages.append({"role": "user", "content": message.strip()}) + return messages + + +def parse_vla_assistant_response( + content: str, catalog_ids: set[UUID] +) -> dict[str, Any]: + """Parse and validate a catalog-backed VLA assistant response.""" + try: + response = _load_json_object(content) + except json.JSONDecodeError as exc: + raise AssistantResponseError("The assistant returned invalid JSON.") from exc + if not isinstance(response, dict) or not isinstance(response.get("message"), str): + raise AssistantResponseError("The assistant response is missing its message.") + + metadata = response.get("metadata") + if metadata is not None and not isinstance(metadata, dict): + raise AssistantResponseError("The assistant metadata is not an object.") + + missing_templates = response.get("missingTemplates", []) + if missing_templates is None: + missing_templates = [] + elif isinstance(missing_templates, dict): + missing_templates = [] if not missing_templates else [missing_templates] + elif isinstance(missing_templates, str): + missing_templates = ( + [] + if not missing_templates.strip() or missing_templates.strip() == "[]" + else [{"reason": missing_templates}] + ) + elif isinstance(missing_templates, list): + missing_templates = [ + item if isinstance(item, dict) else {"reason": str(item)} + for item in missing_templates + ] + if not isinstance(missing_templates, list) or not all( + isinstance(item, dict) for item in missing_templates + ): + raise AssistantResponseError("The assistant missing-template list is invalid.") + + requirements = response.get("requirements", []) + if not isinstance(requirements, list): + raise AssistantResponseError("The assistant requirements are not a list.") + validated_requirements = [] + for item in requirements: + if not isinstance(item, dict): + raise AssistantResponseError("The assistant requirement is not an object.") + try: + template_id = UUID(str(item.get("templateId"))) + except (ValueError, TypeError, AttributeError) as exc: + raise AssistantResponseError( + "The assistant returned an invalid template ID." + ) from exc + if template_id not in catalog_ids: + raise AssistantResponseError( + "The assistant selected a template that is not available." + ) + model = item.get("model") + reason = item.get("reason") + if not isinstance(model, dict) or not isinstance(reason, str): + missing_templates.append( + { + "templateId": str(template_id), + "reason": "The assistant did not provide all values for this template.", + } + ) + continue + validated_requirements.append( + {"templateId": str(template_id), "model": model, "reason": reason} + ) + + return { + "message": response["message"], + "metadata": metadata, + "requirements": validated_requirements, + "missingTemplates": missing_templates, + } + + def _provider_config() -> tuple[str, str, str, str]: provider = cfg.ai_provider if provider not in _DEFAULT_URLS: @@ -325,7 +456,7 @@ async def complete_assistant(messages: list[dict[str, str]]) -> str: def parse_assistant_response(content: str) -> dict[str, Any]: """Parse model JSON and require the top-level fields used by the UI.""" try: - response = json.loads(content) + response = _load_json_object(content) except json.JSONDecodeError as exc: raise AssistantResponseError("The assistant returned invalid JSON.") from exc if not isinstance(response, dict) or not isinstance(response.get("message"), str): @@ -378,3 +509,19 @@ def _validate_implementation_template(proposal: dict[str, Any]) -> None: raise AssistantResponseError( "The assistant returned a SCHEMA implementation that is not a JSON Schema object." ) + + +def _load_json_object(content: str) -> Any: + """Decode an object when a provider wraps JSON in a fence or short prose.""" + text = content.strip() + if text.startswith("```"): + lines = text.splitlines() + text = "\n".join(lines[1:-1]).strip() + try: + return json.loads(text) + except json.JSONDecodeError: + start = text.find("{") + end = text.rfind("}") + if start < 0 or end <= start: + raise + return json.loads(text[start : end + 1]) diff --git a/vla-manager-api/src/vla_manager_api/assistant_routes.py b/vla-manager-api/src/vla_manager_api/assistant_routes.py index 581d8e84..9c24493d 100644 --- a/vla-manager-api/src/vla_manager_api/assistant_routes.py +++ b/vla-manager-api/src/vla_manager_api/assistant_routes.py @@ -3,6 +3,7 @@ from __future__ import annotations from typing import Any +from uuid import UUID from fastapi import APIRouter, Depends, status from pydantic import BaseModel, Field, field_validator @@ -11,8 +12,10 @@ AssistantResponseError, AssistantUnavailable, build_assistant_messages, + build_vla_assistant_messages, complete_assistant, parse_assistant_response, + parse_vla_assistant_response, ) from .dependencies import get_template_repo from .errors import http_error @@ -49,6 +52,48 @@ class AssistantReply(BaseModel): examples: dict[str, Any] | None = None +class BuilderContext(BaseModel): + model_config = _CAMEL_OPEN + + metadata: dict[str, Any] = Field(default_factory=dict) + sample_data: Any | None = None + selected_path: str | None = None + fragments: list[dict[str, Any]] = Field(default_factory=list) + + +class AssistantVLARequest(BaseModel): + model_config = _CAMEL_OPEN + + message: str = Field(min_length=1, max_length=4000) + conversation: list[dict[str, str]] = Field(default_factory=list, max_length=20) + builder_context: BuilderContext = Field(default_factory=BuilderContext) + + @field_validator("message") + @classmethod + def message_must_contain_text(cls, value: str) -> str: + value = value.strip() + if not value: + raise ValueError("Message must contain text") + return value + + +class VLAAssistantRequirement(BaseModel): + model_config = _CAMEL_OPEN + + template_id: str + model: dict[str, Any] + reason: str + + +class AssistantVLAReply(BaseModel): + model_config = _CAMEL_OPEN + + message: str + metadata: dict[str, Any] | None = None + requirements: list[VLAAssistantRequirement] = Field(default_factory=list) + missing_templates: list[dict[str, Any]] = Field(default_factory=list) + + @router.post("/assistant/template", response_model=AssistantReply) async def assist_template( request: AssistantRequest, @@ -91,3 +136,50 @@ async def assist_template( has_examples=reply.examples is not None, ) return reply + + +@router.post("/assistant/vla", response_model=AssistantVLAReply) +async def assist_vla( + request: AssistantVLARequest, + repo: TemplateRepo = Depends(get_template_repo), +) -> AssistantVLAReply: + try: + templates = [ + item.model_dump(by_alias=True, mode="json") for item in await repo.all() + ] + messages = build_vla_assistant_messages( + request.message, + request.builder_context.model_dump(by_alias=True, mode="json"), + templates, + request.conversation, + ) + content = await complete_assistant(messages) + catalog_ids = {UUID(str(item["id"])) for item in templates} + parsed = parse_vla_assistant_response(content, catalog_ids) + reply = AssistantVLAReply.model_validate(parsed) + except AssistantUnavailable as exc: + logger.warning("VLA assistant unavailable", error=str(exc)) + raise http_error( + status.HTTP_503_SERVICE_UNAVAILABLE, + str(exc), + type="ASSISTANT_UNAVAILABLE", + ) from exc + except AssistantResponseError as exc: + logger.warning( + "VLA assistant response rejected", + error=str(exc), + cause=str(exc.__cause__) if exc.__cause__ else None, + ) + raise http_error( + status.HTTP_502_BAD_GATEWAY, + str(exc), + type="ASSISTANT_INVALID_RESPONSE", + ) from exc + + logger.info( + "VLA assistant replied", + requirements=len(reply.requirements), + missing_templates=len(reply.missing_templates), + has_metadata=reply.metadata is not None, + ) + return reply diff --git a/vla-manager-api/tests/test_assistant_routes.py b/vla-manager-api/tests/test_assistant_routes.py index 81b2bcba..11ab06c7 100644 --- a/vla-manager-api/tests/test_assistant_routes.py +++ b/vla-manager-api/tests/test_assistant_routes.py @@ -13,6 +13,7 @@ from vla_manager_api.dependencies import get_repo, get_template_repo from vla_manager_api.main import create_app +from vla_manager_api.models import TemplateNew from vla_manager_api.template_repo import FakeTemplateRepo from vla_manager_api.vla_repo import FakeVLARepo @@ -172,6 +173,150 @@ def test_assistant_reports_missing_model_configuration( assert response.json()["type"] == "ASSISTANT_UNAVAILABLE" +def test_vla_assistant_returns_catalog_backed_requirements( + client: TestClient, + fake_template_repo: FakeTemplateRepo, + monkeypatch: pytest.MonkeyPatch, +) -> None: + import asyncio + + # The repo mints the id, so the seeded template is referred to by what + # `add` hands back rather than by one fixed here. + template_id = str( + asyncio.run( + fake_template_repo.add( + TemplateNew.model_validate( + { + "name": "JSON schema", + "description": "Checks the record shape", + "criterionType": "VALID_INVALID", + "targetAspect": "SYNTAX", + "evaluationMethod": { + "engine": "SCHEMA", + "variableSchema": { + "type": "object", + "properties": {"schema": {"type": "object"}}, + "required": ["schema"], + }, + "implementationTemplate": "{{ schema }}", + }, + } + ) + ) + ) + ) + + async def fake_complete(_messages: list[dict[str, str]]) -> str: + return json.dumps( + { + "message": "I found a schema template for this sample.", + "metadata": {"name": "Energy records", "tags": ["energy"]}, + "requirements": [ + { + "templateId": template_id, + "model": {"schema": {"type": "object"}}, + "reason": "The sample shape is a JSON object.", + } + ], + "missingTemplates": [], + } + ) + + monkeypatch.setattr("vla_manager_api.assistant_routes.complete_assistant", fake_complete) + response = client.post( + "/assistant/vla", + json={ + "message": "Use the sample schema.", + "builderContext": {"sampleData": {"timestamp": "now"}}, + }, + ) + + assert response.status_code == 200, response.text + assert response.json()["requirements"][0]["templateId"] == template_id + assert response.json()["metadata"]["name"] == "Energy records" + + +def test_vla_assistant_rejects_unknown_template_ids( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + async def fake_complete(_messages: list[dict[str, str]]) -> str: + return json.dumps( + { + "message": "Draft", + "requirements": [ + { + "templateId": "22222222-2222-2222-2222-222222222222", + "model": {}, + "reason": "match", + } + ], + } + ) + + monkeypatch.setattr("vla_manager_api.assistant_routes.complete_assistant", fake_complete) + response = client.post("/assistant/vla", json={"message": "Use a schema template."}) + + assert response.status_code == 502 + assert response.json()["type"] == "ASSISTANT_INVALID_RESPONSE" + + +def test_vla_assistant_treats_empty_missing_template_object_as_empty_list( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + async def fake_complete(_messages: list[dict[str, str]]) -> str: + return '{"message":"No matching template is available.","requirements":[],"missingTemplates":{}}' + + monkeypatch.setattr("vla_manager_api.assistant_routes.complete_assistant", fake_complete) + response = client.post("/assistant/vla", json={"message": "Check freshness."}) + + assert response.status_code == 200 + assert response.json()["missingTemplates"] == [] + + +def test_vla_assistant_keeps_string_missing_template_reasons_actionable( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + async def fake_complete(_messages: list[dict[str, str]]) -> str: + return '{"message":"A template is needed.","requirements":[],"missingTemplates":["No freshness template is available."]}' + + monkeypatch.setattr("vla_manager_api.assistant_routes.complete_assistant", fake_complete) + response = client.post("/assistant/vla", json={"message": "Check freshness."}) + + assert response.status_code == 200 + assert response.json()["missingTemplates"] == [{"reason": "No freshness template is available."}] + + +def test_assistant_accepts_json_wrapped_in_a_markdown_fence() -> None: + from vla_manager_api.assistant import parse_assistant_response + + response = parse_assistant_response('```json\n{"message":"Draft ready."}\n```') + + assert response == {"message": "Draft ready.", "proposal": None, "examples": None} + + +def test_vla_assistant_prompt_contains_catalog_and_bounded_sample() -> None: + from vla_manager_api.assistant import build_vla_assistant_messages + + messages = build_vla_assistant_messages( + "Match the uploaded sample.", + {"metadata": {}, "sampleData": {"field": "value"}, "selectedPath": None, "fragments": []}, + [ + { + "id": "template-1", + "name": "JSON schema", + "evaluationMethod": {"engine": "SCHEMA", "variableSchema": {"type": "object"}}, + } + ], + [], + ) + + prompt = messages[0]["content"] + assert "template-1" in prompt + assert "variableSchema" in prompt + assert "return template IDs from the catalog" in prompt + assert "Every required variable" in prompt + + def test_assistant_prompt_describes_the_template_enums_and_examples_shape() -> None: from vla_manager_api.assistant import build_assistant_messages @@ -232,7 +377,7 @@ def fake_urlopen(request: object, timeout: float) -> FakeResponse: assert request.headers["Authorization"] == "Bearer gemini-test-key" assert request.headers["X-goog-api-client"] == "prometheus-x-data-veracity/0.1" body = json.loads(request.data) - assert body["model"] == "gemini-3.1-flash-lite" + assert body["model"] == "gemini-3.5-flash-lite" def test_openrouter_uses_its_openai_compatible_endpoint( diff --git a/vla-manager/src/api/assistant.js b/vla-manager/src/api/assistant.js index 72991e96..2071c7ef 100644 --- a/vla-manager/src/api/assistant.js +++ b/vla-manager/src/api/assistant.js @@ -27,3 +27,12 @@ export async function askTemplateAssistant (payload, client = axios) { throw normaliseError(error) } } + +export async function askVlaBuilderAssistant (payload, client = axios) { + try { + const response = await client.post('/api/assistant/vla', payload) + return response.data + } catch (error) { + throw normaliseError(error) + } +} diff --git a/vla-manager/src/api/vlaBuilderAssistant.js b/vla-manager/src/api/vlaBuilderAssistant.js new file mode 100644 index 00000000..7b8ee516 --- /dev/null +++ b/vla-manager/src/api/vlaBuilderAssistant.js @@ -0,0 +1,118 @@ +const MAX_SAMPLE_BYTES = 32 * 1024 +const METADATA_FIELDS = ['name', 'description', 'dataReference', 'participants', 'tags'] + +function clone (value) { + if (value === undefined) return undefined + return JSON.parse(JSON.stringify(value)) +} + +function serialisedSize (value) { + return new TextEncoder().encode(JSON.stringify(value)).length +} + +export function createBuilderAssistantContext ({ metadata = {}, sampleData = null, selectedPath = null, fragments = [] } = {}) { + let boundedSample = clone(sampleData) + if (boundedSample !== null && serialisedSize(boundedSample) > MAX_SAMPLE_BYTES) { + const json = JSON.stringify(boundedSample) + const bytes = new TextEncoder().encode(json).slice(0, MAX_SAMPLE_BYTES) + boundedSample = { + truncated: true, + preview: new TextDecoder().decode(bytes) + } + } + return { + metadata: clone(metadata) || {}, + sampleData: boundedSample, + selectedPath: selectedPath || null, + fragments: (fragments || []).map(fragment => ({ + data: clone(fragment.data) || {}, + requirement: fragment.requirement + ? { id: fragment.requirement.id, name: fragment.requirement.name } + : undefined + })) + } +} + +function requiredVariables (template) { + const schema = template?.evaluationMethod?.variableSchema || {} + return Array.isArray(schema.required) ? schema.required : [] +} + +function hasValue (value) { + return value !== undefined && value !== null && value !== '' +} + +export function normaliseAssistantRequest (value) { + return typeof value === 'string' ? value.trim() : '' +} + +export function normaliseVlaAssistantReply (reply = {}, templates = []) { + const byId = new Map((templates || []).map(template => [String(template.id), template])) + const missingTemplates = Array.isArray(reply.missingTemplates) ? clone(reply.missingTemplates) : [] + const requirements = Array.isArray(reply.requirements) ? reply.requirements : [] + const matched = [] + for (const requirement of requirements) { + const template = byId.get(String(requirement?.templateId || '')) + if (!template) { + missingTemplates.push({ + templateId: requirement?.templateId, + reason: 'This template is no longer available.' + }) + continue + } + matched.push({ + templateId: String(template.id), + template, + model: clone(requirement.model) || {}, + reason: String(requirement.reason || 'Selected for this requirement.') + }) + } + return { + message: String(reply.message || ''), + metadata: reply.metadata && typeof reply.metadata === 'object' ? clone(reply.metadata) : {}, + requirements: matched, + missingTemplates + } +} + +function duplicateKey (templateId, model) { + return `${templateId}:${JSON.stringify(model)}` +} + +export function applyVlaAssistantDraft (state, draft, templates = []) { + const current = state || { metadata: {}, fragments: [] } + const available = new Map((templates || []).map(template => [String(template.id), template])) + const requirements = Array.isArray(draft?.requirements) ? draft.requirements : [] + const prepared = [] + const seen = new Set((current.fragments || []).map(fragment => duplicateKey(fragment?.data?.id, fragment?.data?.model || {}))) + + for (const requirement of requirements) { + const template = available.get(String(requirement?.templateId || '')) + if (!template) throw new Error('A selected template is no longer available.') + const model = clone(requirement.model) || {} + const missing = requiredVariables(template).filter(name => !hasValue(model[name])) + if (missing.length) { + throw new Error(`${template.name || 'The selected template'} is missing: ${missing.join(', ')}.`) + } + const key = duplicateKey(template.id, model) + if (seen.has(key)) continue + seen.add(key) + prepared.push({ + data: { id: template.id, model }, + requirement: template, + reason: String(requirement.reason || '') + }) + } + + const metadata = { ...(clone(current.metadata) || {}) } + for (const field of METADATA_FIELDS) { + const value = draft?.metadata?.[field] + if (hasValue(value) && (!Array.isArray(value) || value.length > 0)) metadata[field] = clone(value) + } + return { + metadata, + fragments: [...(clone(current.fragments) || []), ...prepared] + } +} + +export { MAX_SAMPLE_BYTES } diff --git a/vla-manager/src/components/CreateView.vue b/vla-manager/src/components/CreateView.vue index ab335af7..c88f0f50 100644 --- a/vla-manager/src/components/CreateView.vue +++ b/vla-manager/src/components/CreateView.vue @@ -12,6 +12,13 @@ ref="reqModal" @req-added="handleReqAdded" /> +
@@ -28,8 +28,11 @@ You are creating one of the templates the VLA builder assistant asked for. Saving it takes you back to that conversation, where you tick it as created and recheck. + + You are fixing a template used in the VLA builder. Its assistant has a report of the failed test ready to send; saving the fix takes you back to the builder. +
- +
@@ -73,6 +76,10 @@ const activeMode = computed(() => String(route.query.mode || '')) // template it was missing; `brief` describes the rule that needs one. const fromBuilder = computed(() => route.query.from === 'builder') const brief = computed(() => String(route.query.brief || '')) +// Creating a missing template returns to the assistant's conversation; +// fixing a template after a failed test returns to the builder itself. +const reopenAssistant = computed(() => route.query.reopen === 'assistant') +const returnTo = computed(() => fromBuilder.value ? (reopenAssistant.value ? 'the VLA conversation' : 'the VLA builder') : '') const activeTemplate = computed(() => templates.value.find(template => String(template.id) === String(route.query.id))) const pageTitle = computed(() => ({ create: 'Create template', edit: 'Edit template', test: 'Test template' }[activeMode.value] || 'VLA templates')) const pageSubtitle = computed(() => activeMode.value ? 'Work on one reusable quality requirement at a time.' : 'Create reusable quality requirements for new agreements') @@ -111,7 +118,7 @@ async function handleSaved (template) { } // Back to the builder with its assistant drawer open on the conversation // this sub-session was started from. -function returnToBuilder () { router.push({ path: '/create', query: { assistant: 'open' } }) } +function returnToBuilder () { router.push({ path: '/create', query: reopenAssistant.value ? { assistant: 'open' } : {} }) } function closeWorkspace () { if (fromBuilder.value) returnToBuilder() else router.replace({ path: '/templates' }) diff --git a/vla-manager/src/mocks/handlers.js b/vla-manager/src/mocks/handlers.js index 1dda61b5..b369a8e3 100644 --- a/vla-manager/src/mocks/handlers.js +++ b/vla-manager/src/mocks/handlers.js @@ -172,10 +172,13 @@ export const handlers = [ }, required: ['field', 'max'] }, - implementationTemplate: '{success: (.[] | {{{field}}} <= {{{max}}})}' + implementationTemplate: '([.[] | {{{field}}} <= {{{max}}}] | all) as $ok | {success: $ok, details: (if $ok then "all values are at most {{{max}}}" else "a value exceeds {{{max}}}" end)}' } }, - examples: { passing: [{ value: 1 }], failing: [{ value: 1e9 }] } + // Complete input documents, as the real assistant is asked for. + examples: { passing: [[{ value: 1 }, { value: 5 }]], failing: [[{ value: 1e9 }]] }, + exampleModel: { field: '.value', max: 10 }, + selfTest: { status: 'passed', problems: [], attempts: 1 } }) }), diff --git a/vla-manager/test/template-presentation.test.mjs b/vla-manager/test/template-presentation.test.mjs index 25cb9142..ab8d4fe4 100644 --- a/vla-manager/test/template-presentation.test.mjs +++ b/vla-manager/test/template-presentation.test.mjs @@ -1,7 +1,7 @@ import test from 'node:test' import assert from 'node:assert/strict' -import { describeVariables, engineInfo } from '../src/api/templatePresentation.js' +import { describeVariables, engineInfo, templateBugReport } from '../src/api/templatePresentation.js' test('names each quality engine and says what it checks', () => { assert.equal(engineInfo('SCHEMA').label, 'JSON Schema') @@ -52,3 +52,31 @@ test('has nothing to list for an empty or missing schema', () => { assert.deepEqual(describeVariables({ type: 'object' }), []) assert.deepEqual(describeVariables({ type: 'object', properties: {} }), []) }) + +test('reports a failed template test for the assistant to fix', () => { + const report = templateBugReport({ + model: { fields: ['foo'], minRequired: 1 }, + data: { foo: 1 }, + outcome: { tone: 'error', message: 'Cannot check whether number has a string key' } + }) + + assert.equal(report, [ + 'Testing this template gave a wrong result.', + 'Template variables: {"fields":["foo"],"minRequired":1}', + 'Test data: {"foo":1}', + 'Result: It could not be evaluated: Cannot check whether number has a string key', + 'Please find the cause in the implementation template and fix it.' + ].join('\n')) +}) + +test('reports a wrong verdict and truncates large test data', () => { + const report = templateBugReport({ + model: {}, + data: { text: 'x'.repeat(5000) }, + outcome: { tone: 'failed', message: 'value is empty' } + }) + + assert.match(report, /Result: It reported that the data does not satisfy it \(details: value is empty\), which I believe is wrong\./) + assert.match(report, /… \(truncated\)\n/) + assert.ok(report.length < 2000) +}) From 6cfcd3d78df896c313503ed6ad64377e25ac247d Mon Sep 17 00:00:00 2001 From: bzp99 Date: Fri, 25 Sep 2026 01:45:33 +0200 Subject: [PATCH 12/14] fix(vla): try a VLA on sample data without attesting it Co-Authored-By: Claude Opus 5.5 --- docs/spec/vla-manager-api.yaml | 59 ++++++ vla-manager-api/README.md | 1 + vla-manager-api/src/vla_manager_api/models.py | 8 + vla-manager-api/src/vla_manager_api/routes.py | 82 +++++++- vla-manager-api/tests/test_vla_evaluation.py | 177 ++++++++++++++++++ vla-manager/src/api/templates.js | 6 + vla-manager/src/components/ListView.vue | 140 +++++++++----- vla-manager/src/mocks/handlers.js | 34 ++-- 8 files changed, 445 insertions(+), 62 deletions(-) create mode 100644 vla-manager-api/tests/test_vla_evaluation.py diff --git a/docs/spec/vla-manager-api.yaml b/docs/spec/vla-manager-api.yaml index 4d6640bb..8ec8b57c 100644 --- a/docs/spec/vla-manager-api.yaml +++ b/docs/spec/vla-manager-api.yaml @@ -434,6 +434,57 @@ paths: application/json: schema: $ref: '#/components/schemas/Error' + /vla/{id}/evaluate: + parameters: + - $ref: '#/components/parameters/idParam' + post: + tags: [VLA] + summary: Try a VLA out on sample data + description: >- + Runs every requirement in the VLA's `quality` array over `data` with + DVA Processing's `/evaluate`, as `/evaluate/from-template` runs a + single one. Nothing is attested: this is for checking a VLA before + it is used in an exchange. A requirement the data fails, or one the + engine could not run, is a result with `success` false (and `error` + set in the latter case), so this stays a `200`. + operationId: evaluateVLA + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/VLAEvaluationRequest' + responses: + '200': + description: >- + One result per requirement, in the order the requirements appear + in the VLA's `quality` array; a VLA without requirements yields + an empty array. + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/EvaluationResult' + '404': + description: No VLA with the given ID exists. + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '422': + description: Invalid request body. + content: + application/json: + schema: + $ref: '#/components/schemas/ValidationError' + '503': + description: >- + DVA Processing could not be reached (`EVALUATION_UNAVAILABLE`). + content: + application/json: + schema: + $ref: '#/components/schemas/Error' /evaluate/from-template: post: tags: [Templates] @@ -1027,6 +1078,14 @@ components: $ref: ./components.yaml#/schemas/QualityEngine EvaluationResult: $ref: ./components.yaml#/schemas/EvaluationResult + VLAEvaluationRequest: + type: object + description: The sample data to try a VLA out on. + required: [data] + additionalProperties: false + properties: + data: + description: The sample data to run the VLA's requirements over. EvaluationFromTemplateRequest: type: object description: >- diff --git a/vla-manager-api/README.md b/vla-manager-api/README.md index f4026f1b..6d7fc88e 100644 --- a/vla-manager-api/README.md +++ b/vla-manager-api/README.md @@ -28,6 +28,7 @@ deployment: this service is not yet in `test-env/compose.yml`. | `GET /vla/{id}` | DVA API, UI | Retrieve a VLA by its UUID — used during VLA resolution in the synchronous attestation flow | | `POST /vla` | VLA Manager UI | Create a VLA from a partial ODCS payload | | `POST /vla/from-templates` | VLA Manager UI | Create a VLA by rendering and validating templates (as `/template/{id}/validate` does) and merging the results into its `quality` array | +| `POST /vla/{id}/evaluate` | VLA Manager UI | Have DVA Processing run each of a VLA's requirements over sample data, attesting nothing ("Try with Sample") | | `DELETE /vla` | Admin only | Wipe all VLAs | | `GET /template` | VLA Manager UI | List all VLA templates | | `GET /template/{id}` | VLA Manager UI | Retrieve a template by its UUID | diff --git a/vla-manager-api/src/vla_manager_api/models.py b/vla-manager-api/src/vla_manager_api/models.py index 4ba6d9c5..b326ee23 100644 --- a/vla-manager-api/src/vla_manager_api/models.py +++ b/vla-manager-api/src/vla_manager_api/models.py @@ -109,6 +109,14 @@ class EvaluationFromTemplate(BaseModel): data: Any +class VLAEvaluation(BaseModel): + """Body of ``POST /vla/{id}/evaluate``: the sample data to try it on.""" + + model_config = ConfigDict(extra="forbid") + + data: Any + + class TemplateInstantiation(BaseModel): """ One entry in ``VLANewFromTemplates.qualityTemplates`` — a template id diff --git a/vla-manager-api/src/vla_manager_api/routes.py b/vla-manager-api/src/vla_manager_api/routes.py index a790c572..26464288 100644 --- a/vla-manager-api/src/vla_manager_api/routes.py +++ b/vla-manager-api/src/vla_manager_api/routes.py @@ -8,18 +8,39 @@ from __future__ import annotations +from datetime import datetime, timezone from typing import Any from uuid import UUID from fastapi import APIRouter, Depends, status -from .dependencies import get_repo, get_requirement_validator, get_template_repo +from .dependencies import ( + get_repo, + get_requirement_evaluator, + get_requirement_validator, + get_template_repo, +) from .errors import http_error -from .models import IDDTO, ValidationFailureReason, VLANew, VLANewFromTemplates +from .log import get_logger +from .models import ( + IDDTO, + QualityEngine, + ValidationFailureReason, + VLAEvaluation, + VLANew, + VLANewFromTemplates, +) from .template_repo import TemplateRepo -from .validation import RequirementValidator, check_requirement +from .validation import ( + ProcessingError, + RequirementEvaluator, + RequirementValidator, + check_requirement, +) from .vla_repo import VLARepo +logger = get_logger(__name__) + router = APIRouter() @@ -49,6 +70,61 @@ async def get_vla(id: UUID, repo: VLARepo = Depends(get_repo)) -> dict[str, Any] return vla +def _as_result(engine: QualityEngine, code: int, body: Any) -> dict[str, Any]: + """ + Read processing's answer for one requirement as an ``EvaluationResult``. + + A run the engine could not finish is already one (a ``500`` with + ``error``); anything else processing refuses with is a problem detail, + which becomes a failed result, as ``/evaluate-batch`` reports it. + """ + if isinstance(body, dict) and isinstance(body.get("success"), bool): + return body + reason = (body.get("title") or body.get("detail")) if isinstance(body, dict) else None + return { + "engine": engine.value, + "timestamp": datetime.now(timezone.utc).isoformat(), + "success": False, + "error": str(reason or f"DVA Processing answered with status {code}"), + } + + +@router.post("/vla/{id}/evaluate") +async def evaluate_vla( + id: UUID, + request: VLAEvaluation, + repo: VLARepo = Depends(get_repo), + evaluator: RequirementEvaluator = Depends(get_requirement_evaluator), +) -> list[dict[str, Any]]: + """ + Try a stored VLA out on sample data, without attesting anything. + + Each requirement in the VLA's ``quality`` is run through DVA + Processing's ``/evaluate``, as "Test fragment" runs one in the builder. + """ + vla = await repo.by_id(id) + if vla is None: + raise http_error(status.HTTP_404_NOT_FOUND, "No VLA with the given ID exists") + + results: list[dict[str, Any]] = [] + for requirement in vla.get("quality") or []: + # Stored requirements came through `DataQuality`, so this holds. + engine = QualityEngine(requirement["engine"]) + try: + code, body = await evaluator.evaluate( + engine, requirement["implementation"], request.data + ) + except ProcessingError as exc: + logger.warning("Could not evaluate a VLA requirement", error=str(exc)) + raise http_error( + status.HTTP_503_SERVICE_UNAVAILABLE, + f"The evaluation service is unavailable: {exc}", + type="EVALUATION_UNAVAILABLE", + ) from exc + results.append(_as_result(engine, code, body)) + return results + + @router.post("/vla", status_code=status.HTTP_201_CREATED, response_model=IDDTO) async def create_vla(vla_req: VLANew, repo: VLARepo = Depends(get_repo)) -> IDDTO: vla = _wrap_vla(vla_req) diff --git a/vla-manager-api/tests/test_vla_evaluation.py b/vla-manager-api/tests/test_vla_evaluation.py new file mode 100644 index 00000000..791f184f --- /dev/null +++ b/vla-manager-api/tests/test_vla_evaluation.py @@ -0,0 +1,177 @@ +""" +Tests for trying a stored VLA out on sample data. + +``POST /vla/{id}/evaluate`` runs each of the VLA's requirements through DVA +Processing's ``/evaluate`` and attests nothing; processing is faked. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from typing import Any + +import pytest +from fastapi.testclient import TestClient + +from vla_manager_api.dependencies import get_repo, get_requirement_evaluator +from vla_manager_api.main import create_app +from vla_manager_api.models import QualityEngine +from vla_manager_api.validation import ProcessingError +from vla_manager_api.vla_repo import FakeVLARepo + +QUALITY = [ + {"engine": "JQ", "implementation": "{success: (.id != null), details: \"id\"}"}, + {"engine": "SCHEMA", "implementation": '{"type": "object"}'}, +] + + +def _result(engine: str, success: bool, **extra: Any) -> dict[str, Any]: + return { + "engine": engine, + "timestamp": "2026-09-25T12:00:00Z", + "success": success, + **extra, + } + + +class FakeEvaluator: + """Answers each implementation as told, as processing's client would.""" + + def __init__(self) -> None: + self.calls: list[dict[str, Any]] = [] + self.answers: dict[str, tuple[int, Any]] = {} + self.unavailable = False + + async def evaluate( + self, engine: QualityEngine, implementation: str, data: Any + ) -> tuple[int, Any]: + self.calls.append( + {"engine": engine, "implementation": implementation, "data": data} + ) + if self.unavailable: + raise ProcessingError("processing service is unavailable") + return self.answers.get( + implementation, (200, _result(engine.value, True, details="ok")) + ) + + +@pytest.fixture +def evaluator() -> FakeEvaluator: + return FakeEvaluator() + + +@pytest.fixture +def client(evaluator: FakeEvaluator) -> Iterator[TestClient]: + repo = FakeVLARepo() + app = create_app() + app.dependency_overrides[get_repo] = lambda: repo + app.dependency_overrides[get_requirement_evaluator] = lambda: evaluator + with TestClient(app) as test_client: + yield test_client + + +def _vla(client: TestClient, quality: list[dict[str, Any]] = QUALITY) -> str: + response = client.post("/vla", json={"name": "Sample VLA", "quality": quality}) + assert response.status_code == 201, response.text + return response.json()["id"] + + +def test_runs_every_requirement_over_the_data_in_order( + client: TestClient, evaluator: FakeEvaluator +) -> None: + vla_id = _vla(client) + + response = client.post(f"/vla/{vla_id}/evaluate", json={"data": {"id": 1}}) + + assert response.status_code == 200, response.text + assert [r["success"] for r in response.json()] == [True, True] + assert evaluator.calls == [ + { + "engine": QualityEngine.JQ, + "implementation": QUALITY[0]["implementation"], + "data": {"id": 1}, + }, + { + "engine": QualityEngine.SCHEMA, + "implementation": QUALITY[1]["implementation"], + "data": {"id": 1}, + }, + ] + + +def test_reports_a_failed_and_a_broken_requirement_as_results( + client: TestClient, evaluator: FakeEvaluator +) -> None: + vla_id = _vla(client) + failed = _result("JQ", False, details="no id") + broken = _result("SCHEMA", False, error="engine crashed") + evaluator.answers = { + QUALITY[0]["implementation"]: (200, failed), + QUALITY[1]["implementation"]: (500, broken), + } + + response = client.post(f"/vla/{vla_id}/evaluate", json={"data": {}}) + + assert response.status_code == 200 + assert response.json() == [failed, broken] + + +def test_a_requirement_processing_refuses_becomes_a_failed_result( + client: TestClient, evaluator: FakeEvaluator +) -> None: + vla_id = _vla(client, QUALITY[:1]) + evaluator.answers = { + QUALITY[0]["implementation"]: ( + 400, + {"type": "BAD_REQUEST", "title": "Unsupported engine"}, + ) + } + + response = client.post(f"/vla/{vla_id}/evaluate", json={"data": {}}) + + assert response.status_code == 200 + [result] = response.json() + assert result["success"] is False + assert result["engine"] == "JQ" + assert result["error"] == "Unsupported engine" + assert result["timestamp"] + + +def test_a_vla_without_requirements_yields_no_results( + client: TestClient, evaluator: FakeEvaluator +) -> None: + vla_id = _vla(client, []) + + response = client.post(f"/vla/{vla_id}/evaluate", json={"data": {}}) + + assert response.status_code == 200 + assert response.json() == [] + assert evaluator.calls == [] + + +def test_reports_an_unknown_vla(client: TestClient) -> None: + response = client.post( + "/vla/00000000-0000-0000-0000-000000000099/evaluate", json={"data": {}} + ) + + assert response.status_code == 404 + + +def test_reports_processing_being_unreachable( + client: TestClient, evaluator: FakeEvaluator +) -> None: + vla_id = _vla(client) + evaluator.unavailable = True + + response = client.post(f"/vla/{vla_id}/evaluate", json={"data": {}}) + + assert response.status_code == 503 + assert response.json()["type"] == "EVALUATION_UNAVAILABLE" + + +def test_requires_the_data(client: TestClient) -> None: + vla_id = _vla(client) + + response = client.post(f"/vla/{vla_id}/evaluate", json={}) + + assert response.status_code == 422 diff --git a/vla-manager/src/api/templates.js b/vla-manager/src/api/templates.js index c59820bf..37453ffd 100644 --- a/vla-manager/src/api/templates.js +++ b/vla-manager/src/api/templates.js @@ -51,6 +51,12 @@ export function evaluateTemplate (templateID, templateModel, data) { return request({ method: 'post', url: '/api/evaluate/from-template', data: { templateID, templateModel, data } }) } +// Runs every requirement of a stored VLA over sample data – a dry run that +// attests nothing – and answers one `EvaluationResult` per requirement. +export function evaluateVla (id, data) { + return request({ method: 'post', url: `/api/vla/${encodeURIComponent(id)}/evaluate`, data: { data } }) +} + export { normaliseError } // Validation keeps the raw axios error rather than going through diff --git a/vla-manager/src/components/ListView.vue b/vla-manager/src/components/ListView.vue index 442acd5f..929d8872 100644 --- a/vla-manager/src/components/ListView.vue +++ b/vla-manager/src/components/ListView.vue @@ -19,10 +19,34 @@ - - Attestation request accepted for {{ lastAttestation.name }}. - Evaluation is processed asynchronously. Request ID: {{ lastAttestation.id }} - + + +
+ {{ trial.error }} + +
+ +
@@ -109,13 +133,15 @@