Templates: redesign to the v1 API contract + visual layout support

alembic/versions/004_form_templates.py

@@ -0,0 +1,78 @@
+"""form templates registry and pdf upload drafts
+
+Revision ID: 004
+Revises: 003
+Create Date: 2026-08-10
+
+Two tables, both new in the contract Layer 6 template work.
+
+`form_templates` is the registry, keyed by `form_type` and distinct from the
+legacy `template` table (int PK + uploaded PDF). Its `fields` JSON column holds
+the TemplateField list, each with a nested `layout`. `form_type` stays a plain
+VARCHAR rather than a Postgres enum, because jurisdictions register their own
+form types and those are not part of the built-in FormType enum.
+
+`template_uploads` holds the drafts behind the PDF authoring flow: the stored
+blank PDF, its page geometry, and the fields commonforms detected. Rows here
+are working state, not templates. Registering a template copies the edited
+fields across and keeps only the `pdf_template_ref` pointing back.
+
+Both JSON columns use sa.JSON for consistency with migrations 001-003 and for
+the SQLite test harness.
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+import sqlmodel
+
+
+# revision identifiers, used by Alembic.
+revision: str = '004'
+down_revision: Union[str, None] = '003'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('form_templates',
+    sa.Column('template_id', sa.Uuid(), nullable=False),
+    sa.Column('form_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('display_name', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('jurisdiction', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+    sa.Column('agency_type', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+    sa.Column('fields', sa.JSON(), nullable=False),
+    sa.Column('source_standard', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+    sa.Column('pdf_template_ref', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+    sa.Column('version', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('status', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.Column('updated_at', sa.DateTime(), nullable=False),
+    sa.PrimaryKeyConstraint('template_id')
+    )
+    op.create_index(op.f('ix_form_templates_form_type'), 'form_templates', ['form_type'], unique=True)
+    op.create_table('template_uploads',
+    sa.Column('upload_id', sa.Uuid(), nullable=False),
+    sa.Column('status', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('pdf_path', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('pdf_template_ref', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('original_filename', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+    sa.Column('page_count', sa.Integer(), nullable=False),
+    sa.Column('pages', sa.JSON(), nullable=False),
+    sa.Column('detected_fields', sa.JSON(), nullable=True),
+    sa.Column('detection_error', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+    sa.Column('job_id', sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.Column('updated_at', sa.DateTime(), nullable=False),
+    sa.PrimaryKeyConstraint('upload_id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('template_uploads')
+    op.drop_index(op.f('ix_form_templates_form_type'), table_name='form_templates')
+    op.drop_table('form_templates')
+    # ### end Alembic commands ###

app/api/router.py

@@ -1,10 +1,10 @@
 from fastapi import APIRouter
 
-from app.api.routes import extraction, forms, input, jobs, system, templates, weather, zipcode
+from app.api.routes import extraction, forms, form_templates, input, jobs, system, weather, zipcode
 from app.core.config import API_PREFIX
 
 api_router = APIRouter()
-api_router.include_router(templates.router, prefix=API_PREFIX)
+api_router.include_router(form_templates.router, prefix=API_PREFIX)
 api_router.include_router(forms.router, prefix=API_PREFIX)
 api_router.include_router(system.router, prefix=API_PREFIX)
 api_router.include_router(jobs.router, prefix=API_PREFIX)

app/api/routes/__init__.py

@@ -1,3 +1,3 @@
-from . import templates, forms
+from . import form_templates, forms
 
-__all__ = ["templates", "forms"]
+__all__ = ["form_templates", "forms"]

app/api/routes/form_templates.py

@@ -0,0 +1,127 @@
+"""Contract Layer 6 template registry endpoints (contracts/path/templates.yaml).
+
+Serves the form-template registry at /api/v1/templates, backed by the
+UUID-keyed `FormTemplate` model, plus the PDF-authoring flow that feeds it:
+upload a blank PDF, poll the detection draft, register the edited fields.
+Handlers are thin, business logic lives in app/services/form_templates.py. The
+legacy prototype template routes (upload / create / make-fillable / preview /
+delete) were removed in the contract migration; the legacy int-PK `Template`
+model survives only as the lookup target of the fill pipeline (forms.py /
+jobs.py / tasks/fill.py).
+"""
+
+from pathlib import Path
+from uuid import UUID
+
+from fastapi import APIRouter, Depends, File, Form, Query, UploadFile
+from fastapi.responses import FileResponse
+from sqlmodel import Session
+
+from app.api.deps import get_db
+from app.api.schemas.templates import (
+    CreateTemplateRequest,
+    TemplateDetail,
+    TemplateDraft,
+    TemplateDraftAccepted,
+    TemplateFieldsResponse,
+    TemplateSummary,
+)
+from app.core.config import MAX_TEMPLATE_PDF_BYTES
+from app.core.errors.base import AppError
+from app.services import form_templates as service
+
+router = APIRouter(prefix="/templates", tags=["templates"])
+
+_PDF_MAGIC = b"%PDF-"
+
+
+def _reject_if_too_large(size: int | None) -> None:
+    """Guard the 50MB cap. Checked once on the declared size before the body is
+    read into memory, and again on what actually arrived."""
+    if size is None or size <= MAX_TEMPLATE_PDF_BYTES:
+        return
+    raise AppError(
+        "PDF exceeds maximum size of 50MB",
+        status_code=413,
+        error_code="FILE_TOO_LARGE",
+        detail={
+            "max_size_bytes": MAX_TEMPLATE_PDF_BYTES,
+            "received_size_bytes": size,
+        },
+    )
+
+
+@router.get("", response_model=list[TemplateSummary])
+def list_templates(db: Session = Depends(get_db)):
+    return service.list_templates(db)
+
+
+@router.post("", response_model=TemplateDetail, status_code=201)
+def create_template(body: CreateTemplateRequest, db: Session = Depends(get_db)):
+    return service.create_template(db, body)
+
+
+# The two /pdf routes are declared before /{template_id} on purpose. FastAPI
+# matches in declaration order, so the literal path has to come first or "pdf"
+# gets read as a template id.
+@router.post("/pdf", response_model=TemplateDraftAccepted, status_code=202)
+def upload_template_pdf(
+    pdf_file: UploadFile = File(...),
+    detect_fields: bool = Form(default=True),
+    db: Session = Depends(get_db),
+):
+    filename = pdf_file.filename or ""
+
+    _reject_if_too_large(pdf_file.size)
+
+    content = pdf_file.file.read()
+    if not content:
+        raise AppError(
+            "No PDF file was uploaded",
+            status_code=400,
+            error_code="MISSING_FILE",
+        )
+    _reject_if_too_large(len(content))
+    # Trust the bytes, not the extension or the declared content type.
+    if not content.startswith(_PDF_MAGIC):
+        raise AppError(
+            "Uploaded file is not a PDF",
+            status_code=415,
+            error_code="UNSUPPORTED_FORMAT",
+            detail={"accepted_formats": ["pdf"]},
+        )
+
+    upload, job = service.store_upload(db, content, filename or None, detect_fields)
+    return service.draft_response(upload, job)
+
+
+@router.get("/pdf/{upload_id}", response_model=TemplateDraft)
+def get_template_draft(upload_id: UUID, db: Session = Depends(get_db)):
+    return service.get_draft(db, upload_id)
+
+
+@router.get("/{template_id}", response_model=TemplateDetail)
+def get_template(template_id: UUID, db: Session = Depends(get_db)):
+    return service.get_template(db, template_id)
+
+
+@router.put("/{template_id}", response_model=TemplateDetail)
+def replace_template(
+    template_id: UUID, body: CreateTemplateRequest, db: Session = Depends(get_db)
+):
+    return service.replace_template(db, template_id, body)
+
+
+@router.get("/{template_id}/fields", response_model=TemplateFieldsResponse)
+def get_template_fields(
+    template_id: UUID,
+    required_only: bool = Query(False, description="Return only required fields"),
+    db: Session = Depends(get_db),
+):
+    return service.get_template_fields(db, template_id, required_only)
+
+
+@router.get("/{template_id}/pdf", response_class=FileResponse)
+def download_template_pdf(template_id: UUID, db: Session = Depends(get_db)):
+    path: Path = service.resolve_template_pdf(db, template_id)
+    return FileResponse(path, media_type="application/pdf", filename=path.name)

app/api/routes/system.py

@@ -7,7 +7,7 @@
 import time
 
 import requests
-from fastapi import APIRouter
+from fastapi import APIRouter, Query
 from fastapi.responses import JSONResponse
 from sqlalchemy import text
 
@@ -16,7 +16,10 @@
     HealthComponents,
     HealthStatus,
     ModelInfo,
+    SchemaFieldEntry,
+    SchemaFieldSearchResponse,
 )
+from app.services import field_catalog
 from app.core.config import APP_VERSION, DATA_DIR, OLLAMA_HOST, WHISPER_HOST
 from app.db.database import engine
 
@@ -179,3 +182,35 @@ def get_schema_versions():
             "message": "Schema version history not yet available — see issue #555",
         },
     )
+
+
+@router.get(
+    "/schema/fields",
+    response_model=SchemaFieldSearchResponse,
+    summary="Search or list the incident-contract field catalog",
+)
+def search_schema_fields(
+    q: str | None = Query(None, description="Search text, matched against names, aliases and descriptions"),
+    section: str | None = Query(None, description="Restrict to one top-level contract section"),
+    limit: int = Query(20, ge=1, le=100, description="Caps search results, ignored when q is omitted"),
+):
+    hits = field_catalog.search(q, section, limit)
+    return SchemaFieldSearchResponse(
+        query=q,
+        total=len(hits),
+        schema_version=field_catalog.schema_version(),
+        fields=[
+            SchemaFieldEntry(
+                path=entry.path,
+                label=entry.label,
+                field_type=entry.field_type,
+                section=entry.section,
+                description=entry.description,
+                enum_values=list(entry.enum_values) if entry.enum_values else None,
+                pii=entry.pii,
+                aliases=list(entry.aliases),
+                score=score,
+            )
+            for entry, score in hits
+        ],
+    )