diff --git a/alembic/versions/004_form_templates.py b/alembic/versions/004_form_templates.py new file mode 100644 index 00000000..ef23be52 --- /dev/null +++ b/alembic/versions/004_form_templates.py @@ -0,0 +1,78 @@ +"""form templates registry and pdf upload drafts + +Revision ID: 004 +Revises: 003 +Create Date: 2026-08-10 + +Two tables, both new in the contract Layer 6 template work. + +`form_templates` is the registry, keyed by `form_type` and distinct from the +legacy `template` table (int PK + uploaded PDF). Its `fields` JSON column holds +the TemplateField list, each with a nested `layout`. `form_type` stays a plain +VARCHAR rather than a Postgres enum, because jurisdictions register their own +form types and those are not part of the built-in FormType enum. + +`template_uploads` holds the drafts behind the PDF authoring flow: the stored +blank PDF, its page geometry, and the fields commonforms detected. Rows here +are working state, not templates. Registering a template copies the edited +fields across and keeps only the `pdf_template_ref` pointing back. + +Both JSON columns use sa.JSON for consistency with migrations 001-003 and for +the SQLite test harness. +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel + + +# revision identifiers, used by Alembic. +revision: str = '004' +down_revision: Union[str, None] = '003' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('form_templates', + sa.Column('template_id', sa.Uuid(), nullable=False), + sa.Column('form_type', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('display_name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('jurisdiction', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('agency_type', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('fields', sa.JSON(), nullable=False), + sa.Column('source_standard', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('pdf_template_ref', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('version', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('status', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint('template_id') + ) + op.create_index(op.f('ix_form_templates_form_type'), 'form_templates', ['form_type'], unique=True) + op.create_table('template_uploads', + sa.Column('upload_id', sa.Uuid(), nullable=False), + sa.Column('status', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('pdf_path', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('pdf_template_ref', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('original_filename', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('page_count', sa.Integer(), nullable=False), + sa.Column('pages', sa.JSON(), nullable=False), + sa.Column('detected_fields', sa.JSON(), nullable=True), + sa.Column('detection_error', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('job_id', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint('upload_id') + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('template_uploads') + op.drop_index(op.f('ix_form_templates_form_type'), table_name='form_templates') + op.drop_table('form_templates') + # ### end Alembic commands ### diff --git a/app/api/router.py b/app/api/router.py index f182c942..54415c18 100644 --- a/app/api/router.py +++ b/app/api/router.py @@ -1,10 +1,10 @@ from fastapi import APIRouter -from app.api.routes import extraction, forms, input, jobs, system, templates, weather, zipcode +from app.api.routes import extraction, forms, form_templates, input, jobs, system, weather, zipcode from app.core.config import API_PREFIX api_router = APIRouter() -api_router.include_router(templates.router, prefix=API_PREFIX) +api_router.include_router(form_templates.router, prefix=API_PREFIX) api_router.include_router(forms.router, prefix=API_PREFIX) api_router.include_router(system.router, prefix=API_PREFIX) api_router.include_router(jobs.router, prefix=API_PREFIX) diff --git a/app/api/routes/__init__.py b/app/api/routes/__init__.py index 2264aee0..17c68a8e 100644 --- a/app/api/routes/__init__.py +++ b/app/api/routes/__init__.py @@ -1,3 +1,3 @@ -from . import templates, forms +from . import form_templates, forms -__all__ = ["templates", "forms"] +__all__ = ["form_templates", "forms"] diff --git a/app/api/routes/form_templates.py b/app/api/routes/form_templates.py new file mode 100644 index 00000000..eec93076 --- /dev/null +++ b/app/api/routes/form_templates.py @@ -0,0 +1,127 @@ +"""Contract Layer 6 template registry endpoints (contracts/path/templates.yaml). + +Serves the form-template registry at /api/v1/templates, backed by the +UUID-keyed `FormTemplate` model, plus the PDF-authoring flow that feeds it: +upload a blank PDF, poll the detection draft, register the edited fields. +Handlers are thin, business logic lives in app/services/form_templates.py. The +legacy prototype template routes (upload / create / make-fillable / preview / +delete) were removed in the contract migration; the legacy int-PK `Template` +model survives only as the lookup target of the fill pipeline (forms.py / +jobs.py / tasks/fill.py). +""" + +from pathlib import Path +from uuid import UUID + +from fastapi import APIRouter, Depends, File, Form, Query, UploadFile +from fastapi.responses import FileResponse +from sqlmodel import Session + +from app.api.deps import get_db +from app.api.schemas.templates import ( + CreateTemplateRequest, + TemplateDetail, + TemplateDraft, + TemplateDraftAccepted, + TemplateFieldsResponse, + TemplateSummary, +) +from app.core.config import MAX_TEMPLATE_PDF_BYTES +from app.core.errors.base import AppError +from app.services import form_templates as service + +router = APIRouter(prefix="/templates", tags=["templates"]) + +_PDF_MAGIC = b"%PDF-" + + +def _reject_if_too_large(size: int | None) -> None: + """Guard the 50MB cap. Checked once on the declared size before the body is + read into memory, and again on what actually arrived.""" + if size is None or size <= MAX_TEMPLATE_PDF_BYTES: + return + raise AppError( + "PDF exceeds maximum size of 50MB", + status_code=413, + error_code="FILE_TOO_LARGE", + detail={ + "max_size_bytes": MAX_TEMPLATE_PDF_BYTES, + "received_size_bytes": size, + }, + ) + + +@router.get("", response_model=list[TemplateSummary]) +def list_templates(db: Session = Depends(get_db)): + return service.list_templates(db) + + +@router.post("", response_model=TemplateDetail, status_code=201) +def create_template(body: CreateTemplateRequest, db: Session = Depends(get_db)): + return service.create_template(db, body) + + +# The two /pdf routes are declared before /{template_id} on purpose. FastAPI +# matches in declaration order, so the literal path has to come first or "pdf" +# gets read as a template id. +@router.post("/pdf", response_model=TemplateDraftAccepted, status_code=202) +def upload_template_pdf( + pdf_file: UploadFile = File(...), + detect_fields: bool = Form(default=True), + db: Session = Depends(get_db), +): + filename = pdf_file.filename or "" + + _reject_if_too_large(pdf_file.size) + + content = pdf_file.file.read() + if not content: + raise AppError( + "No PDF file was uploaded", + status_code=400, + error_code="MISSING_FILE", + ) + _reject_if_too_large(len(content)) + # Trust the bytes, not the extension or the declared content type. + if not content.startswith(_PDF_MAGIC): + raise AppError( + "Uploaded file is not a PDF", + status_code=415, + error_code="UNSUPPORTED_FORMAT", + detail={"accepted_formats": ["pdf"]}, + ) + + upload, job = service.store_upload(db, content, filename or None, detect_fields) + return service.draft_response(upload, job) + + +@router.get("/pdf/{upload_id}", response_model=TemplateDraft) +def get_template_draft(upload_id: UUID, db: Session = Depends(get_db)): + return service.get_draft(db, upload_id) + + +@router.get("/{template_id}", response_model=TemplateDetail) +def get_template(template_id: UUID, db: Session = Depends(get_db)): + return service.get_template(db, template_id) + + +@router.put("/{template_id}", response_model=TemplateDetail) +def replace_template( + template_id: UUID, body: CreateTemplateRequest, db: Session = Depends(get_db) +): + return service.replace_template(db, template_id, body) + + +@router.get("/{template_id}/fields", response_model=TemplateFieldsResponse) +def get_template_fields( + template_id: UUID, + required_only: bool = Query(False, description="Return only required fields"), + db: Session = Depends(get_db), +): + return service.get_template_fields(db, template_id, required_only) + + +@router.get("/{template_id}/pdf", response_class=FileResponse) +def download_template_pdf(template_id: UUID, db: Session = Depends(get_db)): + path: Path = service.resolve_template_pdf(db, template_id) + return FileResponse(path, media_type="application/pdf", filename=path.name) diff --git a/app/api/routes/system.py b/app/api/routes/system.py index eadd1d9f..4ce9275d 100644 --- a/app/api/routes/system.py +++ b/app/api/routes/system.py @@ -7,7 +7,7 @@ import time import requests -from fastapi import APIRouter +from fastapi import APIRouter, Query from fastapi.responses import JSONResponse from sqlalchemy import text @@ -16,7 +16,10 @@ HealthComponents, HealthStatus, ModelInfo, + SchemaFieldEntry, + SchemaFieldSearchResponse, ) +from app.services import field_catalog from app.core.config import APP_VERSION, DATA_DIR, OLLAMA_HOST, WHISPER_HOST from app.db.database import engine @@ -179,3 +182,35 @@ def get_schema_versions(): "message": "Schema version history not yet available — see issue #555", }, ) + + +@router.get( + "/schema/fields", + response_model=SchemaFieldSearchResponse, + summary="Search or list the incident-contract field catalog", +) +def search_schema_fields( + q: str | None = Query(None, description="Search text, matched against names, aliases and descriptions"), + section: str | None = Query(None, description="Restrict to one top-level contract section"), + limit: int = Query(20, ge=1, le=100, description="Caps search results, ignored when q is omitted"), +): + hits = field_catalog.search(q, section, limit) + return SchemaFieldSearchResponse( + query=q, + total=len(hits), + schema_version=field_catalog.schema_version(), + fields=[ + SchemaFieldEntry( + path=entry.path, + label=entry.label, + field_type=entry.field_type, + section=entry.section, + description=entry.description, + enum_values=list(entry.enum_values) if entry.enum_values else None, + pii=entry.pii, + aliases=list(entry.aliases), + score=score, + ) + for entry, score in hits + ], + ) diff --git a/app/api/routes/templates.py b/app/api/routes/templates.py deleted file mode 100644 index ee7189aa..00000000 --- a/app/api/routes/templates.py +++ /dev/null @@ -1,240 +0,0 @@ -import re -from datetime import datetime, timezone -from pathlib import Path - -from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, UploadFile -from fastapi.responses import FileResponse -from sqlmodel import Session - -from app.api.deps import get_db, verify_api_key -from app.api.schemas.templates import ( - TemplateCreate, - TemplateResponse, - TemplateUploadResponse, - MakeFillableRequest, - MakeFillableResponse, -) -from app.core.config import BASE_DIR, DEFAULT_TEMPLATE_DIR -from app.db.repositories import create_template, list_templates, get_template, delete_template -from app.models import Template, FormSubmission, Job -from app.services.controller import Controller -from sqlmodel import select - -router = APIRouter(prefix="/templates", tags=["templates"]) -PROJECT_ROOT = BASE_DIR - - -def _resolve_target_directory(directory: str) -> Path: - dir_value = (directory or DEFAULT_TEMPLATE_DIR).strip() - if not dir_value: - raise HTTPException(status_code=400, detail="Directory is required.") - - candidate = Path(dir_value) - if not candidate.is_absolute(): - candidate = (PROJECT_ROOT / candidate).resolve() - else: - candidate = candidate.resolve() - - if candidate != PROJECT_ROOT and PROJECT_ROOT not in candidate.parents: - raise HTTPException(status_code=400, detail="Directory must be inside the project.") - - return candidate - - -def _resolve_project_file(file_path: str) -> Path: - raw_path = (file_path or "").strip() - if not raw_path: - raise HTTPException(status_code=400, detail="Path is required.") - - candidate = Path(raw_path) - if not candidate.is_absolute(): - candidate = (PROJECT_ROOT / candidate).resolve() - else: - candidate = candidate.resolve() - - if candidate != PROJECT_ROOT and PROJECT_ROOT not in candidate.parents: - raise HTTPException(status_code=400, detail="Path must be inside the project.") - - return candidate - - -@router.post("/upload", response_model=TemplateUploadResponse) -async def upload_template_pdf( - file: UploadFile = File(...), - directory: str = Form(DEFAULT_TEMPLATE_DIR), -): - filename = Path(file.filename or "").name - if not filename: - raise HTTPException(status_code=400, detail="A PDF filename is required.") - - if not filename.lower().endswith(".pdf"): - raise HTTPException(status_code=400, detail="Only PDF files are supported.") - - target_dir = _resolve_target_directory(directory) - target_dir.mkdir(parents=True, exist_ok=True) - - target_path = target_dir / filename - if target_path.exists(): - timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") - target_path = target_dir / f"{target_path.stem}_{timestamp}{target_path.suffix}" - - content = await file.read() - with target_path.open("wb") as output_file: - output_file.write(content) - - relative_path = target_path.relative_to(PROJECT_ROOT).as_posix() - extracted = _extract_pdf_fields(relative_path) - return TemplateUploadResponse( - filename=target_path.name, - pdf_path=relative_path, - field_count=None if extracted is None else len(extracted), - fields=extracted or [], - ) - - -# PDF field-type codes -> the type values the frontend field builder uses. -_FIELD_TYPE_BY_FT = {"/Tx": "string", "/Btn": "checkbox", "/Ch": "list", "/Sig": "signature"} - - -def _pdf_text(value) -> str: - """Decode a pdfrw string (field name / tooltip) to plain text.""" - if value is None: - return "" - if hasattr(value, "to_unicode"): - return value.to_unicode().strip() - return str(value).strip() - - -def _humanize(name: str) -> str: - """Turn a raw field name into a readable description (JobTitle -> Job Title).""" - text = re.sub(r"_+", " ", name) - text = re.sub(r"(?<=[a-z])(?=[A-Z])", " ", text) - return re.sub(r"\s+", " ", text).strip() - - -def _extract_pdf_fields(pdf_path: str) -> list[dict] | None: - """Fillable widgets in the same order Filler.fill_form writes them - (top-to-bottom, left-to-right per page), so seeded rows line up with the - fill order. Returns None if the PDF can't be read.""" - try: - from pdfrw import PdfReader - candidate = Path(pdf_path) - if not candidate.is_absolute(): - candidate = (PROJECT_ROOT / candidate).resolve() - pdf = PdfReader(str(candidate)) - fields: list[dict] = [] - for page in pdf.pages: - widgets = [a for a in (page.Annots or []) if a.Subtype == "/Widget" and a.T] - widgets.sort(key=lambda a: (-float(a.Rect[1]), float(a.Rect[0]))) - for annot in widgets: - name = _pdf_text(annot.T) - fields.append({ - "name": name, - "description": _pdf_text(annot.TU) or _humanize(name), - "type": _FIELD_TYPE_BY_FT.get(str(annot.FT), "string"), - }) - return fields - except Exception: - return None - - -def _count_pdf_widgets(pdf_path: str) -> int | None: - """Number of fillable widgets in a PDF, or None if unreadable.""" - fields = _extract_pdf_fields(pdf_path) - return None if fields is None else len(fields) - - -@router.get("", response_model=list[TemplateResponse]) -def get_templates(db: Session = Depends(get_db)): - return list_templates(db) - - -@router.get("/preview") -def preview_template_pdf(path: str = Query(..., description="Project-relative PDF path")): - resolved_path = _resolve_project_file(path) - - if not resolved_path.exists() or not resolved_path.is_file(): - raise HTTPException(status_code=404, detail="PDF file not found.") - - if resolved_path.suffix.lower() != ".pdf": - raise HTTPException(status_code=400, detail="Only PDF files can be previewed.") - - return FileResponse( - resolved_path, - media_type="application/pdf", - filename=resolved_path.name, - content_disposition_type="inline", - ) - - -@router.post("/create", response_model=TemplateResponse) -def create(template: TemplateCreate, db: Session = Depends(get_db)): - tpl = Template(**template.model_dump()) - created = create_template(db, tpl) - return TemplateResponse( - id=created.id, - name=created.name, - pdf_path=created.pdf_path, - fields=created.fields, - field_count=_count_pdf_widgets(created.pdf_path), - ) - - -@router.post("/make-fillable", response_model=MakeFillableResponse) -def make_fillable(req: MakeFillableRequest): - # Validate the path stays inside the project root. - resolved = _resolve_project_file(req.pdf_path) - if not resolved.exists() or not resolved.is_file(): - raise HTTPException(status_code=404, detail="PDF file not found.") - - controller = Controller() - new_absolute = controller.prepare_fillable(str(resolved)) - new_path = Path(new_absolute) - if not new_path.is_absolute(): - new_path = (PROJECT_ROOT / new_path).resolve() - relative_path = new_path.relative_to(PROJECT_ROOT).as_posix() - - return MakeFillableResponse( - pdf_path=relative_path, - field_count=_count_pdf_widgets(relative_path), - ) - - -@router.delete("/{template_id}", dependencies=[Depends(verify_api_key)]) -def delete_template_endpoint(template_id: int, db: Session = Depends(get_db)): - template = get_template(db, template_id) - if not template: - raise HTTPException(status_code=404, detail="Template not found") - - # 1. Clean up associated submissions and their generated PDFs - sub_stmt = select(FormSubmission).where(FormSubmission.template_id == template_id) - submissions = list(db.exec(sub_stmt)) - for sub in submissions: - if sub.output_pdf_path: - try: - resolved_out = _resolve_project_file(sub.output_pdf_path) - if resolved_out.exists() and resolved_out.is_file(): - resolved_out.unlink() - except Exception: - pass - db.delete(sub) - - # 2. Clean up associated jobs - job_stmt = select(Job).where(Job.template_id == template_id) - jobs = list(db.exec(job_stmt)) - for job in jobs: - db.delete(job) - - # 3. Delete template PDF file - if template.pdf_path: - try: - resolved_pdf = _resolve_project_file(template.pdf_path) - if resolved_pdf.exists() and resolved_pdf.is_file(): - resolved_pdf.unlink() - except Exception: - pass - - # 4. Delete the template itself - delete_template(db, template) - return {"status": "success", "message": "Template and all associated data deleted"} - diff --git a/app/api/schemas/enums.py b/app/api/schemas/enums.py index d02fcf81..4a2a3438 100644 --- a/app/api/schemas/enums.py +++ b/app/api/schemas/enums.py @@ -47,6 +47,7 @@ class JobType(str, Enum): form_generation = "form_generation" batch_form_generation = "batch_form_generation" report_generation = "report_generation" + template_field_detection = "template_field_detection" class FormType(str, Enum): @@ -72,6 +73,40 @@ class FormType(str, Enum): state_new_york = "state_new_york" +class DetectionStatus(str, Enum): + """Field-detection state of an uploaded template PDF. The PDF itself is + stored before any of this runs, so a failed detection is recoverable.""" + + processing = "processing" + completed = "completed" + failed = "failed" + + +class TemplateStatus(str, Enum): + active = "active" + legacy = "legacy" + draft = "draft" + + +class TextAlign(str, Enum): + left = "left" + center = "center" + right = "right" + + +class TemplateFieldType(str, Enum): + string = "string" + integer = "integer" + number = "number" + boolean = "boolean" + date = "date" + datetime = "datetime" + time = "time" + enum = "enum" + text = "text" + array = "array" + + class IncidentCategory(str, Enum): fire = "fire" overpressure_explosion = "overpressure_explosion" diff --git a/app/api/schemas/system.py b/app/api/schemas/system.py index bafa5cfc..24419cc6 100644 --- a/app/api/schemas/system.py +++ b/app/api/schemas/system.py @@ -43,3 +43,32 @@ class SchemaVersion(BaseModel): released_at: str changelog: str | None = None breaking_changes: bool | None = None + + +class SchemaFieldEntry(BaseModel): + """One leaf field of the incident contract, as the catalog exposes it + (contracts/schemas/template-record.yaml#/SchemaFieldEntry). + + Array hops show as `[]`, so a person's address reads + `persons_involved[].address`. + """ + + path: str + label: str | None = None + field_type: str + section: str + description: str | None = None + enum_values: list[str] | None = None + pii: bool = False + aliases: list[str] = [] + # Only set on search results, absent when the whole catalog is listed. + score: float | None = None + + +class SchemaFieldSearchResponse(BaseModel): + """GET /schema/fields response (path/system.yaml#/schema_fields).""" + + query: str | None = None + total: int + schema_version: str | None = None + fields: list[SchemaFieldEntry] diff --git a/app/api/schemas/templates.py b/app/api/schemas/templates.py index df39832b..a192aaf4 100644 --- a/app/api/schemas/templates.py +++ b/app/api/schemas/templates.py @@ -1,38 +1,266 @@ -from pydantic import BaseModel +import re +from datetime import date, datetime +from uuid import UUID -class TemplateCreate(BaseModel): - name: str - pdf_path: str - fields: dict +from pydantic import BaseModel, Field, field_validator, model_validator +from app.api.schemas.enums import ( + DetectionStatus, + FieldSource, + TemplateFieldType, + TemplateStatus, + TextAlign, +) -class MakeFillableRequest(BaseModel): - pdf_path: str +_HEX_COLOR = re.compile(r"^#[0-9A-Fa-f]{6}$") +_FORM_TYPE = re.compile(r"^[a-z0-9_-]+$") -class MakeFillableResponse(BaseModel): - pdf_path: str - field_count: int | None = None +# --------------------------------------------------------------------------- +# Contract Layer 6 schemas (contracts/schemas/template-record.yaml) +# --------------------------------------------------------------------------- +class TemplateFieldLayout(BaseModel): + """Visual placement of a field on the PDF (schemas/template-record.yaml#/TemplateFieldLayout). -class TemplateResponse(BaseModel): - id: int - name: str - pdf_path: str - fields: dict - field_count: int | None = None + Coordinates are PDF points with the origin at the bottom-left of the page: + box = start (x, y) .. end (x + width, y + height). + """ - class Config: - from_attributes = True + page: int = Field(ge=0, description="Zero-based page index") + x: float = Field(ge=0, description="Lower-left X in PDF points") + y: float = Field(ge=0, description="Lower-left Y in PDF points (origin bottom-left)") + width: float = Field(gt=0) + height: float = Field(gt=0) + font: str = "Helvetica" + font_size: float = Field(default=10, gt=0) + color: str = "#000000" + align: TextAlign = TextAlign.left + @field_validator("font") + @classmethod + def _font_not_blank(cls, v: str) -> str: + if not v.strip(): + raise ValueError("font must not be blank") + return v -class ExtractedField(BaseModel): - name: str - description: str - type: str + @field_validator("color") + @classmethod + def _color_is_hex(cls, v: str) -> str: + if not _HEX_COLOR.match(v): + raise ValueError('color must be a hex string like "#000000"') + return v -class TemplateUploadResponse(BaseModel): - filename: str - pdf_path: str - field_count: int | None = None - fields: list[ExtractedField] = [] +class TemplateField(BaseModel): + """One field definition within a template (schemas/template-record.yaml#/TemplateField). + + `source` decides where the value comes from, and each source brings its own + requirement: schema needs an `incident_mapping`, static needs `static_text`, + open needs a `description` (it is the instruction the extractor is given), + and manual needs neither. `layout` places the box on the PDF. + """ + + field_name: str + field_type: TemplateFieldType + source: FieldSource + required: bool + description: str | None = None + max_length: int | None = Field(default=None, gt=0) + # No bound on min/max themselves: contract values legitimately go negative + # (temperatures, elevations). Only their order is checked below. + min_value: float | None = None + max_value: float | None = None + allowed_values: list[str] | None = None + incident_mapping: str | None = None + static_text: str | None = None + default_value: object | None = None + unit: str | None = None + layout: TemplateFieldLayout | None = None + + @field_validator("field_name") + @classmethod + def _name_not_blank(cls, v: str) -> str: + v = v.strip() + if not v: + raise ValueError("field_name must not be empty") + return v + + @model_validator(mode="after") + def _check_field(self) -> "TemplateField": + has_mapping = bool(self.incident_mapping and self.incident_mapping.strip()) + has_static = self.static_text is not None + has_description = bool(self.description and self.description.strip()) + + if self.source is FieldSource.schema and not has_mapping: + raise ValueError( + f"field '{self.field_name}': incident_mapping is required when source is 'schema'" + ) + if self.source is not FieldSource.schema and has_mapping: + raise ValueError( + f"field '{self.field_name}': incident_mapping is only allowed when source is 'schema'" + ) + if self.source is FieldSource.static and not has_static: + raise ValueError( + f"field '{self.field_name}': static_text is required when source is 'static'" + ) + if self.source is not FieldSource.static and has_static: + raise ValueError( + f"field '{self.field_name}': static_text is only allowed when source is 'static'" + ) + if self.source is FieldSource.open and not has_description: + raise ValueError( + f"field '{self.field_name}': description is required when source is 'open'. " + "It is the instruction the extractor is given." + ) + if self.field_type == TemplateFieldType.enum and not self.allowed_values: + raise ValueError( + f"field '{self.field_name}': allowed_values is required when field_type is 'enum'" + ) + if ( + self.min_value is not None + and self.max_value is not None + and self.min_value > self.max_value + ): + raise ValueError( + f"field '{self.field_name}': min_value must be <= max_value" + ) + return self + + +class CreateTemplateRequest(BaseModel): + """POST/PUT request body (schemas/template-record.yaml#/CreateTemplateRequest).""" + + form_type: str + display_name: str + jurisdiction: str | None = None + agency_type: str | None = None + fields: list[TemplateField] = Field(min_length=1) + source_standard: str | None = None + pdf_template_ref: str | None = None + + @field_validator("form_type") + @classmethod + def _form_type_slug(cls, v: str) -> str: + v = v.strip() + if not _FORM_TYPE.match(v): + raise ValueError( + "form_type must contain only lowercase letters, digits, underscores or hyphens" + ) + return v + + @field_validator("display_name") + @classmethod + def _display_not_blank(cls, v: str) -> str: + if not v.strip(): + raise ValueError("display_name must not be empty") + return v + + @model_validator(mode="after") + def _unique_field_names(self) -> "CreateTemplateRequest": + names = [f.field_name for f in self.fields] + dupes = sorted({n for n in names if names.count(n) > 1}) + if dupes: + raise ValueError(f"duplicate field_name(s): {', '.join(dupes)}") + return self + + +class TemplateSummary(BaseModel): + """List item (schemas/template-record.yaml#/TemplateSummary).""" + + template_id: UUID + form_type: str + display_name: str + jurisdiction: str | None = None + agency_type: str | None = None + version: str + last_updated: date + field_count: int + status: TemplateStatus + + +class TemplateDetail(CreateTemplateRequest): + """Full template definition (schemas/template-record.yaml#/Template). + + Server-generated fields layered on top of CreateTemplateRequest. + """ + + template_id: UUID + version: str + last_updated: date + field_count: int + status: TemplateStatus + created_at: datetime + updated_at: datetime + + +class TemplateFieldsResponse(BaseModel): + """GET /templates/{id}/fields response (path/templates.yaml#/template_fields).""" + + template_id: UUID + form_type: str + total_fields: int + required_fields: int + optional_fields: int + fields: list[TemplateField] + + +# --------------------------------------------------------------------------- +# PDF upload and field detection (path/templates.yaml#/templates_pdf) +# --------------------------------------------------------------------------- +class PageGeometry(BaseModel): + """One page's size in PDF points, the unit every layout box uses.""" + + page: int = Field(ge=0, description="Zero-based page index") + width: float + height: float + + +class MappingSuggestion(BaseModel): + """One ranked incident-contract mapping guess for a detected box.""" + + path: str + label: str | None = None + field_type: str | None = None + section: str | None = None + description: str | None = None + score: float + + +class DraftField(BaseModel): + """A detected box as an editable TemplateField, plus what it was guessed from.""" + + field: TemplateField + # Kept verbatim, not normalized, so the editor can show the user what the + # suggestion was based on. + detected_label: str | None = None + suggestions: list[MappingSuggestion] = Field(default_factory=list) + + +class TemplateDraft(BaseModel): + """Everything the visual editor needs after a PDF upload. + + The PDF is stored the moment the upload returns; only detection is async, + so `status` describes detection alone. A failed detection still leaves a + usable upload, the user just draws every box by hand. + """ + + upload_id: UUID + status: DetectionStatus + pdf_template_ref: str + original_filename: str | None = None + page_count: int + pages: list[PageGeometry] + detected_fields: list[DraftField] | None = None + detection_error: str | None = None + retry_after_seconds: int | None = None + + +class TemplateDraftAccepted(TemplateDraft): + """202 body of POST /templates/pdf: the draft plus where to poll it. + + `job_id` is absent when detection was skipped, since there is no background + work to follow. `poll_url` still resolves, it just answers straight away. + """ + + job_id: str | None = None + poll_url: str diff --git a/app/core/celery.py b/app/core/celery.py index 01111667..fdb24df7 100644 --- a/app/core/celery.py +++ b/app/core/celery.py @@ -16,7 +16,13 @@ result_expires=86400, ) -celery_app.conf.include = ["app.tasks.fill", "app.tasks.purge", "app.tasks.transcribe", "app.tasks.extract"] +celery_app.conf.include = [ + "app.tasks.fill", + "app.tasks.purge", + "app.tasks.transcribe", + "app.tasks.extract", + "app.tasks.detect_fields", +] # Optional Celery Beat schedule — runs purge_old_submissions once a day. # Enable by running: celery -A app.core.celery beat diff --git a/app/core/config.py b/app/core/config.py index 09bcb701..28617742 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -109,6 +109,26 @@ # --- Data Retention -------------------------------------------------------- RETENTION_PERIOD_DAYS = int(os.getenv("RETENTION_PERIOD_DAYS", "30")) +# --- Template PDF storage and field detection ----------------------------- +# Blank agency PDFs uploaded for template authoring land here as +# {TEMPLATE_UPLOAD_DIR}/{upload_id}.pdf. The reference handed back to clients +# is that path taken relative to DATA_DIR, so the two can never drift apart. +TEMPLATE_UPLOAD_DIR = DATA_DIR / "templates" / "uploads" + +MAX_TEMPLATE_PDF_BYTES = 50 * 1024 * 1024 + +# Polling hint returned by the draft endpoints while detection is running. +TEMPLATE_DETECTION_POLL_INTERVAL_SECONDS = 5 + +# Two thresholds govern the mapping suggester. Below the floor nothing is +# offered at all: an empty list next to a good search box beats a wrong guess, +# because a pre-filled mapping is trusted far more than it deserves. At or above +# the auto-apply mark the top hit is written straight into the field as a +# schema mapping; between the two the suggestions are listed and the user picks. +MAPPING_SUGGESTION_FLOOR = float(os.getenv("MAPPING_SUGGESTION_FLOOR", "0.5")) +MAPPING_AUTO_APPLY_SCORE = float(os.getenv("MAPPING_AUTO_APPLY_SCORE", "0.85")) +MAX_MAPPING_SUGGESTIONS = 5 + # --- Audio storage -------------------------------------------------------- # Voice input audio files land here: {AUDIO_DIR}/{input_id}.{ext} AUDIO_DIR = DATA_DIR / "audio" diff --git a/app/db/repositories.py b/app/db/repositories.py index e2f565ba..87aae30a 100644 --- a/app/db/repositories.py +++ b/app/db/repositories.py @@ -2,24 +2,72 @@ from sqlmodel import Session, select -from app.models import Template, FormSubmission, Job, Input, Extraction, Incident +from app.models import ( + Template, + FormSubmission, + FormTemplate, + Job, + Input, + Extraction, + Incident, + TemplateUpload, +) from app.api.schemas.enums import ReportStatus -# Templates -def create_template(session: Session, template: Template) -> Template: +# Templates (legacy fill pipeline - read-only lookup, consumed by forms/jobs/tasks) +def get_template(session: Session, template_id: int) -> Template | None: + return session.get(Template, template_id) + +# Form templates (contract Layer 6 registry) +def create_form_template(session: Session, template: FormTemplate) -> FormTemplate: session.add(template) session.commit() session.refresh(template) return template -def get_template(session: Session, template_id: int) -> Template | None: - return session.get(Template, template_id) + +def get_form_template(session: Session, template_id: UUID) -> FormTemplate | None: + return session.get(FormTemplate, template_id) + + +def get_form_template_by_form_type(session: Session, form_type: str) -> FormTemplate | None: + statement = select(FormTemplate).where(FormTemplate.form_type == form_type) + return session.exec(statement).first() -def list_templates(session: Session) -> list[Template]: - statement = select(Template).order_by(Template.created_at.desc(), Template.id.desc()) +def list_form_templates(session: Session) -> list[FormTemplate]: + statement = select(FormTemplate).order_by( + FormTemplate.created_at.desc(), FormTemplate.template_id + ) return list(session.exec(statement)) + +def update_form_template(session: Session, template: FormTemplate) -> FormTemplate: + session.add(template) + session.commit() + session.refresh(template) + return template + + +# Template PDF uploads (field-detection drafts) +def create_template_upload(session: Session, upload: TemplateUpload) -> TemplateUpload: + session.add(upload) + session.commit() + session.refresh(upload) + return upload + + +def get_template_upload(session: Session, upload_id: UUID) -> TemplateUpload | None: + return session.get(TemplateUpload, upload_id) + + +def update_template_upload(session: Session, upload: TemplateUpload) -> TemplateUpload: + session.add(upload) + session.commit() + session.refresh(upload) + return upload + + # Forms def create_form(session: Session, form: FormSubmission) -> FormSubmission: session.add(form) @@ -57,11 +105,6 @@ def update_job(session: Session, job: Job) -> Job: return job -def delete_template(session: Session, template: Template) -> None: - session.delete(template) - session.commit() - - def get_form_submission(session: Session, submission_id: int) -> FormSubmission | None: return session.get(FormSubmission, submission_id) diff --git a/app/models/__init__.py b/app/models/__init__.py index bba2eecb..af645112 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -4,15 +4,19 @@ Extraction, Form, FormSubmission, + FormTemplate, Incident, Input, Job, Report, Template, + TemplateUpload, ) __all__ = [ "Template", + "FormTemplate", + "TemplateUpload", "FormSubmission", "Job", "Input", diff --git a/app/models/models.py b/app/models/models.py index b4396f59..08ddafa9 100644 --- a/app/models/models.py +++ b/app/models/models.py @@ -7,6 +7,7 @@ from sqlmodel.sql.sqltypes import AutoString from app.api.schemas.enums import ( + DetectionStatus, ExtractionStatus, FormStatus, FormType, @@ -17,6 +18,7 @@ OutputFormat, PeriodType, ReportStatus, + TemplateStatus, ) @@ -172,6 +174,66 @@ class Form(SQLModel, table=True): updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) +class TemplateUpload(SQLModel, table=True): + """A blank PDF uploaded for template authoring, plus its detection draft. + + The PDF is stored and its page geometry read synchronously, so a row exists + with `page_count`/`pages` filled before detection starts. `status` tracks + detection alone: a failed detection still leaves a usable upload, the user + just draws every box by hand. Rows are drafts, not templates. Registering a + template copies the edited fields into `form_templates` and keeps only the + `pdf_template_ref` pointing back here. + """ + + __tablename__ = "template_uploads" + + upload_id: UUID = Field(default_factory=uuid4, primary_key=True) + status: DetectionStatus = Field( + default=DetectionStatus.processing, sa_column=Column(AutoString, nullable=False) + ) + # Path on disk, and the DATA_DIR-relative reference handed to clients. + pdf_path: str + pdf_template_ref: str + original_filename: str | None = None + page_count: int = Field(default=0) + # List of {page, width, height} in PDF points. + pages: list = Field(default_factory=list, sa_column=Column(JSON, nullable=False)) + # List of DraftField objects (see app/api/schemas/templates.py). + detected_fields: list | None = Field(default=None, sa_column=Column(JSON)) + detection_error: str | None = None + job_id: str | None = None + created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + + +class FormTemplate(SQLModel, table=True): + """Contract Layer 6 form template registry (path/templates.yaml). + + Distinct from the legacy prototype `Template` (int PK + uploaded PDF): this + is the standards registry keyed by `form_type`, holding incident-schema field + definitions plus their visual `layout`. `field_count` and `last_updated` are + derived in the response schemas (len(fields) / updated_at.date()), not stored. + """ + + __tablename__ = "form_templates" + + template_id: UUID = Field(default_factory=uuid4, primary_key=True) + form_type: str = Field(sa_column=Column(AutoString, nullable=False, unique=True, index=True)) + display_name: str + jurisdiction: str | None = None + agency_type: str | None = None + # List of TemplateField objects (see app/api/schemas/templates.py). + fields: list = Field(sa_column=Column(JSON, nullable=False)) + source_standard: str | None = None + pdf_template_ref: str | None = None + version: str = Field(default="1.0") + status: TemplateStatus = Field( + default=TemplateStatus.active, sa_column=Column(AutoString, nullable=False) + ) + created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + + class Report(SQLModel, table=True): __tablename__ = "reports" diff --git a/app/services/field_catalog.py b/app/services/field_catalog.py new file mode 100644 index 00000000..00ba17ce --- /dev/null +++ b/app/services/field_catalog.py @@ -0,0 +1,309 @@ +"""The incident-contract field catalog and its matcher. + +Everything here is built by flattening `contracts/schemas/incident-contract.yaml` +at first use: dotted paths, types, sections, descriptions, enum values, the +`x-pii` flag and the `x-aliases` list. The contract is the only place any of +that is declared, so a schema change moves search and mapping suggestions with +it and nothing is restated in code. + +Two callers share this one index: GET /api/v1/schema/fields (the mapping picker +in the template editor) and the suggester that runs after commonforms field +detection. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from difflib import SequenceMatcher +from functools import lru_cache +from typing import Any + +import yaml + +from app.core.config import INCIDENT_CONTRACT_PATH +from app.core.logging import get_logger + +logger = get_logger(__name__) + +_ROOT = "IncidentContract" +_PUNCTUATION = re.compile(r"[^a-z0-9]+") + +# Form labels are written for people, not matchers. These are the shortenings +# that show up on nearly every printed incident form. +_ABBREVIATIONS = { + "no": "number", + "num": "number", + "nbr": "number", + "dt": "date", + "addr": "address", + "tel": "phone", + "ph": "phone", + "dob": "date of birth", + "amt": "amount", + "qty": "quantity", + "desc": "description", + "dept": "department", + "apt": "apartment", + "st": "street", + "yr": "year", + "veh": "vehicle", + "inj": "injury", +} + + +@dataclass(frozen=True) +class CatalogEntry: + """One leaf field of the contract, ready to search.""" + + path: str + label: str + field_type: str + section: str + description: str | None = None + enum_values: tuple[str, ...] | None = None + pii: bool = False + aliases: tuple[str, ...] = () + tokens: frozenset[str] = field(default_factory=frozenset, compare=False) + + +# --------------------------------------------------------------------------- +# Text normalization +# --------------------------------------------------------------------------- +def normalize(text: str) -> str: + """Lowercase, drop punctuation, collapse whitespace.""" + return _PUNCTUATION.sub(" ", text.lower()).strip() + + +def normalize_label(text: str) -> str: + """Normalize a label read off a PDF, expanding the usual form shorthand. + + Detected labels are messy ("Incident No.:", "Dt of Loss"), so the words are + expanded before they ever reach the matcher. + """ + words = normalize(text).split() + return " ".join(_ABBREVIATIONS.get(word, word) for word in words) + + +def _humanize(name: str) -> str: + words = name.replace("_", " ").strip() + return words[:1].upper() + words[1:] if words else name + + +def _tokens(*values: str) -> frozenset[str]: + out: set[str] = set() + for value in values: + out.update(normalize(value).split()) + return frozenset(out) + + +# --------------------------------------------------------------------------- +# Catalog construction +# --------------------------------------------------------------------------- +@lru_cache(maxsize=1) +def _contract_doc() -> dict[str, Any]: + return yaml.safe_load(INCIDENT_CONTRACT_PATH.read_text()) + + +@lru_cache(maxsize=1) +def _enums_doc() -> dict[str, Any]: + """The shared enum file the contract points at for closed value lists.""" + enums_path = INCIDENT_CONTRACT_PATH.parent / "enums.yaml" + try: + return yaml.safe_load(enums_path.read_text()) or {} + except OSError: + logger.warning("enum file %s is missing, enum values will be empty", enums_path) + return {} + + +def _resolve(spec: dict[str, Any]) -> dict[str, Any]: + """Follow a $ref one hop, inside the contract or into enums.yaml. + + Anything the ref does not carry (a description written at the reference + site, for example) stays, so both halves survive. + """ + ref = spec.get("$ref") + if not ref: + return spec + + file_part, _, name = ref.partition("#/") + if "enums.yaml" in file_part: + target = _enums_doc().get(name) + elif file_part in ("", "#"): + target = _contract_doc().get(name) + else: + target = None + + if not isinstance(target, dict): + logger.warning("unresolved $ref %s in the incident contract", ref) + return {k: v for k, v in spec.items() if k != "$ref"} + + merged = dict(target) + for key, value in spec.items(): + if key != "$ref": + merged.setdefault(key, value) + return merged + + +def _field_type(spec: dict[str, Any]) -> str: + declared = spec.get("type") + if isinstance(declared, str): + return declared + if spec.get("enum"): + return "string" + if spec.get("properties"): + return "object" + return "string" + + +def _walk( + spec: dict[str, Any], + path: str, + section: str, + name: str, + seen: frozenset[str], + out: list[CatalogEntry], +) -> None: + spec = _resolve(spec) + + properties = spec.get("properties") + if properties: + # A recursive shape would otherwise walk forever. Stop the second time + # the same object type appears on one branch. + marker = spec.get("title") or path + if marker in seen: + return + seen = seen | {marker} + for child_name, child_spec in properties.items(): + if not isinstance(child_spec, dict): + continue + child_path = f"{path}.{child_name}" if path else child_name + _walk(child_spec, child_path, section, child_name, seen, out) + return + + if spec.get("type") == "array": + items = spec.get("items") + if isinstance(items, dict): + resolved = _resolve(items) + if resolved.get("properties"): + _walk(items, f"{path}[]", section, name, seen, out) + return + + enum_values = spec.get("enum") + out.append( + CatalogEntry( + path=path, + label=_humanize(name), + field_type=_field_type(spec), + section=section, + description=(spec.get("description") or "").strip() or None, + enum_values=tuple(str(v) for v in enum_values) if enum_values else None, + pii=bool(spec.get("x-pii")), + aliases=tuple(spec.get("x-aliases") or ()), + tokens=_tokens(name, *(spec.get("x-aliases") or ())), + ) + ) + + +@lru_cache(maxsize=1) +def catalog() -> tuple[CatalogEntry, ...]: + """Every leaf field in the contract, in contract order.""" + properties = _contract_doc()[_ROOT].get("properties", {}) + entries: list[CatalogEntry] = [] + for section, spec in properties.items(): + if isinstance(spec, dict): + _walk(spec, section, section, section, frozenset(), entries) + return tuple(entries) + + +@lru_cache(maxsize=1) +def schema_version() -> str | None: + """The contract version the catalog was built from.""" + properties = _contract_doc()[_ROOT].get("properties", {}) + version = properties.get("schema_version") or {} + example = version.get("example") + return str(example) if example else None + + +# --------------------------------------------------------------------------- +# Matching +# --------------------------------------------------------------------------- +# Scores are banded rather than blended, so the ranking the contract describes +# holds no matter how the fuzzy ratio lands: an exact name beats an exact alias, +# both beat a prefix, and a description-only hit always comes last. +_EXACT_NAME = 1.0 +_EXACT_ALIAS = 0.95 +_NAME_PREFIX = 0.85 +_ALIAS_PREFIX = 0.8 +_FUZZY_CEILING = 0.75 +_FUZZY_FLOOR = 0.6 +_DESCRIPTION_HIT = 0.35 + + +def _leaf_name(path: str) -> str: + return path.rsplit(".", 1)[-1].removesuffix("[]") + + +def score_entry(entry: CatalogEntry, query: str) -> float: + """How well one catalog entry answers a query, 0 when it does not. + + `query` is expected already normalized, since `search` normalizes once and + then scores the whole catalog with it. + """ + if not query: + return 0.0 + + name = normalize(_leaf_name(entry.path)) + aliases = [normalize(a) for a in entry.aliases] + + if query == name: + return _EXACT_NAME + if query in aliases: + return _EXACT_ALIAS + if name.startswith(query): + return _NAME_PREFIX + if any(alias.startswith(query) for alias in aliases): + return _ALIAS_PREFIX + + query_tokens = set(query.split()) + if query_tokens and query_tokens <= entry.tokens: + return _FUZZY_CEILING + + best = max( + (SequenceMatcher(None, query, candidate).ratio() for candidate in [name, *aliases]), + default=0.0, + ) + if best >= _FUZZY_FLOOR: + return round(best * _FUZZY_CEILING, 4) + + if entry.description and query_tokens: + description_tokens = set(normalize(entry.description).split()) + if query_tokens <= description_tokens: + return _DESCRIPTION_HIT + + return 0.0 + + +def search( + query: str | None = None, + section: str | None = None, + limit: int = 20, +) -> list[tuple[CatalogEntry, float | None]]: + """Rank the catalog against `query`, or list it when no query is given. + + `limit` caps search results only. A bare listing returns the whole catalog, + because the editor caches it once per session and filters it locally, and a + truncated catalog would silently hide fields from the mapping picker. + + Ties break toward the shorter path, so the plainest field wins. + """ + entries = [e for e in catalog() if section is None or e.section == section] + + if not query or not query.strip(): + return [(entry, None) for entry in entries] + + normalized = normalize(query) + scored = [(entry, score_entry(entry, normalized)) for entry in entries] + hits = [(entry, score) for entry, score in scored if score > 0] + hits.sort(key=lambda pair: (-pair[1], len(pair[0].path), pair[0].path)) + return [(entry, score) for entry, score in hits[:limit]] diff --git a/app/services/form_templates.py b/app/services/form_templates.py new file mode 100644 index 00000000..ff6af5f4 --- /dev/null +++ b/app/services/form_templates.py @@ -0,0 +1,310 @@ +"""Business logic for the contract Layer 6 template registry. + +Sits between the route handlers (app/api/routes/form_templates.py) and the +repositories. No FastAPI imports here — handlers do HTTP, this does the work: +validation/conflict checks, ORM construction, and ORM -> response mapping +(including the derived `field_count` / `last_updated`). +""" + +from datetime import datetime, timezone +from pathlib import Path +from uuid import UUID, uuid4 + +from sqlmodel import Session + +from app.api.schemas.enums import DetectionStatus, JobType +from app.api.schemas.templates import ( + CreateTemplateRequest, + DraftField, + PageGeometry, + TemplateDetail, + TemplateDraft, + TemplateDraftAccepted, + TemplateField, + TemplateFieldsResponse, + TemplateSummary, +) +from app.core.config import ( + DATA_DIR, + TEMPLATE_DETECTION_POLL_INTERVAL_SECONDS, + TEMPLATE_UPLOAD_DIR, +) +from app.core.errors.base import AppError +from app.db.repositories import ( + create_form_template, + create_job, + create_template_upload, + get_form_template, + get_form_template_by_form_type, + get_template_upload, + list_form_templates, + update_form_template, + update_job, +) +from app.models import FormTemplate, Job, TemplateUpload +from app.services.template_detection import read_pages +from app.tasks.detect_fields import detect_template_fields_task + + +# --------------------------------------------------------------------------- +# Mapping helpers (ORM -> response schema). field_count / last_updated are +# derived here rather than stored on the model. +# --------------------------------------------------------------------------- +def _field_count(template: FormTemplate) -> int: + return len(template.fields or []) + + +def _to_summary(template: FormTemplate) -> TemplateSummary: + return TemplateSummary( + template_id=template.template_id, + form_type=template.form_type, + display_name=template.display_name, + jurisdiction=template.jurisdiction, + agency_type=template.agency_type, + version=template.version, + last_updated=template.updated_at.date(), + field_count=_field_count(template), + status=template.status, + ) + + +def _to_detail(template: FormTemplate) -> TemplateDetail: + return TemplateDetail( + template_id=template.template_id, + form_type=template.form_type, + display_name=template.display_name, + jurisdiction=template.jurisdiction, + agency_type=template.agency_type, + fields=template.fields, + source_standard=template.source_standard, + pdf_template_ref=template.pdf_template_ref, + version=template.version, + last_updated=template.updated_at.date(), + field_count=_field_count(template), + status=template.status, + created_at=template.created_at, + updated_at=template.updated_at, + ) + + +def _require_template(db: Session, template_id: UUID) -> FormTemplate: + template = get_form_template(db, template_id) + if not template: + raise AppError( + f"Template {template_id} not found", + status_code=404, + error_code="TEMPLATE_NOT_FOUND", + ) + return template + + +# --------------------------------------------------------------------------- +# Operations +# --------------------------------------------------------------------------- +def list_templates(db: Session) -> list[TemplateSummary]: + return [_to_summary(t) for t in list_form_templates(db)] + + +def create_template(db: Session, body: CreateTemplateRequest) -> TemplateDetail: + if get_form_template_by_form_type(db, body.form_type): + raise AppError( + f"Template with form_type '{body.form_type}' already exists", + status_code=409, + error_code="TEMPLATE_EXISTS", + ) + + template = FormTemplate( + form_type=body.form_type, + display_name=body.display_name, + jurisdiction=body.jurisdiction, + agency_type=body.agency_type, + fields=[f.model_dump(mode="json") for f in body.fields], + source_standard=body.source_standard, + pdf_template_ref=body.pdf_template_ref, + ) + return _to_detail(create_form_template(db, template)) + + +def get_template(db: Session, template_id: UUID) -> TemplateDetail: + return _to_detail(_require_template(db, template_id)) + + +def replace_template( + db: Session, template_id: UUID, body: CreateTemplateRequest +) -> TemplateDetail: + template = _require_template(db, template_id) + + # form_type is unique in the DB, so a rename onto a form_type another + # template already holds has to be answered here. Without this the insert + # fails deep in the session and the client gets a bare 500. + clash = get_form_template_by_form_type(db, body.form_type) + if clash and clash.template_id != template_id: + raise AppError( + f"Template with form_type '{body.form_type}' already exists", + status_code=409, + error_code="TEMPLATE_EXISTS", + ) + + # Contract defines a 409 TEMPLATE_IN_USE when submitted incidents reference + # this template. The contract forms/incidents layers tie records to + # extract_id + form_type, never template_id, so there is no linkage to query + # yet. Once a form_type<->submission link exists, gate the update here. + # TODO(contract): enforce 409 TEMPLATE_IN_USE. + + template.form_type = body.form_type + template.display_name = body.display_name + template.jurisdiction = body.jurisdiction + template.agency_type = body.agency_type + template.fields = [f.model_dump(mode="json") for f in body.fields] + template.source_standard = body.source_standard + template.pdf_template_ref = body.pdf_template_ref + template.updated_at = datetime.now(timezone.utc) + return _to_detail(update_form_template(db, template)) + + +def resolve_template_pdf(db: Session, template_id: UUID) -> Path: + """On-disk path of a template's source PDF. + + `pdf_template_ref` is client-supplied, so the resolved path is checked to + still sit under the data directory before anything is served from it. + """ + template = _require_template(db, template_id) + if not template.pdf_template_ref: + raise AppError( + f"Template {template_id} has no source PDF", + status_code=404, + error_code="TEMPLATE_PDF_NOT_FOUND", + ) + + path = (DATA_DIR / template.pdf_template_ref).resolve() + if not path.is_relative_to(DATA_DIR) or not path.is_file(): + raise AppError( + f"Source PDF for template {template_id} is missing", + status_code=404, + error_code="TEMPLATE_PDF_NOT_FOUND", + ) + return path + + +# --------------------------------------------------------------------------- +# PDF upload and field detection +# --------------------------------------------------------------------------- +def _to_draft(upload: TemplateUpload) -> TemplateDraft: + return TemplateDraft( + upload_id=upload.upload_id, + status=upload.status, + pdf_template_ref=upload.pdf_template_ref, + original_filename=upload.original_filename, + page_count=upload.page_count, + pages=[PageGeometry(**page) for page in upload.pages], + detected_fields=( + [DraftField(**field) for field in upload.detected_fields] + if upload.detected_fields is not None + else None + ), + detection_error=upload.detection_error, + retry_after_seconds=( + TEMPLATE_DETECTION_POLL_INTERVAL_SECONDS + if upload.status == DetectionStatus.processing + else None + ), + ) + + +def store_upload( + db: Session, content: bytes, filename: str | None, detect_fields: bool +) -> tuple[TemplateUpload, Job | None]: + """Store a blank PDF, read its page geometry, and queue field detection. + + The PDF and its geometry are saved before returning, so the editor can + render pages immediately. Only detection runs in the background, and with + `detect_fields` off the draft is complete the moment it is created. + """ + upload_id = uuid4() + TEMPLATE_UPLOAD_DIR.mkdir(parents=True, exist_ok=True) + pdf_path = TEMPLATE_UPLOAD_DIR / f"{upload_id}.pdf" + pdf_path.write_bytes(content) + + try: + pages = read_pages(pdf_path) + except Exception as exc: + pdf_path.unlink(missing_ok=True) + raise AppError( + "Uploaded file could not be read as a PDF", + status_code=415, + error_code="INVALID_PDF", + detail={"reason": str(exc)}, + ) + + upload = TemplateUpload( + upload_id=upload_id, + status=DetectionStatus.processing if detect_fields else DetectionStatus.completed, + pdf_path=str(pdf_path), + pdf_template_ref=str(pdf_path.relative_to(DATA_DIR)), + original_filename=filename, + page_count=len(pages), + pages=[page.model_dump() for page in pages], + detected_fields=None if detect_fields else [], + ) + + if not detect_fields: + return create_template_upload(db, upload), None + + # Same order as the voice upload: create the job, dispatch, then backfill + # the celery id. A failure anywhere after the file write takes the file + # with it rather than leaving an upload nobody can finish. + try: + job = create_job( + db, + Job(celery_task_id="", job_type=JobType.template_field_detection, status="queued"), + ) + upload.job_id = job.job_id + upload = create_template_upload(db, upload) + result = detect_template_fields_task.delay(str(upload.upload_id), job.job_id) + job.celery_task_id = result.id + update_job(db, job) + except Exception: + pdf_path.unlink(missing_ok=True) + raise + + return upload, job + + +def get_draft(db: Session, upload_id: UUID) -> TemplateDraft: + upload = get_template_upload(db, upload_id) + if not upload: + raise AppError( + f"Upload {upload_id} not found", + status_code=404, + error_code="UPLOAD_NOT_FOUND", + ) + return _to_draft(upload) + + +def draft_response(upload: TemplateUpload, job: Job | None) -> TemplateDraftAccepted: + """The 202 body: the draft the editor can already render, plus where to poll.""" + draft = _to_draft(upload) + return TemplateDraftAccepted( + **draft.model_dump(), + job_id=job.job_id if job else None, + poll_url=f"/api/v1/templates/pdf/{upload.upload_id}", + ) + + +def get_template_fields( + db: Session, template_id: UUID, required_only: bool +) -> TemplateFieldsResponse: + template = _require_template(db, template_id) + + fields = [TemplateField(**f) for f in template.fields] + required = [f for f in fields if f.required] + selected = required if required_only else fields + + return TemplateFieldsResponse( + template_id=template.template_id, + form_type=template.form_type, + total_fields=len(fields), + required_fields=len(required), + optional_fields=len(fields) - len(required), + fields=selected, + ) diff --git a/app/services/template_detection.py b/app/services/template_detection.py new file mode 100644 index 00000000..634bf220 --- /dev/null +++ b/app/services/template_detection.py @@ -0,0 +1,401 @@ +"""Field detection for an uploaded template PDF. + +commonforms finds the boxes, this turns them into editable template fields. +Widget rectangles come out of the PDF already in points with a bottom-left +origin, which is exactly what `TemplateFieldLayout` stores, so no coordinate +conversion happens anywhere on the backend. The editor converts to pixels for +display and back again on save. + +Detection is best effort by design. A box whose label cannot be read still +comes back with its geometry, and geometry alone is a fine starting point for +someone drawing the rest by hand. +""" + +from __future__ import annotations + +import re +from datetime import datetime, timezone +from pathlib import Path +from uuid import UUID + +from pypdf import PdfReader, PdfWriter +from sqlmodel import Session + +from app.api.schemas.enums import DetectionStatus, FieldSource, TemplateFieldType +from app.api.schemas.templates import ( + DraftField, + MappingSuggestion, + PageGeometry, + TemplateField, + TemplateFieldLayout, +) +from app.core.config import ( + MAPPING_AUTO_APPLY_SCORE, + MAPPING_SUGGESTION_FLOOR, + MAX_MAPPING_SUGGESTIONS, +) +from app.core.logging import get_logger +from app.db.repositories import ( + get_job_by_uuid, + get_template_upload, + update_job, + update_template_upload, +) +from app.services import field_catalog + +logger = get_logger(__name__) + +_NAME_CLEANUP = re.compile(r"[^a-z0-9]+") + +# How far from a box a piece of text can sit and still be taken as its label. +# Both are in points: roughly two characters of slack to the left, and a little +# over one line height above. +_LABEL_GAP_LEFT = 160.0 +_LABEL_GAP_ABOVE = 26.0 + + +# --------------------------------------------------------------------------- +# Reading the PDF +# --------------------------------------------------------------------------- +def read_pages(pdf_path: str | Path) -> list[PageGeometry]: + """Per-page size in PDF points, index 0 first.""" + reader = PdfReader(str(pdf_path)) + pages = [] + for index, page in enumerate(reader.pages): + box = page.mediabox + pages.append( + PageGeometry( + page=index, + width=float(box.width), + height=float(box.height), + ) + ) + return pages + + +def _text_positions(page) -> list[tuple[str, float, float]]: + """Every text fragment on the page as (text, x, y) in points. + + pypdf hands the text matrix to the visitor, whose last two entries are the + drawing position. Fragments are kept whole rather than merged into lines: + form labels are nearly always drawn in one go, and merging risks gluing a + neighbouring column onto the label. + """ + found: list[tuple[str, float, float]] = [] + + def visit(text, _cm, tm, _font_dict, _font_size): + cleaned = text.strip() + if cleaned: + found.append((cleaned, float(tm[4]), float(tm[5]))) + + try: + page.extract_text(visitor_text=visit) + except Exception as exc: + # A PDF whose content stream will not parse still has usable widgets. + logger.warning("could not read text for label detection: %s", exc) + return found + + +def _nearest_label( + layout: TemplateFieldLayout, texts: list[tuple[str, float, float]] +) -> str | None: + """The text most likely to be this box's label. + + Printed forms put the label to the left of the box, or directly above it. + Left wins when both exist, because a label above is often the column + heading for a whole run of boxes. + """ + top = layout.y + layout.height + + left = [ + (layout.x - x, text) + for text, x, y in texts + if x < layout.x + and layout.x - x <= _LABEL_GAP_LEFT + and layout.y - 2 <= y <= top + 2 + ] + if left: + return min(left)[1] + + above = [ + (y - top, text) + for text, x, y in texts + if top <= y <= top + _LABEL_GAP_ABOVE + and layout.x - 4 <= x <= layout.x + layout.width + ] + if above: + return min(above)[1] + return None + + +def _widgets(pdf_path: str | Path) -> list[tuple[TemplateFieldLayout, str | None, list]]: + """Every form widget in the PDF as (layout, widget name, page texts). + + The page index lives on the layout, so it is not repeated in the tuple. + """ + reader = PdfReader(str(pdf_path)) + out = [] + for index, page in enumerate(reader.pages): + annotations = page.get("/Annots") + if not annotations: + continue + texts = _text_positions(page) + # MediaBox does not have to start at the origin. Subtracting its corner + # keeps every stored coordinate relative to the page the editor draws. + origin_x = float(page.mediabox.left) + origin_y = float(page.mediabox.bottom) + + for annotation in annotations: + try: + obj = annotation.get_object() + except Exception: + continue + if obj.get("/Subtype") != "/Widget": + continue + rect = obj.get("/Rect") + if not rect or len(rect) != 4: + continue + x0, y0, x1, y1 = (float(v) for v in rect) + width, height = abs(x1 - x0) or 1.0, abs(y1 - y0) or 1.0 + layout = TemplateFieldLayout( + page=index, + x=max(min(x0, x1) - origin_x, 0), + y=max(min(y0, y1) - origin_y, 0), + width=width, + height=height, + ) + name = obj.get("/T") + out.append((layout, str(name) if name else None, texts)) + + # Reading order: top of the page down, then left to right. + out.sort(key=lambda item: (item[0].page, -item[0].y, item[0].x)) + return out + + +# --------------------------------------------------------------------------- +# Turning widgets into draft fields +# --------------------------------------------------------------------------- +def suggest_mappings(label: str | None) -> list[MappingSuggestion]: + """Ranked contract paths for a detected label, best first. + + Returns nothing when the label is missing or nothing scores above the + floor. An empty list is the honest answer, and the editor's search box + covers it. + """ + if not label: + return [] + + query = field_catalog.normalize_label(label) + hits = field_catalog.search(query, limit=MAX_MAPPING_SUGGESTIONS) + return [ + MappingSuggestion( + path=entry.path, + label=entry.label, + field_type=entry.field_type, + section=entry.section, + description=entry.description, + score=round(score, 4), + ) + for entry, score in hits + if score is not None and score >= MAPPING_SUGGESTION_FLOOR + ] + + +def _field_name(raw: str | None, used: set[str], position: int) -> str: + """A unique, slug-shaped name for a detected box.""" + base = _NAME_CLEANUP.sub("_", (raw or "").lower()).strip("_") + if not base: + base = f"field_{position}" + name = base + suffix = 2 + while name in used: + name = f"{base}_{suffix}" + suffix += 1 + used.add(name) + return name + + +def _apply_suggestion( + field: TemplateField, suggestions: list[MappingSuggestion] +) -> TemplateField: + """Pre-apply the top suggestion when it clears the auto-apply mark.""" + if not suggestions or suggestions[0].score < MAPPING_AUTO_APPLY_SCORE: + return field + + top = suggestions[0] + update = {"source": FieldSource.schema, "incident_mapping": top.path} + + entry = next((e for e in field_catalog.catalog() if e.path == top.path), None) + if entry and entry.enum_values: + update["field_type"] = TemplateFieldType.enum + update["allowed_values"] = list(entry.enum_values) + + return field.model_copy(update=update) + + +def _drafts_from_widgets(widgets: list) -> list[DraftField]: + used: set[str] = set() + drafts: list[DraftField] = [] + + for position, (layout, widget_name, texts) in enumerate(widgets, start=1): + label = _nearest_label(layout, texts) + # A widget's own name is often the best clue a fillable PDF gives + # ("incident_number"), so it stands in when no text sits near the box. + # `detected_label` still reports only what was read off the page. + suggestions = suggest_mappings(label or widget_name) + field = TemplateField( + field_name=_field_name(widget_name or label, used, position), + field_type=TemplateFieldType.string, + # Nothing is assumed about an unmapped box: manual means a person + # types the value, which is always safe to change to something else. + source=FieldSource.manual, + required=False, + layout=layout, + ) + drafts.append( + DraftField( + field=_apply_suggestion(field, suggestions), + detected_label=label, + suggestions=suggestions, + ) + ) + return drafts + + +def build_draft_fields(pdf_path: str | Path) -> list[DraftField]: + """Read a PDF's form widgets and turn each into a draft field.""" + return _drafts_from_widgets(_widgets(pdf_path)) + + +def _pad_with_blank_page(pdf_path: Path) -> Path: + """Write a copy of a one-page PDF with a blank second page appended. + + commonforms cannot read a single-page document. It wraps the detector's + output a second time when the page count is 1 (inference.py), and the + rfdetr versions it now installs with already hand back a list, so the run + dies with "'list' object has no attribute 'with_nms'". One blank page keeps + us off that branch. Detections on the padding are dropped afterwards. + """ + reader = PdfReader(str(pdf_path)) + page = reader.pages[0] + writer = PdfWriter() + writer.add_page(page) + writer.add_blank_page(width=page.mediabox.width, height=page.mediabox.height) + + padded = pdf_path.with_name(f"{pdf_path.stem}_padded.pdf") + with padded.open("wb") as handle: + writer.write(handle) + return padded + + +def detect_fields(pdf_path: str | Path) -> list[DraftField]: + """Run commonforms over a PDF, then draft a field per detected box. + + A PDF that already carries form widgets is used as-is. commonforms only + has to run on flat scans, and it is the slow part. + """ + # Imported here, not at module scope: the Controller pulls in the whole + # detection stack, which is far too heavy for an API process to import. + from app.services.controller import Controller + + pdf_path = Path(pdf_path) + widgets = _widgets(pdf_path) + if widgets: + logger.info("%s already has form widgets, skipping detection", pdf_path) + return _drafts_from_widgets(widgets) + + padded = None + if len(PdfReader(str(pdf_path)).pages) == 1: + padded = _pad_with_blank_page(pdf_path) + logger.info("%s is one page, padding it for commonforms", pdf_path) + + fillable_path = None + try: + fillable_path = Path(Controller().prepare_fillable(str(padded or pdf_path))) + drafts = build_draft_fields(fillable_path) + finally: + # Both files are scratch. The boxes live in the draft from here on, and + # the upload the user later registers points at the original PDF. + if padded: + padded.unlink(missing_ok=True) + if fillable_path: + fillable_path.unlink(missing_ok=True) + + if padded: + # Nothing should land on the blank page, but a stray box there would + # otherwise become a field on a page the real PDF does not have. + drafts = [d for d in drafts if d.field.layout and d.field.layout.page == 0] + return drafts + + +# --------------------------------------------------------------------------- +# The background run +# --------------------------------------------------------------------------- +def _finish_job(session: Session, job_id: str | None, status: str, error: dict | None = None) -> None: + if not job_id: + return + job = get_job_by_uuid(session, job_id) + if not job: + return + job.status = status + if status == "completed": + job.progress_percent = 100 + job.error = error + job.updated_at = datetime.now(timezone.utc) + update_job(session, job) + + +def run_detection(session: Session, upload_id: UUID, job_id: str | None = None) -> dict: + """Detect an upload's fields and write the draft back. Returns a summary. + + Failure is not exceptional here. The PDF and its page geometry are already + stored, so a detection that falls over still leaves the editor able to draw + every box by hand, and that is what the failed status tells it to do. + """ + upload = get_template_upload(session, upload_id) + if upload is None: + logger.warning("upload %s vanished before detection ran", upload_id) + _finish_job( + session, + job_id, + "failed", + {"error_code": "UPLOAD_NOT_FOUND", "message": "Upload no longer exists"}, + ) + return {"upload_id": str(upload_id), "status": "failed"} + + job = get_job_by_uuid(session, job_id) if job_id else None + if job: + job.status = "processing" + job.updated_at = datetime.now(timezone.utc) + update_job(session, job) + + now = datetime.now(timezone.utc) + try: + drafts = detect_fields(upload.pdf_path) + except Exception as exc: + logger.exception("field detection failed for upload %s", upload_id) + upload.status = DetectionStatus.failed + upload.detection_error = str(exc) + upload.updated_at = now + update_template_upload(session, upload) + _finish_job( + session, job_id, "failed", {"error_code": "DETECTION_FAILED", "message": str(exc)} + ) + return {"upload_id": str(upload_id), "status": "failed"} + + upload.detected_fields = [draft.model_dump(mode="json") for draft in drafts] + upload.status = DetectionStatus.completed + upload.detection_error = None + upload.updated_at = now + update_template_upload(session, upload) + + if job: + job.result_url = f"/api/v1/templates/pdf/{upload_id}" + update_job(session, job) + _finish_job(session, job_id, "completed") + + return { + "upload_id": str(upload_id), + "status": "completed", + "detected_fields": len(drafts), + } diff --git a/app/tasks/detect_fields.py b/app/tasks/detect_fields.py new file mode 100644 index 00000000..21598a69 --- /dev/null +++ b/app/tasks/detect_fields.py @@ -0,0 +1,29 @@ +"""Celery glue for template field detection. + +The work lives in app/services/template_detection.py. Detection runs here +rather than in the request because commonforms loads a vision model and can +take minutes on a scanned form. The upload row already holds the stored PDF +and its page geometry, so the editor is usable the whole time this runs. +""" + +import logging +from uuid import UUID + +from app.core.celery import celery_app +from app.db.database import get_session + +logger = logging.getLogger(__name__) + + +@celery_app.task(name="detect_template_fields") +def detect_template_fields_task(upload_id_str: str, job_id_str: str | None = None) -> dict: + """Detect the fields of one uploaded template PDF.""" + # Imported inside the task so the API process never pulls the detection + # stack in just by importing this module. + from app.services.template_detection import run_detection + + session = next(get_session()) + try: + return run_detection(session, UUID(upload_id_str), job_id_str) + finally: + session.close() diff --git a/tests/conftest.py b/tests/conftest.py index 62fae518..b1b20a0c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,7 +4,6 @@ (Controller → LLM / commonforms) so tests run fast without Docker or Ollama. """ -import io from unittest.mock import patch, MagicMock import pytest @@ -14,7 +13,18 @@ from app.main import app from app.api.deps import get_db -from app.models import Template, FormSubmission, Job, Input, Extraction, Incident, Form, Report # noqa: F401 — registers tables +from app.models import ( # noqa: F401 — importing these registers their tables + Extraction, + Form, + FormSubmission, + FormTemplate, + Incident, + Input, + Job, + Report, + Template, + TemplateUpload, +) # --------------------------------------------------------------------------- # In-memory database @@ -79,29 +89,30 @@ def pdf_bytes(): return _MINIMAL_PDF -@pytest.fixture -def pdf_upload(pdf_bytes): - """A tuple suitable for httpx/TestClient file upload.""" - return ("file", ("test_form.pdf", io.BytesIO(pdf_bytes), "application/pdf")) - - # --------------------------------------------------------------------------- # Controller mock — patches the heavy dependencies at the route level # --------------------------------------------------------------------------- @pytest.fixture def mock_controller(): - """Patch Controller so create_template / fill_form don't touch the FS or LLM.""" - with patch("app.api.routes.templates.Controller") as tpl_cls, \ - patch("app.api.routes.forms.Controller") as form_cls: - tpl_instance = MagicMock() - tpl_instance.create_template.return_value = "src/inputs/test_template.pdf" - tpl_cls.return_value = tpl_instance - + """Patch the forms Controller so fill_form doesn't touch the FS or LLM.""" + with patch("app.api.routes.forms.Controller") as form_cls: form_instance = MagicMock() form_instance.fill_form.return_value = "src/outputs/filled_output.pdf" form_cls.return_value = form_instance - yield { - "template_ctrl": tpl_instance, - "form_ctrl": form_instance, - } + yield {"form_ctrl": form_instance} + + +@pytest.fixture +def seed_template(): + """Insert a legacy Template row directly (the /templates/create endpoint was + removed in the contract migration). Returns a factory -> template id.""" + def _make(name: str = "T", pdf_path: str = "src/inputs/t.pdf", fields: dict | None = None) -> int: + with Session(_engine) as session: + tpl = Template(name=name, pdf_path=pdf_path, fields=fields if fields is not None else {"name": "string"}) + session.add(tpl) + session.commit() + session.refresh(tpl) + return tpl.id + + return _make diff --git a/tests/test_api.py b/tests/test_api.py index 32104ae7..3c8cbc4d 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -84,104 +84,21 @@ def test_list_templates_ordering(self, db): class TestTemplateEndpoints: def test_list_templates_empty(self, client): + """Contract registry list is empty until a template is registered.""" resp = client.get(f"{API_PREFIX}/templates") assert resp.status_code == 200 assert resp.json() == [] - def test_create_template(self, client, mock_controller): - payload = { - "name": "Fire Report", - "pdf_path": "src/inputs/fire_report.pdf", - "fields": { - "Name": "string", - "Date": "string", - "Location": "string", - }, - } - resp = client.post(f"{API_PREFIX}/templates/create", json=payload) - assert resp.status_code == 200 - - data = resp.json() - assert data["id"] is not None - assert data["name"] == "Fire Report" - assert data["fields"]["Location"] == "string" - # Plain create just persists the row; commonforms only runs via - # the separate /make-fillable endpoint. - mock_controller["template_ctrl"].create_template.assert_not_called() - - def test_create_then_list(self, client, mock_controller): - """Creating a template should make it appear in the list.""" - client.post(f"{API_PREFIX}/templates/create", json={ - "name": "T1", - "pdf_path": "a.pdf", - "fields": {"f": "string"}, - }) - resp = client.get(f"{API_PREFIX}/templates") - assert resp.status_code == 200 - assert len(resp.json()) == 1 - assert resp.json()[0]["name"] == "T1" - - def test_upload_pdf(self, client, pdf_upload, tmp_path, monkeypatch): - """Upload a valid PDF file.""" - # Point the upload directory inside tmp_path (which is inside the project - # for the path-safety check — we monkeypatch the check). - monkeypatch.setattr( - "app.api.routes.templates.PROJECT_ROOT", - tmp_path, - ) - resp = client.post( - f"{API_PREFIX}/templates/upload", - files=[pdf_upload], - data={"directory": str(tmp_path)}, - ) - assert resp.status_code == 200 - data = resp.json() - assert data["filename"] == "test_form.pdf" - assert data["pdf_path"].endswith(".pdf") - - def test_upload_non_pdf_rejected(self, client): - import io - bad_file = ("file", ("notes.txt", io.BytesIO(b"hello"), "text/plain")) - resp = client.post(f"{API_PREFIX}/templates/upload", files=[bad_file]) - assert resp.status_code == 400 - assert "PDF" in resp.json()["detail"] - - def test_preview_missing_file(self, client): - resp = client.get(f"{API_PREFIX}/templates/preview", params={"path": "src/inputs/nonexistent.pdf"}) - assert resp.status_code == 404 - - def test_directory_traversal_blocked(self, client): - import io - pdf = ("file", ("evil.pdf", io.BytesIO(b"%PDF-1.4"), "application/pdf")) - resp = client.post( - f"{API_PREFIX}/templates/upload", - files=[pdf], - data={"directory": "/etc"}, - ) - assert resp.status_code == 400 - assert "inside the project" in resp.json()["detail"] - # ═══════════════════════════════════════════════════════════════════════════ -# Form fill endpoints +# Form fill endpoints (legacy pipeline — templates seeded directly in the DB +# since the /templates/create endpoint was removed in the contract migration) # ═══════════════════════════════════════════════════════════════════════════ class TestFormEndpoints: - def _seed_template(self, client, mock_controller): - """Helper: create a template and return its ID.""" - resp = client.post(f"{API_PREFIX}/templates/create", json={ - "name": "Employee Form", - "pdf_path": "src/inputs/employee.pdf", - "fields": { - "Employee's name": "string", - "Employee's email": "string", - }, - }) - return resp.json()["id"] - - def test_fill_form_success(self, client, mock_controller): - tpl_id = self._seed_template(client, mock_controller) + def test_fill_form_success(self, client, mock_controller, seed_template): + tpl_id = seed_template() resp = client.post(f"{API_PREFIX}/forms/fill", json={ "template_id": tpl_id, @@ -202,8 +119,8 @@ def test_fill_form_missing_template(self, client, mock_controller): }) assert resp.status_code == 404 - def test_fill_form_template_file_not_found(self, client, mock_controller): - tpl_id = self._seed_template(client, mock_controller) + def test_fill_form_template_file_not_found(self, client, mock_controller, seed_template): + tpl_id = seed_template() mock_controller["form_ctrl"].fill_form.side_effect = FileNotFoundError("PDF template not found") resp = client.post(f"{API_PREFIX}/forms/fill", json={ @@ -283,9 +200,9 @@ def boom(*a, **k): assert resp.status_code == 200 assert resp.json()["models"] == ["qwen2.5:1.5b"] - def test_fill_form_passes_model_override(self, client, mock_controller): + def test_fill_form_passes_model_override(self, client, mock_controller, seed_template): """A `model` in the request reaches Controller.fill_form but isn't persisted.""" - tpl_id = self._seed_template(client, mock_controller) + tpl_id = seed_template() resp = client.post(f"{API_PREFIX}/forms/fill", json={ "template_id": tpl_id, "input_text": "John Doe", @@ -316,45 +233,27 @@ def fake_post(*args, **kwargs): class TestE2EPipeline: """ - Full pipeline: upload PDF → create template → fill form → verify DB state. - This is the critical path that the product depends on. + Legacy fill pipeline: seed template → fill form → verify DB state. + Template registration via API was removed in the contract migration, so the + template is seeded directly in the DB. """ - def test_full_flow(self, client, mock_controller, pdf_upload, tmp_path, monkeypatch, db): - # -- Step 1: Upload a PDF -- - monkeypatch.setattr("app.api.routes.templates.PROJECT_ROOT", tmp_path) - upload_resp = client.post( - f"{API_PREFIX}/templates/upload", - files=[pdf_upload], - data={"directory": str(tmp_path)}, - ) - assert upload_resp.status_code == 200 - uploaded_path = upload_resp.json()["pdf_path"] - assert uploaded_path.endswith(".pdf") - - # -- Step 2: Create a template from the uploaded PDF -- - create_resp = client.post(f"{API_PREFIX}/templates/create", json={ - "name": "Incident Report", - "pdf_path": uploaded_path, - "fields": { + def test_full_flow(self, client, mock_controller, seed_template, db): + # -- Step 1: Seed a template -- + template_id = seed_template( + name="Incident Report", + pdf_path="src/inputs/incident.pdf", + fields={ "Officer name": "string", "Badge number": "string", "Incident date": "string", "Location": "string", "Description": "string", }, - }) - assert create_resp.status_code == 200 - template_id = create_resp.json()["id"] + ) assert template_id is not None - # -- Step 3: Verify template appears in list -- - list_resp = client.get(f"{API_PREFIX}/templates") - assert list_resp.status_code == 200 - templates = list_resp.json() - assert any(t["id"] == template_id for t in templates) - - # -- Step 4: Fill the form -- + # -- Step 2: Fill the form -- fill_resp = client.post(f"{API_PREFIX}/forms/fill", json={ "template_id": template_id, "input_text": ( diff --git a/tests/test_deletion.py b/tests/test_deletion.py index 0d036dcb..cb03084b 100644 --- a/tests/test_deletion.py +++ b/tests/test_deletion.py @@ -1,5 +1,5 @@ -"""Tests for DELETE /api/v1/templates/{id}, DELETE /api/v1/forms/{id}, -POST /api/v1/forms/purge, and API-key access control. +"""Tests for DELETE /api/v1/forms/{id}, POST /api/v1/forms/purge, and API-key +access control. """ from datetime import datetime, timedelta, timezone @@ -13,14 +13,14 @@ # Helpers # --------------------------------------------------------------------------- -def _seed_template(client, name="T1", pdf_path="src/inputs/t.pdf"): - resp = client.post(f"{API_PREFIX}/templates/create", json={ - "name": name, - "pdf_path": pdf_path, - "fields": {"name": "string"}, - }) - assert resp.status_code == 200, resp.json() - return resp.json()["id"] +def _seed_template(db, name="T1", pdf_path="src/inputs/t.pdf"): + """Insert a legacy Template row directly — the /templates/create endpoint was + removed in the contract migration.""" + tpl = Template(name=name, pdf_path=pdf_path, fields={"name": "string"}) + db.add(tpl) + db.commit() + db.refresh(tpl) + return tpl.id def _seed_submission(db, template_id, output_pdf_path="src/outputs/out.pdf"): @@ -35,64 +35,6 @@ def _seed_submission(db, template_id, output_pdf_path="src/outputs/out.pdf"): return sub.id -# =========================================================================== -# DELETE /api/v1/templates/{template_id} -# =========================================================================== - -class TestDeleteTemplate: - - def test_delete_template_no_key_required_when_unconfigured(self, client): - """No API key needed when FIREFORM_API_KEY is empty (default).""" - tpl_id = _seed_template(client) - resp = client.delete(f"{API_PREFIX}/templates/{tpl_id}") - assert resp.status_code == 200 - body = resp.json() - assert body["status"] == "success" - - def test_delete_template_removes_from_db(self, client, db): - tpl_id = _seed_template(client) - client.delete(f"{API_PREFIX}/templates/{tpl_id}") - assert db.get(Template, tpl_id) is None - - def test_delete_template_not_found(self, client): - resp = client.delete(f"{API_PREFIX}/templates/99999") - assert resp.status_code == 404 - - def test_delete_template_cascades_submissions(self, client, db): - tpl_id = _seed_template(client) - sub_id = _seed_submission(db, tpl_id) - - client.delete(f"{API_PREFIX}/templates/{tpl_id}") - - assert db.get(FormSubmission, sub_id) is None - assert db.get(Template, tpl_id) is None - - def test_delete_template_deletes_pdf_file(self, client, tmp_path, monkeypatch): - """Verify the template PDF file is removed from disk on delete.""" - monkeypatch.setattr("app.api.routes.templates.PROJECT_ROOT", tmp_path) - pdf_file = tmp_path / "myform.pdf" - pdf_file.write_bytes(b"%PDF-1.4 fake") - - relative_path = "myform.pdf" - tpl_id = _seed_template(client, pdf_path=relative_path) - - client.delete(f"{API_PREFIX}/templates/{tpl_id}") - assert not pdf_file.exists() - - def test_delete_template_deletes_submission_output_pdfs(self, client, db, tmp_path, monkeypatch): - """Output PDFs of related submissions should be wiped on template deletion.""" - monkeypatch.setattr("app.api.routes.templates.PROJECT_ROOT", tmp_path) - - out_pdf = tmp_path / "filled.pdf" - out_pdf.write_bytes(b"%PDF-1.4 filled") - - tpl_id = _seed_template(client, pdf_path="tpl.pdf") - _seed_submission(db, tpl_id, output_pdf_path="filled.pdf") - - client.delete(f"{API_PREFIX}/templates/{tpl_id}") - assert not out_pdf.exists() - - # =========================================================================== # DELETE /api/v1/forms/{submission_id} # =========================================================================== @@ -100,14 +42,14 @@ def test_delete_template_deletes_submission_output_pdfs(self, client, db, tmp_pa class TestDeleteSubmission: def test_delete_submission_no_key_when_unconfigured(self, client, db): - tpl_id = _seed_template(client) + tpl_id = _seed_template(db) sub_id = _seed_submission(db, tpl_id) resp = client.delete(f"{API_PREFIX}/forms/{sub_id}") assert resp.status_code == 200 assert resp.json()["status"] == "success" def test_delete_submission_removes_from_db(self, client, db): - tpl_id = _seed_template(client) + tpl_id = _seed_template(db) sub_id = _seed_submission(db, tpl_id) client.delete(f"{API_PREFIX}/forms/{sub_id}") assert db.get(FormSubmission, sub_id) is None @@ -121,7 +63,7 @@ def test_delete_submission_removes_output_pdf(self, client, db, tmp_path, monkey out_pdf = tmp_path / "filled_out.pdf" out_pdf.write_bytes(b"%PDF-1.4") - tpl_id = _seed_template(client) + tpl_id = _seed_template(db) sub_id = _seed_submission(db, tpl_id, output_pdf_path="filled_out.pdf") client.delete(f"{API_PREFIX}/forms/{sub_id}") @@ -148,7 +90,7 @@ def _seed_old_submission(self, db, tpl_id, days_old=40): return sub.id def test_purge_removes_old_submissions(self, client, db): - tpl_id = _seed_template(client) + tpl_id = _seed_template(db) old_id = self._seed_old_submission(db, tpl_id, days_old=40) new_id = _seed_submission(db, tpl_id) # recent @@ -160,7 +102,7 @@ def test_purge_removes_old_submissions(self, client, db): assert db.get(FormSubmission, new_id) is not None def test_purge_nothing_to_remove(self, client, db): - tpl_id = _seed_template(client) + tpl_id = _seed_template(db) _seed_submission(db, tpl_id) # recent resp = client.post(f"{API_PREFIX}/forms/purge?days=30") assert resp.status_code == 200 @@ -171,7 +113,7 @@ def test_purge_removes_output_pdf_file(self, client, db, tmp_path, monkeypatch): out_pdf = tmp_path / "old_filled.pdf" out_pdf.write_bytes(b"%PDF-1.4") - tpl_id = _seed_template(client) + tpl_id = _seed_template(db) old_ts = datetime.now(timezone.utc) - timedelta(days=50) sub = FormSubmission( template_id=tpl_id, @@ -197,43 +139,14 @@ class TestApiKeyAccessControl: def _set_api_key(self, monkeypatch): monkeypatch.setattr("app.api.deps.FIREFORM_API_KEY", "secret-test-key") - def test_delete_template_requires_key_when_set(self, client): - tpl_id = _seed_template(client) - resp = client.delete(f"{API_PREFIX}/templates/{tpl_id}") - assert resp.status_code == 401 - - def test_delete_template_with_valid_x_api_key(self, client): - tpl_id = _seed_template(client) - resp = client.delete( - f"{API_PREFIX}/templates/{tpl_id}", - headers={"X-API-Key": "secret-test-key"}, - ) - assert resp.status_code == 200 - - def test_delete_template_with_bearer_token(self, client): - tpl_id = _seed_template(client) - resp = client.delete( - f"{API_PREFIX}/templates/{tpl_id}", - headers={"Authorization": "Bearer secret-test-key"}, - ) - assert resp.status_code == 200 - - def test_delete_template_wrong_key_rejected(self, client): - tpl_id = _seed_template(client) - resp = client.delete( - f"{API_PREFIX}/templates/{tpl_id}", - headers={"X-API-Key": "wrong-key"}, - ) - assert resp.status_code == 401 - def test_delete_submission_requires_key_when_set(self, client, db): - tpl_id = _seed_template(client) + tpl_id = _seed_template(db) sub_id = _seed_submission(db, tpl_id) resp = client.delete(f"{API_PREFIX}/forms/{sub_id}") assert resp.status_code == 401 def test_delete_submission_with_valid_key(self, client, db): - tpl_id = _seed_template(client) + tpl_id = _seed_template(db) sub_id = _seed_submission(db, tpl_id) resp = client.delete( f"{API_PREFIX}/forms/{sub_id}", diff --git a/tests/test_field_catalog.py b/tests/test_field_catalog.py new file mode 100644 index 00000000..33ff2388 --- /dev/null +++ b/tests/test_field_catalog.py @@ -0,0 +1,129 @@ +"""Tests for the incident-contract field catalog and GET /schema/fields. + +The catalog is built from contracts/schemas/incident-contract.yaml, so these +assert on fields that have been in the contract since it was written rather +than on exact counts, which move with every schema change. +""" + +from app.core.config import API_PREFIX +from app.services import field_catalog + +FIELDS_URL = f"{API_PREFIX}/schema/fields" + + +def _by_path(path): + return next((e for e in field_catalog.catalog() if e.path == path), None) + + +# --------------------------------------------------------------------------- +# Building the catalog +# --------------------------------------------------------------------------- +def test_catalog_flattens_nested_objects(): + entry = _by_path("location.postal_code") + assert entry is not None + assert entry.section == "location" + assert entry.field_type == "string" + assert entry.label == "Postal code" + + +def test_catalog_marks_array_hops(): + paths = [e.path for e in field_catalog.catalog()] + assert any(p.startswith("persons_involved[].") for p in paths) + + +def test_catalog_reads_aliases_from_the_contract(): + entry = _by_path("location.postal_code") + assert "zip" in entry.aliases + + +def test_catalog_reads_the_pii_flag(): + pii_paths = [e.path for e in field_catalog.catalog() if e.pii] + assert pii_paths, "the contract marks several fields x-pii" + + +def test_catalog_resolves_enum_references(): + with_enums = [e for e in field_catalog.catalog() if e.enum_values] + assert with_enums, "enum-typed fields should carry their values" + + +def test_schema_version_comes_from_the_contract(): + assert field_catalog.schema_version() + + +# --------------------------------------------------------------------------- +# Matching +# --------------------------------------------------------------------------- +def test_exact_name_wins(): + top = field_catalog.search("postal_code", limit=3)[0] + assert top[0].path == "location.postal_code" + assert top[1] == 1.0 + + +def test_alias_finds_the_field(): + top = field_catalog.search("zip", limit=3)[0] + assert top[0].path == "location.postal_code" + + +def test_an_exact_name_outranks_an_alias(): + entry = _by_path("location.postal_code") + # score_entry takes an already normalized query, the way search does. + name_score = field_catalog.score_entry(entry, field_catalog.normalize("postal_code")) + alias_score = field_catalog.score_entry(entry, "zip") + assert name_score > alias_score + + +def test_nonsense_query_returns_nothing(): + assert field_catalog.search("qwertyuiop asdf", limit=5) == [] + + +def test_search_respects_the_limit(): + assert len(field_catalog.search("date", limit=3)) <= 3 + + +def test_listing_returns_the_whole_catalog(): + # The editor caches the catalog and filters locally, so a bare listing + # must not be truncated by the search limit. + assert len(field_catalog.search(limit=5)) == len(field_catalog.catalog()) + + +def test_section_filter(): + hits = field_catalog.search(section="location") + assert hits + assert {entry.section for entry, _ in hits} == {"location"} + + +def test_label_normalization_expands_form_shorthand(): + assert field_catalog.normalize_label("Incident No.:") == "incident number" + assert field_catalog.normalize_label("Dt of Loss") == "date of loss" + + +# --------------------------------------------------------------------------- +# GET /schema/fields +# --------------------------------------------------------------------------- +def test_endpoint_searches(client): + resp = client.get(FIELDS_URL, params={"q": "zip", "limit": 5}) + assert resp.status_code == 200 + body = resp.json() + assert body["query"] == "zip" + assert body["schema_version"] + assert body["total"] == len(body["fields"]) + first = body["fields"][0] + assert first["path"] == "location.postal_code" + assert first["score"] > 0 + assert "zip" in first["aliases"] + + +def test_endpoint_lists_without_a_query(client): + body = client.get(FIELDS_URL).json() + assert body["query"] is None + assert body["total"] > 100 + assert body["fields"][0]["score"] is None + + +def test_endpoint_filters_by_section(client): + body = client.get(FIELDS_URL, params={"section": "location"}).json() + assert {f["section"] for f in body["fields"]} == {"location"} + + +def test_endpoint_rejects_an_oversized_limit(client): + assert client.get(FIELDS_URL, params={"q": "date", "limit": 500}).status_code == 422 diff --git a/tests/test_jobs.py b/tests/test_jobs.py index c79e4a42..d1c5504b 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -6,21 +6,13 @@ class TestJobEndpoints: - def _seed_template(self, client): - resp = client.post(f"{API_PREFIX}/templates/create", json={ - "name": "Test Template", - "pdf_path": "test.pdf", - "fields": {"name": "string"}, - }) - return resp.json()["id"] - @patch("app.api.routes.jobs.fill_form_task") - def test_submit_async_single(self, mock_task, client): + def test_submit_async_single(self, mock_task, client, seed_template): mock_result = MagicMock() mock_result.id = "celery-task-id-1" mock_task.delay.return_value = mock_result - tpl_id = self._seed_template(client) + tpl_id = seed_template() resp = client.post(f"{API_PREFIX}/forms/jobs", json={ "template_ids": [tpl_id], "input_text": "John Doe firefighter", @@ -34,14 +26,14 @@ def test_submit_async_single(self, mock_task, client): mock_task.delay.assert_called_once_with(tpl_id, "John Doe firefighter", None) @patch("app.api.routes.jobs.fill_form_task") - def test_submit_async_batch(self, mock_task, client): + def test_submit_async_batch(self, mock_task, client, seed_template): mock_task.delay.side_effect = [ MagicMock(id="task-1"), MagicMock(id="task-2"), ] - t1 = self._seed_template(client) - t2 = self._seed_template(client) + t1 = seed_template() + t2 = seed_template() resp = client.post(f"{API_PREFIX}/forms/jobs", json={ "template_ids": [t1, t2], "input_text": "batch input", @@ -62,10 +54,10 @@ def test_submit_async_missing_template(self, mock_task, client): mock_task.delay.assert_not_called() @patch("app.api.routes.jobs.fill_form_task") - def test_get_job_status(self, mock_task, client): + def test_get_job_status(self, mock_task, client, seed_template): mock_task.delay.return_value = MagicMock(id="celery-abc") - tpl_id = self._seed_template(client) + tpl_id = seed_template() submit_resp = client.post(f"{API_PREFIX}/forms/jobs", json={ "template_ids": [tpl_id], "input_text": "test input", @@ -85,10 +77,10 @@ def test_get_job_not_found(self, client): assert resp.status_code == 404 @patch("app.api.routes.jobs.fill_form_task") - def test_submit_with_model_override(self, mock_task, client): + def test_submit_with_model_override(self, mock_task, client, seed_template): mock_task.delay.return_value = MagicMock(id="celery-xyz") - tpl_id = self._seed_template(client) + tpl_id = seed_template() resp = client.post(f"{API_PREFIX}/forms/jobs", json={ "template_ids": [tpl_id], "input_text": "test", diff --git a/tests/test_migrations.py b/tests/test_migrations.py index b4860548..7e5927ea 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -41,6 +41,7 @@ def test_upgrade_head(alembic_cfg, alembic_engine): assert "incidents" in tables assert "forms" in tables assert "reports" in tables + assert "form_templates" in tables assert "alembic_version" in tables @@ -312,8 +313,8 @@ def test_reports_no_fk(alembic_cfg, alembic_engine): assert len(fks) == 0 -def test_downgrade_v1_tables(alembic_cfg, alembic_engine): - """Downgrade to 001 removes the v1 tables, leaving the 001 tables intact.""" +def test_downgrade_002(alembic_cfg, alembic_engine): + """Downgrade to 001 removes the 002, 003 and 004 tables, leaving 001 intact.""" command.upgrade(alembic_cfg, "head") command.downgrade(alembic_cfg, "001") @@ -324,6 +325,92 @@ def test_downgrade_v1_tables(alembic_cfg, alembic_engine): assert "incidents" not in tables assert "forms" not in tables assert "reports" not in tables + assert "form_templates" not in tables + assert "template_uploads" not in tables assert "template" in tables assert "formsubmission" in tables assert "job" in tables + + +# --------------------------------------------------------------------------- +# 004 — form_templates registry and template_uploads drafts +# --------------------------------------------------------------------------- + +def test_form_templates_columns(alembic_cfg, alembic_engine): + command.upgrade(alembic_cfg, "head") + + inspector = inspect(alembic_engine) + columns = {c["name"] for c in inspector.get_columns("form_templates")} + assert columns == { + "template_id", + "form_type", + "display_name", + "jurisdiction", + "agency_type", + "fields", + "source_standard", + "pdf_template_ref", + "version", + "status", + "created_at", + "updated_at", + } + + +def test_form_templates_unique_form_type(alembic_cfg, alembic_engine): + command.upgrade(alembic_cfg, "head") + + inspector = inspect(alembic_engine) + indexes = {ix["name"]: ix for ix in inspector.get_indexes("form_templates")} + assert indexes["ix_form_templates_form_type"]["unique"] + + +def test_form_templates_no_fk(alembic_cfg, alembic_engine): + """form_templates is a standalone registry — no FK to legacy template/incidents.""" + command.upgrade(alembic_cfg, "head") + + inspector = inspect(alembic_engine) + assert inspector.get_foreign_keys("form_templates") == [] + + +def test_template_uploads_columns(alembic_cfg, alembic_engine): + command.upgrade(alembic_cfg, "head") + + inspector = inspect(alembic_engine) + columns = {c["name"] for c in inspector.get_columns("template_uploads")} + assert columns == { + "upload_id", + "status", + "pdf_path", + "pdf_template_ref", + "original_filename", + "page_count", + "pages", + "detected_fields", + "detection_error", + "job_id", + "created_at", + "updated_at", + } + + +def test_template_uploads_no_fk(alembic_cfg, alembic_engine): + """Uploads are drafts, not templates, so nothing points at them yet.""" + command.upgrade(alembic_cfg, "head") + + inspector = inspect(alembic_engine) + assert inspector.get_foreign_keys("template_uploads") == [] + + +def test_downgrade_004(alembic_cfg, alembic_engine): + """Downgrade by one step removes both 004 tables, leaving 003 intact.""" + command.upgrade(alembic_cfg, "head") + command.downgrade(alembic_cfg, "-1") + + inspector = inspect(alembic_engine) + tables = inspector.get_table_names() + assert "form_templates" not in tables + assert "template_uploads" not in tables + assert "inputs" in tables + assert "forms" in tables + assert "reports" in tables diff --git a/tests/test_template_detection.py b/tests/test_template_detection.py new file mode 100644 index 00000000..01d13d5b --- /dev/null +++ b/tests/test_template_detection.py @@ -0,0 +1,360 @@ +"""Tests for template field detection (app/services/template_detection.py). + +commonforms is mocked throughout. What is exercised here is everything around +it: reading widget rectangles into layout boxes, naming fields, picking the +label next to a box, scoring mapping suggestions, and writing the draft back. +""" + +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from pypdf import PdfReader, PdfWriter +from pypdf.generic import ( + ArrayObject, + DictionaryObject, + FloatObject, + NameObject, + TextStringObject, +) +from sqlmodel import Session + +from app.api.schemas.enums import DetectionStatus, FieldSource +from app.api.schemas.templates import MappingSuggestion, TemplateFieldLayout +from app.db.repositories import create_job, create_template_upload, get_job_by_uuid +from app.models import Job, TemplateUpload +from app.services import template_detection as detection + + +# --------------------------------------------------------------------------- +# Fixtures: a small PDF carrying real form widgets +# --------------------------------------------------------------------------- +def _widget(name, rect): + return DictionaryObject( + { + NameObject("/Type"): NameObject("/Annot"), + NameObject("/Subtype"): NameObject("/Widget"), + NameObject("/FT"): NameObject("/Tx"), + NameObject("/Rect"): ArrayObject([FloatObject(v) for v in rect]), + NameObject("/T"): TextStringObject(name), + } + ) + + +@pytest.fixture +def widget_pdf(tmp_path): + """A one-page PDF with two text widgets, the lower one listed first.""" + writer = PdfWriter() + writer.add_blank_page(612, 792) + page = writer.pages[0] + annotations = ArrayObject() + for name, rect in ( + ("fire_cause", (188.33, 560.0, 388.33, 578.0)), + ("Incident No.", (188.33, 621.33, 315.66, 650.0)), + ): + annotations.append(writer._add_object(_widget(name, rect))) + page[NameObject("/Annots")] = annotations + + path = tmp_path / "form.pdf" + with path.open("wb") as handle: + writer.write(handle) + return path + + +@pytest.fixture +def flat_pdf(tmp_path): + """A page with no widgets at all, the case commonforms exists for.""" + writer = PdfWriter() + writer.add_blank_page(612, 792) + path = tmp_path / "flat.pdf" + with path.open("wb") as handle: + writer.write(handle) + return path + + +# --------------------------------------------------------------------------- +# Geometry +# --------------------------------------------------------------------------- +def test_read_pages_returns_points(widget_pdf): + pages = detection.read_pages(widget_pdf) + assert [p.model_dump() for p in pages] == [{"page": 0, "width": 612.0, "height": 792.0}] + + +def test_widget_rects_become_layout_boxes(widget_pdf): + drafts = detection.build_draft_fields(widget_pdf) + layout = drafts[0].field.layout + assert layout.page == 0 + assert layout.x == pytest.approx(188.33) + assert layout.y == pytest.approx(621.33) + assert layout.width == pytest.approx(127.33, abs=0.01) + assert layout.height == pytest.approx(28.67, abs=0.01) + + +def test_boxes_come_back_in_reading_order(widget_pdf): + drafts = detection.build_draft_fields(widget_pdf) + # The higher box on the page comes first even though it is second in the + # annotation array. + assert [d.field.layout.y for d in drafts] == sorted( + [d.field.layout.y for d in drafts], reverse=True + ) + + +# --------------------------------------------------------------------------- +# Field naming +# --------------------------------------------------------------------------- +def test_widget_names_are_turned_into_slugs(widget_pdf): + names = [d.field.field_name for d in detection.build_draft_fields(widget_pdf)] + assert names == ["incident_no", "fire_cause"] + + +def test_repeated_names_are_made_unique(): + used = set() + assert detection._field_name("Date", used, 1) == "date" + assert detection._field_name("Date", used, 2) == "date_2" + assert detection._field_name("Date", used, 3) == "date_3" + + +def test_an_unnamed_box_falls_back_to_its_position(): + assert detection._field_name(None, set(), 7) == "field_7" + + +# --------------------------------------------------------------------------- +# Labels +# --------------------------------------------------------------------------- +def _layout(**over): + base = {"page": 0, "x": 200.0, "y": 600.0, "width": 120.0, "height": 18.0} + base.update(over) + return TemplateFieldLayout(**base) + + +def test_label_to_the_left_is_preferred(): + texts = [("Incident No.", 120.0, 604.0), ("Section B", 200.0, 622.0)] + assert detection._nearest_label(_layout(), texts) == "Incident No." + + +def test_label_above_is_used_when_nothing_sits_to_the_left(): + texts = [("Incident No.", 200.0, 622.0)] + assert detection._nearest_label(_layout(), texts) == "Incident No." + + +def test_far_away_text_is_not_a_label(): + texts = [("Unrelated", 5.0, 604.0)] + assert detection._nearest_label(_layout(), texts) is None + + +def test_no_text_means_no_label(widget_pdf): + # The fixture PDF has widgets but no page text. + assert all(d.detected_label is None for d in detection.build_draft_fields(widget_pdf)) + + +# --------------------------------------------------------------------------- +# Mapping suggestions +# --------------------------------------------------------------------------- +def test_a_clear_label_gets_suggestions(): + suggestions = detection.suggest_mappings("Incident No.") + assert suggestions + assert suggestions[0].path == "report_metadata.incident_number" + assert suggestions[0].section == "report_metadata" + + +def test_a_meaningless_label_gets_nothing(): + assert detection.suggest_mappings("qwertyuiop") == [] + + +def test_a_missing_label_gets_nothing(): + assert detection.suggest_mappings(None) == [] + + +def test_a_confident_suggestion_is_pre_applied(widget_pdf): + drafts = detection.build_draft_fields(widget_pdf) + incident = drafts[0].field + # The widget is named "Incident No.", which scores high enough to apply. + assert incident.source == FieldSource.schema + assert incident.incident_mapping == "report_metadata.incident_number" + + +def test_a_weak_suggestion_is_only_offered(monkeypatch): + from app.api.schemas.templates import TemplateField + from app.api.schemas.enums import TemplateFieldType + + field = TemplateField( + field_name="box_1", + field_type=TemplateFieldType.string, + source=FieldSource.manual, + required=False, + ) + weak = [MappingSuggestion(path="location.postal_code", score=0.6)] + applied = detection._apply_suggestion(field, weak) + assert applied.source == FieldSource.manual + assert applied.incident_mapping is None + + +def test_an_applied_enum_mapping_brings_its_values(): + from app.api.schemas.templates import TemplateField + from app.api.schemas.enums import TemplateFieldType + + entry = next(e for e in detection.field_catalog.catalog() if e.enum_values) + field = TemplateField( + field_name="box_1", + field_type=TemplateFieldType.string, + source=FieldSource.manual, + required=False, + ) + applied = detection._apply_suggestion( + field, [MappingSuggestion(path=entry.path, score=0.99)] + ) + assert applied.field_type == TemplateFieldType.enum + assert applied.allowed_values == list(entry.enum_values) + + +# --------------------------------------------------------------------------- +# Running commonforms +# --------------------------------------------------------------------------- +def test_detection_skips_commonforms_when_widgets_exist(widget_pdf): + with patch("app.services.controller.Controller") as controller: + drafts = detection.detect_fields(widget_pdf) + controller.assert_not_called() + assert len(drafts) == 2 + + +def test_a_flat_pdf_goes_through_commonforms(flat_pdf, widget_pdf): + instance = MagicMock() + instance.prepare_fillable.return_value = str(widget_pdf) + with patch("app.services.controller.Controller", return_value=instance): + drafts = detection.detect_fields(flat_pdf) + assert len(drafts) == 2 + + +def test_a_one_page_pdf_is_padded_before_commonforms(flat_pdf, widget_pdf): + """commonforms cannot read a one-page document, so it never sees one.""" + seen = {} + + def capture(path): + seen["path"] = path + seen["pages"] = len(PdfReader(path).pages) + return str(widget_pdf) + + instance = MagicMock() + instance.prepare_fillable.side_effect = capture + with patch("app.services.controller.Controller", return_value=instance): + detection.detect_fields(flat_pdf) + + assert seen["path"] != str(flat_pdf) + assert seen["pages"] == 2 + # The padded copy and the fillable it produced are scratch, not artefacts. + assert not Path(seen["path"]).exists() + assert not widget_pdf.exists() + assert flat_pdf.exists() + + +def test_a_multi_page_pdf_is_passed_through_as_is(tmp_path, widget_pdf): + writer = PdfWriter() + writer.add_blank_page(612, 792) + writer.add_blank_page(612, 792) + flat = tmp_path / "two_pages.pdf" + with flat.open("wb") as handle: + writer.write(handle) + + instance = MagicMock() + instance.prepare_fillable.return_value = str(widget_pdf) + with patch("app.services.controller.Controller", return_value=instance): + detection.detect_fields(flat) + instance.prepare_fillable.assert_called_once_with(str(flat)) + + +def test_boxes_found_on_the_padding_are_dropped(flat_pdf, tmp_path): + """A box detected on the blank page belongs to no page of the real PDF.""" + writer = PdfWriter() + writer.add_blank_page(612, 792) + writer.add_blank_page(612, 792) + for page_ix, name in ((0, "real_box"), (1, "padding_box")): + page = writer.pages[page_ix] + annots = ArrayObject() + annots.append(writer._add_object(_widget(name, (100.0, 100.0, 200.0, 118.0)))) + page[NameObject("/Annots")] = annots + fillable = tmp_path / "detected.pdf" + with fillable.open("wb") as handle: + writer.write(handle) + + instance = MagicMock() + instance.prepare_fillable.return_value = str(fillable) + with patch("app.services.controller.Controller", return_value=instance): + drafts = detection.detect_fields(flat_pdf) + + assert [d.field.layout.page for d in drafts] == [0] + + +# --------------------------------------------------------------------------- +# The background run +# --------------------------------------------------------------------------- +def _seed_upload(session, pdf_path): + upload = TemplateUpload( + pdf_path=str(pdf_path), + pdf_template_ref="templates/uploads/x.pdf", + page_count=1, + pages=[{"page": 0, "width": 612.0, "height": 792.0}], + ) + return create_template_upload(session, upload) + + +def _seed_job(session): + return create_job( + session, Job(celery_task_id="t", job_type="template_field_detection", status="queued") + ) + + +def test_run_detection_writes_the_draft(test_engine, widget_pdf): + with Session(test_engine) as session: + upload = _seed_upload(session, widget_pdf) + job = _seed_job(session) + + result = detection.run_detection(session, upload.upload_id, job.job_id) + + assert result["status"] == "completed" + assert result["detected_fields"] == 2 + + session.refresh(upload) + assert upload.status == DetectionStatus.completed + assert len(upload.detected_fields) == 2 + assert upload.detected_fields[0]["field"]["layout"]["page"] == 0 + + finished = get_job_by_uuid(session, job.job_id) + assert finished.status == "completed" + assert finished.progress_percent == 100 + assert finished.result_url == f"/api/v1/templates/pdf/{upload.upload_id}" + + +def test_run_detection_records_a_failure_without_losing_the_upload(test_engine, flat_pdf): + with Session(test_engine) as session: + upload = _seed_upload(session, flat_pdf) + job = _seed_job(session) + + with patch.object(detection, "detect_fields", side_effect=RuntimeError("model gone")): + result = detection.run_detection(session, upload.upload_id, job.job_id) + + assert result["status"] == "failed" + session.refresh(upload) + assert upload.status == DetectionStatus.failed + assert upload.detection_error == "model gone" + # Geometry survives, so the editor can still be used by hand. + assert upload.page_count == 1 + + failed_job = get_job_by_uuid(session, job.job_id) + assert failed_job.status == "failed" + assert failed_job.error["error_code"] == "DETECTION_FAILED" + + +def test_run_detection_on_a_vanished_upload(test_engine): + from uuid import uuid4 + + with Session(test_engine) as session: + job = _seed_job(session) + result = detection.run_detection(session, uuid4(), job.job_id) + assert result["status"] == "failed" + assert get_job_by_uuid(session, job.job_id).error["error_code"] == "UPLOAD_NOT_FOUND" + + +def test_run_detection_without_a_job(test_engine, widget_pdf): + with Session(test_engine) as session: + upload = _seed_upload(session, widget_pdf) + assert detection.run_detection(session, upload.upload_id)["status"] == "completed" diff --git a/tests/test_templates_pdf.py b/tests/test_templates_pdf.py new file mode 100644 index 00000000..b6eedc20 --- /dev/null +++ b/tests/test_templates_pdf.py @@ -0,0 +1,250 @@ +"""Tests for the template PDF authoring flow. + +Covers POST /templates/pdf, GET /templates/pdf/{upload_id} and +GET /templates/{template_id}/pdf. commonforms never runs here: detection is +dispatched to Celery, and these check the parts around it. +""" + +import io +from unittest.mock import MagicMock, patch +from uuid import uuid4 + +import pytest +from sqlmodel import Session + +from app.api.schemas.enums import DetectionStatus +from app.core.config import API_PREFIX +from app.models import FormTemplate, TemplateUpload + +TEMPLATES_URL = f"{API_PREFIX}/templates" + + +@pytest.fixture +def upload_dir(tmp_path, monkeypatch): + """Send stored PDFs to a temp directory instead of the real data dir.""" + target = tmp_path / "templates" / "uploads" + monkeypatch.setattr("app.services.form_templates.TEMPLATE_UPLOAD_DIR", target) + monkeypatch.setattr("app.services.form_templates.DATA_DIR", tmp_path) + return target + + +@pytest.fixture +def no_celery(): + """Stand in for the detection task so nothing is dispatched to a broker.""" + with patch("app.services.form_templates.detect_template_fields_task") as task: + task.delay.return_value = MagicMock(id="celery-task-1") + yield task + + +def _files(pdf_bytes, name="texas_sfm.pdf"): + return {"pdf_file": (name, io.BytesIO(pdf_bytes), "application/pdf")} + + +def _upload(client, pdf_bytes, detect=True, name="texas_sfm.pdf"): + return client.post( + f"{TEMPLATES_URL}/pdf", + files=_files(pdf_bytes, name), + data={"detect_fields": str(detect).lower()}, + ) + + +# --------------------------------------------------------------------------- +# POST /templates/pdf +# --------------------------------------------------------------------------- +def test_upload_returns_202_with_geometry_and_poll_url(client, pdf_bytes, upload_dir, no_celery): + resp = _upload(client, pdf_bytes) + assert resp.status_code == 202, resp.json() + body = resp.json() + + assert body["status"] == "processing" + assert body["page_count"] == 1 + assert body["pages"] == [{"page": 0, "width": 612.0, "height": 792.0}] + assert body["original_filename"] == "texas_sfm.pdf" + assert body["pdf_template_ref"].endswith(".pdf") + assert body["poll_url"] == f"/api/v1/templates/pdf/{body['upload_id']}" + assert body["job_id"] + assert body["retry_after_seconds"] == 5 + # Detection has not run, so there is no field list yet. + assert body["detected_fields"] is None + + +def test_upload_stores_the_pdf_on_disk(client, pdf_bytes, upload_dir, no_celery): + body = _upload(client, pdf_bytes).json() + stored = upload_dir / f"{body['upload_id']}.pdf" + assert stored.read_bytes() == pdf_bytes + + +def test_upload_dispatches_detection(client, pdf_bytes, upload_dir, no_celery): + body = _upload(client, pdf_bytes).json() + no_celery.delay.assert_called_once_with(body["upload_id"], body["job_id"]) + + +def test_upload_without_detection_completes_immediately(client, pdf_bytes, upload_dir, no_celery): + body = _upload(client, pdf_bytes, detect=False).json() + assert body["status"] == "completed" + assert body["detected_fields"] == [] + assert body["job_id"] is None + assert body["retry_after_seconds"] is None + no_celery.delay.assert_not_called() + + +def test_upload_rejects_a_non_pdf(client, upload_dir, no_celery): + resp = client.post(f"{TEMPLATES_URL}/pdf", files=_files(b"just some text", "notes.pdf")) + assert resp.status_code == 415 + assert resp.json()["error_code"] == "UNSUPPORTED_FORMAT" + + +def test_upload_rejects_an_empty_file(client, upload_dir, no_celery): + resp = client.post(f"{TEMPLATES_URL}/pdf", files=_files(b"", "empty.pdf")) + assert resp.status_code == 400 + assert resp.json()["error_code"] == "MISSING_FILE" + + +def test_upload_rejects_an_oversized_pdf(client, upload_dir, no_celery, monkeypatch): + monkeypatch.setattr("app.api.routes.form_templates.MAX_TEMPLATE_PDF_BYTES", 10) + resp = client.post(f"{TEMPLATES_URL}/pdf", files=_files(b"%PDF-1.4 padded out here")) + assert resp.status_code == 413 + assert resp.json()["error_code"] == "FILE_TOO_LARGE" + + +def test_upload_without_a_file_is_a_422(client, upload_dir, no_celery): + assert client.post(f"{TEMPLATES_URL}/pdf").status_code == 422 + + +def test_a_pdf_that_cannot_be_read_is_a_415(client, upload_dir, no_celery): + # Right magic bytes, nothing behind them. + resp = client.post(f"{TEMPLATES_URL}/pdf", files=_files(b"%PDF-1.4 truncated")) + assert resp.status_code == 415 + assert resp.json()["error_code"] == "INVALID_PDF" + assert list(upload_dir.glob("*.pdf")) == [] + + +# --------------------------------------------------------------------------- +# GET /templates/pdf/{upload_id} +# --------------------------------------------------------------------------- +def test_draft_poll_returns_the_stored_state(client, pdf_bytes, upload_dir, no_celery): + upload_id = _upload(client, pdf_bytes).json()["upload_id"] + resp = client.get(f"{TEMPLATES_URL}/pdf/{upload_id}") + assert resp.status_code == 200 + body = resp.json() + assert body["upload_id"] == upload_id + assert body["status"] == "processing" + assert body["retry_after_seconds"] == 5 + + +def test_draft_poll_after_detection(client, pdf_bytes, upload_dir, no_celery, test_engine): + upload_id = _upload(client, pdf_bytes).json()["upload_id"] + + with Session(test_engine) as session: + upload = session.get(TemplateUpload, __import__("uuid").UUID(upload_id)) + upload.status = DetectionStatus.completed + upload.detected_fields = [ + { + "field": { + "field_name": "incident_number", + "field_type": "string", + "source": "schema", + "required": False, + "incident_mapping": "report_metadata.incident_number", + "layout": {"page": 0, "x": 10, "y": 20, "width": 100, "height": 18}, + }, + "detected_label": "Incident No.", + "suggestions": [ + {"path": "report_metadata.incident_number", "score": 0.93}, + ], + } + ] + session.add(upload) + session.commit() + + body = client.get(f"{TEMPLATES_URL}/pdf/{upload_id}").json() + assert body["status"] == "completed" + assert body["retry_after_seconds"] is None + field = body["detected_fields"][0] + assert field["detected_label"] == "Incident No." + assert field["field"]["incident_mapping"] == "report_metadata.incident_number" + assert field["suggestions"][0]["score"] == 0.93 + + +def test_draft_poll_reports_a_failed_detection(client, pdf_bytes, upload_dir, no_celery, test_engine): + from uuid import UUID + + upload_id = _upload(client, pdf_bytes).json()["upload_id"] + with Session(test_engine) as session: + upload = session.get(TemplateUpload, UUID(upload_id)) + upload.status = DetectionStatus.failed + upload.detection_error = "model download failed" + session.add(upload) + session.commit() + + body = client.get(f"{TEMPLATES_URL}/pdf/{upload_id}").json() + assert body["status"] == "failed" + assert body["detection_error"] == "model download failed" + # The upload is still usable, geometry and all. + assert body["page_count"] == 1 + + +def test_draft_poll_unknown_upload_is_404(client): + resp = client.get(f"{TEMPLATES_URL}/pdf/{uuid4()}") + assert resp.status_code == 404 + assert resp.json()["error_code"] == "UPLOAD_NOT_FOUND" + + +def test_pdf_path_does_not_shadow_a_template_id(client, pdf_bytes, upload_dir, no_celery): + """The literal /pdf route has to win over /{template_id}.""" + assert _upload(client, pdf_bytes).status_code == 202 + + +# --------------------------------------------------------------------------- +# GET /templates/{template_id}/pdf +# --------------------------------------------------------------------------- +def _seed_template(session, pdf_template_ref): + template = FormTemplate( + form_type="state_texas", + display_name="Texas SFM", + fields=[], + pdf_template_ref=pdf_template_ref, + ) + session.add(template) + session.commit() + session.refresh(template) + return template.template_id + + +def test_download_source_pdf(client, pdf_bytes, upload_dir, no_celery, test_engine, tmp_path): + upload = _upload(client, pdf_bytes).json() + with Session(test_engine) as session: + template_id = _seed_template(session, upload["pdf_template_ref"]) + + resp = client.get(f"{TEMPLATES_URL}/{template_id}/pdf") + assert resp.status_code == 200 + assert resp.headers["content-type"] == "application/pdf" + assert resp.content == pdf_bytes + + +def test_download_without_a_source_pdf_is_404(client, upload_dir, test_engine): + with Session(test_engine) as session: + template_id = _seed_template(session, None) + resp = client.get(f"{TEMPLATES_URL}/{template_id}/pdf") + assert resp.status_code == 404 + assert resp.json()["error_code"] == "TEMPLATE_PDF_NOT_FOUND" + + +def test_download_missing_file_is_404(client, upload_dir, test_engine): + with Session(test_engine) as session: + template_id = _seed_template(session, "templates/uploads/gone.pdf") + assert client.get(f"{TEMPLATES_URL}/{template_id}/pdf").status_code == 404 + + +def test_download_cannot_escape_the_data_directory(client, upload_dir, test_engine, tmp_path): + outside = tmp_path.parent / "secret.pdf" + outside.write_bytes(b"%PDF-1.4 secret") + with Session(test_engine) as session: + template_id = _seed_template(session, "../secret.pdf") + + resp = client.get(f"{TEMPLATES_URL}/{template_id}/pdf") + assert resp.status_code == 404 + + +def test_download_unknown_template_is_404(client): + assert client.get(f"{TEMPLATES_URL}/{uuid4()}/pdf").status_code == 404 diff --git a/tests/test_templates_v1.py b/tests/test_templates_v1.py new file mode 100644 index 00000000..50467a6b --- /dev/null +++ b/tests/test_templates_v1.py @@ -0,0 +1,379 @@ +"""Tests for the contract Layer 6 template registry (app/api/routes/form_templates.py). + +Covers list / create / get / replace / fields against the in-memory DB. +""" + +from app.core.config import API_PREFIX + +TEMPLATES_URL = f"{API_PREFIX}/templates" + + +def _layout(**over) -> dict: + base = {"page": 0, "x": 188.33, "y": 621.33, "width": 127.33, "height": 28.67} + base.update(over) + return base + + +def _payload(form_type: str = "state_texas") -> dict: + return { + "form_type": form_type, + "display_name": "Texas State Fire Marshal Incident Report", + "jurisdiction": "US-TX", + "agency_type": "fire_department", + "fields": [ + { + "field_name": "incident_number", + "field_type": "string", + "source": "schema", + "required": True, + "max_length": 20, + "description": "State-assigned incident number", + "incident_mapping": "report_metadata.incident_number", + "layout": _layout(font="Helvetica", font_size=10, color="#000000", align="left"), + }, + { + "field_name": "fire_cause", + "field_type": "enum", + "source": "schema", + "required": False, + "allowed_values": ["accidental", "natural", "intentional", "undetermined"], + "incident_mapping": "fire.cause_category", + "layout": _layout(y=560.0), + }, + ], + "source_standard": "Texas SFM 2026", + } + + +def _create(client, **overrides): + body = _payload(**overrides) + return client.post(TEMPLATES_URL, json=body) + + +def test_list_empty(client): + resp = client.get(TEMPLATES_URL) + assert resp.status_code == 200 + assert resp.json() == [] + + +def test_create_returns_201_with_server_fields(client): + resp = _create(client) + assert resp.status_code == 201 + body = resp.json() + + assert body["form_type"] == "state_texas" + assert body["display_name"] == "Texas State Fire Marshal Incident Report" + assert body["field_count"] == 2 + assert body["status"] == "active" + assert body["version"] == "1.0" + assert body["template_id"] + assert body["last_updated"] + assert body["created_at"] + assert len(body["fields"]) == 2 + + +def test_create_duplicate_form_type_returns_409(client): + assert _create(client).status_code == 201 + dup = _create(client) + assert dup.status_code == 409 + assert dup.json()["error_code"] == "TEMPLATE_EXISTS" + + +def test_create_then_list(client): + _create(client) + resp = client.get(TEMPLATES_URL) + assert resp.status_code == 200 + items = resp.json() + assert len(items) == 1 + assert items[0]["form_type"] == "state_texas" + assert items[0]["field_count"] == 2 + # Summary is a projection — no full field list. + assert "fields" not in items[0] + + +def test_get_by_id(client): + template_id = _create(client).json()["template_id"] + resp = client.get(f"{TEMPLATES_URL}/{template_id}") + assert resp.status_code == 200 + body = resp.json() + assert body["template_id"] == template_id + first = body["fields"][0] + assert first["incident_mapping"] == "report_metadata.incident_number" + assert first["layout"]["x"] == 188.33 + assert first["layout"]["align"] == "left" + + +def test_get_missing_returns_404(client): + resp = client.get(f"{TEMPLATES_URL}/550e8400-e29b-41d4-a716-446655440099") + assert resp.status_code == 404 + assert resp.json()["error_code"] == "TEMPLATE_NOT_FOUND" + + +def test_get_invalid_uuid_returns_422(client): + assert client.get(f"{TEMPLATES_URL}/not-a-uuid").status_code == 422 + + +def test_replace_updates_fields(client): + template_id = _create(client).json()["template_id"] + + updated = _payload() + updated["display_name"] = "Texas SFM Incident Report v2" + updated["fields"] = [updated["fields"][0]] # drop one field + + resp = client.put(f"{TEMPLATES_URL}/{template_id}", json=updated) + assert resp.status_code == 200 + body = resp.json() + assert body["display_name"] == "Texas SFM Incident Report v2" + assert body["field_count"] == 1 + assert body["template_id"] == template_id + + +def test_replace_onto_a_taken_form_type_returns_409(client): + first = _create(client, form_type="tx_sfm_incident").json()["template_id"] + _create(client, form_type="tx_sfm_casualty") + + resp = client.put( + f"{TEMPLATES_URL}/{first}", json=_payload(form_type="tx_sfm_casualty") + ) + assert resp.status_code == 409 + assert resp.json()["error_code"] == "TEMPLATE_EXISTS" + + +def test_replace_keeping_its_own_form_type_is_not_a_conflict(client): + template_id = _create(client).json()["template_id"] + resp = client.put(f"{TEMPLATES_URL}/{template_id}", json=_payload()) + assert resp.status_code == 200 + + +def test_replace_missing_returns_404(client): + resp = client.put( + f"{TEMPLATES_URL}/550e8400-e29b-41d4-a716-446655440099", json=_payload() + ) + assert resp.status_code == 404 + + +def test_create_missing_required_field_returns_422(client): + body = _payload() + del body["fields"] + assert client.post(TEMPLATES_URL, json=body).status_code == 422 + + +def test_fields_endpoint(client): + template_id = _create(client).json()["template_id"] + resp = client.get(f"{TEMPLATES_URL}/{template_id}/fields") + assert resp.status_code == 200 + body = resp.json() + assert body["total_fields"] == 2 + assert body["required_fields"] == 1 + assert body["optional_fields"] == 1 + assert len(body["fields"]) == 2 + assert body["form_type"] == "state_texas" + + +def test_fields_required_only(client): + template_id = _create(client).json()["template_id"] + resp = client.get(f"{TEMPLATES_URL}/{template_id}/fields?required_only=true") + assert resp.status_code == 200 + body = resp.json() + assert body["total_fields"] == 2 + assert body["required_fields"] == 1 + assert len(body["fields"]) == 1 + assert body["fields"][0]["field_name"] == "incident_number" + + +def test_fields_missing_returns_404(client): + resp = client.get( + f"{TEMPLATES_URL}/550e8400-e29b-41d4-a716-446655440099/fields" + ) + assert resp.status_code == 404 + + +# --------------------------------------------------------------------------- +# Validation (all 422 with the contract error envelope) +# --------------------------------------------------------------------------- +def _assert_422(resp): + assert resp.status_code == 422, resp.json() + body = resp.json() + assert body["error_code"] == "VALIDATION_ERROR" + assert len(body["validation_errors"]) >= 1 + + +def test_jurisdiction_optional(client): + body = _payload() + del body["jurisdiction"] + resp = client.post(TEMPLATES_URL, json=body) + assert resp.status_code == 201 + assert resp.json()["jurisdiction"] is None + + +def test_static_text_field_ok(client): + body = _payload() + body["fields"].append({ + "field_name": "footer", + "field_type": "string", + "source": "static", + "required": False, + "static_text": "Generated by FireForm", + "layout": _layout(y=40.0, align="center"), + }) + resp = client.post(TEMPLATES_URL, json=body) + assert resp.status_code == 201 + assert resp.json()["field_count"] == 3 + + +def test_schema_field_without_mapping_rejected(client): + body = _payload() + del body["fields"][0]["incident_mapping"] + _assert_422(client.post(TEMPLATES_URL, json=body)) + + +def test_static_text_on_a_schema_field_rejected(client): + body = _payload() + body["fields"][0]["static_text"] = "x" # source is schema + _assert_422(client.post(TEMPLATES_URL, json=body)) + + +def test_static_field_without_text_rejected(client): + body = _payload() + body["fields"].append({ + "field_name": "footer", + "field_type": "string", + "source": "static", + "required": False, + "layout": _layout(y=40.0), + }) + _assert_422(client.post(TEMPLATES_URL, json=body)) + + +def test_mapping_on_a_manual_field_rejected(client): + body = _payload() + body["fields"].append({ + "field_name": "marshal_name", + "field_type": "string", + "source": "manual", + "required": True, + "incident_mapping": "report_metadata.incident_number", + "layout": _layout(y=120.0), + }) + _assert_422(client.post(TEMPLATES_URL, json=body)) + + +def test_manual_field_needs_nothing_else(client): + body = _payload() + body["fields"].append({ + "field_name": "marshal_name", + "field_type": "string", + "source": "manual", + "required": True, + "layout": _layout(y=120.0), + }) + resp = client.post(TEMPLATES_URL, json=body) + assert resp.status_code == 201, resp.json() + assert resp.json()["field_count"] == 3 + + +def test_open_field_without_description_rejected(client): + body = _payload() + body["fields"].append({ + "field_name": "insurance_company", + "field_type": "string", + "source": "open", + "required": False, + "layout": _layout(y=520.0), + }) + _assert_422(client.post(TEMPLATES_URL, json=body)) + + +def test_open_field_with_description_ok(client): + body = _payload() + body["fields"].append({ + "field_name": "insurance_company", + "field_type": "string", + "source": "open", + "required": False, + "description": "Name of the insurance company covering the property", + "layout": _layout(y=520.0), + }) + resp = client.post(TEMPLATES_URL, json=body) + assert resp.status_code == 201, resp.json() + + +def test_missing_source_rejected(client): + body = _payload() + del body["fields"][0]["source"] + _assert_422(client.post(TEMPLATES_URL, json=body)) + + +def test_unit_is_kept(client): + body = _payload() + body["fields"][0]["unit"] = "acres" + resp = client.post(TEMPLATES_URL, json=body) + assert resp.status_code == 201 + assert resp.json()["fields"][0]["unit"] == "acres" + + +def test_negative_min_value_allowed(client): + body = _payload() + body["fields"][0]["min_value"] = -40 + body["fields"][0]["max_value"] = 50 + assert client.post(TEMPLATES_URL, json=body).status_code == 201 + + +def test_enum_without_allowed_values_rejected(client): + body = _payload() + del body["fields"][1]["allowed_values"] # fire_cause is enum + _assert_422(client.post(TEMPLATES_URL, json=body)) + + +def test_duplicate_field_names_rejected(client): + body = _payload() + body["fields"][1]["field_name"] = body["fields"][0]["field_name"] + _assert_422(client.post(TEMPLATES_URL, json=body)) + + +def test_empty_fields_rejected(client): + body = _payload() + body["fields"] = [] + _assert_422(client.post(TEMPLATES_URL, json=body)) + + +def test_bad_form_type_rejected(client): + _assert_422(_create(client, form_type="State Texas!")) + + +def test_min_greater_than_max_rejected(client): + body = _payload() + body["fields"][0]["min_value"] = 10 + body["fields"][0]["max_value"] = 5 + _assert_422(client.post(TEMPLATES_URL, json=body)) + + +def test_layout_bad_color_rejected(client): + body = _payload() + body["fields"][0]["layout"]["color"] = "black" + _assert_422(client.post(TEMPLATES_URL, json=body)) + + +def test_layout_missing_required_coord_rejected(client): + body = _payload() + del body["fields"][0]["layout"]["width"] + _assert_422(client.post(TEMPLATES_URL, json=body)) + + +def test_layout_negative_coordinate_rejected(client): + body = _payload() + body["fields"][0]["layout"]["x"] = -5 + _assert_422(client.post(TEMPLATES_URL, json=body)) + + +def test_layout_zero_width_rejected(client): + body = _payload() + body["fields"][0]["layout"]["width"] = 0 + _assert_422(client.post(TEMPLATES_URL, json=body)) + + +def test_replace_validates_body(client): + template_id = _create(client).json()["template_id"] + bad = _payload() + bad["fields"][0]["layout"]["color"] = "nope" + _assert_422(client.put(f"{TEMPLATES_URL}/{template_id}", json=bad))