diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 22edb610..644e5a8e 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -4,7 +4,7 @@ on: push: branches: [main,development] pull_request: - branches: [main,development] + branches: [main, development, 'development-*'] jobs: lint: @@ -23,7 +23,7 @@ jobs: uses: astral-sh/setup-uv@v6 - name: Install linter - run: uv pip install --system ruff + run: uv pip install --system ruff==0.16.1 - name: Run linter run: | diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 281b6b16..af033e0f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -9,7 +9,7 @@ on: - 'alembic.ini' - '.github/workflows/tests.yml' pull_request: - branches: [main,development] + branches: [main, development, 'development-*'] paths: - '**.py' - 'requirements.txt' diff --git a/alembic/versions/003_formsubmission_input_id_fk.py b/alembic/versions/003_formsubmission_input_id_fk.py new file mode 100644 index 00000000..303b0a51 --- /dev/null +++ b/alembic/versions/003_formsubmission_input_id_fk.py @@ -0,0 +1,42 @@ +"""formsubmission.input_id FK to inputs. + +Revision ID: 003 +Revises: 002 +Create Date: 2026-08-07 + +Adds a nullable input_id FK on formsubmission -> inputs.input_id so submissions +can link to the Input they were filled from, instead of only duplicating the +transcript into input_text. input_text is kept for now (read by the +/forms/submissions and analytics endpoints); dropping it is a deferred +follow-up once those readers move to the FK. +""" + +from collections.abc import Sequence + +from alembic import op +import sqlalchemy as sa + +revision: str = "003" +down_revision: str | None = "002" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + # batch_alter_table: SQLite can't ALTER a table to add a FK constraint in + # place (no ALTER-constraint support), so this goes through its + # copy-and-move strategy. On Postgres it emits a plain ALTER TABLE. + with op.batch_alter_table("formsubmission") as batch_op: + batch_op.add_column(sa.Column("input_id", sa.Uuid(), nullable=True)) + batch_op.create_foreign_key( + "fk_formsubmission_input_id_inputs", + "inputs", + ["input_id"], + ["input_id"], + ) + + +def downgrade() -> None: + with op.batch_alter_table("formsubmission") as batch_op: + batch_op.drop_constraint("fk_formsubmission_input_id_inputs", type_="foreignkey") + batch_op.drop_column("input_id") diff --git a/app/api/routes/forms.py b/app/api/routes/forms.py index be160291..24c0ce81 100644 --- a/app/api/routes/forms.py +++ b/app/api/routes/forms.py @@ -1,8 +1,6 @@ -from datetime import datetime, timedelta, timezone -from pathlib import Path import requests from fastapi import APIRouter, Depends, File, UploadFile, Query -from sqlmodel import Session, select +from sqlmodel import Session from app.api.deps import get_db, verify_api_key from app.api.schemas.forms import ( @@ -11,30 +9,12 @@ ModelsResponse, TranscriptionResponse, ) -from app.core.config import OLLAMA_HOST, OLLAMA_MODEL, BASE_DIR, RETENTION_PERIOD_DAYS +from app.core import paths +from app.core.config import OLLAMA_HOST, OLLAMA_MODEL, RETENTION_PERIOD_DAYS from app.services.whisper import call_whisper_asr from app.core.errors.base import AppError -from app.db.repositories import create_form, get_template, get_form_submission, delete_form_submission -from app.models import FormSubmission, Template -from app.services.controller import Controller - -PROJECT_ROOT = BASE_DIR - -def _resolve_project_file(file_path: str) -> Path: - raw_path = (file_path or "").strip() - if not raw_path: - raise AppError("Path is required", status_code=400) - - candidate = Path(raw_path) - if not candidate.is_absolute(): - candidate = (PROJECT_ROOT / candidate).resolve() - else: - candidate = candidate.resolve() - - if candidate != PROJECT_ROOT and PROJECT_ROOT not in candidate.parents: - raise AppError("Path must be inside the project", status_code=400) - - return candidate +from app.db.repositories import get_template, get_form_submission, delete_form_submission +from app.services.form import FormService router = APIRouter(prefix="/forms", tags=["forms"]) @@ -46,20 +26,11 @@ def fill_form(form: FormFill, db: Session = Depends(get_db)): if not fetched_template: raise AppError("Template not found", status_code=404, error_code="TEMPLATE_NOT_FOUND") - controller = Controller() + svc = FormService() try: - path = controller.fill_form( - user_input=form.input_text, - fields=fetched_template.fields, - pdf_form_path=fetched_template.pdf_path, - model=form.model, - ) - - # `model` is a runtime override, not a column — keep it out of the DB row. - submission = FormSubmission( - **form.model_dump(exclude={"model"}), output_pdf_path=path - ) - return create_form(db, submission) + return svc.fill_form(db, template=fetched_template, input_id=form.input_id, model=form.model) + except AppError: + raise except Exception as e: raise AppError(str(e), status_code=500, error_code="FORM_FILL_ERROR") @@ -117,7 +88,7 @@ def delete_submission_endpoint(submission_id: int, db: Session = Depends(get_db) if sub.output_pdf_path: try: - resolved_out = _resolve_project_file(sub.output_pdf_path) + resolved_out = paths._resolve_project_file(sub.output_pdf_path) if resolved_out.exists() and resolved_out.is_file(): resolved_out.unlink() except Exception: @@ -130,99 +101,15 @@ def delete_submission_endpoint(submission_id: int, db: Session = Depends(get_db) @router.post("/purge", dependencies=[Depends(verify_api_key)]) def purge_submissions_endpoint(days: int = Query(default=None), db: Session = Depends(get_db)): retention_days = days if days is not None else RETENTION_PERIOD_DAYS - cutoff_date = datetime.now(timezone.utc) - timedelta(days=retention_days) - - statement = select(FormSubmission).where(FormSubmission.created_at < cutoff_date) - submissions = list(db.exec(statement)) - - purged_count = 0 - for sub in submissions: - if sub.output_pdf_path: - try: - resolved_out = _resolve_project_file(sub.output_pdf_path) - if resolved_out.exists() and resolved_out.is_file(): - resolved_out.unlink() - except Exception: - pass - delete_form_submission(db, sub) - purged_count += 1 - + purged_count = FormService().purge_submissions(db, retention_days) return {"status": "success", "purged_count": purged_count, "retention_days_used": retention_days} - @router.get("/submissions") def get_submissions(db: Session = Depends(get_db)): - from sqlmodel import select - statement = ( - select(FormSubmission, Template.name) - .join(Template, FormSubmission.template_id == Template.id, isouter=True) - .order_by(FormSubmission.created_at.desc(), FormSubmission.id.desc()) - ) - results = db.exec(statement).all() - return [ - { - "id": sub.id, - "template_id": sub.template_id, - "template_name": name or "Unknown Template", - "input_text": sub.input_text, - "output_pdf_path": sub.output_pdf_path, - "created_at": sub.created_at.isoformat() if sub.created_at else None, - } - for sub, name in results - ] + return FormService().list_submissions(db) @router.get("/submissions/analytics") def get_submissions_analytics(db: Session = Depends(get_db)): - from collections import Counter - import re - from sqlmodel import select - - statement = select(FormSubmission, Template.name).join( - Template, FormSubmission.template_id == Template.id, isouter=True - ) - results = db.exec(statement).all() - - total_submissions = len(results) - - template_counts = Counter() - daily_counts = Counter() - words = [] - - stopwords = { - "the", "and", "a", "of", "to", "in", "is", "that", "it", "was", "for", "on", - "as", "with", "by", "at", "an", "be", "this", "are", "from", "or", "have", - "has", "had", "but", "not", "he", "she", "they", "we", "i", "you", "my", "his", - "her", "their", "our", "me", "him", "them", "us", "about", "there", "their", - "were", "been", "would", "could", "should", "will", "can", "no", "yes", "any", - "so", "very", "patient", "presents", "with", "reported", "history", "shows", - "left", "right", "pain", "due", "after", "before", "emergency", "department", - "medical", "clinical" - } - - for sub, name in results: - template_name = name or "Unknown Template" - template_counts[template_name] += 1 - - if sub.created_at: - date_str = sub.created_at.strftime("%Y-%m-%d") - daily_counts[date_str] += 1 - - if sub.input_text: - found_words = re.findall(r"\b[a-zA-Z]{3,15}\b", sub.input_text.lower()) - for w in found_words: - if w not in stopwords: - words.append(w) - - sorted_daily = [{"date": k, "count": v} for k, v in sorted(daily_counts.items())] - sorted_templates = [{"template_name": k, "count": v} for k, v in template_counts.most_common()] - common_terms = [{"word": k, "count": v} for k, v in Counter(words).most_common(12)] - - return { - "total_submissions": total_submissions, - "by_template": sorted_templates, - "by_date": sorted_daily, - "common_terms": common_terms, - } - + return FormService().get_analytics(db) diff --git a/app/api/routes/jobs.py b/app/api/routes/jobs.py index 5af90087..4a4f3826 100644 --- a/app/api/routes/jobs.py +++ b/app/api/routes/jobs.py @@ -11,6 +11,7 @@ from app.core.errors.base import AppError from app.db.repositories import create_job, get_job_by_uuid, get_template from app.models import Job +from app.services.input import InputService from app.tasks.fill import fill_form_task router = APIRouter(tags=["jobs"]) @@ -39,14 +40,16 @@ def submit_async_form_fill(form: AsyncFormFill, db: Session = Depends(get_db)): if not get_template(db, tid): raise AppError(f"Template {tid} not found", status_code=404) + transcript = InputService().resolve_transcript(db, form.input_id) + jobs: list[AsyncJobSubmitResponse] = [] for tid in form.template_ids: - result = fill_form_task.delay(tid, form.input_text, form.model) + result = fill_form_task.delay(tid, transcript, str(form.input_id), form.model) job = Job( celery_task_id=result.id, job_type="form_generation", template_id=tid, - input_text=form.input_text, + input_text=transcript, status="queued", model=form.model, ) diff --git a/app/api/routes/templates.py b/app/api/routes/templates.py index ee7189aa..0997517b 100644 --- a/app/api/routes/templates.py +++ b/app/api/routes/templates.py @@ -1,5 +1,3 @@ -import re -from datetime import datetime, timezone from pathlib import Path from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, UploadFile @@ -14,48 +12,11 @@ MakeFillableRequest, MakeFillableResponse, ) -from app.core.config import BASE_DIR, DEFAULT_TEMPLATE_DIR -from app.db.repositories import create_template, list_templates, get_template, delete_template -from app.models import Template, FormSubmission, Job -from app.services.controller import Controller -from sqlmodel import select +from app.core.config import DEFAULT_TEMPLATE_DIR +from app.db.repositories import get_template +from app.services.template import TemplateService router = APIRouter(prefix="/templates", tags=["templates"]) -PROJECT_ROOT = BASE_DIR - - -def _resolve_target_directory(directory: str) -> Path: - dir_value = (directory or DEFAULT_TEMPLATE_DIR).strip() - if not dir_value: - raise HTTPException(status_code=400, detail="Directory is required.") - - candidate = Path(dir_value) - if not candidate.is_absolute(): - candidate = (PROJECT_ROOT / candidate).resolve() - else: - candidate = candidate.resolve() - - if candidate != PROJECT_ROOT and PROJECT_ROOT not in candidate.parents: - raise HTTPException(status_code=400, detail="Directory must be inside the project.") - - return candidate - - -def _resolve_project_file(file_path: str) -> Path: - raw_path = (file_path or "").strip() - if not raw_path: - raise HTTPException(status_code=400, detail="Path is required.") - - candidate = Path(raw_path) - if not candidate.is_absolute(): - candidate = (PROJECT_ROOT / candidate).resolve() - else: - candidate = candidate.resolve() - - if candidate != PROJECT_ROOT and PROJECT_ROOT not in candidate.parents: - raise HTTPException(status_code=400, detail="Path must be inside the project.") - - return candidate @router.post("/upload", response_model=TemplateUploadResponse) @@ -70,88 +31,18 @@ async def upload_template_pdf( if not filename.lower().endswith(".pdf"): raise HTTPException(status_code=400, detail="Only PDF files are supported.") - target_dir = _resolve_target_directory(directory) - target_dir.mkdir(parents=True, exist_ok=True) - - target_path = target_dir / filename - if target_path.exists(): - timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") - target_path = target_dir / f"{target_path.stem}_{timestamp}{target_path.suffix}" - content = await file.read() - with target_path.open("wb") as output_file: - output_file.write(content) - - relative_path = target_path.relative_to(PROJECT_ROOT).as_posix() - extracted = _extract_pdf_fields(relative_path) - return TemplateUploadResponse( - filename=target_path.name, - pdf_path=relative_path, - field_count=None if extracted is None else len(extracted), - fields=extracted or [], - ) - - -# PDF field-type codes -> the type values the frontend field builder uses. -_FIELD_TYPE_BY_FT = {"/Tx": "string", "/Btn": "checkbox", "/Ch": "list", "/Sig": "signature"} - - -def _pdf_text(value) -> str: - """Decode a pdfrw string (field name / tooltip) to plain text.""" - if value is None: - return "" - if hasattr(value, "to_unicode"): - return value.to_unicode().strip() - return str(value).strip() - - -def _humanize(name: str) -> str: - """Turn a raw field name into a readable description (JobTitle -> Job Title).""" - text = re.sub(r"_+", " ", name) - text = re.sub(r"(?<=[a-z])(?=[A-Z])", " ", text) - return re.sub(r"\s+", " ", text).strip() - - -def _extract_pdf_fields(pdf_path: str) -> list[dict] | None: - """Fillable widgets in the same order Filler.fill_form writes them - (top-to-bottom, left-to-right per page), so seeded rows line up with the - fill order. Returns None if the PDF can't be read.""" - try: - from pdfrw import PdfReader - candidate = Path(pdf_path) - if not candidate.is_absolute(): - candidate = (PROJECT_ROOT / candidate).resolve() - pdf = PdfReader(str(candidate)) - fields: list[dict] = [] - for page in pdf.pages: - widgets = [a for a in (page.Annots or []) if a.Subtype == "/Widget" and a.T] - widgets.sort(key=lambda a: (-float(a.Rect[1]), float(a.Rect[0]))) - for annot in widgets: - name = _pdf_text(annot.T) - fields.append({ - "name": name, - "description": _pdf_text(annot.TU) or _humanize(name), - "type": _FIELD_TYPE_BY_FT.get(str(annot.FT), "string"), - }) - return fields - except Exception: - return None - - -def _count_pdf_widgets(pdf_path: str) -> int | None: - """Number of fillable widgets in a PDF, or None if unreadable.""" - fields = _extract_pdf_fields(pdf_path) - return None if fields is None else len(fields) + return TemplateService().save_uploaded_pdf(directory, filename, content) @router.get("", response_model=list[TemplateResponse]) def get_templates(db: Session = Depends(get_db)): - return list_templates(db) + return TemplateService().list_templates(db) @router.get("/preview") def preview_template_pdf(path: str = Query(..., description="Project-relative PDF path")): - resolved_path = _resolve_project_file(path) + resolved_path = TemplateService().resolve_pdf_path(path) if not resolved_path.exists() or not resolved_path.is_file(): raise HTTPException(status_code=404, detail="PDF file not found.") @@ -169,35 +60,17 @@ def preview_template_pdf(path: str = Query(..., description="Project-relative PD @router.post("/create", response_model=TemplateResponse) def create(template: TemplateCreate, db: Session = Depends(get_db)): - tpl = Template(**template.model_dump()) - created = create_template(db, tpl) - return TemplateResponse( - id=created.id, - name=created.name, - pdf_path=created.pdf_path, - fields=created.fields, - field_count=_count_pdf_widgets(created.pdf_path), - ) + return TemplateService().create_template(db, template) @router.post("/make-fillable", response_model=MakeFillableResponse) def make_fillable(req: MakeFillableRequest): - # Validate the path stays inside the project root. - resolved = _resolve_project_file(req.pdf_path) + svc = TemplateService() + resolved = svc.resolve_pdf_path(req.pdf_path) if not resolved.exists() or not resolved.is_file(): raise HTTPException(status_code=404, detail="PDF file not found.") - controller = Controller() - new_absolute = controller.prepare_fillable(str(resolved)) - new_path = Path(new_absolute) - if not new_path.is_absolute(): - new_path = (PROJECT_ROOT / new_path).resolve() - relative_path = new_path.relative_to(PROJECT_ROOT).as_posix() - - return MakeFillableResponse( - pdf_path=relative_path, - field_count=_count_pdf_widgets(relative_path), - ) + return svc.make_fillable(str(resolved)) @router.delete("/{template_id}", dependencies=[Depends(verify_api_key)]) @@ -206,35 +79,5 @@ def delete_template_endpoint(template_id: int, db: Session = Depends(get_db)): if not template: raise HTTPException(status_code=404, detail="Template not found") - # 1. Clean up associated submissions and their generated PDFs - sub_stmt = select(FormSubmission).where(FormSubmission.template_id == template_id) - submissions = list(db.exec(sub_stmt)) - for sub in submissions: - if sub.output_pdf_path: - try: - resolved_out = _resolve_project_file(sub.output_pdf_path) - if resolved_out.exists() and resolved_out.is_file(): - resolved_out.unlink() - except Exception: - pass - db.delete(sub) - - # 2. Clean up associated jobs - job_stmt = select(Job).where(Job.template_id == template_id) - jobs = list(db.exec(job_stmt)) - for job in jobs: - db.delete(job) - - # 3. Delete template PDF file - if template.pdf_path: - try: - resolved_pdf = _resolve_project_file(template.pdf_path) - if resolved_pdf.exists() and resolved_pdf.is_file(): - resolved_pdf.unlink() - except Exception: - pass - - # 4. Delete the template itself - delete_template(db, template) + TemplateService().delete_template(db, template) return {"status": "success", "message": "Template and all associated data deleted"} - diff --git a/app/api/schemas/forms.py b/app/api/schemas/forms.py index 155b14f0..cdfe8cd3 100644 --- a/app/api/schemas/forms.py +++ b/app/api/schemas/forms.py @@ -1,17 +1,13 @@ +from uuid import UUID + from pydantic import BaseModel, field_validator class FormFill(BaseModel): template_id: int - input_text: str + input_id: UUID model: str | None = None - @field_validator("input_text") - def validate_input_text(cls, value): - if not value or not value.strip(): - raise ValueError("Input text cannot be empty") - return value - class FormFillResponse(BaseModel): id: int @@ -34,15 +30,9 @@ class ModelsResponse(BaseModel): class AsyncFormFill(BaseModel): template_ids: list[int] - input_text: str + input_id: UUID model: str | None = None - @field_validator("input_text") - def validate_input_text(cls, value): - if not value or not value.strip(): - raise ValueError("Input text cannot be empty") - return value - @field_validator("template_ids") def validate_template_ids(cls, value): if not value: diff --git a/app/core/paths.py b/app/core/paths.py new file mode 100644 index 00000000..ddc51412 --- /dev/null +++ b/app/core/paths.py @@ -0,0 +1,41 @@ +from pathlib import Path + +from fastapi import HTTPException + +from app.core.config import BASE_DIR, DEFAULT_TEMPLATE_DIR + +PROJECT_ROOT = BASE_DIR + + +def _resolve_target_directory(directory: str) -> Path: + dir_value = (directory or DEFAULT_TEMPLATE_DIR).strip() + if not dir_value: + raise HTTPException(status_code=400, detail="Directory is required.") + + candidate = Path(dir_value) + if not candidate.is_absolute(): + candidate = (PROJECT_ROOT / candidate).resolve() + else: + candidate = candidate.resolve() + + if candidate != PROJECT_ROOT and PROJECT_ROOT not in candidate.parents: + raise HTTPException(status_code=400, detail="Directory must be inside the project.") + + return candidate + + +def _resolve_project_file(file_path: str) -> Path: + raw_path = (file_path or "").strip() + if not raw_path: + raise HTTPException(status_code=400, detail="Path is required.") + + candidate = Path(raw_path) + if not candidate.is_absolute(): + candidate = (PROJECT_ROOT / candidate).resolve() + else: + candidate = candidate.resolve() + + if candidate != PROJECT_ROOT and PROJECT_ROOT not in candidate.parents: + raise HTTPException(status_code=400, detail="Path must be inside the project.") + + return candidate diff --git a/app/core/pdf_utils.py b/app/core/pdf_utils.py new file mode 100644 index 00000000..1679287a --- /dev/null +++ b/app/core/pdf_utils.py @@ -0,0 +1,55 @@ +import re +from pathlib import Path + +from app.core import paths + +# PDF field-type codes -> the type values the frontend field builder uses. +_FIELD_TYPE_BY_FT = {"/Tx": "string", "/Btn": "checkbox", "/Ch": "list", "/Sig": "signature"} + + +def _pdf_text(value) -> str: + """Decode a pdfrw string (field name / tooltip) to plain text.""" + if value is None: + return "" + if hasattr(value, "to_unicode"): + return value.to_unicode().strip() + return str(value).strip() + + +def _humanize(name: str) -> str: + """Turn a raw field name into a readable description (JobTitle -> Job Title).""" + text = re.sub(r"_+", " ", name) + text = re.sub(r"(?<=[a-z])(?=[A-Z])", " ", text) + return re.sub(r"\s+", " ", text).strip() + + +def _extract_pdf_fields(pdf_path: str) -> list[dict] | None: + """Fillable widgets in the same order Filler.fill_form writes them + (top-to-bottom, left-to-right per page), so seeded rows line up with the + fill order. Returns None if the PDF can't be read.""" + try: + from pdfrw import PdfReader + candidate = Path(pdf_path) + if not candidate.is_absolute(): + candidate = (paths.PROJECT_ROOT / candidate).resolve() + pdf = PdfReader(str(candidate)) + fields: list[dict] = [] + for page in pdf.pages: + widgets = [a for a in (page.Annots or []) if a.Subtype == "/Widget" and a.T] + widgets.sort(key=lambda a: (-float(a.Rect[1]), float(a.Rect[0]))) + for annot in widgets: + name = _pdf_text(annot.T) + fields.append({ + "name": name, + "description": _pdf_text(annot.TU) or _humanize(name), + "type": _FIELD_TYPE_BY_FT.get(str(annot.FT), "string"), + }) + return fields + except Exception: + return None + + +def _count_pdf_widgets(pdf_path: str) -> int | None: + """Number of fillable widgets in a PDF, or None if unreadable.""" + fields = _extract_pdf_fields(pdf_path) + return None if fields is None else len(fields) diff --git a/app/db/repositories.py b/app/db/repositories.py index 0935c133..f20a862f 100644 --- a/app/db/repositories.py +++ b/app/db/repositories.py @@ -1,3 +1,4 @@ +from datetime import datetime from uuid import UUID from sqlmodel import Session, select @@ -27,6 +28,27 @@ def create_form(session: Session, form: FormSubmission) -> FormSubmission: return form +def get_submissions(session: Session) -> list[tuple[FormSubmission, str | None]]: + statement = ( + select(FormSubmission, Template.name) + .join(Template, FormSubmission.template_id == Template.id, isouter=True) + .order_by(FormSubmission.created_at.desc(), FormSubmission.id.desc()) + ) + return list(session.exec(statement).all()) + + +def get_submissions_with_template(session: Session) -> list[tuple[FormSubmission, str | None]]: + statement = select(FormSubmission, Template.name).join( + Template, FormSubmission.template_id == Template.id, isouter=True + ) + return list(session.exec(statement).all()) + + +def get_submissions_before(session: Session, cutoff: datetime) -> list[FormSubmission]: + statement = select(FormSubmission).where(FormSubmission.created_at < cutoff) + return list(session.exec(statement)) + + # Jobs def create_job(session: Session, job: Job) -> Job: session.add(job) @@ -61,6 +83,16 @@ def delete_template(session: Session, template: Template) -> None: session.commit() +def get_submissions_by_template(session: Session, template_id: int) -> list[FormSubmission]: + statement = select(FormSubmission).where(FormSubmission.template_id == template_id) + return list(session.exec(statement)) + + +def get_jobs_by_template(session: Session, template_id: int) -> list[Job]: + statement = select(Job).where(Job.template_id == template_id) + return list(session.exec(statement)) + + def get_form_submission(session: Session, submission_id: int) -> FormSubmission | None: return session.get(FormSubmission, submission_id) diff --git a/app/models/models.py b/app/models/models.py index cb9a5ff7..c1b98492 100644 --- a/app/models/models.py +++ b/app/models/models.py @@ -30,6 +30,7 @@ class Template(SQLModel, table=True): class FormSubmission(SQLModel, table=True): id: int | None = Field(default=None, primary_key=True) template_id: int = Field(foreign_key="template.id") + input_id: UUID | None = Field(default=None, foreign_key="inputs.input_id") input_text: str output_pdf_path: str created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) diff --git a/app/services/filler.py b/app/services/filler.py index aec97560..217abc11 100644 --- a/app/services/filler.py +++ b/app/services/filler.py @@ -1,4 +1,5 @@ from pdfrw import PdfReader, PdfWriter +from app.core.pdf_utils import _pdf_text from app.services.llm import LLM from datetime import datetime @@ -10,7 +11,10 @@ def __init__(self): def fill_form(self, pdf_form: str, llm: LLM): """ Fill a PDF form with values from user_input using LLM. - Fields are filled in the visual order (top-to-bottom, left-to-right). + Each widget is matched to its answer by name (widget name == field-dict + key, since prepare_fillable names widgets after their field and the LLM + keys its answers by field name) — not by position, since the field-dict + order and the PDF's physical widget order aren't guaranteed to match. """ output_pdf = ( pdf_form[:-4] @@ -23,28 +27,18 @@ def fill_form(self, pdf_form: str, llm: LLM): t2j = llm.main_loop() textbox_answers = t2j.get_data() # This is a dictionary - answers_list = list(textbox_answers.values()) - # Read PDF pdf = PdfReader(pdf_form) # Loop through pages - i = 0 for page in pdf.pages: if page.Annots: - sorted_annots = sorted( - page.Annots, key=lambda a: (-float(a.Rect[1]), float(a.Rect[0])) - ) - - for annot in sorted_annots: + for annot in page.Annots: if annot.Subtype == "/Widget" and annot.T: - if i < len(answers_list): - annot.V = f"{answers_list[i]}" + name = _pdf_text(annot.T) + if name in textbox_answers: + annot.V = f"{textbox_answers[name]}" annot.AP = None - i += 1 - else: - # Stop if we run out of answers - break PdfWriter().write(output_pdf, pdf) diff --git a/app/services/form.py b/app/services/form.py new file mode 100644 index 00000000..6ce5f19f --- /dev/null +++ b/app/services/form.py @@ -0,0 +1,130 @@ +import re +from collections import Counter +from datetime import datetime, timedelta, timezone +from uuid import UUID + +from sqlmodel import Session + +from app.core import paths +from app.db.repositories import ( + create_form, + delete_form_submission, + get_submissions, + get_submissions_before, + get_submissions_with_template, +) +from app.models import FormSubmission, Template +from app.services.controller import Controller +from app.services.input import InputService + +_STOPWORDS = { + "the", "and", "a", "of", "to", "in", "is", "that", "it", "was", "for", "on", + "as", "with", "by", "at", "an", "be", "this", "are", "from", "or", "have", + "has", "had", "but", "not", "he", "she", "they", "we", "i", "you", "my", "his", + "her", "their", "our", "me", "him", "them", "us", "about", "there", "their", + "were", "been", "would", "could", "should", "will", "can", "no", "yes", "any", + "so", "very", "patient", "presents", "with", "reported", "history", "shows", + "left", "right", "pain", "due", "after", "before", "emergency", "department", + "medical", "clinical" +} + + +class FormService: + def __init__(self): + self.controller = Controller() + self.input_service = InputService() + + def fill_form( + self, session: Session, template: Template, input_id: UUID, model: str | None = None + ) -> FormSubmission: + transcript = self.input_service.resolve_transcript(session, input_id) + return self.fill_and_persist(session, template, transcript, input_id, model) + + def fill_and_persist( + self, + session: Session, + template: Template, + transcript: str, + input_id: UUID, + model: str | None = None, + ) -> FormSubmission: + path = self.controller.fill_form( + user_input=transcript, + fields=template.fields, + pdf_form_path=template.pdf_path, + model=model, + ) + + submission = FormSubmission( + template_id=template.id, + input_id=input_id, + input_text=transcript, + output_pdf_path=path, + ) + return create_form(session, submission) + + def list_submissions(self, session: Session) -> list[dict]: + results = get_submissions(session) + return [ + { + "id": sub.id, + "template_id": sub.template_id, + "template_name": name or "Unknown Template", + "input_text": sub.input_text, + "output_pdf_path": sub.output_pdf_path, + "created_at": sub.created_at.isoformat() if sub.created_at else None, + } + for sub, name in results + ] + + def get_analytics(self, session: Session) -> dict: + results = get_submissions_with_template(session) + + total_submissions = len(results) + + template_counts = Counter() + daily_counts = Counter() + words = [] + + for sub, name in results: + template_name = name or "Unknown Template" + template_counts[template_name] += 1 + + if sub.created_at: + date_str = sub.created_at.strftime("%Y-%m-%d") + daily_counts[date_str] += 1 + + if sub.input_text: + found_words = re.findall(r"\b[a-zA-Z]{3,15}\b", sub.input_text.lower()) + for w in found_words: + if w not in _STOPWORDS: + words.append(w) + + sorted_daily = [{"date": k, "count": v} for k, v in sorted(daily_counts.items())] + sorted_templates = [{"template_name": k, "count": v} for k, v in template_counts.most_common()] + common_terms = [{"word": k, "count": v} for k, v in Counter(words).most_common(12)] + + return { + "total_submissions": total_submissions, + "by_template": sorted_templates, + "by_date": sorted_daily, + "common_terms": common_terms, + } + + def purge_submissions(self, session: Session, retention_days: int) -> int: + cutoff_date = datetime.now(timezone.utc) - timedelta(days=retention_days) + submissions = get_submissions_before(session, cutoff_date) + + purged_count = 0 + for sub in submissions: + if sub.output_pdf_path: + try: + resolved_out = paths._resolve_project_file(sub.output_pdf_path) + if resolved_out.exists() and resolved_out.is_file(): + resolved_out.unlink() + except Exception: + pass + delete_form_submission(session, sub) + purged_count += 1 + + return purged_count diff --git a/app/services/input.py b/app/services/input.py index bdd51c43..ca6a5af2 100644 --- a/app/services/input.py +++ b/app/services/input.py @@ -1,10 +1,12 @@ from datetime import date, datetime, timezone +from uuid import UUID from sqlmodel import Session from app.api.schemas.enums import InputStatus, InputType from app.core.config import AUDIO_DIR -from app.db.repositories import create_input, create_job, update_job +from app.core.errors.base import AppError +from app.db.repositories import create_input, create_job, get_input, update_job from app.models import Input, Job from app.tasks.transcribe import transcribe_audio_task @@ -66,6 +68,28 @@ def process_voice_upload( return record, job + def resolve_transcript(self, session: Session, input_id: UUID) -> str: + """Look up a stored Input and return its transcript, for form-fill callers. + + Shared by the sync and async fill paths so the input_id -> transcript + resolution (and its 404/409 rules) live in exactly one place. + """ + record = get_input(session, input_id) + if record is None: + raise AppError( + f"Input with ID {input_id} not found", + status_code=404, + error_code="INPUT_NOT_FOUND", + ) + if record.status != InputStatus.ready: + raise AppError( + f"Input {input_id} is not ready (status: {record.status})", + status_code=409, + error_code="INPUT_NOT_READY", + detail={"status": record.status}, + ) + return record.transcript + def build_text_input( self, narrative: str, diff --git a/app/services/template.py b/app/services/template.py new file mode 100644 index 00000000..e4cf76f3 --- /dev/null +++ b/app/services/template.py @@ -0,0 +1,106 @@ +from datetime import datetime, timezone +from pathlib import Path + +from sqlmodel import Session + +from app.api.schemas.templates import ( + MakeFillableResponse, + TemplateCreate, + TemplateResponse, + TemplateUploadResponse, +) +from app.core import paths +from app.core.pdf_utils import _count_pdf_widgets, _extract_pdf_fields +from app.db.repositories import ( + create_template, + delete_template, + get_jobs_by_template, + get_submissions_by_template, + list_templates, +) +from app.models import Template +from app.services.controller import Controller + + +class TemplateService: + def __init__(self): + self.controller = Controller() + + def list_templates(self, session: Session) -> list[Template]: + return list_templates(session) + + def resolve_pdf_path(self, path: str) -> Path: + return paths._resolve_project_file(path) + + def save_uploaded_pdf(self, directory: str, filename: str, content: bytes) -> TemplateUploadResponse: + target_dir = paths._resolve_target_directory(directory) + target_dir.mkdir(parents=True, exist_ok=True) + + target_path = target_dir / filename + if target_path.exists(): + timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") + target_path = target_dir / f"{target_path.stem}_{timestamp}{target_path.suffix}" + + with target_path.open("wb") as output_file: + output_file.write(content) + + relative_path = target_path.relative_to(paths.PROJECT_ROOT).as_posix() + extracted = _extract_pdf_fields(relative_path) + return TemplateUploadResponse( + filename=target_path.name, + pdf_path=relative_path, + field_count=None if extracted is None else len(extracted), + fields=extracted or [], + ) + + def create_template(self, session: Session, template: TemplateCreate) -> TemplateResponse: + tpl = Template(**template.model_dump()) + created = create_template(session, tpl) + return TemplateResponse( + id=created.id, + name=created.name, + pdf_path=created.pdf_path, + fields=created.fields, + field_count=_count_pdf_widgets(created.pdf_path), + ) + + def make_fillable(self, resolved_pdf_path: str) -> MakeFillableResponse: + new_absolute = self.controller.prepare_fillable(resolved_pdf_path) + new_path = Path(new_absolute) + if not new_path.is_absolute(): + new_path = (paths.PROJECT_ROOT / new_path).resolve() + relative_path = new_path.relative_to(paths.PROJECT_ROOT).as_posix() + + return MakeFillableResponse( + pdf_path=relative_path, + field_count=_count_pdf_widgets(relative_path), + ) + + def delete_template(self, session: Session, template: Template) -> None: + # Batched like the original route: only session.delete() per row here, + # single commit at the end (via the delete_template repo call) so the + # cascade stays atomic instead of partially committing on failure. + submissions = get_submissions_by_template(session, template.id) + for sub in submissions: + if sub.output_pdf_path: + try: + resolved_out = paths._resolve_project_file(sub.output_pdf_path) + if resolved_out.exists() and resolved_out.is_file(): + resolved_out.unlink() + except Exception: + pass + session.delete(sub) + + jobs = get_jobs_by_template(session, template.id) + for job in jobs: + session.delete(job) + + if template.pdf_path: + try: + resolved_pdf = paths._resolve_project_file(template.pdf_path) + if resolved_pdf.exists() and resolved_pdf.is_file(): + resolved_pdf.unlink() + except Exception: + pass + + delete_template(session, template) diff --git a/app/tasks/fill.py b/app/tasks/fill.py index fc7b3115..b73bd136 100644 --- a/app/tasks/fill.py +++ b/app/tasks/fill.py @@ -1,17 +1,17 @@ import logging from datetime import datetime, timezone +from uuid import UUID from app.core.celery import celery_app from app.db.database import get_session -from app.db.repositories import get_job_by_celery_id, get_template, update_job, create_form -from app.models import FormSubmission -from app.services.controller import Controller +from app.db.repositories import get_job_by_celery_id, get_template, update_job +from app.services.form import FormService logger = logging.getLogger(__name__) @celery_app.task(bind=True, name="fill_form") -def fill_form_task(self, template_id: int, input_text: str, model: str | None = None): +def fill_form_task(self, template_id: int, input_text: str, input_id_str: str, model: str | None = None): session = next(get_session()) try: job = get_job_by_celery_id(session, self.request.id) @@ -27,21 +27,10 @@ def fill_form_task(self, template_id: int, input_text: str, model: str | None = if not template: raise ValueError(f"Template {template_id} not found") - controller = Controller() - path = controller.fill_form( - user_input=input_text, - fields=template.fields, - pdf_form_path=template.pdf_path, - model=model, + submission = FormService().fill_and_persist( + session, template, input_text, UUID(input_id_str), model ) - submission = FormSubmission( - template_id=template_id, - input_text=input_text, - output_pdf_path=path, - ) - create_form(session, submission) - job.status = "completed" job.progress_percent = 100 job.result_url = f"/api/v1/forms/{submission.id}/download" diff --git a/app/tasks/purge.py b/app/tasks/purge.py index 83256751..e5018e90 100644 --- a/app/tasks/purge.py +++ b/app/tasks/purge.py @@ -9,7 +9,8 @@ from pathlib import Path from app.core.celery import celery_app -from app.core.config import BASE_DIR, RETENTION_PERIOD_DAYS +from app.core.config import RETENTION_PERIOD_DAYS +from app.core.paths import PROJECT_ROOT from app.db.database import get_session from app.db.repositories import delete_form_submission from app.models import FormSubmission @@ -17,8 +18,6 @@ logger = logging.getLogger(__name__) -PROJECT_ROOT = BASE_DIR - def _safe_delete_file(file_path: str) -> bool: """Delete a project-relative or absolute file safely. Returns True if deleted.""" diff --git a/contracts/openapi.yaml b/contracts/openapi.yaml index 16931f98..6ae8a3c6 100644 --- a/contracts/openapi.yaml +++ b/contracts/openapi.yaml @@ -17,14 +17,8 @@ servers: tags: - name: input description: Submit voice or text incident narratives - - name: extraction - description: AI-powered data extraction from narratives into canonical JSON - name: forms description: Generate agency-specific PDF forms from extracted data - - name: incidents - description: Manage incident records linking inputs, extractions, and forms - - name: reporting - description: Aggregate statistics and periodic report generation - name: templates description: Form template configuration and management - name: system @@ -45,41 +39,11 @@ paths: /api/v1/input/{input_id}: $ref: "path/input.yaml#/input_by_id" - # ── Layer 2: AI Extraction ───────────────────────────────────── - /api/v1/extract/{input_id}: - $ref: "path/extraction.yaml#/extract_by_input" - /api/v1/extract/{extract_id}: - $ref: "path/extraction.yaml#/extract_by_id" - /api/v1/extract/{extract_id}/validate: - $ref: "path/extraction.yaml#/validate" - - # ── Layer 3: Form Generation ─────────────────────────────────── - /api/v1/forms/generate/all: - $ref: "path/forms.yaml#/generate_all" - /api/v1/forms/generate/{form_type}: - $ref: "path/forms.yaml#/generate_single" - /api/v1/forms/{form_id}: - $ref: "path/forms.yaml#/form_by_id" - /api/v1/forms/{form_id}/pdf: - $ref: "path/forms.yaml#/form_pdf" - /api/v1/forms/{form_id}/json: - $ref: "path/forms.yaml#/form_json" - /api/v1/forms/batch/{batch_id}: - $ref: "path/forms.yaml#/batch_by_id" - - # ── Layer 4: Incident Management ─────────────────────────────── - /api/v1/incidents: - $ref: "path/incidents.yaml#/incidents" - /api/v1/incidents/{incident_id}: - $ref: "path/incidents.yaml#/incident_by_id" - - # ── Layer 5: Reporting & Analytics ───────────────────────────── - /api/v1/reports/summary: - $ref: "path/reporting.yaml#/summary" - /api/v1/reports/generate: - $ref: "path/reporting.yaml#/generate" - /api/v1/reports/{report_id}: - $ref: "path/reporting.yaml#/report_by_id" + # ── Layer 3: Form Fill ────────────────────────────────────────── + /api/v1/forms/fill: + $ref: "path/forms.yaml#/fill" + /api/v1/forms/jobs: + $ref: "path/forms.yaml#/async_fill" # ── Layer 6: Templates & Configuration ───────────────────────── /api/v1/templates: diff --git a/contracts/path/extraction.yaml b/contracts/path/extraction.yaml deleted file mode 100644 index 865e20c4..00000000 --- a/contracts/path/extraction.yaml +++ /dev/null @@ -1,313 +0,0 @@ -# Layer 2 AI Extraction Endpoints -# POST /api/v1/extract/{input_id} -# GET /api/v1/extract/{extract_id} -# PATCH /api/v1/extract/{extract_id} -# POST /api/v1/extract/{extract_id}/validate - -extract_by_input: - post: - operationId: createExtraction - summary: Start AI extraction from input narrative - description: | - Sends the narrative (from a previously submitted input) to the local Ollama - LLM with a structured prompt to extract all incident fields into the canonical - FireForm JSON schema. This is an asynchronous operation the LLM may take - 30–120 seconds. Returns an extract_id and job_id for polling. - tags: - - extraction - parameters: - - name: input_id - in: path - required: true - description: ID of the input record to extract from - schema: - type: string - format: uuid - requestBody: - required: false - content: - application/json: - schema: - $ref: "../schemas/extraction-record.yaml#/ExtractionRequest" - example: - model_override: "llama3:8b" - extraction_hints: - incident_type: "wildland_fire" - state: "CA" - responses: - "202": - description: Extraction job queued successfully - content: - application/json: - schema: - $ref: "../schemas/common.yaml#/AsyncJobResponse" - example: - extract_id: "550e8400-e29b-41d4-a716-446655440020" - job_id: "550e8400-e29b-41d4-a716-446655440098" - job_type: "extraction" - status: "processing" - input_id: "550e8400-e29b-41d4-a716-446655440001" - queued_at: "2024-07-15T14:30:00Z" - estimated_seconds: 60 - poll_url: "/api/v1/extract/550e8400-e29b-41d4-a716-446655440020" - "404": - description: Input ID not found - content: - application/json: - schema: - $ref: "../schemas/common.yaml#/ErrorResponse" - example: - error_code: "INPUT_NOT_FOUND" - message: "Input with ID 550e8400-e29b-41d4-a716-446655440099 not found" - "409": - description: Conflict input not ready or extraction already exists - content: - application/json: - schema: - $ref: "../schemas/common.yaml#/ErrorResponse" - examples: - not_ready: - summary: Input still transcribing - value: - error_code: "INPUT_NOT_READY" - message: "Input is in 'transcribing' state. Wait until status is 'ready'." - detail: - current_status: "transcribing" - already_exists: - summary: Extraction already initiated for this input - value: - error_code: "EXTRACTION_EXISTS" - message: "An extraction already exists for this input" - detail: - existing_extract_id: "550e8400-e29b-41d4-a716-446655440020" - "503": - description: Ollama LLM service unavailable - content: - application/json: - schema: - $ref: "../schemas/common.yaml#/ErrorResponse" - example: - error_code: "LLM_UNAVAILABLE" - message: "Ollama LLM service is not available" - retry_after_seconds: 30 - detail: - ollama_status: "connection_refused" - -extract_by_id: - get: - operationId: getExtraction - summary: Get extraction result by ID - description: | - Returns the full canonical FireForm JSON when extraction is complete, or - the current job status while still processing. When status is "completed", - the response body contains the entire canonical incident schema. When still - processing, includes a retry_after_seconds hint for polling. - tags: - - extraction - parameters: - - name: extract_id - in: path - required: true - description: Unique identifier of the extraction - schema: - type: string - format: uuid - responses: - "200": - description: Extraction record (completed or in-progress) - content: - application/json: - schema: - oneOf: - - $ref: "../schemas/extraction-record.yaml#/ExtractionCompleted" - - $ref: "../schemas/extraction-record.yaml#/ExtractionProcessing" - examples: - completed: - summary: Extraction completed with canonical JSON - value: - extract_id: "550e8400-e29b-41d4-a716-446655440020" - input_id: "550e8400-e29b-41d4-a716-446655440001" - status: "completed" - completed_at: "2024-07-15T14:31:05Z" - incident_contract: - schema_version: "1.1.0" - schema_name: "fireform_incident_contract" - extraction_metadata: - extract_id: "550e8400-e29b-41d4-a716-446655440020" - confidence_score: 0.91 - incident: - name: "Bear Creek Wildfire" - processing: - summary: Extraction still in progress - value: - extract_id: "550e8400-e29b-41d4-a716-446655440020" - input_id: "550e8400-e29b-41d4-a716-446655440001" - status: "processing" - started_at: "2024-07-15T14:30:00Z" - retry_after_seconds: 5 - "404": - description: Extraction not found - content: - application/json: - schema: - $ref: "../schemas/common.yaml#/ErrorResponse" - example: - error_code: "EXTRACT_NOT_FOUND" - message: "Extraction with ID 550e8400-e29b-41d4-a716-446655440099 not found" - - patch: - operationId: updateExtraction - summary: Manually correct extracted fields - description: | - Allows a responder to correct any field in the canonical JSON after LLM - extraction. Uses JSON Merge Patch (RFC 7396) only send the fields that - changed. The server records an audit trail of all changes vs the original - LLM output and recalculates completeness scores and applicable_forms. - tags: - - extraction - parameters: - - name: extract_id - in: path - required: true - description: Unique identifier of the extraction to update - schema: - type: string - format: uuid - requestBody: - required: true - description: JSON Merge Patch (RFC 7396) only include changed fields - content: - application/merge-patch+json: - schema: - $ref: "../schemas/incident-contract.yaml#/IncidentContract" - example: - fire: - estimated_damage_usd: 250000 - casualties: - total_responder_injuries: 2 - responses: - "200": - description: Extraction updated returns full updated canonical JSON - content: - application/json: - schema: - $ref: "../schemas/extraction-record.yaml#/ExtractionCompleted" - "404": - description: Extraction not found - content: - application/json: - schema: - $ref: "../schemas/common.yaml#/ErrorResponse" - "409": - description: Extraction is locked (already submitted) - content: - application/json: - schema: - $ref: "../schemas/common.yaml#/ErrorResponse" - example: - error_code: "EXTRACT_LOCKED" - message: "Cannot modify extraction incident report has been submitted" - detail: - report_status: "submitted" - submitted_at: "2024-07-15T18:00:00Z" - "422": - description: Invalid field path or value - content: - application/json: - schema: - $ref: "../schemas/common.yaml#/ErrorResponse" - example: - error_code: "VALIDATION_ERROR" - message: "Invalid field path or value in patch" - validation_errors: - - field: "fire.cause_certainty" - issue: "Must be one of: confirmed, probable, suspected, undetermined" - value: "maybe" - -validate: - post: - operationId: validateExtraction - summary: Validate extraction against a form's requirements - description: | - Validates the canonical JSON against a specific form type's field requirements. - Returns whether the extraction has all required fields, which recommended - fields are missing, and any warnings. Useful for checking "can I generate - a NERIS report with what I have?" before triggering form generation. - tags: - - extraction - parameters: - - name: extract_id - in: path - required: true - description: Unique identifier of the extraction to validate - schema: - type: string - format: uuid - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - form_type - properties: - form_type: - $ref: "../schemas/enums.yaml#/FormType" - example: - form_type: "neris" - responses: - "200": - description: Validation result - content: - application/json: - schema: - $ref: "../schemas/extraction-record.yaml#/ValidationResult" - example: - valid: true - form_type: "neris" - extract_id: "550e8400-e29b-41d4-a716-446655440020" - missing_required: [] - missing_recommended: - - "fire.detector_present" - - "fire.detector_operated" - warnings: - - "fire.estimated_damage_usd is null NERIS recommends providing damage estimates" - field_coverage_percent: 94 - "404": - description: Extraction not found - content: - application/json: - schema: - $ref: "../schemas/common.yaml#/ErrorResponse" - "422": - description: Unknown form type - content: - application/json: - schema: - $ref: "../schemas/common.yaml#/ErrorResponse" - example: - error_code: "UNKNOWN_FORM_TYPE" - message: "Form type 'xyz' is not recognized" - detail: - valid_form_types: - - neris - - nemsis_epcr - - nibrs - - nfirs_basic - - nfirs_fire - - nfirs_structure - - nfirs_wildland - - nfirs_ems - - nfirs_hazmat - - nfirs_apparatus - - nfirs_personnel - - nfirs_arson - - nfirs_casualty_civilian - - nfirs_casualty_responder - - cal_fire_ics209 - - osha_301 - - un_ssirs - - state_georgia - - state_california - - state_new_york diff --git a/contracts/path/forms.yaml b/contracts/path/forms.yaml index b920b41a..4e0fb604 100644 --- a/contracts/path/forms.yaml +++ b/contracts/path/forms.yaml @@ -1,20 +1,17 @@ -# Layer 3 Form Generation Endpoints -# POST /api/v1/forms/generate/all -# POST /api/v1/forms/generate/{form_type} -# GET /api/v1/forms/{form_id} -# GET /api/v1/forms/{form_id}/pdf -# GET /api/v1/forms/{form_id}/json -# GET /api/v1/forms/batch/{batch_id} +# Layer 3 Form Fill Endpoints +# POST /api/v1/forms/fill +# POST /api/v1/forms/jobs -generate_all: +fill: post: - operationId: generateAllForms - summary: Generate all applicable forms from an extraction + operationId: fillForm + summary: Fill a single template from a stored transcript description: | - Triggers batch generation of ALL forms listed in the extraction's - extraction_metadata.applicable_forms. This is an async batch job. - Use skip_incomplete to skip forms that fail validation, or force_partial - to generate forms with blank fields where data is missing. + Takes the transcript stored on a prior input (see POST /input/text or + POST /input/voice) plus one template's field dictionary, sends them to + the local Ollama LLM to get values back field-by-field, then draws those + values into the template's PDF boxes. Runs synchronously: one LLM pass + per call. The referenced input must have status "ready". tags: - forms requestBody: @@ -22,327 +19,107 @@ generate_all: content: application/json: schema: - $ref: "../schemas/form-record.yaml#/GenerateAllRequest" + $ref: "../schemas/form-record.yaml#/FormFillRequest" example: - extract_id: "550e8400-e29b-41d4-a716-446655440020" - options: - skip_incomplete: true - force_partial: false + template_id: 3 + input_id: "550e8400-e29b-41d4-a716-446655440001" + model: "llama3.1" responses: - "202": - description: Batch form generation job accepted + "200": + description: Form filled and PDF generated content: application/json: schema: - $ref: "../schemas/form-record.yaml#/BatchGenerateResponse" + $ref: "../schemas/form-record.yaml#/FormFillResponse" example: - batch_id: "550e8400-e29b-41d4-a716-446655440030" - status: "processing" - extract_id: "550e8400-e29b-41d4-a716-446655440020" - forms_queued: - - "neris" - - "nfirs_basic" - - "nfirs_wildland" - forms_skipped: - - form_type: "nemsis_epcr" - reason: "Missing required fields: ems.patients[0].date_of_birth" - estimated_seconds: 45 - poll_url: "/api/v1/forms/batch/550e8400-e29b-41d4-a716-446655440030" + id: 12 + template_id: 3 + input_text: "62 year old male, chest pain, transported to St. Mary's." + output_pdf_path: "output/forms/3_12.pdf" "404": - description: Extract ID not found + description: Template or input not found content: application/json: schema: $ref: "../schemas/common.yaml#/ErrorResponse" + example: + error_code: "TEMPLATE_NOT_FOUND" + message: "Template not found" "409": - description: Extraction not yet completed + description: Input exists but is not ready (still queued/transcribing, or failed) content: application/json: schema: $ref: "../schemas/common.yaml#/ErrorResponse" example: - error_code: "EXTRACT_NOT_COMPLETED" - message: "Extraction is still processing. Wait until status is 'completed'." + error_code: "INPUT_NOT_READY" + message: "Input 550e8400-e29b-41d4-a716-446655440001 is not ready (status: transcribing)" detail: - current_status: "processing" - "422": - description: No applicable forms found + status: "transcribing" + "500": + description: Form fill failed (LLM or PDF generation error) content: application/json: schema: $ref: "../schemas/common.yaml#/ErrorResponse" example: - error_code: "NO_APPLICABLE_FORMS" - message: "No forms are applicable for this extraction's incident type" + error_code: "FORM_FILL_ERROR" + message: "Ollama did not respond within 120 seconds" -generate_single: +async_fill: post: - operationId: generateSingleForm - summary: Generate one specific form type + operationId: submitAsyncFormFill + summary: Queue form fill jobs for one or more templates description: | - Generates a single agency-specific form from the canonical extraction data. - The form_type path parameter specifies which form to generate. If the extraction - is missing required fields for this form, returns 422 unless force_partial is true. + Resolves input_id to its stored transcript up front (so a missing or + not-ready input fails synchronously, before any job is queued), then + queues one Celery job per template_id, each running that transcript + against that template's field dictionary. Poll GET /api/v1/jobs/{job_id} + on each returned poll_url for status. tags: - forms - parameters: - - name: form_type - in: path - required: true - description: Type of form to generate - schema: - $ref: "../schemas/enums.yaml#/FormType" requestBody: required: true content: application/json: schema: - $ref: "../schemas/form-record.yaml#/GenerateSingleRequest" + $ref: "../schemas/form-record.yaml#/AsyncFormFillRequest" example: - extract_id: "550e8400-e29b-41d4-a716-446655440020" - options: - output_format: "pdf" - force_partial: false - responses: - "202": - description: Form generation job accepted - content: - application/json: - schema: - $ref: "../schemas/form-record.yaml#/FormGenerateResponse" - example: - form_id: "550e8400-e29b-41d4-a716-446655440040" - form_type: "neris" - status: "processing" - extract_id: "550e8400-e29b-41d4-a716-446655440020" - job_id: "550e8400-e29b-41d4-a716-446655440097" - estimated_seconds: 15 - poll_url: "/api/v1/forms/550e8400-e29b-41d4-a716-446655440040" - "404": - description: Extract ID not found - content: - application/json: - schema: - $ref: "../schemas/common.yaml#/ErrorResponse" - "422": - description: Validation failure or form not in applicable list - content: - application/json: - schema: - $ref: "../schemas/common.yaml#/ErrorResponse" - example: - error_code: "FORM_VALIDATION_FAILED" - message: "Extraction is missing required fields for NERIS form" - detail: - form_type: "neris" - missing_fields: - - "incident.types[0].neris_code" - - "location.coordinates" - validation_errors: - - field: "incident.types[0].neris_code" - issue: "Required field is null" - -form_by_id: - get: - operationId: getForm - summary: Get form metadata and status - description: | - Returns the form record including generation status, associated extract and - incident IDs, and a field_mapping_summary showing how canonical fields were - mapped to the form's agency-specific fields (for audit/transparency). - tags: - - forms - parameters: - - name: form_id - in: path - required: true - description: Unique identifier of the generated form - schema: - type: string - format: uuid + template_ids: [3, 7] + input_id: "550e8400-e29b-41d4-a716-446655440001" + model: "llama3.1" responses: "200": - description: Form record found + description: Jobs queued, one per template_id content: application/json: schema: - $ref: "../schemas/form-record.yaml#/FormRecord" + $ref: "../schemas/form-record.yaml#/AsyncFormFillResponse" example: - form_id: "550e8400-e29b-41d4-a716-446655440040" - form_type: "neris" - status: "completed" - extract_id: "550e8400-e29b-41d4-a716-446655440020" - incident_id: "550e8400-e29b-41d4-a716-446655440050" - created_at: "2024-07-15T14:32:00Z" - completed_at: "2024-07-15T14:32:12Z" - pdf_ready: true - json_ready: true - field_mapping_summary: - total_form_fields: 85 - fields_filled: 72 - fields_blank: 13 - coverage_percent: 84.7 + jobs: + - job_id: "550e8400-e29b-41d4-a716-446655440098" + status: "queued" + poll_url: "/api/v1/jobs/550e8400-e29b-41d4-a716-446655440098" + - job_id: "550e8400-e29b-41d4-a716-446655440099" + status: "queued" + poll_url: "/api/v1/jobs/550e8400-e29b-41d4-a716-446655440099" "404": - description: Form not found + description: One of the requested template_ids, or the input_id, was not found content: application/json: schema: $ref: "../schemas/common.yaml#/ErrorResponse" - -form_pdf: - get: - operationId: downloadFormPdf - summary: Download filled PDF form - description: | - Returns the filled PDF binary file for the specified form. If the form - generation is still in progress, returns 202 Accepted with a retry_after - hint. The Content-Disposition header is set for browser download. - tags: - - forms - parameters: - - name: form_id - in: path - required: true - description: Unique identifier of the form to download - schema: - type: string - format: uuid - responses: - "200": - description: Filled PDF file - content: - application/pdf: - schema: - type: string - format: binary - headers: - Content-Disposition: - description: Attachment filename for download - schema: - type: string - example: 'attachment; filename="NERIS_FF-2024-CA-0157.pdf"' - "202": - description: Form generation still in progress - content: - application/json: - schema: - type: object - properties: - message: - type: string - status: - type: string - retry_after_seconds: - type: integer example: - message: "Form generation is still in progress" - status: "processing" - retry_after_seconds: 5 - "404": - description: Form not found - content: - application/json: - schema: - $ref: "../schemas/common.yaml#/ErrorResponse" - "500": - description: PDF generation failed + error_code: "NOT_FOUND" + message: "Template 7 not found" + "409": + description: Input exists but is not ready (still queued/transcribing, or failed) content: application/json: schema: $ref: "../schemas/common.yaml#/ErrorResponse" example: - error_code: "PDF_GENERATION_FAILED" - message: "Failed to generate PDF for form" + error_code: "INPUT_NOT_READY" + message: "Input 550e8400-e29b-41d4-a716-446655440001 is not ready (status: transcribing)" detail: - reason: "Template file corrupted or missing" - -form_json: - get: - operationId: getFormJson - summary: Get form-specific field-mapped JSON - description: | - Returns the form-specific JSON with fields mapped to the agency's expected - format not the canonical FireForm schema, but the actual field names and - structure that the target agency system expects. Useful for future direct - API submission to agency systems. - tags: - - forms - parameters: - - name: form_id - in: path - required: true - description: Unique identifier of the form - schema: - type: string - format: uuid - responses: - "200": - description: Form-specific mapped JSON - content: - application/json: - schema: - $ref: "../schemas/form-record.yaml#/FormMappedJson" - example: - form_type: "neris" - form_version: "2.0" - form_id: "550e8400-e29b-41d4-a716-446655440040" - extract_id: "550e8400-e29b-41d4-a716-446655440020" - agency_fields: - incident_type: "wildland-fire" - incident_date: "2024-07-10" - alarm_time: "13:52" - acres_burned: 1247 - cause: "natural" - "404": - description: Form not found - content: - application/json: - schema: - $ref: "../schemas/common.yaml#/ErrorResponse" - -batch_by_id: - get: - operationId: getBatchStatus - summary: Get batch form generation status - description: | - Returns the status of a batch form generation job including progress - for each individual form. Poll this endpoint after POST /forms/generate/all. - tags: - - forms - parameters: - - name: batch_id - in: path - required: true - description: Unique identifier of the batch job - schema: - type: string - format: uuid - responses: - "200": - description: Batch job status - content: - application/json: - schema: - $ref: "../schemas/form-record.yaml#/BatchStatus" - example: - batch_id: "550e8400-e29b-41d4-a716-446655440030" - status: "completed" - total: 3 - completed: 3 - failed: 0 - forms: - - form_id: "550e8400-e29b-41d4-a716-446655440040" - form_type: "neris" - status: "completed" - - form_id: "550e8400-e29b-41d4-a716-446655440041" - form_type: "nfirs_basic" - status: "completed" - - form_id: "550e8400-e29b-41d4-a716-446655440042" - form_type: "nfirs_wildland" - status: "completed" - "404": - description: Batch job not found - content: - application/json: - schema: - $ref: "../schemas/common.yaml#/ErrorResponse" + status: "transcribing" diff --git a/contracts/path/incidents.yaml b/contracts/path/incidents.yaml deleted file mode 100644 index 810a311a..00000000 --- a/contracts/path/incidents.yaml +++ /dev/null @@ -1,313 +0,0 @@ -# Layer 4 Incident Management Endpoints -# POST /api/v1/incidents -# GET /api/v1/incidents -# GET /api/v1/incidents/{incident_id} -# PATCH /api/v1/incidents/{incident_id} -# DELETE /api/v1/incidents/{incident_id} - -incidents: - post: - operationId: createIncident - summary: Create a full incident record - description: | - Creates a permanent incident record that links input, extraction, and - generated forms into a single coherent unit. Assigns a permanent incident_id - and stores the complete incident for future retrieval and reporting. - tags: - - incidents - requestBody: - required: true - content: - application/json: - schema: - $ref: "../schemas/incident-record.yaml#/CreateIncidentRequest" - example: - extract_id: "550e8400-e29b-41d4-a716-446655440020" - incident_number: "CA-SQF-2024-0421" - tags: - - "wildland" - - "mutual_aid" - - "lightning" - responses: - "201": - description: Incident record created - content: - application/json: - schema: - $ref: "../schemas/incident-record.yaml#/IncidentRecord" - example: - incident_id: "550e8400-e29b-41d4-a716-446655440050" - extract_id: "550e8400-e29b-41d4-a716-446655440020" - incident_number: "CA-SQF-2024-0421" - status: "draft" - forms_generated: - - form_id: "550e8400-e29b-41d4-a716-446655440040" - form_type: "neris" - status: "completed" - tags: - - "wildland" - - "mutual_aid" - - "lightning" - created_at: "2024-07-15T14:35:00Z" - "404": - description: Extract ID not found - content: - application/json: - schema: - $ref: "../schemas/common.yaml#/ErrorResponse" - "409": - description: Duplicate incident number - content: - application/json: - schema: - $ref: "../schemas/common.yaml#/ErrorResponse" - example: - error_code: "DUPLICATE_INCIDENT_NUMBER" - message: "Incident number CA-SQF-2024-0421 already exists" - detail: - existing_incident_id: "550e8400-e29b-41d4-a716-446655440049" - - get: - operationId: listIncidents - summary: List incidents with filtering - description: | - Returns a paginated list of incident records with optional filtering by - date range, incident type, and status. Supports sorting by date. - Maximum 100 results per page. - tags: - - incidents - parameters: - - name: date_from - in: query - description: Start date filter (inclusive, ISO 8601) - schema: - type: string - format: date - - name: date_to - in: query - description: End date filter (inclusive, ISO 8601) - schema: - type: string - format: date - - name: incident_type - in: query - description: Filter by incident category - schema: - $ref: "../schemas/enums.yaml#/IncidentCategory" - - name: status - in: query - description: Filter by report status - schema: - $ref: "../schemas/enums.yaml#/ReportStatus" - - name: page - in: query - description: Page number (1-based) - schema: - type: integer - minimum: 1 - default: 1 - - name: per_page - in: query - description: Items per page (max 100) - schema: - type: integer - minimum: 1 - maximum: 100 - default: 20 - - name: sort - in: query - description: Sort order - schema: - type: string - enum: - - date_asc - - date_desc - default: date_desc - responses: - "200": - description: Paginated list of incidents - content: - application/json: - schema: - $ref: "../schemas/incident-record.yaml#/IncidentListResponse" - example: - data: - - incident_id: "550e8400-e29b-41d4-a716-446655440050" - incident_number: "CA-SQF-2024-0421" - status: "draft" - incident_name: "Bear Creek Wildfire" - incident_type: "fire" - incident_date: "2024-07-10" - forms_count: 3 - created_at: "2024-07-15T14:35:00Z" - pagination: - total: 42 - page: 1 - per_page: 20 - total_pages: 3 - has_next: true - has_prev: false - "422": - description: Invalid query parameter format - content: - application/json: - schema: - $ref: "../schemas/common.yaml#/ErrorResponse" - example: - error_code: "VALIDATION_ERROR" - message: "Invalid date format for date_from" - validation_errors: - - field: "date_from" - issue: "Must be ISO 8601 date format (YYYY-MM-DD)" - value: "15/07/2024" - -incident_by_id: - get: - operationId: getIncident - summary: Get full incident record - description: | - Returns the complete incident record including the linked canonical - extraction, all generated forms and their statuses, submission log, - and audit trail. - tags: - - incidents - parameters: - - name: incident_id - in: path - required: true - description: Unique identifier of the incident - schema: - type: string - format: uuid - responses: - "200": - description: Full incident record - content: - application/json: - schema: - $ref: "../schemas/incident-record.yaml#/IncidentRecordFull" - "404": - description: Incident not found - content: - application/json: - schema: - $ref: "../schemas/common.yaml#/ErrorResponse" - - patch: - operationId: updateIncident - summary: Update incident metadata - description: | - Updates incident metadata such as status, tags, incident number, and notes. - Does NOT allow changing the underlying extraction data use PATCH /extract - for that. Submitted incidents cannot be modified. - tags: - - incidents - parameters: - - name: incident_id - in: path - required: true - description: Unique identifier of the incident to update - schema: - type: string - format: uuid - requestBody: - required: true - content: - application/json: - schema: - $ref: "../schemas/incident-record.yaml#/UpdateIncidentRequest" - example: - status: "approved" - tags: - - "wildland" - - "mutual_aid" - - "lightning" - - "reviewed" - notes: "Reviewed by BC Wilson. Ready for submission." - responses: - "200": - description: Incident updated - content: - application/json: - schema: - $ref: "../schemas/incident-record.yaml#/IncidentRecord" - "404": - description: Incident not found - content: - application/json: - schema: - $ref: "../schemas/common.yaml#/ErrorResponse" - "409": - description: Cannot modify a submitted incident - content: - application/json: - schema: - $ref: "../schemas/common.yaml#/ErrorResponse" - example: - error_code: "INCIDENT_SUBMITTED" - message: "Cannot modify an incident that has been submitted" - detail: - submitted_at: "2024-07-15T18:00:00Z" - - delete: - operationId: deleteIncident - summary: Soft-delete an incident - description: | - Performs a soft delete sets a deleted_at timestamp but never removes data. - Submitted incidents cannot be deleted. Soft-deleted incidents are excluded - from list queries by default but can be recovered. - tags: - - incidents - parameters: - - name: incident_id - in: path - required: true - description: Unique identifier of the incident to delete - schema: - type: string - format: uuid - responses: - "200": - description: Incident soft-deleted - content: - application/json: - schema: - type: object - properties: - incident_id: - type: string - format: uuid - deleted_at: - type: string - format: date-time - recoverable: - type: boolean - example: - incident_id: "550e8400-e29b-41d4-a716-446655440050" - deleted_at: "2024-07-16T10:00:00Z" - recoverable: true - "404": - description: Incident not found - content: - application/json: - schema: - $ref: "../schemas/common.yaml#/ErrorResponse" - "409": - description: Cannot delete already deleted or submitted - content: - application/json: - schema: - $ref: "../schemas/common.yaml#/ErrorResponse" - examples: - already_deleted: - summary: Incident already soft-deleted - value: - error_code: "ALREADY_DELETED" - message: "Incident has already been deleted" - detail: - deleted_at: "2024-07-16T09:00:00Z" - submitted: - summary: Submitted incidents cannot be deleted - value: - error_code: "INCIDENT_SUBMITTED" - message: "Submitted incidents cannot be deleted" diff --git a/contracts/path/reporting.yaml b/contracts/path/reporting.yaml deleted file mode 100644 index ad101723..00000000 --- a/contracts/path/reporting.yaml +++ /dev/null @@ -1,191 +0,0 @@ -# Layer 5 Reporting & Analytics Endpoints -# GET /api/v1/reports/summary -# POST /api/v1/reports/generate -# GET /api/v1/reports/{report_id} - -summary: - get: - operationId: getReportSummary - summary: Get aggregate incident statistics - description: | - Returns aggregate statistics for a dashboard view total incidents, - breakdown by type and status, forms generated, average completeness - scores, and average processing times. Supports filtering by date range - and grouping by time period or incident type. - tags: - - reporting - parameters: - - name: date_from - in: query - description: Start date (inclusive, ISO 8601) - schema: - type: string - format: date - - name: date_to - in: query - description: End date (inclusive, ISO 8601) - schema: - type: string - format: date - - name: group_by - in: query - description: Group results by time period or incident type - schema: - type: string - enum: - - day - - week - - month - - incident_type - default: month - responses: - "200": - description: Aggregate statistics - content: - application/json: - schema: - $ref: "../schemas/reporting.yaml#/ReportSummary" - example: - date_from: "2024-01-01" - date_to: "2024-07-15" - total_incidents: 157 - by_type: - fire: 42 - ems: 68 - rescue: 12 - hazardous_conditions: 8 - service_call: 15 - good_intent: 7 - false_alarm: 5 - by_status: - draft: 3 - under_review: 2 - approved: 12 - submitted: 140 - forms_generated: 489 - avg_completeness_score: 88.3 - avg_processing_time_seconds: 47.2 - groups: - - period: "2024-07" - incident_count: 23 - forms_generated: 71 - -generate: - post: - operationId: generateReport - summary: Generate a periodic report - description: | - Generates a periodic (monthly, quarterly, or annual) aggregate report - as PDF or JSON. This includes statistics, incident summaries, and - compliance metrics required by agencies like USFA/NERIS. This is an - async operation returns a report_id for polling. - - Note: USFA encourages monthly submission but requires quarterly at minimum. - This endpoint supports generating reports aligned with those cadences. - tags: - - reporting - requestBody: - required: true - content: - application/json: - schema: - $ref: "../schemas/reporting.yaml#/GenerateReportRequest" - example: - period_type: "monthly" - year: 2024 - month: 7 - format: "pdf" - responses: - "202": - description: Report generation job accepted - content: - application/json: - schema: - $ref: "../schemas/common.yaml#/AsyncJobResponse" - example: - job_id: "550e8400-e29b-41d4-a716-446655440096" - job_type: "report_generation" - status: "processing" - estimated_seconds: 30 - poll_url: "/api/v1/reports/550e8400-e29b-41d4-a716-446655440060" - report_id: "550e8400-e29b-41d4-a716-446655440060" - "422": - description: Invalid period or future date - content: - application/json: - schema: - $ref: "../schemas/common.yaml#/ErrorResponse" - examples: - future_period: - summary: Future period requested - value: - error_code: "FUTURE_PERIOD" - message: "Cannot generate a report for a future period" - invalid_quarter: - summary: Invalid quarter value - value: - error_code: "VALIDATION_ERROR" - message: "Invalid quarter value" - validation_errors: - - field: "quarter" - issue: "Must be 1, 2, 3, or 4" - value: 5 - -report_by_id: - get: - operationId: getReport - summary: Retrieve a generated report - description: | - Returns a previously generated periodic report. If the report is still - being generated, returns 202 Accepted with a retry hint. When complete, - returns the report in the originally requested format (PDF binary or JSON). - tags: - - reporting - parameters: - - name: report_id - in: path - required: true - description: Unique identifier of the generated report - schema: - type: string - format: uuid - responses: - "200": - description: Generated report (PDF or JSON depending on original request) - content: - application/pdf: - schema: - type: string - format: binary - application/json: - schema: - $ref: "../schemas/reporting.yaml#/PeriodicReport" - headers: - Content-Disposition: - description: Attachment filename for PDF downloads - schema: - type: string - example: 'attachment; filename="FireForm_Monthly_2024-07.pdf"' - "202": - description: Report still being generated - content: - application/json: - schema: - type: object - properties: - message: - type: string - status: - type: string - retry_after_seconds: - type: integer - example: - message: "Report generation is still in progress" - status: "processing" - retry_after_seconds: 10 - "404": - description: Report not found - content: - application/json: - schema: - $ref: "../schemas/common.yaml#/ErrorResponse" diff --git a/contracts/path/templates.yaml b/contracts/path/templates.yaml index c2621cac..90644717 100644 --- a/contracts/path/templates.yaml +++ b/contracts/path/templates.yaml @@ -13,8 +13,9 @@ templates: description: | Returns all registered form templates including built-in standard templates (NERIS, NEMSIS, NIBRS, NFIRS modules, OSHA, etc.) and any custom templates - added for specific jurisdictions. Each template defines the fields, validation - rules, and mapping from the FireForm incident schema. + added for specific jurisdictions. Each template defines its own field + dictionary - the field names, types, and validation rules used to fill + that template's PDF at fill time. tags: - templates responses: @@ -27,7 +28,7 @@ templates: items: $ref: "../schemas/template.yaml#/TemplateSummary" example: - - template_id: "550e8400-e29b-41d4-a716-446655440070" + - template_id: 1 form_type: "neris" display_name: "NERIS Incident Report" jurisdiction: "US-Federal" @@ -36,7 +37,7 @@ templates: last_updated: "2026-02-01" field_count: 85 status: "active" - - template_id: "550e8400-e29b-41d4-a716-446655440071" + - template_id: 2 form_type: "nemsis_epcr" display_name: "NEMSIS Electronic Patient Care Report" jurisdiction: "US-Federal" @@ -45,7 +46,7 @@ templates: last_updated: "2024-01-15" field_count: 142 status: "active" - - template_id: "550e8400-e29b-41d4-a716-446655440072" + - template_id: 3 form_type: "nfirs_basic" display_name: "NFIRS Basic Module (Legacy)" jurisdiction: "US-Federal" @@ -61,8 +62,9 @@ templates: description: | Registers a new form template for a jurisdiction or agency not yet supported. This is how FireForm extends to new states, countries, or custom agency forms - without code changes. The template defines all fields, their types, validation - rules, and how each maps from the FireForm incident schema. + without code changes. The template defines its own field dictionary - each + field's name, type, and validation rules - used to fill that template's PDF + at fill time. tags: - templates requestBody: @@ -83,7 +85,6 @@ templates: required: true max_length: 20 description: "State-assigned incident number" - incident_mapping: "report_metadata.incident_number" layout: page: 0 x: 188.33 @@ -102,7 +103,6 @@ templates: - "natural" - "intentional" - "undetermined" - incident_mapping: "fire.cause_category" layout: page: 0 x: 188.33 @@ -113,7 +113,6 @@ templates: field_type: "string" required: false static_text: "Generated by FireForm" - incident_mapping: null layout: page: 0 x: 72.0 @@ -148,8 +147,9 @@ template_by_id: summary: Get full template schema description: | Returns the complete template definition including all fields, their types, - required/optional status, validation rules, and the mapping from the FireForm - incident schema fields. Includes the source standard reference where applicable. + required/optional status, and validation rules that this template's PDF is + filled against at fill time. Includes the source standard reference where + applicable. tags: - templates parameters: @@ -158,8 +158,7 @@ template_by_id: required: true description: Unique identifier of the template schema: - type: string - format: uuid + type: integer responses: "200": description: Full template definition @@ -189,8 +188,7 @@ template_by_id: required: true description: Unique identifier of the template to update schema: - type: string - format: uuid + type: integer requestBody: required: true content: @@ -229,8 +227,8 @@ template_fields: summary: Get template field definitions description: | Returns just the fields list for a template with type, required/optional - status, validation rules, and incident-schema mapping. Useful for building - validation checklists and for the validate endpoint to determine requirements. + status, and validation rules. Useful for building validation checklists and + for the validate endpoint to determine requirements. tags: - templates parameters: @@ -239,8 +237,7 @@ template_fields: required: true description: Unique identifier of the template schema: - type: string - format: uuid + type: integer - name: required_only in: query description: If true, return only required fields @@ -256,10 +253,9 @@ template_fields: type: object properties: template_id: - type: string - format: uuid + type: integer form_type: - $ref: "../schemas/enums.yaml#/FormType" + type: string total_fields: type: integer required_fields: @@ -271,7 +267,7 @@ template_fields: items: $ref: "../schemas/template.yaml#/TemplateField" example: - template_id: "550e8400-e29b-41d4-a716-446655440070" + template_id: 1 form_type: "neris" total_fields: 85 required_fields: 32 @@ -281,7 +277,6 @@ template_fields: field_type: "enum" required: true description: "Primary incident type code" - incident_mapping: "incident.types[0].neris_code" layout: page: 0 x: 120.0 @@ -292,7 +287,6 @@ template_fields: field_type: "date" required: true description: "Date of incident" - incident_mapping: "incident.start_datetime" layout: page: 0 x: 120.0 diff --git a/contracts/schemas/enums.yaml b/contracts/schemas/enums.yaml index b79bedae..98027006 100644 --- a/contracts/schemas/enums.yaml +++ b/contracts/schemas/enums.yaml @@ -18,15 +18,6 @@ ExtractionStatus: - needs_review description: Status of an AI extraction job -FormStatus: - type: string - enum: - - queued - - generating - - completed - - failed - description: Status of a form generation job - ReportStatus: type: string enum: @@ -45,34 +36,6 @@ JobStatus: - failed description: Status of any async job -FormType: - type: string - enum: - - neris - - nemsis_epcr - - nibrs - - nfirs_basic - - nfirs_fire - - nfirs_structure - - nfirs_wildland - - nfirs_ems - - nfirs_hazmat - - nfirs_apparatus - - nfirs_personnel - - nfirs_arson - - nfirs_casualty_civilian - - nfirs_casualty_responder - - cal_fire_ics209 - - osha_301 - - un_ssirs - - state_georgia - - state_california - - state_new_york - description: | - Stable string identifier for a form type. Includes the new NERIS standard - (replacing NFIRS as of Feb 2026), legacy NFIRS modules, NEMSIS, NIBRS, - OSHA, state-specific, and international (UN SSIRS) forms. - IncidentCategory: type: string enum: @@ -120,11 +83,3 @@ PeriodType: - quarterly - annual description: Reporting period type - -OutputFormat: - type: string - enum: - - pdf - - json - - both - description: Output format for generated forms diff --git a/contracts/schemas/extraction-record.yaml b/contracts/schemas/extraction-record.yaml deleted file mode 100644 index ae0fd487..00000000 --- a/contracts/schemas/extraction-record.yaml +++ /dev/null @@ -1,134 +0,0 @@ -# Extraction-related schemas - -ExtractionRequest: - type: object - properties: - model_override: - type: string - description: Override the default LLM model (e.g. "llama3:70b") - extraction_hints: - type: object - description: Optional hints to improve extraction accuracy - properties: - incident_type: - type: string - description: Hint about the incident type (e.g. "wildland_fire", "structure_fire") - state: - type: string - description: US state code to apply state-specific extraction rules - agency_type: - type: string - description: Agency type hint for form selection - additionalProperties: true - -ExtractionCompleted: - type: object - required: - - extract_id - - input_id - - status - - incident_contract - properties: - extract_id: - type: string - format: uuid - input_id: - type: string - format: uuid - status: - type: string - enum: - - completed - completed_at: - type: string - format: date-time - model_used: - type: string - description: LLM model that performed the extraction - processing_time_seconds: - type: number - incident_contract: - $ref: "incident-contract.yaml#/IncidentContract" - corrections: - type: array - description: Audit trail of manual corrections applied via PATCH - items: - type: object - properties: - field_path: - type: string - original_value: {} - corrected_value: {} - corrected_at: - type: string - format: date-time - corrected_by: - type: string - -ExtractionProcessing: - type: object - required: - - extract_id - - input_id - - status - properties: - extract_id: - type: string - format: uuid - input_id: - type: string - format: uuid - status: - type: string - enum: - - processing - - failed - started_at: - type: string - format: date-time - retry_after_seconds: - type: integer - description: Polling hint for clients - error_type: - type: string - nullable: true - description: Present when status is "failed" - error_detail: - type: string - nullable: true - partial_result: - $ref: "incident-contract.yaml#/IncidentContract" - -ValidationResult: - type: object - required: - - valid - - form_type - - extract_id - properties: - valid: - type: boolean - description: Whether all required fields for this form type are present - form_type: - $ref: "enums.yaml#/FormType" - extract_id: - type: string - format: uuid - missing_required: - type: array - items: - type: string - description: JSON paths of required fields that are missing - missing_recommended: - type: array - items: - type: string - description: JSON paths of recommended fields that are missing - warnings: - type: array - items: - type: string - description: Human-readable warnings about data quality - field_coverage_percent: - type: number - description: Percentage of form fields that have values diff --git a/contracts/schemas/form-record.yaml b/contracts/schemas/form-record.yaml index a83ec41a..1f31e59b 100644 --- a/contracts/schemas/form-record.yaml +++ b/contracts/schemas/form-record.yaml @@ -1,198 +1,85 @@ -# Form generation related schemas +# Form fill related schemas -GenerateAllRequest: +FormFillRequest: type: object required: - - extract_id + - template_id + - input_id properties: - extract_id: + template_id: + type: integer + description: ID of the template to fill + input_id: type: string format: uuid - options: - type: object - properties: - skip_incomplete: - type: boolean - default: true - description: Skip forms that fail validation - force_partial: - type: boolean - default: false - description: Generate forms even with missing fields (leaving blanks) - -GenerateSingleRequest: - type: object - required: - - extract_id - properties: - extract_id: + description: ID of a stored input (see POST /input/text, POST /input/voice) + whose transcript is used to fill this template. The input must have + status "ready". + model: type: string - format: uuid - options: - type: object - properties: - output_format: - $ref: "../schemas/enums.yaml#/OutputFormat" - force_partial: - type: boolean - default: false - force: - type: boolean - default: false - description: Allow generation even if form_type is not in applicable_forms + nullable: true + description: Ollama model override. Defaults to the configured default model. -BatchGenerateResponse: +FormFillResponse: type: object required: - - batch_id - - status - - extract_id + - id + - template_id + - input_text + - output_pdf_path properties: - batch_id: - type: string - format: uuid - status: - type: string - enum: [processing] - extract_id: - type: string - format: uuid - forms_queued: - type: array - items: - $ref: "../schemas/enums.yaml#/FormType" - forms_skipped: - type: array - items: - type: object - properties: - form_type: - $ref: "../schemas/enums.yaml#/FormType" - reason: - type: string - estimated_seconds: + id: type: integer - poll_url: - type: string - -FormGenerateResponse: - type: object - required: - - form_id - - form_type - - status - properties: - form_id: - type: string - format: uuid - form_type: - $ref: "../schemas/enums.yaml#/FormType" - status: - type: string - enum: [processing, completed] - extract_id: - type: string - format: uuid - job_id: - type: string - format: uuid - estimated_seconds: + template_id: type: integer - poll_url: - type: string - -FormRecord: - type: object - required: - - form_id - - form_type - - status - properties: - form_id: - type: string - format: uuid - form_type: - $ref: "../schemas/enums.yaml#/FormType" - status: - $ref: "../schemas/enums.yaml#/FormStatus" - extract_id: - type: string - format: uuid - incident_id: - type: string - format: uuid - nullable: true - created_at: + input_text: type: string - format: date-time - completed_at: + output_pdf_path: type: string - format: date-time - nullable: true - pdf_ready: - type: boolean - json_ready: - type: boolean - field_mapping_summary: - type: object - properties: - total_form_fields: - type: integer - fields_filled: - type: integer - fields_blank: - type: integer - coverage_percent: - type: number + description: Path to the filled PDF output file -FormMappedJson: +AsyncFormFillRequest: type: object required: - - form_type - - form_id + - template_ids + - input_id properties: - form_type: - $ref: "../schemas/enums.yaml#/FormType" - form_version: - type: string - form_id: + template_ids: + type: array + items: + type: integer + description: IDs of the templates to fill. One job is queued per template, + each running the same transcript against that template's field dictionary. + input_id: type: string format: uuid - extract_id: + description: ID of a stored input (see POST /input/text, POST /input/voice) + whose transcript is used to fill each template. The input must have + status "ready". + model: type: string - format: uuid - agency_fields: - type: object - additionalProperties: true - description: Agency-specific field names and values as the target system expects + nullable: true + description: Ollama model override. Defaults to the configured default model. -BatchStatus: +AsyncFormFillResponse: type: object required: - - batch_id - - status + - jobs properties: - batch_id: - type: string - format: uuid - status: - type: string - enum: [processing, completed, failed] - total: - type: integer - completed: - type: integer - failed: - type: integer - forms: + jobs: type: array items: type: object + required: + - job_id + - status + - poll_url properties: - form_id: + job_id: type: string format: uuid - form_type: - $ref: "../schemas/enums.yaml#/FormType" status: - $ref: "../schemas/enums.yaml#/FormStatus" + type: string + poll_url: + type: string + description: Path to poll via GET /api/v1/jobs/{job_id} diff --git a/contracts/schemas/incident-contract.yaml b/contracts/schemas/incident-contract.yaml deleted file mode 100644 index 1a129df1..00000000 --- a/contracts/schemas/incident-contract.yaml +++ /dev/null @@ -1,930 +0,0 @@ -# Canonical FireForm Incident Schema -# This is the master superset schema single source of truth for all downstream forms - -IncidentContract: - type: object - description: | - The canonical FireForm incident data model. This is the superset schema containing - every field any downstream form could need. Form-specific mappers select only the - relevant fields for each agency template. - properties: - schema_version: - type: string - description: Schema version identifier - example: "1.1.0" - schema_name: - type: string - description: Schema name identifier - enum: - - fireform_incident_contract - - extraction_metadata: - $ref: "#/ExtractionMetadata" - report_metadata: - $ref: "#/ReportMetadata" - incident: - $ref: "#/Incident" - location: - $ref: "#/Location" - fire: - $ref: "#/Fire" - wildland: - $ref: "#/Wildland" - structure: - $ref: "#/Structure" - casualties: - $ref: "#/Casualties" - ems: - $ref: "#/EMS" - hazmat: - $ref: "#/Hazmat" - arson: - $ref: "#/Arson" - responding_agencies: - $ref: "#/RespondingAgencies" - resources_deployed: - $ref: "#/ResourcesDeployed" - weather: - $ref: "#/Weather" - environmental_impact: - $ref: "#/EnvironmentalImpact" - infrastructure_impact: - $ref: "#/InfrastructureImpact" - near_miss_and_safety: - $ref: "#/NearMissAndSafety" - lessons_learned: - $ref: "#/LessonsLearned" - follow_up: - $ref: "#/FollowUp" - periodic_reporting: - $ref: "#/PeriodicReporting" - attachments: - $ref: "#/Attachments" - -# --- Sub-schemas --- - -ExtractionMetadata: - type: object - properties: - extract_id: - type: string - format: uuid - input_id: - type: string - format: uuid - input_type: - type: string - enum: [voice, text] - extracted_at: - type: string - format: date-time - llm_model: - type: string - example: "llama3:8b" - confidence_score: - type: number - minimum: 0 - maximum: 1 - description: Overall confidence score from the LLM extraction (0.0–1.0) - completeness: - $ref: "#/Completeness" - applicable_forms: - type: array - items: - $ref: "../schemas/enums.yaml#/FormType" - -Completeness: - type: object - description: | - Tells the system which forms can be fully auto-generated vs which need - manual review. Recalculated server-side after every PATCH /extract. - properties: - overall_percent: - type: integer - minimum: 0 - maximum: 100 - missing_fields: - type: array - items: - type: string - description: JSON paths of fields that have no value - low_confidence_fields: - type: array - items: - type: string - description: JSON paths of fields where LLM confidence is low - inferred_fields: - type: array - items: - type: string - description: JSON paths of fields that were inferred (not explicitly stated) - forms_fully_generatable: - type: array - items: - $ref: "../schemas/enums.yaml#/FormType" - forms_needing_review: - type: array - items: - $ref: "../schemas/enums.yaml#/FormType" - forms_missing_data: - type: array - items: - $ref: "../schemas/enums.yaml#/FormType" - -ReportMetadata: - type: object - properties: - report_id: - type: string - example: "FF-2024-CA-0157" - incident_number: - type: string - example: "CA-SQF-2024-0421" - report_date: - type: string - format: date - report_time: - type: string - format: time - report_status: - $ref: "../schemas/enums.yaml#/ReportStatus" - reporting_unit: - $ref: "#/ReportingUnit" - prepared_by: - type: array - items: - $ref: "#/Personnel" - reviewed_by: - type: array - items: - $ref: "#/Reviewer" - submission_log: - type: array - items: - type: object - properties: - form_type: - $ref: "../schemas/enums.yaml#/FormType" - submitted_at: - type: string - format: date-time - submitted_to: - type: string - -ReportingUnit: - type: object - properties: - station_name: - type: string - station_id: - type: string - agency_name: - type: string - agency_id: - type: string - format: uuid - agency_type: - type: string - -Personnel: - type: object - properties: - name: - type: string - badge_number: - type: string - rank: - type: string - role: - type: string - contact_number: - type: string - signature_captured: - type: boolean - -Reviewer: - type: object - properties: - name: - type: string - badge_number: - type: string - rank: - type: string - role: - type: string - reviewed_at: - type: string - format: date-time - approved: - type: boolean - -Incident: - type: object - properties: - name: - type: string - description: Human-readable incident name - types: - type: array - items: - $ref: "#/IncidentType" - start_datetime: - type: string - format: date-time - alarm_datetime: - type: string - format: date-time - first_arrival_datetime: - type: string - format: date-time - containment_datetime: - type: string - format: date-time - nullable: true - controlled_datetime: - type: string - format: date-time - nullable: true - cleared_datetime: - type: string - format: date-time - nullable: true - total_duration_hours: - type: number - nullable: true - narrative: - type: string - description: Free text summary of the incident - raw_transcript: - type: string - description: Original voice or text input verbatim - -IncidentType: - type: object - properties: - primary: - type: boolean - category: - $ref: "../schemas/enums.yaml#/IncidentCategory" - subcategory: - type: string - neris_code: - type: string - description: NERIS incident type code (replaces NFIRS as of Feb 2026) - nfirs_code: - type: string - description: Legacy NFIRS incident type code - -Location: - type: object - properties: - address: - type: string - nullable: true - nearest_landmark: - type: string - nullable: true - nearest_town: - type: string - nullable: true - county: - type: string - nullable: true - state: - type: string - nullable: true - country: - type: string - nullable: true - postal_code: - type: string - nullable: true - coordinates: - $ref: "#/Coordinates" - ignition_point_coordinates: - $ref: "#/CoordinatesBasic" - elevation_range_ft: - type: string - nullable: true - legal_description: - type: string - nullable: true - jurisdiction: - type: object - properties: - federal: - type: boolean - state: - type: boolean - private: - type: boolean - tribal: - type: boolean - property_type: - type: string - nullable: true - property_use: - type: string - nullable: true - dispatch_center: - type: string - nullable: true - -Coordinates: - type: object - properties: - latitude: - type: number - longitude: - type: number - accuracy_meters: - type: number - -CoordinatesBasic: - type: object - properties: - latitude: - type: number - longitude: - type: number - -Fire: - type: object - properties: - cause_category: - type: string - nullable: true - cause_specific: - type: string - nullable: true - cause_certainty: - $ref: "../schemas/enums.yaml#/CauseCertainty" - arson_suspected: - type: boolean - material_first_ignited: - type: string - nullable: true - fuel_types: - type: array - items: - type: string - fire_spread_directions: - type: array - items: - type: string - rate_of_spread: - $ref: "../schemas/enums.yaml#/RateOfSpread" - flame_lengths_ft: - type: string - nullable: true - spotting_distance_miles: - type: number - nullable: true - unusual_behaviors: - type: array - items: - type: string - detector_present: - type: boolean - nullable: true - detector_operated: - type: boolean - nullable: true - suppression_system_present: - type: boolean - nullable: true - suppression_system_operated: - type: boolean - nullable: true - estimated_damage_usd: - type: number - nullable: true - contents_loss_usd: - type: number - nullable: true - -Wildland: - type: object - properties: - is_wildland_incident: - type: boolean - total_acres_burned: - type: number - nullable: true - land_ownership_breakdown: - type: object - properties: - federal_acres: - type: number - state_acres: - type: number - private_acres: - type: number - tribal_acres: - type: number - percent_contained: - type: integer - minimum: 0 - maximum: 100 - fire_lines: - type: object - properties: - primary_line_miles: - type: number - secondary_line_miles: - type: number - dozer_line_miles: - type: number - hand_line_miles: - type: number - aerial_operations: - type: object - properties: - water_drops_gallons: - type: integer - retardant_drops_gallons: - type: integer - total_flight_hours: - type: number - containment_strategies: - type: array - items: - type: string - -Structure: - type: object - properties: - is_structure_involved: - type: boolean - structures_threatened: - type: integer - nullable: true - structures_damaged: - type: integer - nullable: true - structures_destroyed: - type: integer - nullable: true - structures_protected: - type: integer - nullable: true - construction_type: - type: string - nullable: true - stories: - type: integer - nullable: true - area_sqft: - type: number - nullable: true - occupancy_at_time: - type: integer - nullable: true - -Casualties: - type: object - properties: - civilian: - type: array - items: - $ref: "#/CivilianCasualty" - responder: - type: array - items: - $ref: "#/ResponderCasualty" - total_civilian_injuries: - type: integer - total_civilian_fatalities: - type: integer - total_responder_injuries: - type: integer - total_responder_fatalities: - type: integer - -CivilianCasualty: - type: object - properties: - age: - type: integer - nullable: true - sex: - type: string - nullable: true - injury_type: - type: string - severity: - $ref: "../schemas/enums.yaml#/InjurySeverity" - cause: - type: string - nullable: true - location_at_time: - type: string - nullable: true - transported: - type: boolean - hospital: - type: string - nullable: true - -ResponderCasualty: - type: object - properties: - personnel_id: - type: string - agency: - type: string - role: - type: string - injury_type: - type: string - severity: - $ref: "../schemas/enums.yaml#/InjurySeverity" - treatment: - type: string - nullable: true - transported: - type: boolean - hospital: - type: string - nullable: true - return_to_duty_date: - type: string - format: date - nullable: true - osha_recordable: - type: boolean - nfirs_5_required: - type: boolean - -EMS: - type: object - properties: - ems_response_required: - type: boolean - patients: - type: array - items: - $ref: "#/EMSPatient" - total_patients: - type: integer - ems_agency_responded: - type: string - nullable: true - nemsis_report_required: - type: boolean - nemsis_report_ids: - type: array - items: - type: string - -EMSPatient: - type: object - properties: - patient_ref_id: - type: string - age_approx: - type: integer - nullable: true - sex: - type: string - nullable: true - chief_complaint: - type: string - nullable: true - disposition: - type: string - nullable: true - transported: - type: boolean - date_of_birth: - type: string - format: date - nullable: true - nemsis_data_captured: - type: boolean - -Hazmat: - type: object - properties: - involved: - type: boolean - materials: - type: array - items: - type: object - properties: - name: - type: string - un_number: - type: string - nullable: true - quantity: - type: string - nullable: true - epa_reportable_quantity_exceeded: - type: boolean - spill_size_gallons: - type: number - nullable: true - -Arson: - type: object - properties: - suspected: - type: boolean - confirmed: - type: boolean - law_enforcement_notified: - type: boolean - investigation_required: - type: boolean - investigation_agency: - type: string - nullable: true - evidence_collected: - type: boolean - nibrs_report_required: - type: boolean - notes: - type: string - nullable: true - -RespondingAgencies: - type: object - properties: - primary_agency: - type: string - all_agencies: - type: array - items: - $ref: "#/RespondingAgency" - mutual_aid_activated: - type: boolean - mutual_aid_agencies: - type: array - items: - type: string - unified_command: - type: boolean - -RespondingAgency: - type: object - properties: - agency_name: - type: string - agency_type: - type: string - role: - type: string - personnel_count: - type: integer - -ResourcesDeployed: - type: object - properties: - total_personnel: - type: integer - personnel_breakdown: - type: object - properties: - firefighters: - type: integer - crew_supervisors: - type: integer - engineers: - type: integer - incident_command: - type: integer - support_staff: - type: integer - apparatus: - type: array - items: - $ref: "#/Apparatus" - crew_types: - type: array - items: - type: string - -Apparatus: - type: object - properties: - type: - type: string - count: - type: integer - -Weather: - type: object - properties: - on_arrival: - $ref: "#/WeatherReading" - worst_conditions: - $ref: "#/WeatherReadingExtended" - factors_influencing_fire: - type: array - items: - type: string - -WeatherReading: - type: object - properties: - datetime: - type: string - format: date-time - temperature_f: - type: number - relative_humidity_percent: - type: number - wind_speed_mph: - type: number - wind_direction: - type: string - haines_index: - type: integer - nullable: true - -WeatherReadingExtended: - type: object - properties: - datetime: - type: string - format: date-time - temperature_f: - type: number - relative_humidity_percent: - type: number - wind_speed_mph: - type: number - wind_gusts_mph: - type: number - nullable: true - -EnvironmentalImpact: - type: object - properties: - wildlife_habitat_affected_acres: - type: number - nullable: true - watershed_impact: - type: string - nullable: true - soil_erosion_risk: - type: string - nullable: true - sensitive_species_affected: - type: array - items: - type: string - air_quality_impact: - type: string - nullable: true - water_body_affected: - type: boolean - nullable: true - -InfrastructureImpact: - type: object - properties: - items: - type: array - items: - $ref: "#/InfrastructureItem" - -InfrastructureItem: - type: object - properties: - type: - type: string - unit: - type: string - quantity: - type: number - severity: - type: string - -NearMissAndSafety: - type: object - properties: - near_miss_events: - type: array - items: - type: object - properties: - description: - type: string - date: - type: string - format: date - contributing_factors: - type: array - items: - type: string - lessons_learned: - type: string - nullable: true - corrective_action: - type: string - nullable: true - safety_breaches: - type: integer - weather_related_risks: - type: array - items: - type: string - -LessonsLearned: - type: object - properties: - successful_tactics: - type: array - items: - type: string - areas_for_improvement: - type: array - items: - type: string - recommendations: - type: array - items: - type: string - -FollowUp: - type: object - properties: - mop_up: - type: object - properties: - percent_complete: - type: integer - estimated_completion_date: - type: string - format: date - personnel_assigned: - type: integer - rehabilitation: - type: object - properties: - erosion_control_acres: - type: number - reseeding_acres: - type: number - hazard_tree_removal_required: - type: boolean - next_inspection_date: - type: string - format: date - nullable: true - investigation_ongoing: - type: boolean - -PeriodicReporting: - type: object - properties: - contributes_to_monthly_report: - type: boolean - contributes_to_quarterly_report: - type: boolean - contributes_to_annual_report: - type: boolean - neris_submitted: - type: boolean - neris_submitted_at: - type: string - format: date-time - nullable: true - state_submitted: - type: boolean - state_submitted_at: - type: string - format: date-time - nullable: true - -Attachments: - type: object - properties: - maps: - type: boolean - photos_count: - type: integer - weather_charts: - type: boolean - resource_tracking_logs: - type: boolean - incident_action_plans: - type: boolean - attachment_refs: - type: array - items: - type: object - properties: - ref_id: - type: string - format: uuid - filename: - type: string - content_type: - type: string - size_bytes: - type: integer diff --git a/contracts/schemas/incident-record.yaml b/contracts/schemas/incident-record.yaml deleted file mode 100644 index 686e7ce8..00000000 --- a/contracts/schemas/incident-record.yaml +++ /dev/null @@ -1,150 +0,0 @@ -# Incident management schemas - -CreateIncidentRequest: - type: object - required: - - extract_id - properties: - extract_id: - type: string - format: uuid - incident_number: - type: string - description: Optional org-assigned incident number - tags: - type: array - items: - type: string - -UpdateIncidentRequest: - type: object - properties: - status: - $ref: "../schemas/enums.yaml#/ReportStatus" - tags: - type: array - items: - type: string - incident_number: - type: string - notes: - type: string - -IncidentRecord: - type: object - required: - - incident_id - - extract_id - - status - properties: - incident_id: - type: string - format: uuid - extract_id: - type: string - format: uuid - incident_number: - type: string - nullable: true - status: - $ref: "../schemas/enums.yaml#/ReportStatus" - incident_name: - type: string - nullable: true - incident_type: - type: string - nullable: true - incident_date: - type: string - format: date - nullable: true - forms_generated: - type: array - items: - type: object - properties: - form_id: - type: string - format: uuid - form_type: - $ref: "../schemas/enums.yaml#/FormType" - status: - $ref: "../schemas/enums.yaml#/FormStatus" - tags: - type: array - items: - type: string - notes: - type: string - nullable: true - created_at: - type: string - format: date-time - updated_at: - type: string - format: date-time - deleted_at: - type: string - format: date-time - nullable: true - -IncidentRecordFull: - description: Full incident record with linked extraction and forms - allOf: - - $ref: "#/IncidentRecord" - - type: object - properties: - incident_contract: - $ref: "incident-contract.yaml#/IncidentContract" - forms: - type: array - items: - $ref: "form-record.yaml#/FormRecord" - submission_log: - type: array - items: - type: object - properties: - form_type: - $ref: "../schemas/enums.yaml#/FormType" - submitted_at: - type: string - format: date-time - submitted_to: - type: string - status: - type: string - -IncidentListResponse: - type: object - properties: - data: - type: array - items: - type: object - properties: - incident_id: - type: string - format: uuid - incident_number: - type: string - nullable: true - status: - $ref: "../schemas/enums.yaml#/ReportStatus" - incident_name: - type: string - nullable: true - incident_type: - type: string - nullable: true - incident_date: - type: string - format: date - nullable: true - forms_count: - type: integer - created_at: - type: string - format: date-time - pagination: - $ref: "common.yaml#/Pagination" diff --git a/contracts/schemas/reporting.yaml b/contracts/schemas/reporting.yaml deleted file mode 100644 index f352e709..00000000 --- a/contracts/schemas/reporting.yaml +++ /dev/null @@ -1,112 +0,0 @@ -# Reporting & Analytics schemas - -ReportSummary: - type: object - properties: - date_from: - type: string - format: date - date_to: - type: string - format: date - total_incidents: - type: integer - by_type: - type: object - additionalProperties: - type: integer - description: Incident counts grouped by category - by_status: - type: object - additionalProperties: - type: integer - description: Incident counts grouped by report status - forms_generated: - type: integer - avg_completeness_score: - type: number - avg_processing_time_seconds: - type: number - groups: - type: array - items: - type: object - properties: - period: - type: string - description: Period label (e.g. "2024-07" for monthly, "fire" for by type) - incident_count: - type: integer - forms_generated: - type: integer - -GenerateReportRequest: - type: object - required: - - period_type - - year - properties: - period_type: - $ref: "../schemas/enums.yaml#/PeriodType" - year: - type: integer - minimum: 2000 - maximum: 2100 - month: - type: integer - minimum: 1 - maximum: 12 - description: Required when period_type is "monthly" - quarter: - type: integer - minimum: 1 - maximum: 4 - description: Required when period_type is "quarterly" - format: - $ref: "../schemas/enums.yaml#/OutputFormat" - -PeriodicReport: - type: object - properties: - report_id: - type: string - format: uuid - period_type: - $ref: "../schemas/enums.yaml#/PeriodType" - period_label: - type: string - description: Human-readable period (e.g. "July 2024", "Q3 2024", "2024") - generated_at: - type: string - format: date-time - summary: - $ref: "#/ReportSummary" - incidents: - type: array - items: - type: object - properties: - incident_id: - type: string - format: uuid - incident_number: - type: string - incident_date: - type: string - format: date - incident_type: - type: string - status: - type: string - forms_generated: - type: integer - compliance: - type: object - properties: - neris_submission_rate: - type: number - description: Percentage of incidents with NERIS submitted - average_submission_delay_days: - type: number - overdue_incidents: - type: integer diff --git a/contracts/schemas/template.yaml b/contracts/schemas/template.yaml index f9ce9214..0127e5bc 100644 --- a/contracts/schemas/template.yaml +++ b/contracts/schemas/template.yaml @@ -4,8 +4,7 @@ TemplateSummary: type: object properties: template_id: - type: string - format: uuid + type: integer form_type: type: string description: Unique form type identifier (built-in or custom jurisdiction) @@ -70,8 +69,7 @@ Template: - type: object properties: template_id: - type: string - format: uuid + type: integer version: type: string last_updated: @@ -134,22 +132,12 @@ TemplateField: type: string nullable: true description: Valid values for enum fields - incident_mapping: - type: string - nullable: true - description: | - JSON path in the FireForm incident schema this field pulls its value from - (e.g. "fire.cause_category"). Null for a static field. Exactly one of - incident_mapping or static_text is expected. static_text: type: string nullable: true description: | - Fixed text drawn into the field instead of a mapped value. Null for a - data-mapped field. Mutually exclusive with incident_mapping. - default_value: - nullable: true - description: Default value if the mapped incident field is null + Fixed text drawn into the field instead of an LLM-extracted value. Null + for a field whose value comes from the transcript at fill time. layout: nullable: true description: Visual placement of the field on the PDF. Null for fields with diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 00000000..092631f0 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,5 @@ +# Tools used at development time only, not needed to run the server. +# datamodel-code-generator turns the incident contract into Pydantic models +# (see scripts/generate_contract_models.py, run via `make generate-contract-models`). +datamodel-code-generator==0.25.9 +ruff==0.16.1 diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 00000000..c04f5335 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,7 @@ +target-version = "py311" +line-length = 100 + +[lint] +# Pin the rule set explicitly. Ruff's defaults changed in 0.16, so relying on +# them means a version bump silently changes what CI enforces. +select = ["E4", "E7", "E9", "F"] diff --git a/tests/conftest.py b/tests/conftest.py index 62fae518..897a611e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -91,8 +91,8 @@ def pdf_upload(pdf_bytes): @pytest.fixture def mock_controller(): """Patch Controller so create_template / fill_form don't touch the FS or LLM.""" - with patch("app.api.routes.templates.Controller") as tpl_cls, \ - patch("app.api.routes.forms.Controller") as form_cls: + with patch("app.services.template.Controller") as tpl_cls, \ + patch("app.services.form.Controller") as form_cls: tpl_instance = MagicMock() tpl_instance.create_template.return_value = "src/inputs/test_template.pdf" tpl_cls.return_value = tpl_instance diff --git a/tests/test_api.py b/tests/test_api.py index 32104ae7..b2b4538d 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -4,9 +4,12 @@ All heavy dependencies (LLM, commonforms, filesystem) are mocked via conftest. """ +from uuid import uuid4 + from sqlmodel import select -from app.models import Template, FormSubmission +from app.api.schemas.enums import InputStatus, InputType +from app.models import Template, FormSubmission, Input from app.core.config import API_PREFIX @@ -41,8 +44,16 @@ def test_form_submission_roundtrip(self, db): db.commit() db.refresh(tpl) + input_record = Input( + input_type=InputType.text, status=InputStatus.ready, transcript="John Doe, firefighter" + ) + db.add(input_record) + db.commit() + db.refresh(input_record) + sub = FormSubmission( template_id=tpl.id, + input_id=input_record.input_id, input_text="John Doe, firefighter", output_pdf_path="src/outputs/filled.pdf", ) @@ -53,6 +64,7 @@ def test_form_submission_roundtrip(self, db): fetched = db.get(FormSubmission, sub.id) assert fetched is not None assert fetched.template_id == tpl.id + assert fetched.input_id == input_record.input_id assert fetched.input_text == "John Doe, firefighter" assert fetched.created_at is not None @@ -126,7 +138,7 @@ def test_upload_pdf(self, client, pdf_upload, tmp_path, monkeypatch): # Point the upload directory inside tmp_path (which is inside the project # for the path-safety check — we monkeypatch the check). monkeypatch.setattr( - "app.api.routes.templates.PROJECT_ROOT", + "app.core.paths.PROJECT_ROOT", tmp_path, ) resp = client.post( @@ -180,35 +192,87 @@ def _seed_template(self, client, mock_controller): }) return resp.json()["id"] - def test_fill_form_success(self, client, mock_controller): + def _seed_input(self, db, status=InputStatus.ready, transcript="The employee is John Doe, email jdoe@ucsc.edu"): + """Helper: create an Input row directly and return its UUID.""" + record = Input(input_type=InputType.text, status=status, transcript=transcript) + db.add(record) + db.commit() + db.refresh(record) + return record.input_id + + def test_fill_form_success(self, client, mock_controller, db): tpl_id = self._seed_template(client, mock_controller) + input_id = self._seed_input(db) resp = client.post(f"{API_PREFIX}/forms/fill", json={ "template_id": tpl_id, - "input_text": "The employee is John Doe, email jdoe@ucsc.edu", + "input_id": str(input_id), }) assert resp.status_code == 200 data = resp.json() assert data["id"] is not None assert data["template_id"] == tpl_id + assert data["input_text"] == "The employee is John Doe, email jdoe@ucsc.edu" assert data["output_pdf_path"] == "src/outputs/filled_output.pdf" mock_controller["form_ctrl"].fill_form.assert_called_once() + fetched = db.get(FormSubmission, data["id"]) + assert fetched.input_id == input_id + def test_fill_form_missing_template(self, client, mock_controller): resp = client.post(f"{API_PREFIX}/forms/fill", json={ "template_id": 9999, - "input_text": "some text", + "input_id": str(uuid4()), + }) + assert resp.status_code == 404 + + def test_fill_form_missing_input(self, client, mock_controller): + tpl_id = self._seed_template(client, mock_controller) + + resp = client.post(f"{API_PREFIX}/forms/fill", json={ + "template_id": tpl_id, + "input_id": str(uuid4()), }) assert resp.status_code == 404 + assert resp.json()["error_code"] == "INPUT_NOT_FOUND" - def test_fill_form_template_file_not_found(self, client, mock_controller): + def test_fill_form_input_transcribing(self, client, mock_controller, db): + """An input still being transcribed is not usable yet → 409.""" tpl_id = self._seed_template(client, mock_controller) + input_id = self._seed_input(db, status=InputStatus.transcribing, transcript=None) + + resp = client.post(f"{API_PREFIX}/forms/fill", json={ + "template_id": tpl_id, + "input_id": str(input_id), + }) + assert resp.status_code == 409 + body = resp.json() + assert body["error_code"] == "INPUT_NOT_READY" + assert body["detail"]["status"] == "transcribing" + + def test_fill_form_input_failed(self, client, mock_controller, db): + """An input whose transcription failed is never usable → 409.""" + tpl_id = self._seed_template(client, mock_controller) + input_id = self._seed_input(db, status=InputStatus.failed, transcript=None) + + resp = client.post(f"{API_PREFIX}/forms/fill", json={ + "template_id": tpl_id, + "input_id": str(input_id), + }) + assert resp.status_code == 409 + body = resp.json() + assert body["error_code"] == "INPUT_NOT_READY" + assert body["detail"]["status"] == "failed" + + def test_fill_form_template_file_not_found(self, client, mock_controller, db): + tpl_id = self._seed_template(client, mock_controller) + input_id = self._seed_input(db) mock_controller["form_ctrl"].fill_form.side_effect = FileNotFoundError("PDF template not found") resp = client.post(f"{API_PREFIX}/forms/fill", json={ "template_id": tpl_id, - "input_text": "some text", + "input_id": str(input_id), }) assert resp.status_code == 500 assert resp.json()["error_code"] == "FORM_FILL_ERROR" @@ -283,12 +347,13 @@ def boom(*a, **k): assert resp.status_code == 200 assert resp.json()["models"] == ["qwen2.5:1.5b"] - def test_fill_form_passes_model_override(self, client, mock_controller): + def test_fill_form_passes_model_override(self, client, mock_controller, db): """A `model` in the request reaches Controller.fill_form but isn't persisted.""" tpl_id = self._seed_template(client, mock_controller) + input_id = self._seed_input(db, transcript="John Doe") resp = client.post(f"{API_PREFIX}/forms/fill", json={ "template_id": tpl_id, - "input_text": "John Doe", + "input_id": str(input_id), "model": "qwen2.5:3b", }) assert resp.status_code == 200 @@ -322,7 +387,7 @@ class TestE2EPipeline: def test_full_flow(self, client, mock_controller, pdf_upload, tmp_path, monkeypatch, db): # -- Step 1: Upload a PDF -- - monkeypatch.setattr("app.api.routes.templates.PROJECT_ROOT", tmp_path) + monkeypatch.setattr("app.core.paths.PROJECT_ROOT", tmp_path) upload_resp = client.post( f"{API_PREFIX}/templates/upload", files=[pdf_upload], @@ -355,13 +420,22 @@ def test_full_flow(self, client, mock_controller, pdf_upload, tmp_path, monkeypa assert any(t["id"] == template_id for t in templates) # -- Step 4: Fill the form -- - fill_resp = client.post(f"{API_PREFIX}/forms/fill", json={ - "template_id": template_id, - "input_text": ( + input_record = Input( + input_type=InputType.text, + status=InputStatus.ready, + transcript=( "Officer Jane Smith, badge 4521. On January 15 2025 at " "123 Main St, a structure fire was reported. Two engines " "responded, fire contained within 45 minutes." ), + ) + db.add(input_record) + db.commit() + db.refresh(input_record) + + fill_resp = client.post(f"{API_PREFIX}/forms/fill", json={ + "template_id": template_id, + "input_id": str(input_record.input_id), }) assert fill_resp.status_code == 200 fill_data = fill_resp.json() @@ -375,4 +449,5 @@ def test_full_flow(self, client, mock_controller, pdf_upload, tmp_path, monkeypa db_forms = list(db.exec(select(FormSubmission))) assert len(db_forms) == 1 assert db_forms[0].template_id == template_id + assert db_forms[0].input_id == input_record.input_id assert "Jane Smith" in db_forms[0].input_text diff --git a/tests/test_deletion.py b/tests/test_deletion.py index 0d036dcb..633529fe 100644 --- a/tests/test_deletion.py +++ b/tests/test_deletion.py @@ -69,7 +69,7 @@ def test_delete_template_cascades_submissions(self, client, db): def test_delete_template_deletes_pdf_file(self, client, tmp_path, monkeypatch): """Verify the template PDF file is removed from disk on delete.""" - monkeypatch.setattr("app.api.routes.templates.PROJECT_ROOT", tmp_path) + monkeypatch.setattr("app.core.paths.PROJECT_ROOT", tmp_path) pdf_file = tmp_path / "myform.pdf" pdf_file.write_bytes(b"%PDF-1.4 fake") @@ -81,7 +81,7 @@ def test_delete_template_deletes_pdf_file(self, client, tmp_path, monkeypatch): def test_delete_template_deletes_submission_output_pdfs(self, client, db, tmp_path, monkeypatch): """Output PDFs of related submissions should be wiped on template deletion.""" - monkeypatch.setattr("app.api.routes.templates.PROJECT_ROOT", tmp_path) + monkeypatch.setattr("app.core.paths.PROJECT_ROOT", tmp_path) out_pdf = tmp_path / "filled.pdf" out_pdf.write_bytes(b"%PDF-1.4 filled") @@ -117,7 +117,7 @@ def test_delete_submission_not_found(self, client): assert resp.status_code == 404 def test_delete_submission_removes_output_pdf(self, client, db, tmp_path, monkeypatch): - monkeypatch.setattr("app.api.routes.forms.PROJECT_ROOT", tmp_path) + monkeypatch.setattr("app.core.paths.PROJECT_ROOT", tmp_path) out_pdf = tmp_path / "filled_out.pdf" out_pdf.write_bytes(b"%PDF-1.4") @@ -167,7 +167,7 @@ def test_purge_nothing_to_remove(self, client, db): assert resp.json()["purged_count"] == 0 def test_purge_removes_output_pdf_file(self, client, db, tmp_path, monkeypatch): - monkeypatch.setattr("app.api.routes.forms.PROJECT_ROOT", tmp_path) + monkeypatch.setattr("app.core.paths.PROJECT_ROOT", tmp_path) out_pdf = tmp_path / "old_filled.pdf" out_pdf.write_bytes(b"%PDF-1.4") diff --git a/tests/test_filler.py b/tests/test_filler.py new file mode 100644 index 00000000..ef0162ec --- /dev/null +++ b/tests/test_filler.py @@ -0,0 +1,90 @@ +"""Tests for Filler.fill_form's value-to-widget matching (issue #642). + +Widgets are matched to LLM answers by field NAME, not by position — the +field-dict order and the PDF's physical widget order aren't guaranteed to +align. +""" + +from unittest.mock import MagicMock, patch + +from app.services.filler import Filler + + +class _FakeAnnot: + """Stand-in for a pdfrw widget annotation.""" + + def __init__(self, name, rect): + self.Subtype = "/Widget" + self.T = name + self.Rect = rect + self.V = None + self.AP = "placeholder-appearance" + + +class _FakePage: + def __init__(self, annots): + self.Annots = annots + + +class _FakePdf: + def __init__(self, pages): + self.pages = pages + + +class _FakeLLM: + """Stand-in for app.services.llm.LLM — main_loop() returns self, like the real one.""" + + def __init__(self, data: dict): + self._data = data + + def main_loop(self): + return self + + def get_data(self): + return self._data + + +class TestFillerNameMatching: + + def _run(self, annots, answers): + fake_pdf = _FakePdf(pages=[_FakePage(annots)]) + with patch("app.services.filler.PdfReader", return_value=fake_pdf), \ + patch("app.services.filler.PdfWriter") as mock_writer_cls: + mock_writer_cls.return_value = MagicMock() + Filler().fill_form(pdf_form="src/inputs/template.pdf", llm=_FakeLLM(answers)) + + def test_values_match_by_name_when_widget_order_diverges_from_field_order(self): + """Field dict is keyed a, b, c (insertion order) but the widgets sit on + the page in physical order c, b, a (top-to-bottom). Positional matching + would zip answers_list[0]="AAA" onto the first-visited widget (c's box) + and answers_list[2]="CCC" onto the last (a's box) — wrong. Name matching + must put each value in its own-named box regardless of page order.""" + annot_c = _FakeAnnot("c", rect=[0, 300, 100, 320]) # topmost + annot_b = _FakeAnnot("b", rect=[0, 200, 100, 220]) # middle + annot_a = _FakeAnnot("a", rect=[0, 100, 100, 120]) # bottommost + + # Physical/traversal order on the page: c, b, a. + # Field-dict / answer order: a, b, c. + answers = {"a": "AAA", "b": "BBB", "c": "CCC"} + self._run([annot_c, annot_b, annot_a], answers) + + assert annot_a.V == "AAA" + assert annot_b.V == "BBB" + assert annot_c.V == "CCC" + + def test_widget_with_no_matching_answer_is_left_unfilled(self): + annot_a = _FakeAnnot("a", rect=[0, 100, 100, 120]) + annot_d = _FakeAnnot("d", rect=[0, 200, 100, 220]) + + self._run([annot_a, annot_d], {"a": "AAA"}) + + assert annot_a.V == "AAA" + assert annot_d.V is None + + def test_answer_with_no_matching_widget_does_not_crash(self): + annot_a = _FakeAnnot("a", rect=[0, 100, 100, 120]) + + # "extra" has no corresponding widget on the page. + self._run([annot_a], {"a": "AAA", "extra": "unused"}) + + assert annot_a.V == "AAA" diff --git a/tests/test_jobs.py b/tests/test_jobs.py index c79e4a42..3e332148 100644 --- a/tests/test_jobs.py +++ b/tests/test_jobs.py @@ -1,7 +1,11 @@ """Tests for async job submission and status endpoints.""" from unittest.mock import patch, MagicMock +from uuid import uuid4 + +from app.api.schemas.enums import InputStatus, InputType from app.core.config import API_PREFIX +from app.models import Input class TestJobEndpoints: @@ -14,16 +18,24 @@ def _seed_template(self, client): }) return resp.json()["id"] + def _seed_input(self, db, status=InputStatus.ready, transcript="John Doe firefighter"): + record = Input(input_type=InputType.text, status=status, transcript=transcript) + db.add(record) + db.commit() + db.refresh(record) + return record.input_id + @patch("app.api.routes.jobs.fill_form_task") - def test_submit_async_single(self, mock_task, client): + def test_submit_async_single(self, mock_task, client, db): mock_result = MagicMock() mock_result.id = "celery-task-id-1" mock_task.delay.return_value = mock_result tpl_id = self._seed_template(client) + input_id = self._seed_input(db) resp = client.post(f"{API_PREFIX}/forms/jobs", json={ "template_ids": [tpl_id], - "input_text": "John Doe firefighter", + "input_id": str(input_id), }) assert resp.status_code == 200 data = resp.json() @@ -31,10 +43,10 @@ def test_submit_async_single(self, mock_task, client): assert data["jobs"][0]["status"] == "queued" assert "job_id" in data["jobs"][0] assert data["jobs"][0]["poll_url"].startswith(f"{API_PREFIX}/jobs/") - mock_task.delay.assert_called_once_with(tpl_id, "John Doe firefighter", None) + mock_task.delay.assert_called_once_with(tpl_id, "John Doe firefighter", str(input_id), None) @patch("app.api.routes.jobs.fill_form_task") - def test_submit_async_batch(self, mock_task, client): + def test_submit_async_batch(self, mock_task, client, db): mock_task.delay.side_effect = [ MagicMock(id="task-1"), MagicMock(id="task-2"), @@ -42,9 +54,10 @@ def test_submit_async_batch(self, mock_task, client): t1 = self._seed_template(client) t2 = self._seed_template(client) + input_id = self._seed_input(db, transcript="batch input") resp = client.post(f"{API_PREFIX}/forms/jobs", json={ "template_ids": [t1, t2], - "input_text": "batch input", + "input_id": str(input_id), }) assert resp.status_code == 200 jobs = resp.json()["jobs"] @@ -53,22 +66,66 @@ def test_submit_async_batch(self, mock_task, client): assert mock_task.delay.call_count == 2 @patch("app.api.routes.jobs.fill_form_task") - def test_submit_async_missing_template(self, mock_task, client): + def test_submit_async_missing_template(self, mock_task, client, db): + input_id = self._seed_input(db, transcript="some text") resp = client.post(f"{API_PREFIX}/forms/jobs", json={ "template_ids": [9999], - "input_text": "some text", + "input_id": str(input_id), }) assert resp.status_code == 404 mock_task.delay.assert_not_called() @patch("app.api.routes.jobs.fill_form_task") - def test_get_job_status(self, mock_task, client): + def test_submit_async_missing_input(self, mock_task, client): + tpl_id = self._seed_template(client) + resp = client.post(f"{API_PREFIX}/forms/jobs", json={ + "template_ids": [tpl_id], + "input_id": str(uuid4()), + }) + assert resp.status_code == 404 + assert resp.json()["error_code"] == "INPUT_NOT_FOUND" + mock_task.delay.assert_not_called() + + @patch("app.api.routes.jobs.fill_form_task") + def test_submit_async_input_transcribing(self, mock_task, client, db): + """A not-yet-ready input is rejected up front — no job should be queued.""" + tpl_id = self._seed_template(client) + input_id = self._seed_input(db, status=InputStatus.transcribing, transcript=None) + + resp = client.post(f"{API_PREFIX}/forms/jobs", json={ + "template_ids": [tpl_id], + "input_id": str(input_id), + }) + assert resp.status_code == 409 + body = resp.json() + assert body["error_code"] == "INPUT_NOT_READY" + assert body["detail"]["status"] == "transcribing" + mock_task.delay.assert_not_called() + + @patch("app.api.routes.jobs.fill_form_task") + def test_submit_async_input_failed(self, mock_task, client, db): + tpl_id = self._seed_template(client) + input_id = self._seed_input(db, status=InputStatus.failed, transcript=None) + + resp = client.post(f"{API_PREFIX}/forms/jobs", json={ + "template_ids": [tpl_id], + "input_id": str(input_id), + }) + assert resp.status_code == 409 + body = resp.json() + assert body["error_code"] == "INPUT_NOT_READY" + assert body["detail"]["status"] == "failed" + mock_task.delay.assert_not_called() + + @patch("app.api.routes.jobs.fill_form_task") + def test_get_job_status(self, mock_task, client, db): mock_task.delay.return_value = MagicMock(id="celery-abc") tpl_id = self._seed_template(client) + input_id = self._seed_input(db, transcript="test input") submit_resp = client.post(f"{API_PREFIX}/forms/jobs", json={ "template_ids": [tpl_id], - "input_text": "test input", + "input_id": str(input_id), }) job_id = submit_resp.json()["jobs"][0]["job_id"] @@ -85,28 +142,30 @@ def test_get_job_not_found(self, client): assert resp.status_code == 404 @patch("app.api.routes.jobs.fill_form_task") - def test_submit_with_model_override(self, mock_task, client): + def test_submit_with_model_override(self, mock_task, client, db): mock_task.delay.return_value = MagicMock(id="celery-xyz") tpl_id = self._seed_template(client) + input_id = self._seed_input(db, transcript="test") resp = client.post(f"{API_PREFIX}/forms/jobs", json={ "template_ids": [tpl_id], - "input_text": "test", + "input_id": str(input_id), "model": "mistral:latest", }) assert resp.status_code == 200 - mock_task.delay.assert_called_once_with(tpl_id, "test", "mistral:latest") + mock_task.delay.assert_called_once_with(tpl_id, "test", str(input_id), "mistral:latest") - def test_submit_empty_template_ids(self, client): + def test_submit_empty_template_ids(self, client, db): + input_id = self._seed_input(db, transcript="test") resp = client.post(f"{API_PREFIX}/forms/jobs", json={ "template_ids": [], - "input_text": "test", + "input_id": str(input_id), }) assert resp.status_code == 422 - def test_submit_empty_input_text(self, client): + def test_submit_invalid_input_id(self, client): resp = client.post(f"{API_PREFIX}/forms/jobs", json={ "template_ids": [1], - "input_text": "", + "input_id": "not-a-uuid", }) assert resp.status_code == 422 diff --git a/tests/test_migrations.py b/tests/test_migrations.py index 6c404b80..191c025a 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -79,17 +79,19 @@ def test_formsubmission_columns(alembic_cfg, alembic_engine): inspector = inspect(alembic_engine) columns = {c["name"] for c in inspector.get_columns("formsubmission")} - assert columns == {"id", "template_id", "input_text", "output_pdf_path", "created_at"} + assert columns == { + "id", "template_id", "input_id", "input_text", "output_pdf_path", "created_at" + } def test_formsubmission_fk(alembic_cfg, alembic_engine): command.upgrade(alembic_cfg, "head") inspector = inspect(alembic_engine) - fks = inspector.get_foreign_keys("formsubmission") - assert len(fks) == 1 - assert fks[0]["referred_table"] == "template" - assert fks[0]["referred_columns"] == ["id"] + fks = {fk["referred_table"]: fk for fk in inspector.get_foreign_keys("formsubmission")} + assert len(fks) == 2 + assert fks["template"]["referred_columns"] == ["id"] + assert fks["inputs"]["referred_columns"] == ["input_id"] def test_job_columns(alembic_cfg, alembic_engine): @@ -293,9 +295,9 @@ def test_reports_no_fk(alembic_cfg, alembic_engine): def test_downgrade_002(alembic_cfg, alembic_engine): - """Downgrade by one step removes only the 002 tables, leaving 001 tables intact.""" + """Downgrading to 001 removes the 002 tables, leaving 001 tables intact.""" command.upgrade(alembic_cfg, "head") - command.downgrade(alembic_cfg, "-1") + command.downgrade(alembic_cfg, "001") inspector = inspect(alembic_engine) tables = inspector.get_table_names() @@ -307,3 +309,16 @@ def test_downgrade_002(alembic_cfg, alembic_engine): assert "template" in tables assert "formsubmission" in tables assert "job" in tables + + +def test_downgrade_003(alembic_cfg, alembic_engine): + """Downgrade by one step from head removes only the input_id FK/column.""" + command.upgrade(alembic_cfg, "head") + command.downgrade(alembic_cfg, "-1") + + inspector = inspect(alembic_engine) + columns = {c["name"] for c in inspector.get_columns("formsubmission")} + assert "input_id" not in columns + assert "input_text" in columns + tables = inspector.get_table_names() + assert "inputs" in tables