development approach b benchmark

.github/workflows/benchmark.yml

@@ -0,0 +1,57 @@
+name: Run Pipeline Benchmark
+
+on:
+  pull_request:
+    branches: [ main, development ]
+
+jobs:
+  benchmark:
+    runs-on: ubuntu-latest
+
+    services:
+      ollama:
+        image: ollama/ollama:latest
+        ports:
+          - 11434:11434
+
+    steps:
+      - name: Checkout PR Branch
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+
+      - name: Install dependencies
+        run: |
+          pip install -r requirements.txt
+          pip install pytest
+
+      - name: Run PR Branch Benchmark
+        run: |
+          pytest benchmark/test_benchmark.py -v
+          mv benchmark/benchmark_report.json branch_report.json
+
+      - name: Checkout Target Branch
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.base_ref }}
+          clean: false
+
+      - name: Run Target Branch Benchmark
+        run: |
+          pytest benchmark/test_benchmark.py -v
+          mv benchmark/benchmark_report.json target_report.json
+
+      - name: Compare Benchmarks
+        id: compare
+        run: |
+          python benchmark/compare_benchmarks.py branch_report.json target_report.json > comparison.md
+          cat comparison.md
+
+      - name: Comment PR with results
+        uses: thollander/actions-comment-pull-request@v3
+        with:
+          filePath: comparison.md

alembic/env.py

@@ -1,12 +1,20 @@
 from logging.config import fileConfig
 
-from alembic import context
 from sqlalchemy import engine_from_config, pool
-
 from sqlmodel import SQLModel
 
+from alembic import context
 from app.core.config import DATABASE_URL
-from app.models import Extraction, Form, FormSubmission, Incident, Input, Job, Report, Template  # noqa: F401
+from app.models import (  # noqa: F401
+    Extraction,
+    Form,
+    FormSubmission,
+    Incident,
+    Input,
+    Job,
+    Report,
+    Template,
+)
 
 config = context.config
 

alembic/versions/001_initial_schema.py

@@ -7,10 +7,11 @@
 """
 from collections.abc import Sequence
 
-from alembic import op
 import sqlalchemy as sa
 import sqlmodel
 
+from alembic import op
+
 revision: str = "001"
 down_revision: str | None = None
 branch_labels: str | Sequence[str] | None = None

alembic/versions/002_v1_models.py

@@ -16,10 +16,11 @@
 
 from collections.abc import Sequence
 
-from alembic import op
 import sqlalchemy as sa
 import sqlmodel
 
+from alembic import op
+
 revision: str = "002"
 down_revision: str | None = "001"
 branch_labels: str | Sequence[str] | None = None

app/api/deps.py

@@ -1,7 +1,9 @@
-from app.db.database import get_session
 from fastapi import Header, Request
+
 from app.core.config import FIREFORM_API_KEY
 from app.core.errors.base import AppError
+from app.db.database import get_session
+
 
 def get_db():
     yield from get_session()

app/api/routes/__init__.py

@@ -1,3 +1,3 @@
-from . import templates, forms
+from . import forms, templates
 
-__all__ = ["templates", "forms"]
+__all__ = ["forms", "templates"]

app/api/routes/forms.py

@@ -1,6 +1,7 @@
 from pathlib import Path
+
 import requests
-from fastapi import APIRouter, Depends, File, UploadFile, Query
+from fastapi import APIRouter, Depends, File, Query, UploadFile
 from sqlmodel import Session
 
 from app.api.deps import get_db, verify_api_key
@@ -10,11 +11,16 @@
     ModelsResponse,
     TranscriptionResponse,
 )
-from app.core.config import OLLAMA_HOST, OLLAMA_MODEL, BASE_DIR, RETENTION_PERIOD_DAYS
-from app.services.whisper import call_whisper_asr
+from app.core.config import BASE_DIR, OLLAMA_HOST, OLLAMA_MODEL, RETENTION_PERIOD_DAYS
 from app.core.errors.base import AppError
-from app.db.repositories import get_template, get_form_submission, delete_form_submission
+from app.db.repositories import (
+    delete_form_submission,
+    get_form_submission,
+    get_template,
+)
+from app.models import FormSubmission
 from app.services.form import FormService
+from app.services.whisper import call_whisper_asr
 
 PROJECT_ROOT = BASE_DIR
 

app/api/routes/input.py

@@ -21,7 +21,8 @@
     INPUT_POLL_INTERVAL_SECONDS,
 )
 from app.core.errors.base import AppError
-from app.db.repositories import create_input, get_input as repo_get_input
+from app.db.repositories import create_input
+from app.db.repositories import get_input as repo_get_input
 from app.services.input import InputService
 from app.services.whisper import check_whisper_available
 

app/api/routes/templates.py

@@ -6,11 +6,11 @@
 
 from app.api.deps import get_db, verify_api_key
 from app.api.schemas.templates import (
+    MakeFillableRequest,
+    MakeFillableResponse,
     TemplateCreate,
     TemplateResponse,
     TemplateUploadResponse,
-    MakeFillableRequest,
-    MakeFillableResponse,
 )
 from app.core.config import DEFAULT_TEMPLATE_DIR
 from app.db.repositories import get_template

app/api/schemas/templates.py

@@ -1,4 +1,5 @@
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
+
 
 class TemplateCreate(BaseModel):
     name: str
@@ -15,15 +16,14 @@ class MakeFillableResponse(BaseModel):
     field_count: int | None = None
 
 class TemplateResponse(BaseModel):
+    model_config = ConfigDict(from_attributes=True)
+
     id: int
     name: str
     pdf_path: str
     fields: dict
     field_count: int | None = None
 
-    class Config:
-        from_attributes = True
-
 
 class ExtractedField(BaseModel):
     name: str

app/core/celery.py

@@ -1,26 +1,8 @@
 from celery import Celery
+from celery.schedules import crontab
 
-from app.core.config import CELERY_BROKER_URL, CELERY_RESULT_BACKEND
+celery_app = Celery("fireform")
 
-celery_app = Celery(
-    "fireform",
-    broker=CELERY_BROKER_URL,
-    backend=CELERY_RESULT_BACKEND,
-)
-
-celery_app.conf.update(
-    task_serializer="json",
-    result_serializer="json",
-    accept_content=["json"],
-    task_track_started=True,
-    result_expires=86400,
-)
-
-celery_app.conf.include = ["app.tasks.fill", "app.tasks.purge", "app.tasks.transcribe"]
-
-# Optional Celery Beat schedule — runs purge_old_submissions once a day.
-# Enable by running: celery -A app.core.celery beat
-from celery.schedules import crontab  # noqa: E402
 celery_app.conf.beat_schedule = {
     "daily-submission-purge": {
         "task": "purge_old_submissions",

app/db/init_db.py

@@ -2,10 +2,10 @@
 import logging
 from pathlib import Path
 
-from alembic import command
 from alembic.config import Config
 from sqlmodel import Session, select
 
+from alembic import command
 from app.core.config import DEFAULT_TEMPLATE_DIR
 from app.db.database import engine
 from app.models import FormSubmission, Template  # noqa: F401

app/db/repositories.py

@@ -3,7 +3,8 @@
 
 from sqlmodel import Session, select
 
-from app.models import Template, FormSubmission, Job, Input
+from app.models import FormSubmission, Input, Job, Template
+
 
 # Templates
 def create_template(session: Session, template: Template) -> Template:

app/models/__init__.py

@@ -12,12 +12,12 @@
 )
 
 __all__ = [
-    "Template",
-    "FormSubmission",
-    "Job",
-    "Input",
     "Extraction",
-    "Incident",
     "Form",
+    "FormSubmission",
+    "Incident",
+    "Input",
+    "Job",
     "Report",
+    "Template",
 ]

app/models/models.py

@@ -1,9 +1,9 @@
 import uuid as uuid_mod
-from uuid import UUID, uuid4
 from datetime import date, datetime, timezone
+from uuid import UUID, uuid4
 
-from sqlalchemy import Column, JSON
-from sqlmodel import SQLModel, Field
+from sqlalchemy import JSON, Column
+from sqlmodel import Field, SQLModel
 from sqlmodel.sql.sqltypes import AutoString
 
 from app.api.schemas.enums import (
@@ -32,6 +32,7 @@ class FormSubmission(SQLModel, table=True):
     template_id: int = Field(foreign_key="template.id")
     input_id: UUID | None = Field(default=None, foreign_key="inputs.input_id")
     input_text: str
+    extracted_fields: dict | None = Field(default=None, sa_column=Column(JSON))
     output_pdf_path: str
     created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
 

app/services/controller.py

@@ -1,5 +1,5 @@
-from app.services.file_manipulator import FileManipulator
 from app.services.external_apis_coordinator import ExternalAPIsCoordinator
+from app.services.file_manipulator import FileManipulator
 
 
 class Controller:

app/services/external_apis/weather_api.py

@@ -1,5 +1,4 @@
 import openmeteo_requests
-
 import pandas as pd
 import requests_cache
 from retry_requests import retry

app/services/external_apis/zipcode_api.py

@@ -1,5 +1,5 @@
+from geopy.exc import GeocoderServiceError, GeocoderTimedOut
 from geopy.geocoders import Nominatim
-from geopy.exc import GeocoderTimedOut, GeocoderServiceError
 
 from app.core.logging import get_logger
 

app/services/external_apis_coordinator.py

@@ -1,5 +1,6 @@
-from app.services.external_apis.zipcode_api import ZipCodeAPI
 from app.services.external_apis.weather_api import WeatherAPI
+from app.services.external_apis.zipcode_api import ZipCodeAPI
+
 
 class ExternalAPIsCoordinator:
     def __init__(self):

app/services/file_manipulator.py

@@ -1,7 +1,8 @@
 import os
+
+from app.core.logging import get_logger
 from app.services.filler import Filler
 from app.services.llm import LLM
-from app.core.logging import get_logger
 
 logger = get_logger(__name__)
 

app/services/filler.py

@@ -1,6 +1,8 @@
+from datetime import datetime
+
 from pdfrw import PdfReader, PdfWriter
+
 from app.services.llm import LLM
-from datetime import datetime
 
 
 def _pdf_text(value) -> str:

app/services/form.py

@@ -76,10 +76,13 @@ def fill_and_persist(
             model=model,
         )
 
+        extracted_fields = self.controller.file_manipulator.llm._json
+
         submission = FormSubmission(
             template_id=template.id,
             input_id=input_id,
             input_text=transcript,
+            extracted_fields=extracted_fields,
             output_pdf_path=path,
         )
         return create_form(session, submission)

app/services/llm.py

@@ -1,7 +1,8 @@
 import json
 import os
+
 import requests
-from requests.exceptions import Timeout, RequestException
+from requests.exceptions import RequestException, Timeout
 
 from app.core.config import OLLAMA_HOST, OLLAMA_MODEL
 from app.core.logging import get_logger
@@ -92,7 +93,6 @@ def add_response_to_json(self, field: str, value: str):
         else:
             self._json[field] = parsed_value
 
-        return
 
 
     def get_data(self):

app/tasks/purge.py

@@ -8,12 +8,13 @@
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
 
+from sqlmodel import select
+
 from app.core.celery import celery_app
 from app.core.config import BASE_DIR, RETENTION_PERIOD_DAYS
 from app.db.database import get_session
 from app.db.repositories import delete_form_submission
 from app.models import FormSubmission
-from sqlmodel import select
 
 logger = logging.getLogger(__name__)