[FEAT]: Benchmark Module Using pytest-benchmark to Compare Pipeline Performance

.github/workflows/benchmark.yml

@@ -0,0 +1,57 @@
+name: Run Pipeline Benchmark
+
+on:
+  pull_request:
+    branches: [ main, development ]
+
+jobs:
+  benchmark:
+    runs-on: ubuntu-latest
+
+    services:
+      ollama:
+        image: ollama/ollama:latest
+        ports:
+          - 11434:11434
+
+    steps:
+      - name: Checkout PR Branch
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+
+      - name: Install dependencies
+        run: |
+          pip install -r requirements.txt
+          pip install pytest
+
+      - name: Run PR Branch Benchmark
+        run: |
+          pytest benchmark/test_benchmark.py -v
+          mv benchmark/benchmark_report.json branch_report.json
+
+      - name: Checkout Target Branch
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.base_ref }}
+          clean: false
+
+      - name: Run Target Branch Benchmark
+        run: |
+          pytest benchmark/test_benchmark.py -v
+          mv benchmark/benchmark_report.json target_report.json
+
+      - name: Compare Benchmarks
+        id: compare
+        run: |
+          python benchmark/compare_benchmarks.py branch_report.json target_report.json > comparison.md
+          cat comparison.md
+
+      - name: Comment PR with results
+        uses: thollander/actions-comment-pull-request@v3
+        with:
+          filePath: comparison.md

alembic/env.py

@@ -1,12 +1,20 @@
 from logging.config import fileConfig
 
-from alembic import context
 from sqlalchemy import engine_from_config, pool
-
 from sqlmodel import SQLModel
 
+from alembic import context
 from app.core.config import DATABASE_URL
-from app.models import Extraction, Form, FormSubmission, Incident, Input, Job, Report, Template  # noqa: F401
+from app.models import (  # noqa: F401
+    Extraction,
+    Form,
+    FormSubmission,
+    Incident,
+    Input,
+    Job,
+    Report,
+    Template,
+)
 
 config = context.config
 

alembic/versions/001_initial_schema.py

@@ -7,10 +7,11 @@
 """
 from collections.abc import Sequence
 
-from alembic import op
 import sqlalchemy as sa
 import sqlmodel
 
+from alembic import op
+
 revision: str = "001"
 down_revision: str | None = None
 branch_labels: str | Sequence[str] | None = None

alembic/versions/002_v1_models.py

@@ -16,10 +16,11 @@
 
 from collections.abc import Sequence
 
-from alembic import op
 import sqlalchemy as sa
 import sqlmodel
 
+from alembic import op
+
 revision: str = "002"
 down_revision: str | None = "001"
 branch_labels: str | Sequence[str] | None = None

app/api/deps.py

@@ -1,7 +1,9 @@
-from app.db.database import get_session
 from fastapi import Header, Request
+
 from app.core.config import FIREFORM_API_KEY
 from app.core.errors.base import AppError
+from app.db.database import get_session
+
 
 def get_db():
     yield from get_session()

app/api/routes/__init__.py

@@ -1,3 +1,3 @@
-from . import templates, forms
+from . import forms, templates
 
-__all__ = ["templates", "forms"]
+__all__ = ["forms", "templates"]

app/api/routes/forms.py

@@ -1,7 +1,8 @@
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
+
 import requests
-from fastapi import APIRouter, Depends, File, UploadFile, Query
+from fastapi import APIRouter, Depends, File, Query, UploadFile
 from sqlmodel import Session, select
 
 from app.api.deps import get_db, verify_api_key
@@ -11,12 +12,17 @@
     ModelsResponse,
     TranscriptionResponse,
 )
-from app.core.config import OLLAMA_HOST, OLLAMA_MODEL, BASE_DIR, RETENTION_PERIOD_DAYS
-from app.services.whisper import call_whisper_asr
+from app.core.config import BASE_DIR, OLLAMA_HOST, OLLAMA_MODEL, RETENTION_PERIOD_DAYS
 from app.core.errors.base import AppError
-from app.db.repositories import create_form, get_template, get_form_submission, delete_form_submission
+from app.db.repositories import (
+    create_form,
+    delete_form_submission,
+    get_form_submission,
+    get_template,
+)
 from app.models import FormSubmission, Template
 from app.services.controller import Controller
+from app.services.whisper import call_whisper_asr
 
 PROJECT_ROOT = BASE_DIR
 
@@ -40,7 +46,7 @@
 
 
 @router.post("/fill", response_model=FormFillResponse)
 def fill_form(form: FormFill, db: Session = Depends(get_db)):
 
     fetched_template = get_template(db, form.template_id)
     if not fetched_template:
@@ -60,7 +66,7 @@
             **form.model_dump(exclude={"model"}), output_pdf_path=path
         )
         return create_form(db, submission)
     except Exception as e:
         raise AppError(str(e), status_code=500, error_code="FORM_FILL_ERROR")
 
 
@@ -87,7 +93,7 @@
 
 
 @router.post("/transcribe", response_model=TranscriptionResponse)
 def transcribe(audio: UploadFile = File(...)):
     """Forward recorded audio to the local Whisper ASR sidecar and return text.
 
     Mirrors the Ollama wiring: WHISPER_HOST points at the whisper service
@@ -110,7 +116,7 @@
 
 
 @router.delete("/{submission_id}", dependencies=[Depends(verify_api_key)])
 def delete_submission_endpoint(submission_id: int, db: Session = Depends(get_db)):
     sub = get_form_submission(db, submission_id)
     if not sub:
         raise AppError("Submission not found", status_code=404, error_code="SUBMISSION_NOT_FOUND")
@@ -120,15 +126,15 @@
             resolved_out = _resolve_project_file(sub.output_pdf_path)
             if resolved_out.exists() and resolved_out.is_file():
                 resolved_out.unlink()
         except Exception:
             pass
 
     delete_form_submission(db, sub)
     return {"status": "success", "message": "Submission and associated output file deleted"}
 
 
 @router.post("/purge", dependencies=[Depends(verify_api_key)])
 def purge_submissions_endpoint(days: int = Query(default=None), db: Session = Depends(get_db)):
     retention_days = days if days is not None else RETENTION_PERIOD_DAYS
     cutoff_date = datetime.now(timezone.utc) - timedelta(days=retention_days)
 
@@ -142,8 +148,8 @@
                 resolved_out = _resolve_project_file(sub.output_pdf_path)
                 if resolved_out.exists() and resolved_out.is_file():
                     resolved_out.unlink()
             except Exception:
                 pass
         delete_form_submission(db, sub)
         purged_count += 1
 
@@ -152,7 +158,7 @@
 
 
 @router.get("/submissions")
 def get_submissions(db: Session = Depends(get_db)):
     from sqlmodel import select
     statement = (
         select(FormSubmission, Template.name)
@@ -175,8 +181,9 @@
 
 @router.get("/submissions/analytics")
 def get_submissions_analytics(db: Session = Depends(get_db)):
-    from collections import Counter
     import re
+    from collections import Counter
+
     from sqlmodel import select
 
     statement = select(FormSubmission, Template.name).join(
@@ -194,9 +201,8 @@
         "the", "and", "a", "of", "to", "in", "is", "that", "it", "was", "for", "on",
         "as", "with", "by", "at", "an", "be", "this", "are", "from", "or", "have",
         "has", "had", "but", "not", "he", "she", "they", "we", "i", "you", "my", "his",
-        "her", "their", "our", "me", "him", "them", "us", "about", "there", "their",
-        "were", "been", "would", "could", "should", "will", "can", "no", "yes", "any",
-        "so", "very", "patient", "presents", "with", "reported", "history", "shows",
+        "her", "their", "our", "me", "him", "them", "us", "about", "there", "were", "been", "would", "could", "should", "will", "can", "no", "yes", "any",
+        "so", "very", "patient", "presents", "reported", "history", "shows",
         "left", "right", "pain", "due", "after", "before", "emergency", "department",
         "medical", "clinical"
     }

app/api/routes/input.py

@@ -21,7 +21,8 @@
     INPUT_POLL_INTERVAL_SECONDS,
 )
 from app.core.errors.base import AppError
-from app.db.repositories import create_input, get_input as repo_get_input
+from app.db.repositories import create_input
+from app.db.repositories import get_input as repo_get_input
 from app.services.input import InputService
 from app.services.whisper import check_whisper_available
 

app/api/routes/templates.py

@@ -4,21 +4,25 @@
 
 from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, UploadFile
 from fastapi.responses import FileResponse
-from sqlmodel import Session
+from sqlmodel import Session, select
 
 from app.api.deps import get_db, verify_api_key
 from app.api.schemas.templates import (
+    MakeFillableRequest,
+    MakeFillableResponse,
     TemplateCreate,
     TemplateResponse,
     TemplateUploadResponse,
-    MakeFillableRequest,
-    MakeFillableResponse,
 )
 from app.core.config import BASE_DIR, DEFAULT_TEMPLATE_DIR
-from app.db.repositories import create_template, list_templates, get_template, delete_template
-from app.models import Template, FormSubmission, Job
+from app.db.repositories import (
+    create_template,
+    delete_template,
+    get_template,
+    list_templates,
+)
+from app.models import FormSubmission, Job, Template
 from app.services.controller import Controller
-from sqlmodel import select
 
 router = APIRouter(prefix="/templates", tags=["templates"])
 PROJECT_ROOT = BASE_DIR

app/api/schemas/templates.py

@@ -1,5 +1,6 @@
 from pydantic import BaseModel
 
+
 class TemplateCreate(BaseModel):
     name: str
     pdf_path: str

app/core/celery.py

@@ -20,7 +20,8 @@
 
 # Optional Celery Beat schedule — runs purge_old_submissions once a day.
 # Enable by running: celery -A app.core.celery beat
-from celery.schedules import crontab  # noqa: E402
+from celery.schedules import crontab
+
 celery_app.conf.beat_schedule = {
     "daily-submission-purge": {
         "task": "purge_old_submissions",

app/db/init_db.py

@@ -2,10 +2,10 @@
 import logging
 from pathlib import Path
 
-from alembic import command
 from alembic.config import Config
 from sqlmodel import Session, select
 
+from alembic import command
 from app.core.config import DEFAULT_TEMPLATE_DIR
 from app.db.database import engine
 from app.models import FormSubmission, Template  # noqa: F401

app/db/repositories.py

@@ -2,7 +2,8 @@
 
 from sqlmodel import Session, select
 
-from app.models import Template, FormSubmission, Job, Input
+from app.models import FormSubmission, Input, Job, Template
+
 
 # Templates
 def create_template(session: Session, template: Template) -> Template:

app/models/__init__.py

@@ -12,12 +12,12 @@
 )
 
 __all__ = [
-    "Template",
-    "FormSubmission",
-    "Job",
-    "Input",
     "Extraction",
-    "Incident",
     "Form",
+    "FormSubmission",
+    "Incident",
+    "Input",
+    "Job",
     "Report",
+    "Template",
 ]

app/models/models.py

@@ -1,9 +1,9 @@
 import uuid as uuid_mod
-from uuid import UUID, uuid4
 from datetime import date, datetime, timezone
+from uuid import UUID, uuid4
 
-from sqlalchemy import Column, JSON
-from sqlmodel import SQLModel, Field
+from sqlalchemy import JSON, Column
+from sqlmodel import Field, SQLModel
 from sqlmodel.sql.sqltypes import AutoString
 
 from app.api.schemas.enums import (

app/services/controller.py

@@ -1,5 +1,5 @@
-from app.services.file_manipulator import FileManipulator
 from app.services.external_apis_coordinator import ExternalAPIsCoordinator
+from app.services.file_manipulator import FileManipulator
 
 
 class Controller:

app/services/external_apis/weather_api.py

@@ -1,5 +1,4 @@
 import openmeteo_requests
-
 import pandas as pd
 import requests_cache
 from retry_requests import retry

app/services/external_apis/zipcode_api.py

@@ -1,5 +1,5 @@
+from geopy.exc import GeocoderServiceError, GeocoderTimedOut
 from geopy.geocoders import Nominatim
-from geopy.exc import GeocoderTimedOut, GeocoderServiceError
 
 from app.core.logging import get_logger
 

app/services/external_apis_coordinator.py

@@ -1,5 +1,6 @@
-from app.services.external_apis.zipcode_api import ZipCodeAPI
 from app.services.external_apis.weather_api import WeatherAPI
+from app.services.external_apis.zipcode_api import ZipCodeAPI
+
 
 class ExternalAPIsCoordinator:
     def __init__(self):

app/services/file_manipulator.py

@@ -1,7 +1,8 @@
 import os
+
+from app.core.logging import get_logger
 from app.services.filler import Filler
 from app.services.llm import LLM
-from app.core.logging import get_logger
 
 logger = get_logger(__name__)
 

app/services/filler.py

@@ -1,6 +1,8 @@
+from datetime import datetime
+
 from pdfrw import PdfReader, PdfWriter
+
 from app.services.llm import LLM
-from datetime import datetime
 
 
 class Filler:

app/services/llm.py

@@ -1,7 +1,8 @@
 import json
 import os
+
 import requests
-from requests.exceptions import Timeout, RequestException
+from requests.exceptions import RequestException, Timeout
 
 from app.core.config import OLLAMA_HOST, OLLAMA_MODEL
 from app.core.logging import get_logger
@@ -92,7 +93,6 @@ def add_response_to_json(self, field: str, value: str):
         else:
             self._json[field] = parsed_value
 
-        return
 
 
     def get_data(self):

app/tasks/fill.py

@@ -3,7 +3,12 @@
 
 from app.core.celery import celery_app
 from app.db.database import get_session
-from app.db.repositories import get_job_by_celery_id, get_template, update_job, create_form
+from app.db.repositories import (
+    create_form,
+    get_job_by_celery_id,
+    get_template,
+    update_job,
+)
 from app.models import FormSubmission
 from app.services.controller import Controller
 

app/tasks/purge.py

@@ -8,12 +8,13 @@
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
 
+from sqlmodel import select
+
 from app.core.celery import celery_app
 from app.core.config import BASE_DIR, RETENTION_PERIOD_DAYS
 from app.db.database import get_session
 from app.db.repositories import delete_form_submission
 from app.models import FormSubmission
-from sqlmodel import select
 
 logger = logging.getLogger(__name__)