Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/fixtures/QandA.jsonl
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
{"prompt": "This is a synthetic fixture FAQ question used only for CI smoke testing.", "completion": "This is a synthetic fixture answer, not real data."}
{"prompt": "A second synthetic fixture question, distinct wording.", "completion": "A second synthetic fixture answer."}
32 changes: 32 additions & 0 deletions .github/fixtures/finalcases.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
[
{
"case_id": "fixture_001",
"case_title": "CI Fixture Case One",
"case_description": "Synthetic fixture case used only to smoke-test the retrieval pipeline in CI. Not real case data.",
"case_link": "https://example.com/fixture-1",
"sections": [
{
"section_id": "section_001",
"section_title": "Fixture Section",
"section_description": "Synthetic section text used only for CI smoke testing."
}
],
"strong_points": ["Fixture strong point."],
"weak_points": ["Fixture weak point."]
},
{
"case_id": "fixture_002",
"case_title": "CI Fixture Case Two",
"case_description": "A second synthetic fixture case, distinct wording, still only for CI smoke testing.",
"case_link": "https://example.com/fixture-2",
"sections": [
{
"section_id": "section_001",
"section_title": "Another Fixture Section",
"section_description": "More synthetic section text for CI smoke testing."
}
],
"strong_points": ["Another fixture strong point."],
"weak_points": ["Another fixture weak point."]
}
]
40 changes: 40 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
name: CI

on:
pull_request:
push:
branches: [main]

jobs:
pre-commit:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- run: pip install pre-commit
- run: pre-commit run --all-files

retrieval-smoke-test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.12"
cache: "pip"
cache-dependency-path: legalis_api/requirements.txt
- run: pip install -r legalis_api/requirements.txt
# Real case/FAQ data is intentionally not published (see NOTICE.md) and
# is not available in CI. This uses tiny synthetic fixture data to prove
# the retrieval pipeline still imports, starts, and returns
# correctly-shaped results -- it does NOT measure retrieval quality.
# Quality is measured locally against the real dataset via
# eval/run_eval.py; see eval/eval_results_*.json for those numbers.
- name: Stage fixture data
run: |
mkdir -p Data
cp .github/fixtures/finalcases.json Data/finalcases.json
cp .github/fixtures/QandA.jsonl Data/QandA.jsonl
- run: python eval/smoke_test.py
12 changes: 4 additions & 8 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,11 +1,7 @@
cases.json
legal_cases.json
__pycache__
legalis_model
results
finalcases.json
trainer.py
QandA.jsonl
faq_model
Data
Frontend/legalis_ui/node_modules
/legalis_model
/faq_model
/trainer.py
.venv
59 changes: 59 additions & 0 deletions eval/smoke_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
"""
CI smoke test for the retrieval pipeline.

Unlike run_eval.py, this does NOT measure retrieval quality (that requires
the real dataset, which is intentionally not published/available in CI). It
only proves legalis_api/main.py imports, starts, and returns correctly-shaped
results against tiny synthetic fixture data — catching import errors, crashes,
and schema breakage on every PR without exposing real data.

Usage:
python eval/smoke_test.py
"""

import os
import sys
from pathlib import Path

REPO_ROOT = Path(__file__).resolve().parent.parent


def check(condition, message):
if not condition:
print(f"FAIL: {message}")
sys.exit(1)


def main():
os.chdir(REPO_ROOT / "legalis_api")
sys.path.insert(0, str(REPO_ROOT / "legalis_api"))
import main as api # noqa: E402 (import after path/cwd setup, deliberately)

check(len(api.cases_data) > 0, "no fixture case data loaded")
check(len(api.faq_data) > 0, "no fixture FAQ data loaded")

case_results = api.find_relevant_cases("fixture test query", num_results=2)
check(len(case_results) > 0, "find_relevant_cases returned no results")
for key in (
"case_id",
"case_title",
"case_link",
"similarity_score",
"sections",
"strong_points",
"weak_points",
):
check(key in case_results[0], f"case result missing expected key: {key}")

faq_results = api.find_relevant_faq("fixture test query", num_results=2)
check(len(faq_results) > 0, "find_relevant_faq returned no results")
for key in ("faq_prompt", "faq_completion", "similarity_score"):
check(key in faq_results[0], f"FAQ result missing expected key: {key}")

print(
"Smoke test passed: retrieval pipeline imports, starts, and returns correctly-shaped results."
)


if __name__ == "__main__":
main()
Loading