diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 403eac9..02ad1fd 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -8,6 +8,10 @@ updates: time: "07:00" timezone: Europe/London open-pull-requests-limit: 5 + ignore: + - dependency-name: "*" + update-types: + - version-update:semver-major groups: production-dependencies: dependency-type: production @@ -35,3 +39,7 @@ updates: time: "07:30" timezone: Europe/London open-pull-requests-limit: 3 + ignore: + - dependency-name: "node" + update-types: + - version-update:semver-major diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e0c4dad..dc9f706 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,7 +79,7 @@ jobs: - name: Verify LaTeX runtime run: docker exec modelhawk-ci pdflatex --version - - name: Exercise audit and PDF endpoints + - name: Exercise audit, examiner and PDF endpoints shell: bash run: | set -euo pipefail @@ -109,9 +109,45 @@ jobs: if len(audit.get('findings', [])) < 4: raise SystemExit('Expected the bundled regression fixture to produce at least four findings') + PY + + curl --fail --silent --show-error \ + -H 'Content-Type: application/json' \ + --data-binary @/tmp/modelhawk-audit.json \ + http://127.0.0.1:3000/api/audit/ai \ + -o /tmp/modelhawk-ai.json + + python - <<'PY' + import json + + with open('/tmp/modelhawk-audit.json', encoding='utf-8') as handle: + audit = json.load(handle) + with open('/tmp/modelhawk-ai.json', encoding='utf-8') as handle: + examiner = json.load(handle) + + if examiner.get('provider') != 'deterministic': + raise SystemExit(f"Expected deterministic CI examiner, got {examiner.get('provider')}") + + narrative = examiner.get('aiNarrative') + required = { + 'executiveSummary', + 'technicalAssessment', + 'riskInterpretation', + 'portfolioVerdict', + 'improvedRecommendations', + 'suggestedMarkdownCells', + 'vivaAnswers', + 'limitations', + } + + if not isinstance(narrative, dict) or not required.issubset(narrative): + raise SystemExit('Examiner endpoint returned an incomplete controlled narrative') + + if len(narrative.get('vivaAnswers', [])) != len(audit.get('vivaQuestions', [])): + raise SystemExit('Examiner did not preserve one answer per deterministic viva question') with open('/tmp/modelhawk-pdf-payload.json', 'w', encoding='utf-8') as handle: - json.dump({'auditResult': audit}, handle) + json.dump({'auditResult': audit, 'aiNarrative': narrative}, handle) PY curl --fail --silent --show-error \ @@ -137,6 +173,40 @@ jobs: print(f'PDF smoke test passed: {len(report)} bytes') PY + - name: Verify invalid API requests fail closed + shell: bash + run: | + set -euo pipefail + + printf 'not a notebook' >/tmp/modelhawk-invalid.txt + status=$(curl --silent --show-error -o /tmp/modelhawk-invalid-audit.json -w '%{http_code}' \ + -F "file=@/tmp/modelhawk-invalid.txt;filename=invalid.txt" \ + http://127.0.0.1:3000/api/audit) + test "$status" = "400" + grep -q 'Please upload a .ipynb notebook file.' /tmp/modelhawk-invalid-audit.json + + status=$(curl --silent --show-error -o /tmp/modelhawk-invalid-ai.json -w '%{http_code}' \ + -H 'Content-Type: application/json' \ + --data '{"fileName":"invalid.ipynb","scores":{"leakageRisk":999,"metricQuality":1,"reproducibility":1,"portfolioReadiness":1}}' \ + http://127.0.0.1:3000/api/audit/ai) + test "$status" = "400" + + status=$(curl --silent --show-error -o /tmp/modelhawk-invalid-pdf.json -w '%{http_code}' \ + -H 'Content-Type: application/json' \ + --data '{"fileName":"invalid.ipynb"}' \ + http://127.0.0.1:3000/api/report/pdf) + test "$status" = "400" + + - name: Verify runtime security headers + shell: bash + run: | + set -euo pipefail + curl --silent --show-error --head http://127.0.0.1:3000/ >/tmp/modelhawk-headers.txt + grep -qi '^content-security-policy:' /tmp/modelhawk-headers.txt + grep -qi '^strict-transport-security:' /tmp/modelhawk-headers.txt + grep -qi '^x-content-type-options: nosniff' /tmp/modelhawk-headers.txt + grep -qi '^x-frame-options: DENY' /tmp/modelhawk-headers.txt + - name: Show container logs on failure if: failure() run: docker logs modelhawk-ci || true diff --git a/README.md b/README.md index 3824fba..1ccb441 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ Useful routes: - `/upload` — notebook upload workflow - `/api/health` — deployment health and revision endpoint -The public instance currently runs on Render's free web-service tier. It can spin down after inactivity, so the first request after an idle period may be noticeably slower. The application itself is containerised and the same Docker image is exercised in CI before merge. Render is configured to deploy from `main` only after the GitHub checks pass. +The public instance currently runs on Render's free web-service tier. It can spin down after inactivity, so the first request after an idle period may briefly show Render's loading screen while the service wakes. The application itself is containerised and the same Docker image is exercised in CI before merge. Render is configured to deploy from `main` only after the GitHub checks pass. The public deployment does not require a hosted AI-provider key. If no external provider is configured, the examiner layer uses the deterministic fallback path. diff --git a/app/demo/page.tsx b/app/demo/page.tsx index 92e9cf9..82a4a82 100644 --- a/app/demo/page.tsx +++ b/app/demo/page.tsx @@ -166,9 +166,9 @@ export default function DemoPage() {

- The previous sample PDF has been removed from this page because it - was generated before the hardened audit rules. A replacement will - only be published after it is regenerated from the current engine. + PDF reports are generated on demand from the current deterministic + audit result, so exported reports stay aligned with the hardened + rules used by the live application.

diff --git a/app/layout.tsx b/app/layout.tsx index 98648d5..a73b0be 100644 --- a/app/layout.tsx +++ b/app/layout.tsx @@ -1,23 +1,53 @@ import type { Metadata } from "next"; import "./globals.css"; +const description = + "Static ML notebook auditing with deterministic methodology checks, AI-assisted review and production PDF reports."; + export const metadata: Metadata = { - title: "ModelHawk", - description: - "Adversarial examiner for machine learning notebooks, metrics and project defense.", - icons: { - icon: "/modelhawk-favicon.png", + metadataBase: new URL("https://modelhawk.onrender.com"), + title: { + default: "ModelHawk | Static ML Notebook Auditor", + template: "%s | ModelHawk", + }, + description, + applicationName: "ModelHawk", + keywords: [ + "machine learning", + "ML audit", + "Jupyter notebook", + "model evaluation", + "data leakage", + "reproducibility", + ], + authors: [{ name: "Alberto Reyes Castro" }], + creator: "Alberto Reyes Castro", + openGraph: { + type: "website", + url: "/", + title: "ModelHawk | Static ML Notebook Auditor", + description, + siteName: "ModelHawk", + images: [{ + url: "/docs/screenshots/01-landing-page.png", + width: 1440, + height: 1000, + alt: "ModelHawk static ML notebook auditor", + }], + }, + twitter: { + card: "summary_large_image", + title: "ModelHawk | Static ML Notebook Auditor", + description, + images: ["/docs/screenshots/01-landing-page.png"], }, + icons: { icon: "/modelhawk-favicon.png" }, }; -export default function RootLayout({ - children, -}: Readonly<{ - children: React.ReactNode; -}>) { +export default function RootLayout({ children }: Readonly<{ children: React.ReactNode }>) { return ( {children} ); -} \ No newline at end of file +} diff --git a/app/page.tsx b/app/page.tsx index 7eb3ab8..1ca3096 100644 --- a/app/page.tsx +++ b/app/page.tsx @@ -21,10 +21,10 @@ const checks = [ label: "Scoring", }, { - title: "AI Defense Report", + title: "Controlled Examiner Report", description: - "Use AI to explain the findings, generate viva questions and turn the audit into clear portfolio-ready feedback.", - label: "AI layer", + "Use a controlled examiner layer — AI-assisted when configured — to explain findings, generate viva questions and turn the audit into clear portfolio-ready feedback.", + label: "Examiner layer", }, ]; @@ -162,8 +162,8 @@ export default function Home() {

ModelHawk does not simply ask an AI to “review your notebook”. It first extracts objective signals from your project, runs targeted ML - checks, scores the risks and then uses AI to explain the findings - clearly. + checks, scores the risks and then uses a controlled examiner layer + to explain the findings, with external AI only when configured.

@@ -206,15 +206,15 @@ export default function Home() {
- Medium-high risk + High leakage risk
- - - - + + + +
@@ -222,9 +222,9 @@ export default function Home() { Hawk finding

- Accuracy is reported as a main metric, but the dataset appears - imbalanced. Add recall, precision, F1, PR-AUC and threshold - justification. + Multiple candidate model families appear to be evaluated on + held-out test data before final selection. Keep the test set + reserved for one final unbiased evaluation.

@@ -244,16 +244,16 @@ export default function Home() {

- uses accuracy + test-set selection risk - no threshold tuning + test-set threshold tuning risk - imbalanced problem + preprocessing outside CV risk - weak reproducibility + custom composite score
diff --git a/app/upload/page.tsx b/app/upload/page.tsx index b1ab3f6..6e98df1 100644 --- a/app/upload/page.tsx +++ b/app/upload/page.tsx @@ -231,7 +231,7 @@ export default function UploadPage() {
- Static audit + AI examiner + Static audit + controlled examiner

@@ -243,14 +243,15 @@ export default function UploadPage() {

ModelHawk reads your notebook, detects the project profile, applies - technical audit checks and then uses an AI examiner layer to - generate richer project-specific explanations. + technical audit checks and then uses a controlled examiner layer to + generate richer project-specific explanations, with external AI only + when a provider is configured.

- +

@@ -264,8 +265,8 @@ export default function UploadPage() {

Upload a Jupyter notebook file. ModelHawk will parse it, return - technical signals, generate risk scores and add AI examiner - notes. + technical signals, generate risk scores and add controlled + examiner notes.