diff --git a/.dockerignore b/.dockerignore index f55a6e2..ed60729 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,4 +3,6 @@ __pycache__ .pytest_cache .env +.env.* +local_decisions.csv *.log diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d402a8..63f655a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,9 @@ on: push: pull_request: +permissions: + contents: read + jobs: test: runs-on: ubuntu-latest @@ -15,3 +18,21 @@ jobs: - run: pip install -r requirements.txt - run: python scripts/validate_dataset.py - run: pytest -q + + secret-scan: + runs-on: ubuntu-latest + env: + GITLEAKS_VERSION: "8.21.2" + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install gitleaks (checksum verified) + run: | + base="https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}" + file="gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz" + curl -sSfLO "${base}/${file}" + curl -sSfL "${base}/gitleaks_${GITLEAKS_VERSION}_checksums.txt" | grep " ${file}$" | sha256sum --check - + tar -xzf "${file}" gitleaks + - name: Scan full Git history for secrets + run: ./gitleaks git . --redact --verbose \ No newline at end of file diff --git a/app.py b/app.py index 0506dbe..3fa171d 100644 --- a/app.py +++ b/app.py @@ -235,6 +235,7 @@ def load_manifest(): reviewer_label = st.radio( "Final decision", list(LABELS), + index=None, horizontal=False, format_func=lambda value: DECISION_LABELS.get(value, value), ) @@ -245,7 +246,7 @@ def load_manifest(): ) if assessment and assessment.get("label") in LABELS: - if reviewer_label != assessment["label"]: + if reviewer_label and reviewer_label != assessment["label"]: st.info("Your decision differs from the AI recommendation. Explain why in reviewer notes.") if st.button("Record reviewer decision", width="stretch"): diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 7fa6baa..74b435c 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -29,14 +29,14 @@ A valid model label does not make the underlying rationale or citation trustwort Controls: - Vertex uses a response schema with a two-value label enum -- parser ambiguity fails closed to UNPARSEABLE +- parser ambiguity fails closed to UNPARSEABLE; a label word inside prose ("does not show the claim is SUPPORTED") or an invalid JSON label is never treated as a valid label - exact/whitespace-normalized citation grounding is checked independently - rationale is displayed as inert text and the citation as code - model/provider metadata is retained for evaluation ### Human decision is authoritative -The reviewer makes the final SUPPORTED/UNSUPPORTED decision. When the reviewer overrides a valid model label, ProofStack requires a note to preserve the reason for disagreement. +The reviewer makes the final SUPPORTED/UNSUPPORTED decision. No decision is pre-selected, and a decision cannot be recorded until the reviewer explicitly chooses one. When the reviewer overrides a valid model label, ProofStack requires a note to preserve the reason for disagreement. ## Primary threats and controls @@ -123,6 +123,7 @@ Controls: - append-style local logging - Firestore server timestamps in cloud mode - reviewer identity required +- explicit final decision required (no pre-selected default) - override note required - provider/model, rationale, cited excerpt and citation-valid flag retained @@ -145,10 +146,12 @@ Automated tests cover: - role-hijack language - embedded and malformed model output - ambiguous SUPPORTED/UNSUPPORTED text +- label words inside prose, including negation, and invalid JSON labels - hallucinated citation - oversized evidence - stale assessment context - reviewer missing identity +- reviewer missing final decision - human override without explanation - provider configuration and malformed-response failure - Firestore configuration behavior @@ -157,7 +160,7 @@ Automated tests cover: ## Deployment review Before final demo: -- verify no secrets in repo/history +- verify no secrets in repo/history (CI runs gitleaks over the full history on every push and pull request) - confirm IAM remains least privilege - verify synthetic-only dataset - run CI diff --git a/docs/SECURITY_TEST_MATRIX.md b/docs/SECURITY_TEST_MATRIX.md index ac00844..076b616 100644 --- a/docs/SECURITY_TEST_MATRIX.md +++ b/docs/SECURITY_TEST_MATRIX.md @@ -32,7 +32,7 @@ Residual risk: ProofStack does not implement a general user-level rate limiter. | --- | --- | | A01 Broken Access Control | Cloud Run is private by default; runtime identity is separate from deployer identity. | | A02 Security Misconfiguration | deployment script defaults to private access and explicit environment configuration. | -| A03 Software Supply Chain Failures | dependency ranges, CI, and Dependabot are used; production-grade SBOM/signing is outside MVP scope. | +| A03 Software Supply Chain Failures | dependency ranges, CI, and Dependabot are used; CI runs with read-only repository permissions and scans Git history for secrets; production-grade SBOM/signing is outside MVP scope. | | A04 Cryptographic Failures | TLS is provided by Cloud Run; no application-managed cryptographic keys are introduced. | | A05 Injection | evidence is untrusted and model-directed injection is explicitly tested; the app has no SQL or shell execution path from reviewer input. | | A06 Insecure Design | trust boundaries, human authority, fail-closed behavior, and least privilege are design requirements. | @@ -53,6 +53,8 @@ Residual risk: ProofStack does not implement a general user-level rate limiter. 8. Changing question/evidence must invalidate stale assessment state. 9. A reviewer override must require an explanation. 10. Missing provider configuration or empty provider output must fail visibly. +11. A label word inside prose (for example "does not show the claim is SUPPORTED") or an invalid JSON label must become UNPARSEABLE, not a valid label. +12. A reviewer decision must not be recorded until the reviewer explicitly chooses SUPPORTED or UNSUPPORTED. ## Live adversarial check diff --git a/src/proofstack/parser.py b/src/proofstack/parser.py index d98587d..9e2620f 100644 --- a/src/proofstack/parser.py +++ b/src/proofstack/parser.py @@ -4,7 +4,12 @@ from proofstack import LABELS, UNPARSEABLE from proofstack.schema import ModelAssessment -_LABEL_RE = re.compile(r"\b(SUPPORTED|UNSUPPORTED)\b", flags=re.I) +# Non-JSON fallback accepts only an explicit label on its own line ("UNSUPPORTED", +# "Label: SUPPORTED"). A label word inside prose is never used, because +# "the evidence does not show the claim is SUPPORTED" must not become SUPPORTED. +_LABEL_RE = re.compile( + r"^\W*(?:label\W*)?(SUPPORTED|UNSUPPORTED)\W*$", flags=re.I | re.M +) _JSON_BLOCK_RE = re.compile(r"\{[^{}]*\}", flags=re.S) @@ -26,6 +31,16 @@ def _assessment_from_obj(obj: dict, raw: str, parse_status: str): ) +def _invalid_label(raw: str) -> ModelAssessment: + """JSON with a label outside LABELS must not fall through to word matching.""" + return ModelAssessment( + label=UNPARSEABLE, + rationale=raw[:500], + parse_status="invalid_label", + raw=raw, + ) + + def parse_model_output(text: str) -> ModelAssessment: """Parse model output without silently converting ambiguity into a valid label.""" raw = str(text or "").strip() @@ -42,6 +57,8 @@ def parse_model_output(text: str) -> ModelAssessment: parsed = _assessment_from_obj(obj, raw, "json") if parsed: return parsed + if "label" in obj: + return _invalid_label(raw) except json.JSONDecodeError: pass @@ -59,6 +76,8 @@ def parse_model_output(text: str) -> ModelAssessment: parsed = _assessment_from_obj(obj, raw, "json_embedded") if parsed: return parsed + if "label" in obj: + return _invalid_label(raw) labels = {_valid(value) for value in _LABEL_RE.findall(raw)} labels.discard(None) diff --git a/src/proofstack/review_state.py b/src/proofstack/review_state.py index 2439454..929c381 100644 --- a/src/proofstack/review_state.py +++ b/src/proofstack/review_state.py @@ -24,6 +24,8 @@ def reviewer_submission_errors( if not reviewer: errors.append("Reviewer name is required.") + if reviewer_label not in LABELS: + errors.append("Choose a final decision.") model_label = "" if isinstance(assessment, dict): diff --git a/tests/test_parser.py b/tests/test_parser.py index dc779ac..3cfafe6 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -36,6 +36,36 @@ def test_ambiguous_is_unparseable(): assert result.label == "UNPARSEABLE" +def test_label_on_own_line_is_tolerated(): + result = parse_model_output("Label: UNSUPPORTED\nThe export lists two accounts without MFA.") + assert result.label == "UNSUPPORTED" + assert result.parse_status == "label_regex" + + +def test_label_word_inside_prose_is_unparseable(): + # Negation: reading the label word out of this sentence would invert its meaning. + result = parse_model_output("The evidence does not show the claim is SUPPORTED.") + assert result.label == "UNPARSEABLE" + + +def test_conflicting_label_lines_are_unparseable(): + assert parse_model_output("SUPPORTED\nUNSUPPORTED").label == "UNPARSEABLE" + + +def test_invalid_json_label_does_not_fall_back_to_prose(): + result = parse_model_output( + '{"label":"APPROVED","rationale":"the claim is UNSUPPORTED by logs"}' + ) + assert result.label == "UNPARSEABLE" + assert result.parse_status == "invalid_label" + + +def test_invalid_embedded_json_label_does_not_fall_back_to_prose(): + result = parse_model_output('Result: {"label":"COMPLIANT"} so it is SUPPORTED') + assert result.label == "UNPARSEABLE" + assert result.parse_status == "invalid_label" + + def test_empty_is_unparseable(): result = parse_model_output("") assert result.label == "UNPARSEABLE" diff --git a/tests/test_review_state.py b/tests/test_review_state.py index daa564f..2b0a420 100644 --- a/tests/test_review_state.py +++ b/tests/test_review_state.py @@ -50,3 +50,13 @@ def test_matching_decision_does_not_require_note(): "", {"label": "SUPPORTED"}, ) == [] + + +def test_missing_final_decision_is_rejected(): + errors = reviewer_submission_errors("John", None, "", {"label": "SUPPORTED"}) + assert "Choose a final decision." in errors + + +def test_invalid_final_decision_is_rejected(): + errors = reviewer_submission_errors("John", "MAYBE", "", None) + assert "Choose a final decision." in errors diff --git a/tests/test_ui_security.py b/tests/test_ui_security.py index 264c10e..72844b4 100644 --- a/tests/test_ui_security.py +++ b/tests/test_ui_security.py @@ -16,3 +16,9 @@ def test_model_rationale_is_rendered_as_plain_text(): def test_app_does_not_enable_unsafe_html(): assert "unsafe_allow_html=True" not in APP + + +def test_final_decision_has_no_preselected_default(): + start = APP.index('"Final decision"') + radio_call = APP[start:APP.index("\n)", start)] + assert "index=None" in radio_call