From 5ef27991cdd20b83f552cda8df967e141a9891e4 Mon Sep 17 00:00:00 2001 From: pz1130 <61965067+pz1130@users.noreply.github.com> Date: Wed, 2 Sep 2026 20:27:38 +0800 Subject: [PATCH] fix: harden uploads and optimize framework queries --- .env.example | 4 +- .github/workflows/test.yml | 4 +- Dockerfile | 4 +- Makefile | 8 +- README.md | 11 +- constraints.txt | 56 ++++++++ deploy/docker-entrypoint.sh | 4 + docker-compose.yml | 2 +- pyproject.toml | 10 +- src/framework_reader/identity/schema.sql | 3 + src/framework_reader/identity/store.py | 24 +++- src/framework_reader/interpret/batch.py | 5 +- src/framework_reader/query/api.py | 53 ++++++++ src/framework_reader/userframework/extract.py | 29 +++- .../userframework/importer.py | 35 +++-- src/framework_reader/web/app.py | 127 +++++------------- src/framework_reader/web/images.py | 65 +++++++++ src/framework_reader/web/uploads.py | 42 ++++++ tests/identity/test_store.py | 34 +++++ tests/query/test_api.py | 28 ++++ tests/test_docker_optional.py | 8 ++ tests/userframework/test_extract.py | 9 ++ tests/web/test_app.py | 9 ++ tests/web/test_clause_chat.py | 40 ++++++ tests/web/test_documents.py | 10 ++ 25 files changed, 498 insertions(+), 126 deletions(-) create mode 100644 constraints.txt create mode 100644 src/framework_reader/web/images.py create mode 100644 src/framework_reader/web/uploads.py diff --git a/.env.example b/.env.example index 3795f34..5db7bfa 100644 --- a/.env.example +++ b/.env.example @@ -33,8 +33,8 @@ GROQ_API_KEY= # Optional Docker (compose interpolates these; they are not baked into the image) # FR_HTTP_BIND=127.0.0.1 # FR_SITE_ADDRESS=localhost -# FR_BOOTSTRAP_ADMIN_EMAIL=admin@localhost -# FR_BOOTSTRAP_ADMIN_PASSWORD=changeme +FR_BOOTSTRAP_ADMIN_EMAIL=admin@localhost +FR_BOOTSTRAP_ADMIN_PASSWORD= # FR_ENTRA_TENANT_ID= # FR_ENTRA_CLIENT_ID= # FR_ENTRA_CLIENT_SECRET= diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 99bc1ac..edf5b27 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,9 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" - - run: python -m pip install -e ".[dev]" + - run: python -m pip install -c constraints.txt -e ".[dev]" + - run: ruff check src tests + - run: mypy src/framework_reader/web/uploads.py src/framework_reader/web/images.py src/framework_reader/schema/entities.py # Code tests only. No vendor/ fetch, no content-pack build, no signing keys. spec §10.C - run: pytest -v - name: Confirm vendor/ is absent in public CI diff --git a/Dockerfile b/Dockerfile index d509707..e7f4e02 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,9 +8,9 @@ ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update \ && apt-get install -y --no-install-recommends curl ca-certificates \ && rm -rf /var/lib/apt/lists/* -COPY pyproject.toml README.md ./ +COPY pyproject.toml constraints.txt README.md ./ COPY src ./src -RUN pip install --no-cache-dir --root-user-action=ignore . \ +RUN pip install --no-cache-dir --root-user-action=ignore -c constraints.txt . \ && python -c "from pathlib import Path; import framework_reader.identity as i, framework_reader.prompts as p, framework_reader.web as w; \ assert (Path(i.__file__).parent/'schema.sql').is_file(); \ assert (Path(p.__file__).parent/'drafter.md').is_file(); \ diff --git a/Makefile b/Makefile index d9f708f..d5f3731 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,17 @@ .PHONY: install test build clean install: - python -m pip install -e ".[dev]" + python -m pip install -c constraints.txt -e ".[dev]" test: .venv/bin/python -m pytest -v +.PHONY: check + +check: + .venv/bin/python -m ruff check src tests + .venv/bin/python -m mypy src/framework_reader/web/uploads.py src/framework_reader/web/images.py src/framework_reader/schema/entities.py + build: python -m framework_reader.pack.build diff --git a/README.md b/README.md index 3744bd8..fca0d23 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ The content pack does not leave this machine; model calls use your key. ```bash make install # install dependencies (incl. dev) +make check # Ruff + focused type checks make test # run the test suite (no vendor/, no API key needed) ./scripts/fetch_sources.sh # fetch NIST public-domain sources into vendor/ make build # build build/content.sqlite (needs vendor/) @@ -371,6 +372,7 @@ process, HTTP inside the container. TLS is terminated by Caddy in front, not by uvicorn. ```bash +# First set FR_BOOTSTRAP_ADMIN_PASSWORD in .env. # HTTP on http://127.0.0.1:8765 (set FR_HTTP_BIND=0.0.0.0 to listen on the LAN) docker compose up --build @@ -379,11 +381,10 @@ docker compose up --build docker compose -f docker-compose.yml -f deploy/compose.https.yml up --build ``` -Sign in at `/login` with **`admin@localhost` / `changeme`**. Change that -password before exposing the port. Override with `FR_BOOTSTRAP_ADMIN_EMAIL` -and `FR_BOOTSTRAP_ADMIN_PASSWORD`. The first account holds admin + author + -approver so a solo deploy can actually use the workbench; after it exists, -the door locks. +Before the first start, set a strong `FR_BOOTSTRAP_ADMIN_PASSWORD` in `.env`; +Compose refuses to start without it. `FR_BOOTSTRAP_ADMIN_EMAIL` defaults to +`admin@localhost`. The first account holds admin + author + approver so a solo +deploy can actually use the workbench; after it exists, the door locks. User data lives in the `fr-data` volume (`FRAMEWORK_READER_HOME` and a copy of the content pack). `FR_SECRET_KEY` comes from the environment; if you omit it, diff --git a/constraints.txt b/constraints.txt new file mode 100644 index 0000000..5551e29 --- /dev/null +++ b/constraints.txt @@ -0,0 +1,56 @@ +# Tested dependency set for repeatable CI and container builds. +# Refresh deliberately after running the full suite; pyproject.toml remains the +# human-readable declaration of direct dependencies. +annotated-doc==0.0.5 +annotated-types==0.8.0 +anthropic==0.125.0 +anyio==4.14.2 +certifi==2026.7.22 +cffi==2.1.1 +charset-normalizer==3.5.1 +click==8.4.2 +cryptography==50.0.0 +distro==1.9.0 +docstring_parser==0.18.0 +et_xmlfile==2.0.0 +fastapi==0.141.1 +h11==0.16.0 +httpcore==1.0.9 +httpcore2==2.12.0 +httpx==0.28.1 +httpx2==2.12.0 +idna==3.19 +iniconfig==2.3.0 +jiter==0.16.0 +librt==0.15.0 +markdown-it-py==4.2.0 +mdurl==0.1.2 +mypy==1.20.2 +mypy_extensions==1.1.0 +openpyxl==3.1.5 +packaging==26.3 +pathspec==1.1.1 +pillow==12.3.0 +pluggy==1.6.0 +prompt_toolkit==3.0.53 +pycparser==3.0 +pydantic==2.13.4 +pydantic_core==2.46.4 +Pygments==2.21.0 +PyJWT==2.13.0 +pypdf==6.16.2 +pytest==9.1.1 +python-multipart==0.0.32 +PyYAML==6.0.3 +reportlab==5.0.1 +rich==15.0.0 +ruff==0.16.5 +shellingham==1.5.4 +sniffio==1.3.1 +starlette==1.6.0 +typer==0.27.1 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +truststore==0.10.4 +uvicorn==0.52.4 +wcwidth==0.8.2 diff --git a/deploy/docker-entrypoint.sh b/deploy/docker-entrypoint.sh index dd118ec..3e8637e 100755 --- a/deploy/docker-entrypoint.sh +++ b/deploy/docker-entrypoint.sh @@ -46,6 +46,10 @@ if [ -n "${FR_BOOTSTRAP_ADMIN_EMAIL:-}" ]; then echo "FR_BOOTSTRAP_ADMIN_EMAIL is set but FR_BOOTSTRAP_ADMIN_PASSWORD is empty." >&2 exit 1 fi + if [ "$FR_BOOTSTRAP_ADMIN_PASSWORD" = "changeme" ]; then + echo "FR_BOOTSTRAP_ADMIN_PASSWORD must not use the published default 'changeme'." >&2 + exit 1 + fi fr account bootstrap --email "$FR_BOOTSTRAP_ADMIN_EMAIL" --password "$FR_BOOTSTRAP_ADMIN_PASSWORD" fi diff --git a/docker-compose.yml b/docker-compose.yml index 65a1d1f..96c279a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,7 +13,7 @@ services: FRAMEWORK_READER_HOME: /data/home FR_SECRET_KEY: ${FR_SECRET_KEY:-} FR_BOOTSTRAP_ADMIN_EMAIL: ${FR_BOOTSTRAP_ADMIN_EMAIL:-admin@localhost} - FR_BOOTSTRAP_ADMIN_PASSWORD: ${FR_BOOTSTRAP_ADMIN_PASSWORD:-changeme} + FR_BOOTSTRAP_ADMIN_PASSWORD: ${FR_BOOTSTRAP_ADMIN_PASSWORD:?Set a strong bootstrap admin password in .env} FR_ENTRA_TENANT_ID: ${FR_ENTRA_TENANT_ID:-} FR_ENTRA_CLIENT_ID: ${FR_ENTRA_CLIENT_ID:-} FR_ENTRA_CLIENT_SECRET: ${FR_ENTRA_CLIENT_SECRET:-} diff --git a/pyproject.toml b/pyproject.toml index 78dac64..76a7bc8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ dependencies = [ ] [project.optional-dependencies] -dev = ["pytest>=8.0"] +dev = ["pytest>=8.0", "httpx2>=2.12,<3", "ruff>=0.12,<1", "mypy>=1.15,<2"] [project.scripts] fr = "framework_reader.cli.main:app" @@ -47,3 +47,11 @@ framework_reader = ["identity/*.sql", "pack/*.sql", "prompts/*.md", "web/static/ testpaths = ["tests"] # tests/query/test_sample.py and tests/blindtest/test_sample.py share a basename. addopts = ["--import-mode=importlib"] + +[tool.ruff.lint] +select = ["E9", "F63", "F7", "F82"] + +[tool.mypy] +python_version = "3.12" +warn_unused_configs = true +check_untyped_defs = true diff --git a/src/framework_reader/identity/schema.sql b/src/framework_reader/identity/schema.sql index 6bb1a44..4af3012 100644 --- a/src/framework_reader/identity/schema.sql +++ b/src/framework_reader/identity/schema.sql @@ -58,6 +58,9 @@ CREATE TABLE IF NOT EXISTS audit_log ( detail TEXT NOT NULL DEFAULT '' ); +CREATE INDEX IF NOT EXISTS idx_audit_login_failures +ON audit_log(event, actor, at); + -- 运营开关。目前只有一条：allow_self_grant（设计 §4.3） CREATE TABLE IF NOT EXISTS setting ( key TEXT PRIMARY KEY, diff --git a/src/framework_reader/identity/store.py b/src/framework_reader/identity/store.py index 1cbae8b..ea7ccea 100644 --- a/src/framework_reader/identity/store.py +++ b/src/framework_reader/identity/store.py @@ -20,7 +20,10 @@ # 会话：绝对 8 小时、空闲 1 小时（设计 §5.5） ABSOLUTE_TTL = timedelta(hours=8) IDLE_TTL = timedelta(hours=1) +SESSION_TOUCH_INTERVAL = timedelta(minutes=1) INVITE_TTL = timedelta(days=7) +LOGIN_FAILURE_WINDOW = timedelta(minutes=5) +MAX_LOGIN_FAILURES = 5 # 从点「用公司账号登录」到 Entra 把人送回来。十分钟够慢的人输密码加二次验证， # 又短到那串一次性状态不会在库里躺一天。 FLOW_TTL = timedelta(minutes=10) @@ -557,6 +560,9 @@ def take_oidc_flow(self, state: str) -> dict | None: def login(self, email: str, password: str) -> Session: from framework_reader.identity.passwords import verify_password + email = email.strip().lower() + if self._recent_login_failures(email) >= MAX_LOGIN_FAILURES: + raise IdentityError("Wrong email or password.") account = self.by_email(email) conn = self._conn() try: @@ -578,6 +584,17 @@ def login(self, email: str, password: str) -> Session: self.log("login.ok", actor=account.email) return self.start_session(account) + def _recent_login_failures(self, email: str) -> int: + conn = self._conn() + try: + return conn.execute( + "SELECT COUNT(*) FROM audit_log " + "WHERE event = 'login.failed' AND actor = ? AND at >= ?", + (email, (_now() - LOGIN_FAILURE_WINDOW).isoformat()), + ).fetchone()[0] + finally: + conn.close() + def start_session(self, account: Account) -> Session: token = secrets.token_urlsafe(32) csrf = secrets.token_urlsafe(32) @@ -616,9 +633,10 @@ def resume(self, token: str) -> Session | None: conn.execute("DELETE FROM session WHERE id = ?", (row["id"],)) conn.commit() return None - conn.execute("UPDATE session SET last_seen = ? WHERE id = ?", - (now.isoformat(), row["id"])) - conn.commit() + if datetime.fromisoformat(row["last_seen"]) + SESSION_TOUCH_INTERVAL < now: + conn.execute("UPDATE session SET last_seen = ? WHERE id = ?", + (now.isoformat(), row["id"])) + conn.commit() account_row = conn.execute( "SELECT * FROM account WHERE id = ?", (row["account_id"],) ).fetchone() diff --git a/src/framework_reader/interpret/batch.py b/src/framework_reader/interpret/batch.py index 64e35dc..63c84b1 100644 --- a/src/framework_reader/interpret/batch.py +++ b/src/framework_reader/interpret/batch.py @@ -11,6 +11,7 @@ DIFFERENTIATING_FIELDS, Field, Interpretation, + InterviewRecord, InterpretationProvenance, ModelRef, ) @@ -38,7 +39,7 @@ def _is_empty(value) -> bool: def _keep_human_content( store: InterpretationStore, control_id: str, fresh: dict[str, Field], blanks_only: bool = False, -) -> tuple[dict[str, Field], "InterviewRecord"]: +) -> tuple[dict[str, Field], InterviewRecord]: """重跑起草只覆盖 AI 写的部分。作者的原话与他改过的字段一律保留。 闸的方向是对的（W2 spec §6：作者说过的话不能丢），但粒度应当在字段上， @@ -47,8 +48,6 @@ def _keep_human_content( `blanks_only`：只补空格。凡是已经有字的字段一概不动，不管是谁写的—— 用户点「补空缺」的意思就是「别碰我看过的那些」，包括他看过并认可的 AI 初稿。 """ - from framework_reader.interpret.model import InterviewRecord - if not store.exists(control_id): return fresh, InterviewRecord() previous = store.load(control_id) diff --git a/src/framework_reader/query/api.py b/src/framework_reader/query/api.py index b8ae9f1..0264e9e 100644 --- a/src/framework_reader/query/api.py +++ b/src/framework_reader/query/api.py @@ -27,6 +27,11 @@ class ControlView(BaseModel): status: str +class ControlSummary(ControlView): + has_interpretation: bool + interpretation_state: str | None = None + + class SupersessionView(BaseModel): control_id: str label: str @@ -330,6 +335,43 @@ def list_controls( ).fetchall() return [ControlView(**dict(r)) for r in rows] + def framework_progress(self) -> dict[str, tuple[int, int]]: + """每个框架的（叶子条款数，有解读的叶子条款数）。 + + 目录页要的是聚合数，不该先取出所有条款再逐条查解读。 + """ + rows = self._conn.execute( + "SELECT c.framework_id, COUNT(DISTINCT c.id) AS controls, " + " COUNT(DISTINCT i.control_id) AS interpreted " + "FROM all_control c " + "LEFT JOIN all_interpretation i ON i.control_id = c.id " + "WHERE c.status <> 'deprecated' " + " AND c.id NOT IN (SELECT parent_id FROM all_control " + " WHERE parent_id IS NOT NULL) " + "GROUP BY c.framework_id" + ).fetchall() + return { + r["framework_id"]: (r["controls"], r["interpreted"]) + for r in rows + } + + def control_summaries(self, framework_id: str) -> list[ControlSummary]: + """框架详情页所需的数据一次取齐，避免每条控制再查两次。""" + rows = self._conn.execute( + "SELECT c.id, c.framework_id, c.label, c.status, " + " COUNT(i.control_id) > 0 AS has_interpretation, " + " MAX(i.state) AS interpretation_state " + "FROM all_control c " + "LEFT JOIN all_interpretation i ON i.control_id = c.id " + "WHERE c.framework_id = ? AND c.status <> 'deprecated' " + " AND c.id NOT IN (SELECT parent_id FROM all_control " + " WHERE parent_id IS NOT NULL) " + "GROUP BY c.id, c.framework_id, c.label, c.status " + "ORDER BY c.id", + (framework_id,), + ).fetchall() + return [ControlSummary(**dict(r)) for r in rows] + def list_interpreted(self, *, leaf_only: bool = True) -> list[ControlView]: """有解读的条款。首页每天三条从这里抽，没解读的学了也是空壳。""" clauses = ["id IN (SELECT DISTINCT control_id FROM all_interpretation)"] @@ -405,6 +447,17 @@ def interpretation(self, control_id: str, locale: str = "zh-CN") -> dict[str, di for r in rows } + def forbidden_outbound_texts(self) -> list[str]: + """不得进入模型 payload 的内容包原文。 + + 调用方只拿业务含义明确的数据，不拿底层连接去写裸 SQL。 + """ + return [ + r["body"] for r in self._conn.execute( + "SELECT body FROM original_text" + ).fetchall() + ] + def interpretation_state( self, control_id: str, locale: str = "zh-CN" ) -> str | None: diff --git a/src/framework_reader/userframework/extract.py b/src/framework_reader/userframework/extract.py index 4aecd40..35658e9 100644 --- a/src/framework_reader/userframework/extract.py +++ b/src/framework_reader/userframework/extract.py @@ -19,6 +19,9 @@ CHUNK_MAX = 1200 SUPPORTED = (".txt", ".md", ".markdown", ".docx", ".pdf") +MAX_EXTRACTED_CHARS = 5_000_000 +MAX_DOCX_XML_BYTES = 50 * 1024 * 1024 +MAX_PDF_PAGES = 1000 _TAG = re.compile(r"<[^>]+>") _PARA_END = re.compile(r"") @@ -40,6 +43,10 @@ def extract(filename: str, data: bytes) -> str: raw = _from_text(data) else: raise UnsupportedDocument(f"Only {', '.join(SUPPORTED)}.") + if len(raw) > MAX_EXTRACTED_CHARS: + raise UnsupportedDocument( + f"The extracted text is over {MAX_EXTRACTED_CHARS:,} characters." + ) return "\n".join(strip_toc_lines(normalize_cjk(raw).splitlines())) @@ -151,7 +158,20 @@ def pdf_pages(data: bytes) -> list[str]: try: reader = PdfReader(io.BytesIO(data)) - pages = [(page.extract_text() or "").strip() for page in reader.pages] + if len(reader.pages) > MAX_PDF_PAGES: + raise UnsupportedDocument( + f"That PDF has more than {MAX_PDF_PAGES:,} pages." + ) + pages = [] + chars = 0 + for page in reader.pages: + text = (page.extract_text() or "").strip() + chars += len(text) + if chars > MAX_EXTRACTED_CHARS: + raise UnsupportedDocument( + f"The extracted text is over {MAX_EXTRACTED_CHARS:,} characters." + ) + pages.append(text) except (PdfReadError, OSError, ValueError, KeyError, TypeError) as exc: raise UnsupportedDocument( "This PDF won't open - it may not be a PDF, or the file was corrupted in transfer.") from exc @@ -177,7 +197,12 @@ def _from_docx(data: bytes) -> str: try: with zipfile.ZipFile(io.BytesIO(data)) as bundle: - xml = bundle.read("word/document.xml").decode("utf-8", "replace") + member = bundle.getinfo("word/document.xml") + if member.file_size > MAX_DOCX_XML_BYTES: + raise UnsupportedDocument( + f"The Word document expands past {MAX_DOCX_XML_BYTES // (1024 * 1024)} MB." + ) + xml = bundle.read(member).decode("utf-8", "replace") except (zipfile.BadZipFile, KeyError) as exc: raise UnsupportedDocument( "This .docx won't open - it may be an old .doc with a renamed extension. " diff --git a/src/framework_reader/userframework/importer.py b/src/framework_reader/userframework/importer.py index f85cd1b..82f186b 100644 --- a/src/framework_reader/userframework/importer.py +++ b/src/framework_reader/userframework/importer.py @@ -12,12 +12,29 @@ # 用户自己公司的制度原文。他的文档、他的机器、他的 key——可以拿去起草。 # 与 Tier C/D 的受版权标准原文完全是两回事，后者永远不许出网（主 spec §9）。 _BODY_HEADERS = {"正文", "描述", "要求", "内容", "条款正文", "body", "text", "description"} +MAX_SHEETS = 100 +MAX_ROWS = 100_000 +MAX_CELLS = 2_000_000 class ImportError_(Exception): """导入失败。消息要能让用户自己改好表，所以一律带行号或列名。""" +def _bounded_rows(rows) -> list[list[str]]: + out = [] + cells = 0 + for number, row in enumerate(rows, start=1): + if number > MAX_ROWS: + raise ImportError_(f"The file has more than {MAX_ROWS:,} rows") + values = ["" if cell is None else str(cell) for cell in row] + cells += len(values) + if cells > MAX_CELLS: + raise ImportError_(f"The file has more than {MAX_CELLS:,} cells") + out.append(values) + return out + + def read_sheets(path: Path) -> list[tuple[str, list[list[str]]]]: """一个工作簿里的**每一张表**，带名字。 @@ -28,18 +45,20 @@ def read_sheets(path: Path) -> list[tuple[str, list[list[str]]]]: suffix = Path(path).suffix.lower() if suffix == ".csv": with Path(path).open(encoding="utf-8-sig", newline="") as handle: - return [("", [list(row) for row in csv.reader(handle)])] + return [("", _bounded_rows(csv.reader(handle)))] if suffix in (".xlsx", ".xlsm"): from openpyxl import load_workbook book = load_workbook(path, read_only=True, data_only=True) - return [ - (sheet.title, [ - ["" if cell is None else str(cell) for cell in row] - for row in sheet.iter_rows(values_only=True) - ]) - for sheet in book.worksheets - ] + try: + if len(book.worksheets) > MAX_SHEETS: + raise ImportError_(f"The workbook has more than {MAX_SHEETS} sheets") + return [ + (sheet.title, _bounded_rows(sheet.iter_rows(values_only=True))) + for sheet in book.worksheets + ] + finally: + book.close() raise ImportError_(f"Unknown file type {suffix} - supported: .csv and .xlsx") diff --git a/src/framework_reader/web/app.py b/src/framework_reader/web/app.py index e82e09d..2f9d2a2 100644 --- a/src/framework_reader/web/app.py +++ b/src/framework_reader/web/app.py @@ -210,15 +210,13 @@ def _frameworks(reader: QueryAPI) -> list[dict]: from framework_reader.userframework.store import UserFrameworkStore mine = {f.id for f in UserFrameworkStore(_user_db()).list_frameworks()} + progress = reader.framework_progress() out = [] for view in reader.list_frameworks(): - controls = reader.list_controls(view.id, leaf_only=True) + controls, with_interp = progress.get(view.id, (0, 0)) out.append({ "id": view.id, "name": view.name, "mine": view.id in mine, - "controls": len(controls), - "with_interp": sum( - 1 for c in controls if reader.interpretation(c.id) - ), + "controls": controls, "with_interp": with_interp, }) out.sort(key=lambda f: (not f["mine"], f["id"])) return out @@ -425,11 +423,10 @@ def framework(framework_id: str): controls = [ { "id": c.id, "short": _short(c.id), "label": c.label, - "has_interp": bool(reader.interpretation(c.id)), - # 签字状态只有用户库存得下。内置框架不查，省掉 1196 次多余查询。 - "confirmed": reader.interpretation_state(c.id) == "confirmed", + "has_interp": c.has_interpretation, + "confirmed": c.interpretation_state == "confirmed", } - for c in reader.list_controls(framework_id, leaf_only=True) + for c in reader.control_summaries(framework_id) ] # 网页起草一律 overlay 到用户库当工作副本——导入的、内置的都一样， # 不进 git。views.framework 看 pending 决定画不画「起草 N 条」。 @@ -813,9 +810,7 @@ def _chat_store(): def _ask_model(control_id: str, message: str, history) -> str: from framework_reader.llm.client import Message - from framework_reader.llm.guard import ( - PayloadGuard, forbidden_texts_from_db, - ) + from framework_reader.llm.guard import PayloadGuard from framework_reader.prompts import load_prompt reader = api() @@ -833,7 +828,7 @@ def _ask_model(control_id: str, message: str, history) -> str: from framework_reader.userframework.chat import mapping_lines lines += ["", "The mappings for this control in the official mapping (when citing, copy IDs and sources verbatim; " "do not invent entries that are not in the list):"] - lines += mapping_lines(api().neighbors(control_id, exportable_only=True)) + lines += mapping_lines(reader.neighbors(control_id, exportable_only=True)) if history: lines += ["", "Earlier in this conversation:"] lines += [f"{'User' if h.role == 'user' else 'AI'}: {h.text}" @@ -843,8 +838,7 @@ def _ask_model(control_id: str, message: str, history) -> str: client, model = _extractor_client() # 守卫用**真的**受版权原文清单，不是空守卫：这条路径上正文来自 # 用户自己的框架，但守卫是最后一道拦网，不该因为「应该不会有」就撤掉。 - conn = api().connection() if hasattr(api(), "connection") else None - guard = PayloadGuard(forbidden_texts_from_db(conn)) if conn else PayloadGuard([]) + guard = PayloadGuard(reader.forbidden_outbound_texts()) from framework_reader.llm.guard import GuardedClient guarded = GuardedClient(client, guard) @@ -1655,6 +1649,7 @@ async def import_framework( ImportError_, parse_any_sheet, parse_table, read_sheets, ) from framework_reader.userframework.store import UserFrameworkStore + from framework_reader.web.uploads import UploadTooLarge, save_limited def fail(message: str) -> HTMLResponse: return HTMLResponse(views.import_page(error=message)) @@ -1674,7 +1669,7 @@ def fail(message: str) -> HTMLResponse: tmp = Path(raw_path) try: with open(handle, "wb") as sink: - sink.write(await file.read()) + await save_limited(file, sink) if suffix.lower() in _DOCUMENT_SUFFIXES: return _outline_upload( request, framework_id.strip(), name.strip(), @@ -1696,6 +1691,8 @@ def fail(message: str) -> HTMLResponse: except ImportError_ as exc: return fail(f"Import failed: {exc}" + (f" {note}" if note else "")) + except UploadTooLarge as exc: + return HTMLResponse(views.import_page(error=str(exc)), 413) except ImportError_ as exc: return fail(f"Import failed: {exc}") finally: @@ -1997,11 +1994,16 @@ def documents_page(error: str = ""): async def document_upload(request: Request, title: str = Form(""), file: UploadFile = File(...)): from framework_reader.userframework.extract import UnsupportedDocument + from framework_reader.web.uploads import UploadTooLarge, read_limited try: doc = _documents().add( - file.filename or "", await file.read(), + file.filename or "", await read_limited(file), by=_who(request) or _local_user(), title=title.strip()) + except UploadTooLarge as exc: + return HTMLResponse(views.documents( + _documents().list_documents(), can_write=True, + error=str(exc)), 413) except UnsupportedDocument as exc: return HTMLResponse(views.documents( _documents().list_documents(), can_write=True, @@ -2146,80 +2148,6 @@ def _branding_logo() -> Path | None: return candidate return None - def _sniff_image(data: bytes) -> str | None: - """按魔数认栅格图。SVG 单独走 _looks_like_svg + _sanitize_svg—— - 它能夹脚本与事件属性，放行前必须净化（见那条的注释）。""" - if data.startswith(b"\x89PNG\r\n\x1a\n"): - return "png" - if data.startswith(b"\xff\xd8\xff"): - return "jpg" - if data[:6] in (b"GIF87a", b"GIF89a"): - return "gif" - if data[:4] == b"RIFF" and data[8:12] == b"WEBP": - return "webp" - return None - - def _looks_like_svg(data: bytes) -> bool: - head = data[:512].lstrip(b"\xef\xbb\xbf\t\r\n ") - return head.startswith(b"<") - - _SVG_FORBID = {"script", "foreignObject", "iframe", "object", "embed", - "animate", "animateTransform", "animateMotion", "set", - "handler", "audio", "video"} - _SVG_CSS_BAD = ("url(", "expression", "@import", "javascript:") - - def _sanitize_svg(data: bytes) -> bytes: - """SVG 进场前的消毒：剥掉脚本、事件属性、外链引用与危险 CSS。 - - 方式本就不执行 SVG 里的脚本，但这个文件也能被直接在地址栏 - 打开——净化加伺服时的 CSP（script-src 'none'）双保险。形如 - 「先解析成 XML（顺带验证良构），再白名单式拆东西」，不碰正文。 - """ - import xml.etree.ElementTree as ET - - SVG = "http://www.w3.org/2000/svg" - ET.register_namespace("", SVG) - ET.register_namespace("xlink", "http://www.w3.org/1999/xlink") - root = ET.fromstring(data) - if root.tag != f"{{{SVG}}}svg": - raise ValueError("not an SVG document") - - def local(name: str) -> str: - return name.split("}")[-1] - - def scrub_css(text: str) -> str: - low = text.lower() - return text if not any(w in low for w in _SVG_CSS_BAD) else "" - - for node in root.iter(): - if local(node.tag) in _SVG_FORBID: - raise ValueError(f"forbidden element: {local(node.tag)}") - for name in list(node.attrib): - ln = local(name) - if ln.lower().startswith("on"): - del node.attrib[name] - elif ln == "href": - # href 白名单：#内部引用、data:image/ 内嵌位图都放行—— - # 「外壳是 SVG、里面是一张 base64 位图」是设计工具导出 - # logo 的标准形态，剥了它整张图就空白。其余一律剥： - # http(s) 外链是跟踪面，data:image/svg+xml 是嵌套 SVG， - # 会从净化过的外壳里夹带没净化的一份。 - value = str(node.attrib[name]).lstrip() - allowed = value.startswith("#") or any( - value.startswith(prefix) for prefix in - ("data:image/png;base64,", "data:image/jpeg;base64,", - "data:image/webp;base64,", "data:image/gif;base64,")) - if not allowed: - del node.attrib[name] - elif ln == "style": - cleaned = scrub_css(node.attrib[name]) - if cleaned != node.attrib[name]: - node.attrib[name] = cleaned - if local(node.tag) == "style": - node.text = scrub_css(node.text or "") - return (b'\n' - + ET.tostring(root)) - def _product_mark(name: str) -> Path: return Path(__file__).parent / "static" / name @@ -2365,6 +2293,10 @@ def branding_page(): @app.post("/settings/branding") @needs(perm.MEMBER_MANAGE) async def branding_upload(request: Request): + from framework_reader.web.images import ( + looks_like_svg, sanitize_svg, sniff_image, + ) + from framework_reader.web.uploads import UploadTooLarge, read_limited def refuse(message: str): found = _branding_logo() logo = ({"version": int(found.stat().st_mtime)} if found else None) @@ -2374,13 +2306,14 @@ def refuse(message: str): upload = form.get("file") if upload is None or not getattr(upload, "filename", ""): return refuse("Choose a file first.") - data = await upload.read() - if len(data) > 512 * 1024: - return refuse("That file is over 512 KB - a logo does not need to be that big.") - kind = _sniff_image(data) - if kind is None and _looks_like_svg(data): + try: + data = await read_limited(upload, max_bytes=512 * 1024) + except UploadTooLarge as exc: + return refuse(str(exc)) + kind = sniff_image(data) + if kind is None and looks_like_svg(data): try: - data = _sanitize_svg(data) + data = sanitize_svg(data) except ValueError as exc: return refuse(f"That SVG is not accepted: {exc}") except Exception: diff --git a/src/framework_reader/web/images.py b/src/framework_reader/web/images.py new file mode 100644 index 0000000..98edcfe --- /dev/null +++ b/src/framework_reader/web/images.py @@ -0,0 +1,65 @@ +"""管理员上传的品牌图片：识别栅格格式并净化 SVG。""" +import xml.etree.ElementTree as ET + +_SVG_FORBID = { + "script", "foreignObject", "iframe", "object", "embed", "animate", + "animateTransform", "animateMotion", "set", "handler", "audio", "video", +} +_SVG_CSS_BAD = ("url(", "expression", "@import", "javascript:") + + +def sniff_image(data: bytes) -> str | None: + if data.startswith(b"\x89PNG\r\n\x1a\n"): + return "png" + if data.startswith(b"\xff\xd8\xff"): + return "jpg" + if data[:6] in (b"GIF87a", b"GIF89a"): + return "gif" + if data[:4] == b"RIFF" and data[8:12] == b"WEBP": + return "webp" + return None + + +def looks_like_svg(data: bytes) -> bool: + head = data[:512].lstrip(b"\xef\xbb\xbf\t\r\n ") + return head.startswith(b"<") + + +def sanitize_svg(data: bytes) -> bytes: + """剥掉脚本、事件属性、外链引用与危险 CSS。""" + svg = "http://www.w3.org/2000/svg" + ET.register_namespace("", svg) + ET.register_namespace("xlink", "http://www.w3.org/1999/xlink") + root = ET.fromstring(data) + if root.tag != f"{{{svg}}}svg": + raise ValueError("not an SVG document") + + def local(name: str) -> str: + return name.split("}")[-1] + + def scrub_css(value: str) -> str: + low = value.lower() + return value if not any(word in low for word in _SVG_CSS_BAD) else "" + + for node in root.iter(): + if local(node.tag) in _SVG_FORBID: + raise ValueError(f"forbidden element: {local(node.tag)}") + for name in list(node.attrib): + attribute = local(name) + if attribute.lower().startswith("on"): + del node.attrib[name] + elif attribute == "href": + value = str(node.attrib[name]).lstrip() + allowed = value.startswith("#") or any( + value.startswith(prefix) for prefix in ( + "data:image/png;base64,", "data:image/jpeg;base64,", + "data:image/webp;base64,", "data:image/gif;base64,", + ) + ) + if not allowed: + del node.attrib[name] + elif attribute == "style": + node.attrib[name] = scrub_css(node.attrib[name]) + if local(node.tag) == "style": + node.text = scrub_css(node.text or "") + return b'\n' + ET.tostring(root) diff --git a/src/framework_reader/web/uploads.py b/src/framework_reader/web/uploads.py new file mode 100644 index 0000000..2f96ac3 --- /dev/null +++ b/src/framework_reader/web/uploads.py @@ -0,0 +1,42 @@ +"""上传体积边界。路由只决定如何处理文件，不能各自忘记限制大小。""" +from collections.abc import AsyncIterator +from typing import BinaryIO + +from fastapi import UploadFile + +MAX_UPLOAD_BYTES = 20 * 1024 * 1024 +_CHUNK_BYTES = 1024 * 1024 + + +class UploadTooLarge(Exception): + pass + + +def _limit_label(max_bytes: int) -> str: + if max_bytes >= 1024 * 1024 and max_bytes % (1024 * 1024) == 0: + return f"{max_bytes // (1024 * 1024)} MB" + return f"{max_bytes // 1024} KB" + + +async def _chunks( + upload: UploadFile, max_bytes: int | None = None, +) -> AsyncIterator[bytes]: + max_bytes = MAX_UPLOAD_BYTES if max_bytes is None else max_bytes + total = 0 + while chunk := await upload.read(_CHUNK_BYTES): + total += len(chunk) + if total > max_bytes: + raise UploadTooLarge(f"That file is over {_limit_label(max_bytes)}.") + yield chunk + + +async def read_limited( + upload: UploadFile, max_bytes: int | None = None, +) -> bytes: + parts = [chunk async for chunk in _chunks(upload, max_bytes)] + return b"".join(parts) + + +async def save_limited(upload: UploadFile, destination: BinaryIO) -> None: + async for chunk in _chunks(upload): + destination.write(chunk) diff --git a/tests/identity/test_store.py b/tests/identity/test_store.py index bedd65a..2574e34 100644 --- a/tests/identity/test_store.py +++ b/tests/identity/test_store.py @@ -139,6 +139,18 @@ def test_an_sso_only_account_cannot_log_in_with_a_blank_password(store): store.login("sso@acme.cn", "") +def test_repeated_password_failures_are_rate_limited(store): + _admin(store) + for _ in range(5): + with pytest.raises(IdentityError): + store.login(" BOSS@ACME.CN ", "wrong") + before = len([e for e in store.audit() if e["event"] == "login.failed"]) + with pytest.raises(IdentityError): + store.login("boss@acme.cn", "pw-boss") + after = len([e for e in store.audit() if e["event"] == "login.failed"]) + assert after == before + + # ---------- 会话 ---------- def test_a_session_resumes_from_its_token(store): @@ -147,6 +159,28 @@ def test_a_session_resumes_from_its_token(store): assert store.resume(session.token).account.email == "boss@acme.cn" +def test_resume_does_not_write_last_seen_on_every_request(store, monkeypatch): + import sqlite3 + import framework_reader.identity.store as module + + _admin(store) + session = store.login("boss@acme.cn", "pw-boss") + + def last_seen(): + conn = sqlite3.connect(store.path) + try: + return conn.execute("SELECT last_seen FROM session").fetchone()[0] + finally: + conn.close() + + first = last_seen() + store.resume(session.token) + assert last_seen() == first + monkeypatch.setattr(module, "SESSION_TOUCH_INTERVAL", timedelta(seconds=-1)) + store.resume(session.token) + assert last_seen() != first + + def test_the_raw_token_is_never_stored(store): """库泄漏不该等于所有会话被接管。""" import sqlite3 diff --git a/tests/query/test_api.py b/tests/query/test_api.py index 16622d8..d8712ed 100644 --- a/tests/query/test_api.py +++ b/tests/query/test_api.py @@ -102,6 +102,34 @@ def test_stats_reports_counts(db): assert s["exportable_mappings"] == 0 +def test_framework_progress_counts_leaf_controls_and_interpretations(db): + progress = QueryAPI(db).framework_progress() + assert progress["NIST-CSF-2.0"] == (1, 0) + assert progress["ISO-27002-2022"] == (1, 0) + + +def test_control_summaries_return_page_state_in_one_shape(db): + rows = QueryAPI(db).control_summaries("NIST-CSF-2.0") + assert len(rows) == 1 + assert rows[0].id == "NIST-CSF-2.0:DE.CM-01" + assert rows[0].has_interpretation is False + assert rows[0].interpretation_state is None + + +def test_query_api_exposes_forbidden_texts_without_exposing_its_connection(db): + conn = sqlite3.connect(db) + conn.execute( + "INSERT INTO original_text (control_id, locale, body) VALUES (?, ?, ?)", + ("ISO-27002-2022:A.8.16", "zh-CN", "不得发送给模型的标准原文"), + ) + conn.commit() + conn.close() + + api = QueryAPI(db) + assert api.forbidden_outbound_texts() == ["不得发送给模型的标准原文"] + assert not hasattr(api, "connection") + + def _superseded_db(tmp_path): """DE.CM-04（废止）被拆进 DE.CM-01 与 DE.CM-09。""" from framework_reader.pack.db import insert_supersessions diff --git a/tests/test_docker_optional.py b/tests/test_docker_optional.py index d8dbf86..e721e59 100644 --- a/tests/test_docker_optional.py +++ b/tests/test_docker_optional.py @@ -20,3 +20,11 @@ def test_compose_does_not_bind_http_to_all_interfaces_by_default(): text = (ROOT / "docker-compose.yml").read_text(encoding="utf-8") assert "127.0.0.1" in text assert "0.0.0.0:8765" not in text + + +def test_compose_requires_a_non_default_bootstrap_password(): + compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8") + entrypoint = (ROOT / "deploy/docker-entrypoint.sh").read_text(encoding="utf-8") + assert "FR_BOOTSTRAP_ADMIN_PASSWORD:?" in compose + assert "changeme" not in compose + assert 'FR_BOOTSTRAP_ADMIN_PASSWORD" = "changeme"' in entrypoint diff --git a/tests/userframework/test_extract.py b/tests/userframework/test_extract.py index 29cdd7d..03a616c 100644 --- a/tests/userframework/test_extract.py +++ b/tests/userframework/test_extract.py @@ -46,6 +46,15 @@ def test_a_renamed_doc_says_what_to_do(): assert "Re-save it as .docx" in str(caught.value) +def test_a_docx_with_an_excessive_expanded_document_is_rejected(monkeypatch): + import framework_reader.userframework.extract as module + + monkeypatch.setattr(module, "MAX_DOCX_XML_BYTES", 32) + with pytest.raises(UnsupportedDocument) as caught: + extract("制度.docx", _docx(["正文" * 100])) + assert "expands past" in str(caught.value) + + def test_a_pdf_is_refused_with_a_reason(): """从 PDF 切出来的段落是乱的。噪声接地比没有接地更糟。""" with pytest.raises(UnsupportedDocument) as caught: diff --git a/tests/web/test_app.py b/tests/web/test_app.py index b65221c..ba6d4f2 100644 --- a/tests/web/test_app.py +++ b/tests/web/test_app.py @@ -132,6 +132,15 @@ def test_an_unsupported_file_type_is_explained(client): assert "csv" in result.text +def test_an_oversized_framework_upload_is_rejected(client, monkeypatch): + from framework_reader.web import uploads + + monkeypatch.setattr(uploads, "MAX_UPLOAD_BYTES", 8) + result = _upload(client, b"number,title\n1,Account management\n") + assert result.status_code == 413 + assert "over" in result.text + + def test_the_import_needs_an_id_and_a_name(client): body = "编号,标题\n3.1,账号\n".encode() assert _upload(client, body, framework_id="").status_code == 200 diff --git a/tests/web/test_clause_chat.py b/tests/web/test_clause_chat.py index 4032fa6..1b39f4f 100644 --- a/tests/web/test_clause_chat.py +++ b/tests/web/test_clause_chat.py @@ -108,6 +108,46 @@ def test_the_model_does_not_see_more_than_six_turns(env): assert len(history) <= 6 +def test_the_default_chat_path_enforces_the_content_pack_guard( + tmp_path, monkeypatch, +): + from types import SimpleNamespace + + from framework_reader.llm.client import FakeClient + import framework_reader.llm.config as config_module + from framework_reader.web.app import create_app + + env = _make(tmp_path, monkeypatch) + conn = sqlite3.connect(env.db) + forbidden = "这是不得发送给任何模型厂商的受版权标准原文完整片段内容" + conn.execute( + "INSERT INTO original_text (control_id, locale, body) VALUES (?, ?, ?)", + (BUILTIN_CID, "zh-CN", forbidden), + ) + conn.commit() + conn.close() + + inner = FakeClient(['{"reply":"should not run","updates":[]}']) + + class Registry: + def build(self, _role, *, guard, key_lookup): + return inner + + def role(self, _role): + return SimpleNamespace(model="fake") + + monkeypatch.setattr( + config_module, "effective_registry", + lambda config: (Registry(), lambda _name: "key"), + ) + client = TestClient(create_app(env.db), follow_redirects=False) + result = client.post( + f"/c/{BUILTIN_CID}/chat", data={"message": forbidden}) + assert result.status_code == 303 + assert inner.calls == [] + assert "did not go through" in client.get(f"/c/{BUILTIN_CID}").text + + # ---------- 建议要人点头 ---------- def _proposes(*a): diff --git a/tests/web/test_documents.py b/tests/web/test_documents.py index 2df0bac..46d19d2 100644 --- a/tests/web/test_documents.py +++ b/tests/web/test_documents.py @@ -130,6 +130,16 @@ def test_a_pdf_is_refused_with_a_sentence_not_a_stack_trace(env): assert env.docs.list_documents() == [] +def test_an_oversized_document_is_rejected_before_parsing(env, monkeypatch): + from framework_reader.web import uploads + + monkeypatch.setattr(uploads, "MAX_UPLOAD_BYTES", 8) + response = _upload(_as(env, "author"), body="这是一份超过限制的制度") + assert response.status_code == 413 + assert "over" in response.text + assert env.docs.list_documents() == [] + + def test_the_page_warns_that_this_leaves_the_building(env): """上传内部制度 = 把它发给你配置的模型厂商。这句话必须写在传之前。""" page = _as(env, "author").get("/documents").text