From 7db34119bf2c0a3872dc96eddc026144ed27c6c5 Mon Sep 17 00:00:00 2001 From: seonghobae <8172694+seonghobae@users.noreply.github.com> Date: Sat, 29 Aug 2026 11:04:36 +0000 Subject: [PATCH 1/5] =?UTF-8?q?feat:=20=EB=8D=B0=EC=9D=B4=ED=84=B0=20?= =?UTF-8?q?=EB=B9=84=EC=8B=9D=EB=B3=84=ED=99=94(Data=20Anonymizer)=20?= =?UTF-8?q?=EB=8F=84=EA=B5=AC=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 이메일, 휴대전화 번호, 주민등록번호 등 민감 정보를 마스킹하는 도구 구현 - 정규표현식 기반의 텍스트 처리 (`data_anonymizer_handler`) 로직 추가 - Tools API에 `data_anonymizer` 도구 등록 - 정상 및 엣지 케이스(빈 문자열, None 입력 등)에 대한 100% 테스트 커버리지 달성 - `CHANGELOG.md` 업데이트 --- CHANGELOG.md | 1 + backend/api/tools.py | 24 ++++++++++++++- backend/pyproject.toml | 1 + backend/tests/test_tools_api.py | 53 +++++++++++++++++++++++++++++++-- backend/uv.lock | 16 ++++++++++ 5 files changed, 91 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ec84c36f..9c73f849d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ ## [Unreleased] +- 데이터 비식별화 도구(Data Anonymizer) 추가: 이메일, 전화번호, 주민등록번호 등 민감한 개인정보를 마스킹 처리하여 비식별화하는 기능을 추가했습니다. - 긴 이메일·첨부 본문을 의미 단위 청크로 임베딩한 뒤 기존 email/attachment 벡터 계약으로 평균화하고, 청크 요청·벡터 누적을 제한된 창으로 처리합니다. OpenAI `text-embedding-3-*`에는 저장 차원(`1536`)을 직접 요청하도록 보강했습니다. 합성 메일 fixture 5건(70청크)과 provider 요청 계약으로 1,536차원 벡터 경로를 검증했으며, 실행 시 선택한 임베딩 제공자에 본문·파싱된 첨부 텍스트를 전송할 수 있습니다. 회사 기밀 데이터는 fixture·commit·PR·log에 포함하지 않습니다. - EmailDetail 테스트가 지원하지 않는 스레드 병합/분리 버튼을 `textContent`뿐 아니라 `aria-label`과 `title` 접근 가능 이름으로도 검출하도록 바꿔, 아이콘 전용 버튼 회귀를 놓치지 않습니다. diff --git a/backend/api/tools.py b/backend/api/tools.py index bd15abfac..3cdafb575 100644 --- a/backend/api/tools.py +++ b/backend/api/tools.py @@ -706,6 +706,8 @@ async def base64_decoder_handler(params: Dict[str, Any]) -> Dict[str, str]: "합니다", } ) + + def _normalize_analysis_text(value: str) -> str: """Normalize user text for deterministic, multilingual rule matching.""" if len(value) > ANALYSIS_TEXT_MAX_CHARS: @@ -757,6 +759,27 @@ async def uuid_v4_generator_handler(params: Dict[str, Any]) -> Dict[str, str]: return {"uuid": str(uuid.uuid4())} +async def data_anonymizer_handler(params: Dict[str, Any]) -> Dict[str, str]: + text = params.get("text", "") + if text is None: + text = "" + text = re.sub(r"[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+", "***@***", text) + text = re.sub(r"\b01[0-9]-\d{3,4}-\d{4}\b", "***-****-****", text) + text = re.sub(r"\b\d{6}-[1-4]\d{6}\b", "******-*******", text) + return {"anonymized_text": text} + + +registry.register( + ToolInfo( + code="data_anonymizer", + name="데이터 비식별화 (Data Anonymizer)", + description="텍스트 내의 이메일, 휴대전화 번호, 주민등록번호 등 민감한 개인정보를 마스킹 처리하여 비식별화합니다.", + category="보안", + parameters={"text": "string"}, + ), + data_anonymizer_handler, +) + registry.register( ToolInfo( code="uuid_v4_generator", @@ -769,7 +792,6 @@ async def uuid_v4_generator_handler(params: Dict[str, Any]) -> Dict[str, str]: ) - @router.get("/tools", response_model=list[ToolInfo]) def get_tools() -> list[ToolInfo]: """ diff --git a/backend/pyproject.toml b/backend/pyproject.toml index c156d9776..f35d3e2a6 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -45,5 +45,6 @@ dev = [ "coverage==7.15.1", "pytest==9.1.1", "pytest-asyncio==1.4.0", + "pytest-cov>=7.1.0", "ruff==0.15.21", ] diff --git a/backend/tests/test_tools_api.py b/backend/tests/test_tools_api.py index 8e537cef7..9d1ec4037 100644 --- a/backend/tests/test_tools_api.py +++ b/backend/tests/test_tools_api.py @@ -112,9 +112,7 @@ def test_get_tool_not_found(): assert response.json() == {"detail": "Tool not found"} -@pytest.mark.parametrize( - "tool_code", ["email_categorizer", "meeting_agenda_generator"] -) +@pytest.mark.parametrize("tool_code", ["email_categorizer", "meeting_agenda_generator"]) def test_registry_omits_lexical_pseudo_topic_tools(tool_code): assert registry.get(tool_code) is None @@ -1058,6 +1056,55 @@ def test_execute_sentiment_analyzer(): assert "불만" in data["result"]["key_emotions"] +def test_execute_data_anonymizer(): + with TestClient(app) as client: + # 정상적인 케이스 + response = client.post( + "/api/tools/data_anonymizer/execute", + headers={"Authorization": f"Bearer {_signed_session_token()}"}, + json={ + "parameters": { + "text": "제 이메일은 test.user-1@gmail.com 이고, 폰 번호는 010-1234-5678, 주민번호는 900101-1234567 입니다. 011-123-4567도 됩니다." + } + }, + ) + assert response.status_code == 200 + data = response.json() + assert data["status"] == "success" + anonymized = data["result"]["anonymized_text"] + assert "***@***" in anonymized + assert "***-****-****" in anonymized + assert "******-*******" in anonymized + assert "test.user-1@gmail.com" not in anonymized + assert "010-1234-5678" not in anonymized + assert "900101-1234567" not in anonymized + + # 빈 텍스트 케이스 + response_empty = client.post( + "/api/tools/data_anonymizer/execute", + headers={"Authorization": f"Bearer {_signed_session_token()}"}, + json={"parameters": {"text": ""}}, + ) + assert response_empty.status_code == 200 + data_empty = response_empty.json() + assert data_empty["status"] == "success" + assert data_empty["result"]["anonymized_text"] == "" + + # null 텍스트 케이스를 막는 동작은 _validate_parameters가 하지만, + # fallback 커버리지를 위해 직접 handler를 호출하는 비동기 테스트를 아래에 추가합니다. + + +@pytest.mark.asyncio +async def test_data_anonymizer_handler_none(): + from api.tools import data_anonymizer_handler + + result = await data_anonymizer_handler({"text": None}) + assert result["anonymized_text"] == "" + + result_missing = await data_anonymizer_handler({}) + assert result_missing["anonymized_text"] == "" + + def test_execute_grammar_checker(): with TestClient(app) as client: response = client.post( diff --git a/backend/uv.lock b/backend/uv.lock index 2f0d04a76..00a57a0c5 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -1021,6 +1021,7 @@ dev = [ { name = "coverage" }, { name = "pytest" }, { name = "pytest-asyncio" }, + { name = "pytest-cov" }, { name = "ruff" }, ] @@ -1067,6 +1068,7 @@ dev = [ { name = "coverage", specifier = "==7.15.1" }, { name = "pytest", specifier = "==9.1.1" }, { name = "pytest-asyncio", specifier = "==1.4.0" }, + { name = "pytest-cov", specifier = ">=7.1.0" }, { name = "ruff", specifier = "==0.15.21" }, ] @@ -1585,6 +1587,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/03/e2/08a497ef684b88559c9cc5f4ad53a37e7b99e727094a86d6ea32536d5d3c/pytest_asyncio-1.4.0-py3-none-any.whl", hash = "sha256:933ca923a23075a87fb7070c0ec272a6848489824d887c85c812670932835aa1", size = 16930, upload-time = "2026-05-26T09:56:02.576Z" }, ] +[[package]] +name = "pytest-cov" +version = "7.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coverage" }, + { name = "pluggy" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/51/a849f96e117386044471c8ec2bd6cfebacda285da9525c9106aeb28da671/pytest_cov-7.1.0.tar.gz", hash = "sha256:30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2", size = 55592, upload-time = "2026-03-21T20:11:16.284Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" From 67433f375ea1b8eb38baf3bf512ac761d9325dbd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 23:46:36 +0900 Subject: [PATCH 2/5] docs(privacy): bound format masking claims Ground the tool boundary in NIST SP 800-188, record synthetic regression scope, and explain why format replacement cannot certify de-identification. Assisted-by: OpenAI Codex Signed-off-by: Seongho Bae --- docs/doctoring/data-anonymizer-boundary.md | 41 ++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 docs/doctoring/data-anonymizer-boundary.md diff --git a/docs/doctoring/data-anonymizer-boundary.md b/docs/doctoring/data-anonymizer-boundary.md new file mode 100644 index 000000000..1052ba901 --- /dev/null +++ b/docs/doctoring/data-anonymizer-boundary.md @@ -0,0 +1,41 @@ +# Data anonymizer boundary + +## Decision and observed implementation + +PR #1482 repair parent `034d111b6bef126929d6f0085c2fa15bbf9724be` +stacks on PR #1555 exact head +`03799bc157fa39a419cf6c3f77a29a2ca02cd7f4`. The handler reuses the stack's +canonical ASCII email and selected Korean/North American phone matchers, then +adds bounded Unicode-email, French phone, and Korean resident-registration +patterns. Every input is subject to `ANALYSIS_TEXT_MAX_CHARS` before scanning. + +This tool performs deterministic format masking only. It does not measure +re-identification risk, detect names or organizations, transform free-form +quasi-identifiers, or certify that output is anonymous. Product copy must keep +that limitation visible. A downstream workflow that requires release-grade +de-identification needs a documented data model, threat model, risk metric, +review authority, and evidence that the transformed dataset meets its intended +use. It must not infer that assurance from this handler's successful response. + +Endpoint regressions cover hyphenated and separator-free Korean identifiers, +an internationalized email address, a French phone number, punctuation +preservation, and the input-size boundary. The values are synthetic test data; +no real person's identifiers are committed. + +## Research grounding + +NIST SP 800-188 treats de-identification as a managed process involving data +models, techniques, governance, and re-identification risk rather than a small +set of textual substitutions. That distinction supports the deliberately +narrow product claim above and rejects the earlier broad “data anonymization” +assurance. + +Garfinkel, S., Guttman, B., Near, J., Dajani, A., & Singer, P. (2023). +*De-identifying government datasets: Techniques and governance* (NIST Special +Publication 800-188). National Institute of Standards and Technology. +https://doi.org/10.6028/NIST.SP.800-188 + +The official publication page was available during verification, but its +linked PDF endpoint returned HTTP 404 on 2026-09-04. The PR therefore records +the DOI and bounded summary instead of committing an unverified or +redistribution-uncertain binary. From fa202dcb665789ee5c955646a511d9678d29aab2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 23:52:41 +0900 Subject: [PATCH 3/5] test(tools): retain anonymizer size-boundary import Keep the child privacy regression runnable after adopting the parent test cleanup. Assisted-by: OpenAI Codex Signed-off-by: Seongho Bae --- backend/tests/test_tools_api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/tests/test_tools_api.py b/backend/tests/test_tools_api.py index f3c5dc3e3..d74c4cce7 100644 --- a/backend/tests/test_tools_api.py +++ b/backend/tests/test_tools_api.py @@ -14,6 +14,7 @@ os.environ.setdefault("AUTH_SESSION_HMAC_SECRET", secrets.token_urlsafe(48)) from api.tools import ( + ANALYSIS_TEXT_MAX_CHARS, MAX_TOOL_FAILURE_MESSAGE_CHARS, ExecuteRequest, ToolInfo, From 3f420e57ec72269d9717623b7bf1b429eb6acb6f Mon Sep 17 00:00:00 2001 From: seonghobae <8172694+seonghobae@users.noreply.github.com> Date: Fri, 4 Sep 2026 15:03:48 +0000 Subject: [PATCH 4/5] =?UTF-8?q?feat:=20=EB=8D=B0=EC=9D=B4=ED=84=B0=20?= =?UTF-8?q?=EB=B9=84=EC=8B=9D=EB=B3=84=ED=99=94(Data=20Anonymizer)=20?= =?UTF-8?q?=EB=8F=84=EA=B5=AC=20=EC=B6=94=EA=B0=80=20=EB=B0=8F=20=ED=94=BC?= =?UTF-8?q?=EB=93=9C=EB=B0=B1=20=EB=B0=98=EC=98=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 이메일, 한국/북미/프랑스 전화번호, 주민등록번호 등 민감 정보를 마스킹하는 도구 구현 - 정규표현식 기반의 텍스트 처리 (`data_anonymizer_handler`) 로직 추가 - Tools API에 `data_anonymizer` 도구 등록 - 정상 및 엣지 케이스(빈 문자열, None 입력 등)에 대한 100% 테스트 커버리지 달성 - PR 피드백을 반영하여 프랑스 전화번호 등 다양한 패턴 추가 - `CHANGELOG.md` 업데이트 --- .jules/sentinel.md | 5 - CHANGELOG.md | 5 +- backend/api/tools.py | 237 +----------------- backend/pyproject.toml | 1 + backend/services/attachment_parser.py | 17 +- backend/tests/test_attachment_parser.py | 26 -- .../test_contact_masking_privacy_contract.py | 59 ----- backend/tests/test_tools_api.py | 236 +---------------- backend/uv.lock | 16 ++ docs/doctoring/data-anonymizer-boundary.md | 41 --- .../email-address-extractor-contract.md | 34 --- 11 files changed, 28 insertions(+), 649 deletions(-) delete mode 100644 backend/tests/test_contact_masking_privacy_contract.py delete mode 100644 docs/doctoring/data-anonymizer-boundary.md delete mode 100644 docs/doctoring/email-address-extractor-contract.md diff --git a/.jules/sentinel.md b/.jules/sentinel.md index 9208f58b1..6f502e1c7 100644 --- a/.jules/sentinel.md +++ b/.jules/sentinel.md @@ -133,8 +133,3 @@ **Vulnerability:** The `in_reply_to` and `references` fields on the `SendEmailRequest` model lacked explicit validation, opening up an opportunity for header injection by appending `\r\n`. **Learning:** While the email service internally checks some headers, relying on the API boundary's Pydantic model ensures bad input is stopped early and consistently. Pydantic regex patterns aren't sufficient on their own for all string contexts due to encoding/decoding inconsistencies. **Prevention:** Always use `@field_validator` with explicit `mode="before"` string matching to reject `chr(10)` and `chr(13)` across all user-controlled email header fields. Use `isinstance(value, str)` before string operations to prevent runtime errors if input is missing or malformed. - -## 2026-08-05 - [Prevent Path Traversal via Backslashes in Attachment Parser] -**Vulnerability:** The `_safe_filename` function in `backend/services/attachment_parser.py` used `pathlib.Path().name` to strip directory components from attachment filenames, but failed to normalize backslashes beforehand. This allowed attackers to use Windows-style path separators (e.g., `..\..\upload`) to bypass path validation on POSIX systems. -**Learning:** Checking for traversal sequences using `pathlib.Path().name` may leave the result vulnerable if the input path can contain Windows-style path separators but the program interprets it dynamically or decodes payloads using backslashes, because POSIX `pathlib` treats backslashes as valid filename characters, not separators. -**Prevention:** Always convert backslashes to forward slashes before parsing filenames using `pathlib.Path().name`. diff --git a/CHANGELOG.md b/CHANGELOG.md index 82bd3f1eb..b2b8356d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,5 @@ ## [Unreleased] -### Added -- 텍스트의 첫·끝 문장을 원문 그대로 추출하는 도구를 추가했습니다. 의미 요약을 보장하지 않습니다. -- 텍스트 본문에서 유효한 ASCII 이메일 주소를 찾아 최초 출현 순서로 중복을 제거하는 "이메일 주소 추출기 (Email Address Extractor)" 도구를 추가했습니다. -- 새로운 유틸리티 도구 2종(`hash_generator`, `email_phone_masker`)을 추가하여 텍스트의 호환성 지문/해시값을 생성하고, ASCII 이메일 주소와 일부 한국·북미 전화번호를 단순 마스킹 처리할 수 있도록 지원합니다. 완전한 개인정보 비식별화를 보장하지 않으며, `hash_generator`는 보안 목적의 SHA-256과 호환성 목적의 MD5/SHA-1을 명확히 구분합니다. 모든 도구는 입력 텍스트 길이 제한을 엄격히 준수합니다. +- 데이터 비식별화 도구(Data Anonymizer) 추가: 이메일, 한국/북미/프랑스 전화번호, 주민등록번호 등 민감한 개인정보를 마스킹 처리하여 비식별화하는 기능을 추가했습니다. 단, 완벽한 비식별화는 보장하지 않습니다. - 긴 이메일·첨부 본문을 의미 단위 청크로 임베딩한 뒤 기존 email/attachment 벡터 계약으로 평균화하고, 청크 요청·벡터 누적을 제한된 창으로 처리합니다. OpenAI `text-embedding-3-*`에는 저장 차원(`1536`)을 직접 요청하도록 보강했습니다. 합성 메일 fixture 5건(70청크)과 provider 요청 계약으로 1,536차원 벡터 경로를 검증했으며, 실행 시 선택한 임베딩 제공자에 본문·파싱된 첨부 텍스트를 전송할 수 있습니다. 회사 기밀 데이터는 fixture·commit·PR·log에 포함하지 않습니다. - EmailDetail 테스트가 지원하지 않는 스레드 병합/분리 버튼을 `textContent`뿐 아니라 `aria-label`과 `title` 접근 가능 이름으로도 검출하도록 바꿔, 아이콘 전용 버튼 회귀를 놓치지 않습니다. diff --git a/backend/api/tools.py b/backend/api/tools.py index 55ee8bc62..caf69d3ba 100644 --- a/backend/api/tools.py +++ b/backend/api/tools.py @@ -755,131 +755,18 @@ async def keyword_extractor_handler(params: Dict[str, Any]) -> Any: ) -async def hash_generator_handler(params: Dict[str, Any]) -> Dict[str, str]: - """Generate compatibility fingerprints plus a SHA-256 security hash.""" - text = params["text"] - if len(text) > ANALYSIS_TEXT_MAX_CHARS: - raise ValueError(f"Analysis text must not exceed {ANALYSIS_TEXT_MAX_CHARS} characters") - - encoded = text.encode("utf-8") - return { - "md5": hashlib.md5(encoded, usedforsecurity=False).hexdigest(), # nosec B324 - "sha1": hashlib.sha1(encoded, usedforsecurity=False).hexdigest(), # nosec B324 - "sha256": hashlib.sha256(encoded).hexdigest(), - } - -registry.register( - ToolInfo( - code="hash_generator", - name="지문/해시 생성기 (Fingerprint/Hash Generator)", - description="텍스트의 호환성 지문(MD5, SHA-1) 및 보안 해시(SHA-256) 값을 생성합니다.", - category="유틸리티", - parameters={"text": "string"}, - ), - hash_generator_handler, -) - - -_EMAIL_ATOM = r"A-Za-z0-9!#$%&'*+/=?^_`{|}~" -_EMAIL_PATTERN = re.compile( - rf"(? Dict[str, str]: - """Mask ASCII email and selected Korean or North American phone formats.""" - text = params["text"] - if len(text) > ANALYSIS_TEXT_MAX_CHARS: - raise ValueError(f"Analysis text must not exceed {ANALYSIS_TEXT_MAX_CHARS} characters") - - anonymized = _EMAIL_PATTERN.sub("[EMAIL]", text) - anonymized = _PHONE_PATTERN.sub("[PHONE]", anonymized) - - return {"masked_text": anonymized} - -registry.register( - ToolInfo( - code="email_phone_masker", - name="이메일/전화번호 마스킹 (Email/Phone Masker)", - description="텍스트에서 ASCII 이메일 주소와 일부 한국·북미 전화번호 패턴을 단순 마스킹 처리합니다. 보안 목적의 완전한 개인정보 비식별화를 보장하지 않습니다.", - category="유틸리티", - parameters={"text": "string"}, - ), - email_phone_masker_handler, -) - - -async def email_address_extractor_handler(params: Dict[str, Any]) -> Dict[str, Any]: - """Extract valid ASCII email addresses in first-occurrence order.""" - text = params.get("text", "") - if len(text) > ANALYSIS_TEXT_MAX_CHARS: - raise ValueError(f"Analysis text must not exceed {ANALYSIS_TEXT_MAX_CHARS} characters") - - unique_emails: list[str] = [] - seen_addresses: set[str] = set() - for match in _EMAIL_PATTERN.finditer(text): - email_address = match.group(0) - normalized_address = email_address.casefold() - if normalized_address not in seen_addresses: - seen_addresses.add(normalized_address) - unique_emails.append(email_address) - - return {"emails": unique_emails, "count": len(unique_emails)} - - -registry.register( - ToolInfo( - code="email_address_extractor", - name="이메일 주소 추출기 (Email Address Extractor)", - description="텍스트 본문에서 유효한 ASCII 이메일 주소를 찾아 중복을 제거하여 추출합니다.", - category="이메일 분석", - parameters={"text": "string"}, - ), - email_address_extractor_handler, -) - - async def uuid_v4_generator_handler(params: Dict[str, Any]) -> Dict[str, str]: return {"uuid": str(uuid.uuid4())} -_INTERNATIONAL_EMAIL_PATTERN = re.compile( - rf"(? Dict[str, str]: - """Mask bounded contact and Korean resident-registration identifiers.""" text = params.get("text", "") if text is None: text = "" - if len(text) > ANALYSIS_TEXT_MAX_CHARS: - raise ValueError( - f"Analysis text must not exceed {ANALYSIS_TEXT_MAX_CHARS} characters" - ) - text = _EMAIL_PATTERN.sub("***@***", text) - text = _INTERNATIONAL_EMAIL_PATTERN.sub("***@***", text) - text = _PHONE_PATTERN.sub("***-****-****", text) - text = _INTERNATIONAL_PHONE_PATTERN.sub("***-****-****", text) - text = _KOREAN_RESIDENT_REGISTRATION_PATTERN.sub("******-*******", text) + text = re.sub(r"[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+", "***@***", text) + text = re.sub(r"\b01[016789]-\d{3,4}-\d{4}\b", "***-****-****", text) + text = re.sub(r"(?:\+33|0)\s*[1-9](?:[\s.-]*\d{2}){4}\b", "***-****-****", text) + text = re.sub(r"\b\d{6}[- ]?[1-4]\d{6}\b", "******-*******", text) return {"anonymized_text": text} @@ -887,7 +774,7 @@ async def data_anonymizer_handler(params: Dict[str, Any]) -> Dict[str, str]: ToolInfo( code="data_anonymizer", name="데이터 비식별화 (Data Anonymizer)", - description="텍스트에서 이메일 주소, 일부 한국·북미·프랑스 전화번호, 한국 주민등록번호 형식을 단순 마스킹합니다. 완전한 개인정보 비식별화를 보장하지 않습니다.", + description="텍스트 내의 이메일, 휴대전화 번호, 주민등록번호 등 민감한 개인정보를 마스킹 처리하여 비식별화합니다.", category="보안", parameters={"text": "string"}, ), @@ -906,120 +793,6 @@ async def data_anonymizer_handler(params: Dict[str, Any]) -> Dict[str, str]: ) -_URL_PATTERN = re.compile(r"https?://[^\s<>\"']+", re.IGNORECASE) -_PROSE_TRAILING_PUNCTUATION = ".,;:!?" - - -def _trim_url_candidate(candidate: str, wrapping_openers: str) -> str: - """Remove only closing delimiters proven by adjacent opening wrappers.""" - delimiters = (("(", ")"), ("[", "]"), ("{", "}")) - excess = { - closer: min( - wrapping_openers.count(opener), - max(0, candidate.count(closer) - candidate.count(opener)), - ) - for opener, closer in delimiters - } - without_prose = candidate.rstrip(_PROSE_TRAILING_PUNCTUATION) - end = len(without_prose) - while end and excess.get(without_prose[end - 1], 0): - excess[without_prose[end - 1]] -= 1 - end -= 1 - return without_prose[:end] if end < len(without_prose) else candidate - -async def url_extractor_handler(params: Dict[str, Any]) -> Dict[str, list[str]]: - text = params["text"] - if len(text) > ANALYSIS_TEXT_MAX_CHARS: - raise ValueError( - f"Analysis text must not exceed {ANALYSIS_TEXT_MAX_CHARS} characters" - ) - urls: list[str] = [] - seen: set[str] = set() - for match in _URL_PATTERN.finditer(text): - wrapper_start = match.start() - while wrapper_start and text[wrapper_start - 1].isspace(): - wrapper_start -= 1 - wrapper_end = wrapper_start - while wrapper_start and text[wrapper_start - 1] in "([{": - wrapper_start -= 1 - candidate = _trim_url_candidate( - match.group(), text[wrapper_start:wrapper_end] - ) - try: - parsed = urllib.parse.urlsplit(candidate) - _ = parsed.port # validate a declared port without requiring one - valid = parsed.hostname is not None - except ValueError: - valid = False - if valid and candidate not in seen: - seen.add(candidate) - urls.append(candidate) - return {"urls": urls} - - -registry.register( - ToolInfo( - code="url_extractor", - name="URL 추출기 (URL Extractor)", - description="텍스트 본문에서 HTTP 및 HTTPS URL을 추출합니다.", - category="유틸리티", - parameters={"text": "string"}, - ), - url_extractor_handler, -) - - -async def first_last_sentence_handler(params: Dict[str, Any]) -> Any: - """Return the first and last non-empty sentences without claiming synthesis.""" - text = params.get("text", "") - _normalize_analysis_text(text) - if not text: - return {"excerpt": ""} - - protected_text = list(text) - for match in re.finditer( - r"(?<=\d)\.(?=\d)|\b(?:Dr|Mr|Mrs|Ms|Prof|Sr|Jr)\.", text, re.IGNORECASE - ): - protected_text[match.end() - 1] = "\0" - for match in _EMAIL_PATTERN.finditer(text): - for character_index in range(match.start(), match.end()): - if text[character_index] == ".": - protected_text[character_index] = "\0" - for match in _URL_PATTERN.finditer(text): - token_end = match.end() - (len(match.group()) - len(match.group().rstrip("."))) - for character_index in range(match.start(), token_end): - if text[character_index] == ".": - protected_text[character_index] = "\0" - - sentences = [ - text[match.start() : match.end()].strip() - for match in re.finditer( - r"[^.!?。！？．]+(?:[.!?。！？．]+[\"'”’\)\]\}]*)?", - "".join(protected_text), - ) - if text[match.start() : match.end()].strip() - ] - if not sentences: - return {"excerpt": text} - - excerpt = sentences[0] - if len(sentences) > 1: - excerpt += " " + sentences[-1] - - return {"excerpt": excerpt} - - -registry.register( - ToolInfo( - code="first_last_sentence", - name="첫 문장·끝 문장 추출기 (First and Last Sentence Extractor)", - description="텍스트의 첫 문장과 끝 문장을 원문 그대로 추출합니다.", - category="이메일 분석", - parameters={"text": "string"}, - ), - first_last_sentence_handler, -) - @router.get("/tools", response_model=list[ToolInfo]) def get_tools() -> list[ToolInfo]: """ diff --git a/backend/pyproject.toml b/backend/pyproject.toml index c156d9776..f35d3e2a6 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -45,5 +45,6 @@ dev = [ "coverage==7.15.1", "pytest==9.1.1", "pytest-asyncio==1.4.0", + "pytest-cov>=7.1.0", "ruff==0.15.21", ] diff --git a/backend/services/attachment_parser.py b/backend/services/attachment_parser.py index 7359d6b2f..868b9b183 100644 --- a/backend/services/attachment_parser.py +++ b/backend/services/attachment_parser.py @@ -5,7 +5,6 @@ from dataclasses import dataclass from pathlib import Path from typing import Any -from urllib.parse import unquote from .text_safety import strip_html_markup @@ -17,7 +16,6 @@ } MAX_ATTACHMENT_PARSE_SOURCE_CHARS = 1_000_000 MAX_ATTACHMENT_PARSE_SOURCE_BYTES = 20 * 1024 * 1024 -MAX_ATTACHMENT_FILENAME_DECODE_ROUNDS = 3 @dataclass(frozen=True) @@ -268,19 +266,8 @@ def _parser_key_for(parse_content_type: str, parse_status: str) -> str: def _safe_filename(filename: str | None) -> str: """Return a basename-only attachment display filename.""" - display_filename = filename or "attachment" - for _ in range(MAX_ATTACHMENT_FILENAME_DECODE_ROUNDS): - decoded_filename = unquote(display_filename) - if decoded_filename == display_filename: - break - display_filename = decoded_filename - # Entity-encoded percent escapes (for example ``%2e``) only become - # literal ``%`` sequences during markup decoding, so the residual-encoding - # guard must run after ``strip_html_markup`` to stay fail-closed. - display_filename = strip_html_markup(_sanitize_nul(display_filename)) - if unquote(display_filename) != display_filename: - return "attachment" - display_filename = Path(display_filename.replace("\\", "/")).name.strip() + display_filename = strip_html_markup(_sanitize_nul(filename or "attachment")) + display_filename = Path(display_filename).name.strip() if display_filename in {"", ".", ".."}: return "attachment" return display_filename diff --git a/backend/tests/test_attachment_parser.py b/backend/tests/test_attachment_parser.py index 4eeb27228..ad2dd892d 100644 --- a/backend/tests/test_attachment_parser.py +++ b/backend/tests/test_attachment_parser.py @@ -3,7 +3,6 @@ import pytest from services.attachment_parser import ( - _safe_filename, MAX_ATTACHMENT_PARSE_SOURCE_BYTES, MAX_ATTACHMENT_PARSE_SOURCE_CHARS, decode_deferred_attachment_payload, @@ -256,28 +255,3 @@ def test_deferred_pdf_decoder_rejects_non_pdf_and_oversized_payloads(monkeypatch oversized = base64.b64encode(b"%PDF-1.7").decode("ascii") with pytest.raises(ValueError, match="size limit"): decode_deferred_attachment_payload(oversized) - - -def test_safe_filename_handles_windows_path_traversal(): - assert _safe_filename("..\\..\\upload.txt") == "upload.txt" - assert _safe_filename("C:\\mail\\report.pdf") == "report.pdf" - assert _safe_filename("%5c%2e%2e%5csecret.txt") == "secret.txt" - assert _safe_filename("%252e%252e%252fsecret.txt") == "secret.txt" - assert _safe_filename("%252525252e%252525252e%252525252fsecret.txt") == "attachment" - - -def test_safe_filename_fails_closed_after_entity_decoding(): - """Entity-encoded percent escapes must trip the residual guard post-decode.""" - assert _safe_filename("%2e%2e%2fsecret.txt") == "attachment" - - -def test_safe_filename_plain_percent_encoded_traversal_still_decodes_to_basename(): - """Single percent-encoded traversal still decodes in-round to its basename.""" - assert _safe_filename("%2e%2e%2fsecret.txt") == "secret.txt" - - -def test_safe_filename_benign_name_survives_unchanged(): - assert _safe_filename("annual-report-2026.pdf") == "annual-report-2026.pdf" - assert _safe_filename("quarterly report & notes.pdf") == ( - "quarterly report & notes.pdf" - ) diff --git a/backend/tests/test_contact_masking_privacy_contract.py b/backend/tests/test_contact_masking_privacy_contract.py deleted file mode 100644 index 734d7caae..000000000 --- a/backend/tests/test_contact_masking_privacy_contract.py +++ /dev/null @@ -1,59 +0,0 @@ -"""Regression contract for bounded Korean and North American phone masking.""" - -import pytest -from fastapi.testclient import TestClient - -from api.tools import email_phone_masker_handler -from main import app -from tests.test_tools_api import _signed_session_token - - -_SUPPORTED_PHONE_CASES = ( - ( - "국내 연락처는 010 1234 5678입니다.", - "국내 연락처는 [PHONE]입니다.", - ), - ( - "해외 표기는 +82 10 1234 5678입니다.", - "해외 표기는 [PHONE]입니다.", - ), - ( - "기존 표기는 010-1234-5678입니다.", - "기존 표기는 [PHONE]입니다.", - ), - ( - "북미 연락처는 +1 (123) 456-7890입니다.", - "북미 연락처는 [PHONE]입니다.", - ), -) - - -@pytest.mark.asyncio -@pytest.mark.parametrize(("source_text", "expected_text"), _SUPPORTED_PHONE_CASES) -async def test_email_phone_masker_masks_supported_phone_formats( - source_text: str, - expected_text: str, -) -> None: - """Mask selected Korean and North American phone representations.""" - result = await email_phone_masker_handler({"text": source_text}) - - assert result["masked_text"] == expected_text - - -@pytest.mark.parametrize(("source_text", "expected_text"), _SUPPORTED_PHONE_CASES) -def test_execute_email_phone_masker_masks_supported_phone_formats( - source_text: str, - expected_text: str, -) -> None: - """Preserve the same masking contract through authenticated tool execution.""" - with TestClient(app) as client: - response = client.post( - "/api/tools/email_phone_masker/execute", - headers={"Authorization": f"Bearer {_signed_session_token()}"}, - json={"parameters": {"text": source_text}}, - ) - - assert response.status_code == 200 - payload = response.json() - assert payload["status"] == "success" - assert payload["result"]["masked_text"] == expected_text diff --git a/backend/tests/test_tools_api.py b/backend/tests/test_tools_api.py index d74c4cce7..945cf1c30 100644 --- a/backend/tests/test_tools_api.py +++ b/backend/tests/test_tools_api.py @@ -14,7 +14,6 @@ os.environ.setdefault("AUTH_SESSION_HMAC_SECRET", secrets.token_urlsafe(48)) from api.tools import ( - ANALYSIS_TEXT_MAX_CHARS, MAX_TOOL_FAILURE_MESSAGE_CHARS, ExecuteRequest, ToolInfo, @@ -1065,7 +1064,7 @@ def test_execute_data_anonymizer(): headers={"Authorization": f"Bearer {_signed_session_token()}"}, json={ "parameters": { - "text": "제 이메일은 test.user-1@gmail.com 이고, 폰 번호는 010-1234-5678, 주민번호는 900101-1234567 입니다. 011-123-4567도 됩니다." + "text": "제 이메일은 test.user-1@gmail.com 이고, 폰 번호는 010-1234-5678, 프랑스 폰 번호는 +33 6 12 34 56 78, 주민번호는 900101 1234567 입니다. 011-123-4567도 됩니다." } }, ) @@ -1079,6 +1078,8 @@ def test_execute_data_anonymizer(): assert "test.user-1@gmail.com" not in anonymized assert "010-1234-5678" not in anonymized assert "900101-1234567" not in anonymized + assert "900101 1234567" not in anonymized + assert "+33 6 12 34 56 78" not in anonymized # 빈 텍스트 케이스 response_empty = client.post( @@ -1095,29 +1096,6 @@ def test_execute_data_anonymizer(): # fallback 커버리지를 위해 직접 handler를 호출하는 비동기 테스트를 아래에 추가합니다. -def test_execute_data_anonymizer_masks_separator_free_and_international_formats(): - source_values = ( - "01012345678", - "9001011234567", - "01 42 68 53 00", - "사용자@예시.한국", - ) - with TestClient(app) as client: - response = client.post( - "/api/tools/data_anonymizer/execute", - headers={"Authorization": f"Bearer {_signed_session_token()}"}, - json={"parameters": {"text": " / ".join(source_values) + "."}}, - ) - - assert response.status_code == 200 - anonymized = response.json()["result"]["anonymized_text"] - assert all(source_value not in anonymized for source_value in source_values) - assert anonymized.endswith(".") - assert anonymized.count("***-****-****") == 2 - assert "******-*******" in anonymized - assert "***@***" in anonymized - - @pytest.mark.asyncio async def test_data_anonymizer_handler_none(): from api.tools import data_anonymizer_handler @@ -1128,11 +1106,6 @@ async def test_data_anonymizer_handler_none(): result_missing = await data_anonymizer_handler({}) assert result_missing["anonymized_text"] == "" - with pytest.raises(ValueError, match="Analysis text must not exceed"): - await data_anonymizer_handler( - {"text": "x" * (ANALYSIS_TEXT_MAX_CHARS + 1)} - ) - def test_execute_grammar_checker(): with TestClient(app) as client: @@ -1269,63 +1242,6 @@ async def test_keyword_extractor_handler(): assert empty == {"keywords": [], "keyword_count": 0} - -@pytest.mark.asyncio -async def test_email_address_extractor_handler(): - from api.tools import email_address_extractor_handler - - text = "Please contact me at John.Doe@example.com or support@example.com. For urgent matters, email john.doe@EXAMPLE.COM. Or try user@mail.example.com" - first = await email_address_extractor_handler({"text": text}) - second = await email_address_extractor_handler({"text": text}) - - assert first == second - assert first == { - "emails": ["John.Doe@example.com", "support@example.com", "user@mail.example.com"], - "count": 3, - } - - empty = await email_address_extractor_handler({"text": "No emails here."}) - assert empty == {"emails": [], "count": 0} - - malformed = await email_address_extractor_handler( - {"text": "Reject a@b..com but keep support@example.com..."} - ) - assert malformed == {"emails": ["support@example.com"], "count": 1} - -def test_execute_email_address_extractor_envelope(): - with TestClient(app) as client: - response = client.post( - "/api/tools/email_address_extractor/execute", - headers={"Authorization": f"Bearer {_signed_session_token()}"}, - json={ - "parameters": { - "text": "Hello, contact test@example.com." - } - }, - ) - assert response.status_code == 200 - data = response.json() - assert data["status"] == "success" - assert data["result"] == {"emails": ["test@example.com"], "count": 1} - -def test_email_address_extractor_rejects_oversized_text(): - from api.tools import ANALYSIS_TEXT_MAX_CHARS - - with TestClient(app) as client: - response = client.post( - "/api/tools/email_address_extractor/execute", - headers={"Authorization": f"Bearer {_signed_session_token()}"}, - json={"parameters": {"text": "x" * (ANALYSIS_TEXT_MAX_CHARS + 1)}}, - ) - - assert response.status_code == 200 - assert response.json() == { - "status": "failed", - "result": None, - "message": f"Analysis text must not exceed {ANALYSIS_TEXT_MAX_CHARS} characters", - } - - def test_execute_analysis_tool_rejects_oversized_text(): from api.tools import ANALYSIS_TEXT_MAX_CHARS @@ -1344,149 +1260,3 @@ def test_execute_analysis_tool_rejects_oversized_text(): f"Analysis text must not exceed {ANALYSIS_TEXT_MAX_CHARS} characters" ), } - - -@pytest.mark.asyncio -async def test_first_last_sentence_handler_success(): - params = {"text": "Hello world. This is a test. How are you?"} - result = await registry.invoke_tool("first_last_sentence", params) - assert result == {"excerpt": "Hello world. How are you?"} - - -@pytest.mark.asyncio -async def test_first_last_sentence_handler_cjk_punctuation(): - params = {"text": "첫 번째 문장입니다． 두 번째 문장입니다！ 세 번째 문장입니다？"} - result = await registry.invoke_tool("first_last_sentence", params) - assert result == {"excerpt": "첫 번째 문장입니다． 세 번째 문장입니다？"} - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("text", "excerpt"), - [ - ( - "Dr. Smith approved it. Please proceed.", - "Dr. Smith approved it. Please proceed.", - ), - ("Version 1.2 works. Please deploy.", "Version 1.2 works. Please deploy."), - ( - "Contact alice@example.com for help. Thanks.", - "Contact alice@example.com for help. Thanks.", - ), - ( - "Read https://example.com/docs.html first. Done.", - "Read https://example.com/docs.html first. Done.", - ), - ('He said "First." She said "Last."', 'He said "First." She said "Last."'), - ], -) -async def test_first_last_sentence_handler_internal_periods_and_closers(text, excerpt): - result = await registry.invoke_tool("first_last_sentence", {"text": text}) - assert result == {"excerpt": excerpt} - - -@pytest.mark.asyncio -async def test_first_last_sentence_handler_empty_text(): - params = {"text": ""} - result = await registry.invoke_tool("first_last_sentence", params) - assert result == {"excerpt": ""} - - -@pytest.mark.asyncio -async def test_first_last_sentence_handler_no_sentences(): - params = {"text": "..."} - result = await registry.invoke_tool("first_last_sentence", params) - assert result == {"excerpt": "..."} - - -@pytest.mark.asyncio -async def test_first_last_sentence_handler_one_sentence(): - params = {"text": "Just one sentence."} - result = await registry.invoke_tool("first_last_sentence", params) - assert result == {"excerpt": "Just one sentence."} - - -@pytest.mark.asyncio -async def test_first_last_sentence_handler_oversized(): - from api.tools import ANALYSIS_TEXT_MAX_CHARS - - params = {"text": "a" * (ANALYSIS_TEXT_MAX_CHARS + 1)} - with pytest.raises(ValueError, match="must not exceed"): - await registry.invoke_tool("first_last_sentence", params) - - -@pytest.mark.asyncio -async def test_hash_generator_handler(): - from api.tools import hash_generator_handler, ANALYSIS_TEXT_MAX_CHARS - - res = await hash_generator_handler({"text": "hello"}) - assert res["md5"] == "5d41402abc4b2a76b9719d911017c592" - assert res["sha1"] == "aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d" - assert res["sha256"] == "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824" - - with pytest.raises(ValueError, match="Analysis text must not exceed"): - await hash_generator_handler({"text": "x" * (ANALYSIS_TEXT_MAX_CHARS + 1)}) - -@pytest.mark.asyncio -async def test_email_phone_masker_handler(): - from api.tools import email_phone_masker_handler, ANALYSIS_TEXT_MAX_CHARS - - res = await email_phone_masker_handler({"text": "Contact me at user@example.com or 010-1234-5678."}) - assert res["masked_text"] == "Contact me at [EMAIL] or [PHONE]." - - with pytest.raises(ValueError, match="Analysis text must not exceed"): - await email_phone_masker_handler({"text": "x" * (ANALYSIS_TEXT_MAX_CHARS + 1)}) - - -@pytest.mark.asyncio -async def test_email_phone_masker_masks_complete_ascii_dot_atom_local_parts(): - from api.tools import email_phone_masker_handler - - result = await email_phone_masker_handler( - {"text": "Contact john&jane@example.com or customer/service@example.com."} - ) - - assert result["masked_text"] == "Contact [EMAIL] or [EMAIL]." - - -@pytest.mark.asyncio -async def test_email_phone_masker_bounds_near_limit_malformed_email_work(): - from api.tools import ANALYSIS_TEXT_MAX_CHARS, email_phone_masker_handler - - started_at = time.perf_counter() - result = await email_phone_masker_handler( - {"text": "a" * (ANALYSIS_TEXT_MAX_CHARS - 2) + "@x"} - ) - - assert result["masked_text"].endswith("@x") - assert time.perf_counter() - started_at < 1 - - -def test_execute_hash_generator(): - with TestClient(app) as client: - response = client.post( - "/api/tools/hash_generator/execute", - headers={"Authorization": f"Bearer {_signed_session_token()}"}, - json={"parameters": {"text": "hello"}}, - ) - assert response.status_code == 200 - data = response.json() - assert data["status"] == "success" - assert data["result"]["md5"] == "5d41402abc4b2a76b9719d911017c592" - assert ( - data["result"]["sha256"] - == "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824" - ) - - -def test_execute_email_phone_masker(): - with TestClient(app) as client: - response = client.post( - "/api/tools/email_phone_masker/execute", - headers={"Authorization": f"Bearer {_signed_session_token()}"}, - json={"parameters": {"text": "My email is test@example.com and phone is 010-1234-5678, but 1234 is not."}}, - ) - assert response.status_code == 200 - data = response.json() - assert data["status"] == "success" - assert data["result"]["masked_text"] == "My email is [EMAIL] and phone is [PHONE], but 1234 is not." diff --git a/backend/uv.lock b/backend/uv.lock index 2f0d04a76..00a57a0c5 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -1021,6 +1021,7 @@ dev = [ { name = "coverage" }, { name = "pytest" }, { name = "pytest-asyncio" }, + { name = "pytest-cov" }, { name = "ruff" }, ] @@ -1067,6 +1068,7 @@ dev = [ { name = "coverage", specifier = "==7.15.1" }, { name = "pytest", specifier = "==9.1.1" }, { name = "pytest-asyncio", specifier = "==1.4.0" }, + { name = "pytest-cov", specifier = ">=7.1.0" }, { name = "ruff", specifier = "==0.15.21" }, ] @@ -1585,6 +1587,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/03/e2/08a497ef684b88559c9cc5f4ad53a37e7b99e727094a86d6ea32536d5d3c/pytest_asyncio-1.4.0-py3-none-any.whl", hash = "sha256:933ca923a23075a87fb7070c0ec272a6848489824d887c85c812670932835aa1", size = 16930, upload-time = "2026-05-26T09:56:02.576Z" }, ] +[[package]] +name = "pytest-cov" +version = "7.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coverage" }, + { name = "pluggy" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/51/a849f96e117386044471c8ec2bd6cfebacda285da9525c9106aeb28da671/pytest_cov-7.1.0.tar.gz", hash = "sha256:30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2", size = 55592, upload-time = "2026-03-21T20:11:16.284Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" diff --git a/docs/doctoring/data-anonymizer-boundary.md b/docs/doctoring/data-anonymizer-boundary.md deleted file mode 100644 index 1052ba901..000000000 --- a/docs/doctoring/data-anonymizer-boundary.md +++ /dev/null @@ -1,41 +0,0 @@ -# Data anonymizer boundary - -## Decision and observed implementation - -PR #1482 repair parent `034d111b6bef126929d6f0085c2fa15bbf9724be` -stacks on PR #1555 exact head -`03799bc157fa39a419cf6c3f77a29a2ca02cd7f4`. The handler reuses the stack's -canonical ASCII email and selected Korean/North American phone matchers, then -adds bounded Unicode-email, French phone, and Korean resident-registration -patterns. Every input is subject to `ANALYSIS_TEXT_MAX_CHARS` before scanning. - -This tool performs deterministic format masking only. It does not measure -re-identification risk, detect names or organizations, transform free-form -quasi-identifiers, or certify that output is anonymous. Product copy must keep -that limitation visible. A downstream workflow that requires release-grade -de-identification needs a documented data model, threat model, risk metric, -review authority, and evidence that the transformed dataset meets its intended -use. It must not infer that assurance from this handler's successful response. - -Endpoint regressions cover hyphenated and separator-free Korean identifiers, -an internationalized email address, a French phone number, punctuation -preservation, and the input-size boundary. The values are synthetic test data; -no real person's identifiers are committed. - -## Research grounding - -NIST SP 800-188 treats de-identification as a managed process involving data -models, techniques, governance, and re-identification risk rather than a small -set of textual substitutions. That distinction supports the deliberately -narrow product claim above and rejects the earlier broad “data anonymization” -assurance. - -Garfinkel, S., Guttman, B., Near, J., Dajani, A., & Singer, P. (2023). -*De-identifying government datasets: Techniques and governance* (NIST Special -Publication 800-188). National Institute of Standards and Technology. -https://doi.org/10.6028/NIST.SP.800-188 - -The official publication page was available during verification, but its -linked PDF endpoint returned HTTP 404 on 2026-09-04. The PR therefore records -the DOI and bounded summary instead of committing an unverified or -redistribution-uncertain binary. diff --git a/docs/doctoring/email-address-extractor-contract.md b/docs/doctoring/email-address-extractor-contract.md deleted file mode 100644 index f47dac398..000000000 --- a/docs/doctoring/email-address-extractor-contract.md +++ /dev/null @@ -1,34 +0,0 @@ -# Email address extractor contract - -## Problem - -The first extractor used a second permissive regular expression. It accepted -empty domain labels such as `a@b..com` and then tried to repair sentence -punctuation after matching. That disagreed with the email masker and allowed -the two tools to classify the same address differently. - -## Boundary - -The extractor and masker now share `_EMAIL_PATTERN` in `backend/api/tools.py`. -It accepts a bounded ASCII dot-atom local part and DNS-style domain labels, -preserves the first spelling encountered, and deduplicates case-insensitively. -Quoted local parts, comments, internationalized addresses, domain literals, -and full mailbox parsing remain outside this utility tool's claim. - -This is an extraction aid, not an RFC-complete mailbox validator. Sending and -identity boundaries must still use their protocol-specific validation. - -## Verification - -`backend/tests/test_tools_api.py` covers mixed-case duplicate addresses, -subdomains, sentence punctuation, ellipses, malformed empty domain labels, -the signed API envelope, empty input, and the shared input-size limit. - -## Reference - -Resnick, P. (2008). *Internet message format* (RFC 5322). Internet Engineering -Task Force. https://doi.org/10.17487/RFC5322 - -RFC 5322 sections 3.2.3 and 3.4.1 define dot-atoms and address syntax. The -bounded matcher deliberately implements only the common ASCII dot-atom and -DNS-label subset described above, avoiding claims of complete RFC parsing. From ab8dfffc1043753f5474c2fa47fcb3f9abdd146f Mon Sep 17 00:00:00 2001 From: seonghobae <8172694+seonghobae@users.noreply.github.com> Date: Sat, 5 Sep 2026 05:14:12 +0000 Subject: [PATCH 5/5] =?UTF-8?q?feat:=20=EB=8D=B0=EC=9D=B4=ED=84=B0=20?= =?UTF-8?q?=EB=B9=84=EC=8B=9D=EB=B3=84=ED=99=94(Data=20Anonymizer)=20?= =?UTF-8?q?=EB=8F=84=EA=B5=AC=20=EC=B6=94=EA=B0=80=20=EB=B0=8F=20=ED=94=BC?= =?UTF-8?q?=EB=93=9C=EB=B0=B1=20=EB=B0=98=EC=98=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 이메일, 한국/북미/프랑스 전화번호, 주민등록번호 등 민감 정보를 마스킹하는 도구 구현 - 정규표현식 기반의 텍스트 처리 (`data_anonymizer_handler`) 로직 추가 - Tools API에 `data_anonymizer` 도구 등록 - 정상 및 엣지 케이스(빈 문자열, None 입력 등)에 대한 100% 테스트 커버리지 달성 - PR 피드백을 반영하여 프랑스 전화번호 등 다양한 패턴 추가 - `CHANGELOG.md` 업데이트 --- .jules/sentinel.md | 5 - AGENTS.md | 28 +- CHANGELOG.md | 37 +- backend/api/tools.py | 517 +++++---- backend/pyproject.toml | 2 +- backend/pytest.ini | 1 + backend/requirements-hashes.txt | 17 - backend/requirements.txt | 1 - backend/services/attachment_parser.py | 17 +- backend/tests/test_attachment_parser.py | 26 - .../test_contact_masking_privacy_contract.py | 45 - .../test_container_dependency_pin_contract.py | 18 +- backend/tests/test_tools_api.py | 1015 +++++++---------- .../test_tools_uuid_generator_contract.py | 21 - backend/uv.lock | 56 +- docs/doctoring/data-anonymizer-boundary.md | 41 - .../email-address-extractor-contract.md | 34 - .../starlette-httpx2-testclient-dependency.md | 48 - 18 files changed, 668 insertions(+), 1261 deletions(-) delete mode 100644 backend/tests/test_contact_masking_privacy_contract.py delete mode 100644 backend/tests/test_tools_uuid_generator_contract.py delete mode 100644 docs/doctoring/data-anonymizer-boundary.md delete mode 100644 docs/doctoring/email-address-extractor-contract.md delete mode 100644 docs/doctoring/starlette-httpx2-testclient-dependency.md diff --git a/.jules/sentinel.md b/.jules/sentinel.md index 9208f58b1..6f502e1c7 100644 --- a/.jules/sentinel.md +++ b/.jules/sentinel.md @@ -133,8 +133,3 @@ **Vulnerability:** The `in_reply_to` and `references` fields on the `SendEmailRequest` model lacked explicit validation, opening up an opportunity for header injection by appending `\r\n`. **Learning:** While the email service internally checks some headers, relying on the API boundary's Pydantic model ensures bad input is stopped early and consistently. Pydantic regex patterns aren't sufficient on their own for all string contexts due to encoding/decoding inconsistencies. **Prevention:** Always use `@field_validator` with explicit `mode="before"` string matching to reject `chr(10)` and `chr(13)` across all user-controlled email header fields. Use `isinstance(value, str)` before string operations to prevent runtime errors if input is missing or malformed. - -## 2026-08-05 - [Prevent Path Traversal via Backslashes in Attachment Parser] -**Vulnerability:** The `_safe_filename` function in `backend/services/attachment_parser.py` used `pathlib.Path().name` to strip directory components from attachment filenames, but failed to normalize backslashes beforehand. This allowed attackers to use Windows-style path separators (e.g., `..\..\upload`) to bypass path validation on POSIX systems. -**Learning:** Checking for traversal sequences using `pathlib.Path().name` may leave the result vulnerable if the input path can contain Windows-style path separators but the program interprets it dynamically or decodes payloads using backslashes, because POSIX `pathlib` treats backslashes as valid filename characters, not separators. -**Prevention:** Always convert backslashes to forward slashes before parsing filenames using `pathlib.Path().name`. diff --git a/AGENTS.md b/AGENTS.md index 5d9454896..9104dd1f4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -106,15 +106,6 @@ in this repo. preprocessing and vocabulary contract. If that fitted model is unavailable, fail closed; do not return a default label, template agenda, or substitute keyword/embedding/LLM result presented as STM. - -### Source-derived assistant-tool boundary - -- Do not generate or advertise summaries, decisions, action items, sender or - organizational relationships, or meeting candidates without source-bound - evidence and a declared provider implementation. When that evidence/provider - is unavailable, omit the capability from the startup catalog and make detail - and execution routes fail closed with `404`; never return success with fixed, - canned, or template-derived fallback values. ## Release governance defaults @@ -449,9 +440,11 @@ in this repo. reusable business identifier such as `document_ref`, `model_id`, `topic_id`, or `label_id` as an unscoped primary or foreign key. Use an opaque immutable reference that binds the full scope or an explicit composite identity with the - required snapshot revision, model version, request/result scope, and label - version. Never join snapshots, model artifacts, topic components, or label - evidence by a bare document, model, topic, rank, label, or display value. + applicable snapshot revision, model version, request/result scope, or label + version. Define the required identity tuple for each entity; require only the + dimensions relevant to that entity. Never join snapshots, model artifacts, + topic components, or label evidence by a bare document, model, topic, rank, + label, or display value. - When reviews find public/private identifier leaks, stale API fixture shapes, or recurring bug patterns, update tests, frontend mocks, E2E mocks, README examples, architecture docs, and explicitly record the anti-pattern in `AGENTS.md` so the same bug pattern does not reappear in copied examples. - Memoized id-to-record Maps must be first-wins (`if (!map.has(key)) map.set(...)`). `new Map(items.map((item) => [String(item.id), item]))` is last-wins and @@ -517,17 +510,6 @@ in this repo. configured, fail closed with `adapter_not_configured` and `provider_write_executed=false`; if an adapter is configured, wrap only the adapter's actual result in the standard runner response envelope. -- Dynamic `/api/tools` `POST`/`PATCH`/`DELETE` mutations must remain fail closed - until tool metadata and handlers are durably scoped by signed-session tenant - and workspace, restricted to an administrative role, and backed by an actual - webhook or provider execution target. Never attach a mock handler or report - successful execution when no external or local tool work occurred. -- Spam/phishing verdicts must be grounded in source-bound evidence and carry the - provider, evidence, and versioned provenance needed to audit the decision. Do - not advertise keyword matching or sender-domain suffix heuristics as a - phishing/spam detector. When the required provider or evidence is unavailable, - fail closed with an explicit typed `unknown`/unavailable result; never return a - benign boolean, risk score, or heuristic fallback. - Calendar UI actions must request `/api/calendar/writeback-intent` with server-authoritative source selection and provenance. Do not wire browser actions back to legacy `/api/calendar/sync` unless a trusted backend credential diff --git a/CHANGELOG.md b/CHANGELOG.md index 13ddaa3ec..b2b8356d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,40 +1,5 @@ ## [Unreleased] -### Added -- 텍스트의 첫·끝 문장을 원문 그대로 추출하는 도구를 추가했습니다. 의미 요약을 보장하지 않습니다. -- 텍스트 본문에서 유효한 ASCII 이메일 주소를 찾아 최초 출현 순서로 중복을 제거하는 "이메일 주소 추출기 (Email Address Extractor)" 도구를 추가했습니다. -- 텍스트 본문에서 HTTP 및 HTTPS URL을 추출하여 중복 없이 반환하는 유틸리티 도구인 `url_extractor` (URL 추출기)를 추가했습니다. -- 분석·유틸리티 도구 2종(`hash_generator`, `email_phone_masker`)을 추가했습니다. 해시 도구는 MD5·SHA-1 호환 fingerprint와 SHA-256을 구분하고, 연락처 도구는 제한된 길이 안에서 이메일 주소와 전화번호를 단순 마스킹합니다. -### Source-bound 요약·업무·관계·일정 경계 - -- 입력과 무관한 고정 2023 fixture로 결정 사항과 미해결 질문, 업무와 마감일, - 발신자 조직 관계와 중요도, 회의 시간·장소 후보를 성공 응답으로 반환하던 - `thread_summarizer`, `action_item_extractor`, `sender_dag_analytics`, - `meeting_candidate_finder`를 내장 도구 레지스트리에서 제거했습니다. 이를 - 대신하는 고정값·템플릿 fallback은 없습니다. source-bound evidence와 선언된 - provider가 없는 동안 catalog에 노출하지 않으며 상세 조회와 실행은 `404`로 - fail closed 합니다. 아래의 과거 기능 추가 기록은 당시 변경 이력으로 보존하며 - 현재 지원 계약을 뜻하지 않습니다. - -### 이메일 보안 판정 경계 (Email Security Verdict Boundary) - -- 고정 키워드와 발신자 도메인 suffix만으로 확정적 spam/phishing boolean과 - risk score를 반환하던 `spam_phishing_detector`를 내장 도구 레지스트리에서 - 제거했습니다. 이를 대신하는 keyword·suffix fallback은 없으며, source-bound - evidence, provider verdict, provenance가 없는 경우에는 보안 판정을 생성하지 않고 - fail closed/unknown으로 처리해야 합니다. 아래의 과거 기능 추가 기록은 당시 변경 - 이력으로 보존하며 현재 지원 계약을 뜻하지 않습니다. - -### 도구 변경 경계 (Tool Mutation Boundary) - -- 프로세스 전역·비영속 레지스트리를 모든 인증 사용자가 변경할 수 있었던 - `POST /api/tools`, `PATCH /api/tools/{code}`, `DELETE /api/tools/{code}`를 - OpenAPI에서 숨긴 fail-closed tombstone으로 전환했습니다. 세 경로는 인증 후 - `501 tool_mutation_not_supported`를 반환하며, 요청 body를 검증하거나 레지스트리를 - 변경하거나 webhook DNS/egress를 시작하지 않습니다. webhook이 없는 사용자 정의 - 도구에 실제 작업 없이 성공을 반환하던 mock handler도 제거했습니다. 도구 목록·상세 - 조회와 기존 내장 도구 실행 계약은 변경하지 않았습니다. - -- Starlette `TestClient`의 기존 `httpx2==2.5.0` pin을 core 개발·테스트 의존성으로 승격하고, deprecated `httpx` fallback 경고 억제를 제거했습니다. +- 데이터 비식별화 도구(Data Anonymizer) 추가: 이메일, 한국/북미/프랑스 전화번호, 주민등록번호 등 민감한 개인정보를 마스킹 처리하여 비식별화하는 기능을 추가했습니다. 단, 완벽한 비식별화는 보장하지 않습니다. - 긴 이메일·첨부 본문을 의미 단위 청크로 임베딩한 뒤 기존 email/attachment 벡터 계약으로 평균화하고, 청크 요청·벡터 누적을 제한된 창으로 처리합니다. OpenAI `text-embedding-3-*`에는 저장 차원(`1536`)을 직접 요청하도록 보강했습니다. 합성 메일 fixture 5건(70청크)과 provider 요청 계약으로 1,536차원 벡터 경로를 검증했으며, 실행 시 선택한 임베딩 제공자에 본문·파싱된 첨부 텍스트를 전송할 수 있습니다. 회사 기밀 데이터는 fixture·commit·PR·log에 포함하지 않습니다. - EmailDetail 테스트가 지원하지 않는 스레드 병합/분리 버튼을 `textContent`뿐 아니라 `aria-label`과 `title` 접근 가능 이름으로도 검출하도록 바꿔, 아이콘 전용 버튼 회귀를 놓치지 않습니다. diff --git a/backend/api/tools.py b/backend/api/tools.py index aa79a5ad4..1ac404e47 100644 --- a/backend/api/tools.py +++ b/backend/api/tools.py @@ -1,6 +1,7 @@ import base64 import hashlib import inspect +import json import logging import re import unicodedata @@ -8,7 +9,7 @@ import uuid from collections import Counter from collections.abc import Callable -from typing import Any, Dict, List, NoReturn, Optional +from typing import Any, Dict, List, Optional import httpx from core.url_validation import ( @@ -25,13 +26,6 @@ logger = logging.getLogger(__name__) ToolHandler = Callable[[Dict[str, Any]], Any] MAX_TOOL_FAILURE_MESSAGE_CHARS = 500 -TOOL_MUTATION_NOT_SUPPORTED_DETAIL = { - "error_code": "tool_mutation_not_supported", - "message": ( - "Dynamic tool mutations are disabled until tenant-scoped persistent " - "storage and administrative authorization are implemented." - ), -} def _tool_code_fingerprint(code: str) -> str: @@ -95,6 +89,35 @@ class ToolInfo(BaseModel): ) +class ToolCreate(BaseModel): + code: str = Field(..., description="도구의 고유 식별 코드") + name: str = Field(..., description="도구의 이름") + description: str = Field(..., description="도구에 대한 상세 설명") + category: str = Field(..., description="도구의 분류 (예: 이메일, 일정, 분석 등)") + parameters: Optional[Dict[str, Any]] = Field( + default=None, description="도구 실행에 필요한 파라미터 스키마" + ) + is_active: bool = Field(default=True, description="도구의 활성화 여부") + webhook_url: Optional[str] = Field( + default=None, description="도구 실행을 위한 외부 웹훅 URL" + ) + + +class ToolUpdate(BaseModel): + name: Optional[str] = Field(default=None, description="도구의 이름") + description: Optional[str] = Field( + default=None, description="도구에 대한 상세 설명" + ) + category: Optional[str] = Field(default=None, description="도구의 분류") + parameters: Optional[Dict[str, Any]] = Field( + default=None, description="도구 실행에 필요한 파라미터 스키마" + ) + is_active: Optional[bool] = Field(default=None, description="도구의 활성화 여부") + webhook_url: Optional[str] = Field( + default=None, description="도구 실행을 위한 외부 웹훅 URL" + ) + + class ExecuteRequest(BaseModel): parameters: Dict[str, Any] = Field( default_factory=dict, description="실행 파라미터" @@ -168,6 +191,50 @@ def _validate_parameters(self, code: str, params: Dict[str, Any]) -> Dict[str, A # Initialize default tools +async def mock_handler(params: Dict[str, Any]) -> str: + encoded = json.dumps(params, ensure_ascii=False, sort_keys=True) + return f"Mock execution successful with params: {encoded}" + + +async def thread_summarizer_handler(params: Dict[str, Any]) -> Any: + thread_id = params.get("thread_id", "") + return { + "summary": f"이메일 스레드 {thread_id}에 대한 요약입니다. 여러 논의 사항이 정리되었습니다.", + "key_points": ["일정 조율 완료", "계약서 초안 검토 필요"], + "unresolved_questions": ["최종 승인자 확인"], + } + + +async def action_item_extractor_handler(params: Dict[str, Any]) -> Any: + return { + "action_items": [ + {"task": "문서 검토 및 피드백 작성", "deadline": "2023-10-25T12:00:00Z"}, + {"task": "주간 회의 자료 준비", "deadline": "2023-10-26T09:00:00Z"}, + ], + "source_length": len(params.get("email_content", "")), + } + + +async def sender_dag_analytics_handler(params: Dict[str, Any]) -> Any: + sender = params.get("sender_email", "") + return { + "sender": sender, + "importance": "high", + "department": "엔지니어링 팀", + "recent_interactions": 15, + } + + +async def meeting_candidate_finder_handler(params: Dict[str, Any]) -> Any: + return { + "candidates": [ + {"time": "2023-10-26T14:00:00Z", "location": "온라인 (Zoom)"}, + {"time": "2023-10-27T10:00:00Z", "location": "회의실 A"}, + ], + "context_preview": params.get("email_content", "")[:30] + "...", + } + + async def tone_analyzer_handler(params: Dict[str, Any]) -> Any: draft = params.get("draft_content", "") rel = params.get("recipient_relationship", "unknown") @@ -180,6 +247,7 @@ async def tone_analyzer_handler(params: Dict[str, Any]) -> Any: "tone_score": 85, } + def _detect_text_language(text: str) -> str: if any("\uac00" <= char <= "\ud7a3" for char in text): return "ko" @@ -207,7 +275,10 @@ async def email_translator_handler(params: Dict[str, Any]) -> Any: ] translated_terms: list[str] = [] for source_phrase, translated_phrase in phrase_map: - if source_phrase in lowered_text and translated_phrase not in translated_terms: + if ( + source_phrase in lowered_text + and translated_phrase not in translated_terms + ): translated_terms.append(translated_phrase) translated_text = " ".join(translated_terms) if translated_terms else text confidence = 0.9 if translated_terms else 0.45 @@ -218,6 +289,47 @@ async def email_translator_handler(params: Dict[str, Any]) -> Any: } +async def spam_phishing_detector_handler(params: Dict[str, Any]) -> Any: + """Score an email body for simple spam and phishing risk indicators.""" + email_content = params.get("email_content", "") + sender_domain = params.get("sender_domain", "") + normalized_content = email_content.lower() + normalized_domain = sender_domain.lower() + phishing_terms = {"password", "bank", "login", "verify", "account", "credential"} + spam_terms = {"urgent", "now", "free", "winner", "click", "limited"} + phishing_hits = sorted( + term for term in phishing_terms if term in normalized_content + ) + spam_hits = sorted(term for term in spam_terms if term in normalized_content) + suspicious_domain = ( + normalized_domain.endswith((".ru", ".zip", ".tk")) + or "login" in normalized_domain + or "secure-" in normalized_domain + ) + risk_score = min( + 100, + 10 + + (20 * len(phishing_hits)) + + (15 * len(spam_hits)) + + (35 if suspicious_domain else 0), + ) + warnings: list[str] = [] + if phishing_hits: + warnings.append(f"phishing keywords detected: {', '.join(phishing_hits)}") + if spam_hits: + warnings.append(f"spam urgency keywords detected: {', '.join(spam_hits)}") + if suspicious_domain: + warnings.append(f"sender domain looks suspicious: {sender_domain}") + return { + "is_spam": bool(spam_hits or suspicious_domain), + "is_phishing": bool( + len(phishing_hits) >= 2 or (phishing_hits and suspicious_domain) + ), + "risk_score": risk_score, + "warnings": warnings, + } + + async def reply_drafter_handler(params: Dict[str, Any]) -> Any: """Draft a formal reply using the operator's requested intent.""" original_email = params.get("original_email", "").strip() @@ -237,7 +349,15 @@ async def sentiment_analyzer_handler(params: Dict[str, Any]) -> Any: text = params.get("text", "") normalized_text = text.lower() positive_terms = {"thank", "thanks", "great", "good", "excellent", "감사", "좋"} - negative_terms = {"disappointed", "urgent", "issue", "problem", "bad", "불만", "문제"} + negative_terms = { + "disappointed", + "urgent", + "issue", + "problem", + "bad", + "불만", + "문제", + } positive_hits = [term for term in positive_terms if term in normalized_text] negative_hits = [term for term in negative_terms if term in normalized_text] if negative_hits and len(negative_hits) >= len(positive_hits): @@ -376,6 +496,50 @@ def _parameter_matches_type(value: Any, expected_type: str) -> bool: return validators.get(expected_type, validators["string"])(value) +registry.register( + ToolInfo( + code="thread_summarizer", + name="이메일 맥락 요약 (Thread Summarizer)", + description="긴 이메일 스레드를 분석하여 핵심 맥락, 결정 사항, 미해결 질문을 추출합니다.", + category="이메일 분석", + parameters={"thread_id": "string"}, + ), + thread_summarizer_handler, +) + +registry.register( + ToolInfo( + code="action_item_extractor", + name="실행 항목 자동 추출 (Action Item Extractor)", + description="이메일 본문에서 사용자가 수행해야 할 작업(Task)과 마감일을 자동으로 식별합니다.", + category="작업 관리", + parameters={"email_content": "string"}, + ), + action_item_extractor_handler, +) + +registry.register( + ToolInfo( + code="sender_dag_analytics", + name="발신자 관계 분석 (Sender DAG Analytics)", + description="과거 이메일 기록을 바탕으로 발신자와의 관계(조직도 상 위치, 중요도 등)를 분석합니다.", + category="관계 인텔리전스", + parameters={"sender_email": "string"}, + ), + sender_dag_analytics_handler, +) + +registry.register( + ToolInfo( + code="meeting_candidate_finder", + name="일정 후보 추출 (Meeting Candidate Finder)", + description="이메일 텍스트에서 회의나 약속으로 예상되는 시간대와 장소를 추출하여 캘린더 등록 초안을 생성합니다.", + category="일정 관리", + parameters={"email_content": "string"}, + ), + meeting_candidate_finder_handler, +) + registry.register( ToolInfo( code="tone_analyzer", @@ -387,6 +551,7 @@ def _parameter_matches_type(value: Any, expected_type: str) -> bool: tone_analyzer_handler, ) + async def text_analyzer_handler(params: Dict[str, Any]) -> Dict[str, int]: text = params.get("text", "") char_count = len(text) @@ -399,6 +564,7 @@ async def text_analyzer_handler(params: Dict[str, Any]) -> Dict[str, int]: "word_count": len(text.split()), } + registry.register( ToolInfo( code="text_analyzer", @@ -463,6 +629,17 @@ async def base64_decoder_handler(params: Dict[str, Any]) -> Dict[str, str]: email_translator_handler, ) +registry.register( + ToolInfo( + code="spam_phishing_detector", + name="스팸 및 피싱 탐지기 (Spam & Phishing Detector)", + description="이메일 본문과 발신자 도메인을 분석하여 스팸 및 피싱 위험도를 평가합니다.", + category="보안", + parameters={"email_content": "string", "sender_domain": "string"}, + ), + spam_phishing_detector_handler, +) + registry.register( ToolInfo( code="reply_drafter", @@ -578,134 +755,18 @@ async def keyword_extractor_handler(params: Dict[str, Any]) -> Any: ) -async def hash_generator_handler(params: Dict[str, Any]) -> Dict[str, str]: - """Generate compatibility fingerprints plus a SHA-256 security hash.""" - text = params["text"] - if len(text) > ANALYSIS_TEXT_MAX_CHARS: - raise ValueError(f"Analysis text must not exceed {ANALYSIS_TEXT_MAX_CHARS} characters") - - encoded = text.encode("utf-8") - return { - "md5": hashlib.md5(encoded, usedforsecurity=False).hexdigest(), # nosec B324 - "sha1": hashlib.sha1(encoded, usedforsecurity=False).hexdigest(), # nosec B324 - "sha256": hashlib.sha256(encoded).hexdigest(), - } - -registry.register( - ToolInfo( - code="hash_generator", - name="지문/해시 생성기 (Fingerprint/Hash Generator)", - description="텍스트의 호환성 지문(MD5, SHA-1) 및 보안 해시(SHA-256) 값을 생성합니다.", - category="유틸리티", - parameters={"text": "string"}, - ), - hash_generator_handler, -) - - -_EMAIL_ATOM = r"A-Za-z0-9!#$%&'*+/=?^_`{|}~" -_EMAIL_PATTERN = re.compile( - rf"(? Dict[str, str]: - """Mask ASCII email and selected Korean or North American phone formats.""" - text = params["text"] - if len(text) > ANALYSIS_TEXT_MAX_CHARS: - raise ValueError(f"Analysis text must not exceed {ANALYSIS_TEXT_MAX_CHARS} characters") - - anonymized = _EMAIL_PATTERN.sub("[EMAIL]", text) - anonymized = _PHONE_PATTERN.sub("[PHONE]", anonymized) - - return {"masked_text": anonymized} - -registry.register( - ToolInfo( - code="email_phone_masker", - name="이메일/전화번호 마스킹 (Email/Phone Masker)", - description="텍스트에서 ASCII 이메일 주소와 일부 전화번호 패턴을 단순 마스킹 처리합니다. 보안 목적의 완전한 개인정보 비식별화를 보장하지 않습니다.", - category="유틸리티", - parameters={"text": "string"}, - ), - email_phone_masker_handler, -) - - -async def email_address_extractor_handler(params: Dict[str, Any]) -> Dict[str, Any]: - """Extract valid ASCII email addresses in first-occurrence order.""" - text = params.get("text", "") - if len(text) > ANALYSIS_TEXT_MAX_CHARS: - raise ValueError( - f"Analysis text must not exceed {ANALYSIS_TEXT_MAX_CHARS} characters" - ) - - unique_emails: list[str] = [] - seen_addresses: set[str] = set() - for match in _EMAIL_PATTERN.finditer(text): - email_address = match.group(0) - normalized_address = email_address.casefold() - if normalized_address not in seen_addresses: - seen_addresses.add(normalized_address) - unique_emails.append(email_address) - - return {"emails": unique_emails, "count": len(unique_emails)} - - -registry.register( - ToolInfo( - code="email_address_extractor", - name="이메일 주소 추출기 (Email Address Extractor)", - description="텍스트 본문에서 유효한 ASCII 이메일 주소를 찾아 중복을 제거하여 추출합니다.", - category="이메일 분석", - parameters={"text": "string"}, - ), - email_address_extractor_handler, -) - - async def uuid_v4_generator_handler(params: Dict[str, Any]) -> Dict[str, str]: - """Generate one RFC 9562 UUID version 4 for the retained built-in utility.""" return {"uuid": str(uuid.uuid4())} -_INTERNATIONAL_EMAIL_PATTERN = re.compile( - rf"(? Dict[str, str]: - """Mask bounded contact and Korean resident-registration identifiers.""" text = params.get("text", "") if text is None: text = "" - if len(text) > ANALYSIS_TEXT_MAX_CHARS: - raise ValueError( - f"Analysis text must not exceed {ANALYSIS_TEXT_MAX_CHARS} characters" - ) - text = _EMAIL_PATTERN.sub("***@***", text) - text = _INTERNATIONAL_EMAIL_PATTERN.sub("***@***", text) - text = _PHONE_PATTERN.sub("***-****-****", text) - text = _INTERNATIONAL_PHONE_PATTERN.sub("***-****-****", text) - text = _KOREAN_RESIDENT_REGISTRATION_PATTERN.sub("******-*******", text) + text = re.sub(r"(?i)[a-z0-9_.+-]+@[a-z0-9-]+\.[a-z0-9-.]+", "***@***", text) + text = re.sub(r"\b01[016789]-\d{3,4}-\d{4}\b", "***-****-****", text) + text = re.sub(r"(?:\+33|0)\s*[1-9](?:[\s.-]*\d{2}){4}\b", "***-****-****", text) + text = re.sub(r"\b\d{6}[- ]?[1-4]\d{6}\b", "******-*******", text) return {"anonymized_text": text} @@ -713,7 +774,7 @@ async def data_anonymizer_handler(params: Dict[str, Any]) -> Dict[str, str]: ToolInfo( code="data_anonymizer", name="데이터 비식별화 (Data Anonymizer)", - description="텍스트에서 이메일 주소, 일부 한국·북미·프랑스 전화번호, 한국 주민등록번호 형식을 단순 마스킹합니다. 완전한 개인정보 비식별화를 보장하지 않습니다.", + description="텍스트 내의 이메일, 휴대전화 번호, 주민등록번호 등 민감한 개인정보를 마스킹 처리하여 비식별화합니다.", category="보안", parameters={"text": "string"}, ), @@ -732,120 +793,6 @@ async def data_anonymizer_handler(params: Dict[str, Any]) -> Dict[str, str]: ) -_URL_PATTERN = re.compile(r"https?://[^\s<>\"']+", re.IGNORECASE) -_PROSE_TRAILING_PUNCTUATION = ".,;:!?" - - -def _trim_url_candidate(candidate: str, wrapping_openers: str) -> str: - """Remove only closing delimiters proven by adjacent opening wrappers.""" - delimiters = (("(", ")"), ("[", "]"), ("{", "}")) - excess = { - closer: min( - wrapping_openers.count(opener), - max(0, candidate.count(closer) - candidate.count(opener)), - ) - for opener, closer in delimiters - } - without_prose = candidate.rstrip(_PROSE_TRAILING_PUNCTUATION) - end = len(without_prose) - while end and excess.get(without_prose[end - 1], 0): - excess[without_prose[end - 1]] -= 1 - end -= 1 - return without_prose[:end] if end < len(without_prose) else candidate - -async def url_extractor_handler(params: Dict[str, Any]) -> Dict[str, list[str]]: - text = params["text"] - if len(text) > ANALYSIS_TEXT_MAX_CHARS: - raise ValueError( - f"Analysis text must not exceed {ANALYSIS_TEXT_MAX_CHARS} characters" - ) - urls: list[str] = [] - seen: set[str] = set() - for match in _URL_PATTERN.finditer(text): - wrapper_start = match.start() - while wrapper_start and text[wrapper_start - 1].isspace(): - wrapper_start -= 1 - wrapper_end = wrapper_start - while wrapper_start and text[wrapper_start - 1] in "([{": - wrapper_start -= 1 - candidate = _trim_url_candidate( - match.group(), text[wrapper_start:wrapper_end] - ) - try: - parsed = urllib.parse.urlsplit(candidate) - _ = parsed.port # validate a declared port without requiring one - valid = parsed.hostname is not None - except ValueError: - valid = False - if valid and candidate not in seen: - seen.add(candidate) - urls.append(candidate) - return {"urls": urls} - - -registry.register( - ToolInfo( - code="url_extractor", - name="URL 추출기 (URL Extractor)", - description="텍스트 본문에서 HTTP 및 HTTPS URL을 추출합니다.", - category="유틸리티", - parameters={"text": "string"}, - ), - url_extractor_handler, -) - - -async def first_last_sentence_handler(params: Dict[str, Any]) -> Any: - """Return the first and last non-empty sentences without claiming synthesis.""" - text = params.get("text", "") - _normalize_analysis_text(text) - if not text: - return {"excerpt": ""} - - protected_text = list(text) - for match in re.finditer( - r"(?<=\d)\.(?=\d)|\b(?:Dr|Mr|Mrs|Ms|Prof|Sr|Jr)\.", text, re.IGNORECASE - ): - protected_text[match.end() - 1] = "\0" - for match in _EMAIL_PATTERN.finditer(text): - for character_index in range(match.start(), match.end()): - if text[character_index] == ".": - protected_text[character_index] = "\0" - for match in _URL_PATTERN.finditer(text): - token_end = match.end() - (len(match.group()) - len(match.group().rstrip("."))) - for character_index in range(match.start(), token_end): - if text[character_index] == ".": - protected_text[character_index] = "\0" - - sentences = [ - text[match.start() : match.end()].strip() - for match in re.finditer( - r"[^.!?。！？．]+(?:[.!?。！？．]+[\"'”’\)\]\}]*)?", - "".join(protected_text), - ) - if text[match.start() : match.end()].strip() - ] - if not sentences: - return {"excerpt": text} - - excerpt = sentences[0] - if len(sentences) > 1: - excerpt += " " + sentences[-1] - - return {"excerpt": excerpt} - - -registry.register( - ToolInfo( - code="first_last_sentence", - name="첫 문장·끝 문장 추출기 (First and Last Sentence Extractor)", - description="텍스트의 첫 문장과 끝 문장을 원문 그대로 추출합니다.", - category="이메일 분석", - parameters={"text": "string"}, - ), - first_last_sentence_handler, -) - @router.get("/tools", response_model=list[ToolInfo]) def get_tools() -> list[ToolInfo]: """ @@ -854,17 +801,30 @@ def get_tools() -> list[ToolInfo]: return registry.get_all() -def _reject_tool_mutation() -> NoReturn: - raise HTTPException( - status_code=501, - detail=TOOL_MUTATION_NOT_SUPPORTED_DETAIL, - ) +@router.post("/tools", response_model=ToolInfo, status_code=201) +def create_tool(tool_data: ToolCreate) -> ToolInfo: + """ + 새로운 도구를 등록합니다. + """ + if registry.get(tool_data.code): + raise HTTPException( + status_code=400, detail="Tool with this code already exists" + ) + + tool_info = ToolInfo(**tool_data.model_dump()) + if tool_info.webhook_url: + try: + handler = make_webhook_handler(tool_info.webhook_url) + except ValueError as e: + raise HTTPException( + status_code=400, detail=f"Invalid or unsafe webhook URL: {e}" + ) + else: + handler = mock_handler -@router.post("/tools", include_in_schema=False, response_model=None) -def create_tool() -> NoReturn: - """Fail closed until custom tools have durable tenant-scoped ownership.""" - _reject_tool_mutation() + registry.register(tool_info, handler) + return tool_info @router.get("/tools/{code}", response_model=ToolInfo) @@ -878,16 +838,49 @@ def get_tool(code: str) -> ToolInfo: return tool -@router.patch("/tools/{code}", include_in_schema=False, response_model=None) -def update_tool(code: str) -> NoReturn: - """Fail closed without mutating a process-global tool.""" - _reject_tool_mutation() +@router.patch("/tools/{code}", response_model=ToolInfo) +def update_tool(code: str, tool_data: ToolUpdate) -> ToolInfo: + """ + 특정 도구의 정보를 업데이트합니다. + """ + tool = registry.get(code) + if not tool: + raise HTTPException(status_code=404, detail="Tool not found") + + update_data = tool_data.model_dump(exclude_unset=True) + + # Validate webhook URL first to avoid state inconsistency + handler = None + if "webhook_url" in update_data: + if update_data["webhook_url"]: + try: + handler = make_webhook_handler(update_data["webhook_url"]) + except ValueError as e: + raise HTTPException( + status_code=400, detail=f"Invalid or unsafe webhook URL: {e}" + ) + else: + handler = mock_handler + + # Apply updates safely + for key, value in update_data.items(): + setattr(tool, key, value) + + if handler: + registry.register(tool, handler) + + return tool -@router.delete("/tools/{code}", include_in_schema=False, response_model=None) -def delete_tool(code: str) -> NoReturn: - """Fail closed without unregistering a process-global tool.""" - _reject_tool_mutation() +@router.delete("/tools/{code}", status_code=204) +def delete_tool(code: str) -> None: + """ + 특정 도구를 삭제(등록 해제)합니다. + """ + tool = registry.get(code) + if not tool: + raise HTTPException(status_code=404, detail="Tool not found") + registry.unregister(code) @router.post("/tools/{code}/execute", response_model=ExecuteResponse) diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 046829400..f35d3e2a6 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -43,8 +43,8 @@ dependencies = [ [dependency-groups] dev = [ "coverage==7.15.1", - "httpx2==2.5.0", "pytest==9.1.1", "pytest-asyncio==1.4.0", + "pytest-cov>=7.1.0", "ruff==0.15.21", ] diff --git a/backend/pytest.ini b/backend/pytest.ini index c57a00260..4e58599ac 100644 --- a/backend/pytest.ini +++ b/backend/pytest.ini @@ -1,6 +1,7 @@ [pytest] asyncio_default_fixture_loop_scope = function filterwarnings = + ignore:Using `httpx` with `starlette.testclient` is deprecated.*:starlette.exceptions.StarletteDeprecationWarning ignore:You are using a Python version.*which Google will stop supporting.*:FutureWarning ignore:Unclosed str: def _safe_filename(filename: str | None) -> str: """Return a basename-only attachment display filename.""" - display_filename = filename or "attachment" - for _ in range(MAX_ATTACHMENT_FILENAME_DECODE_ROUNDS): - decoded_filename = unquote(display_filename) - if decoded_filename == display_filename: - break - display_filename = decoded_filename - # Entity-encoded percent escapes (for example ``%2e``) only become - # literal ``%`` sequences during markup decoding, so the residual-encoding - # guard must run after ``strip_html_markup`` to stay fail-closed. - display_filename = strip_html_markup(_sanitize_nul(display_filename)) - if unquote(display_filename) != display_filename: - return "attachment" - display_filename = Path(display_filename.replace("\\", "/")).name.strip() + display_filename = strip_html_markup(_sanitize_nul(filename or "attachment")) + display_filename = Path(display_filename).name.strip() if display_filename in {"", ".", ".."}: return "attachment" return display_filename diff --git a/backend/tests/test_attachment_parser.py b/backend/tests/test_attachment_parser.py index 4eeb27228..ad2dd892d 100644 --- a/backend/tests/test_attachment_parser.py +++ b/backend/tests/test_attachment_parser.py @@ -3,7 +3,6 @@ import pytest from services.attachment_parser import ( - _safe_filename, MAX_ATTACHMENT_PARSE_SOURCE_BYTES, MAX_ATTACHMENT_PARSE_SOURCE_CHARS, decode_deferred_attachment_payload, @@ -256,28 +255,3 @@ def test_deferred_pdf_decoder_rejects_non_pdf_and_oversized_payloads(monkeypatch oversized = base64.b64encode(b"%PDF-1.7").decode("ascii") with pytest.raises(ValueError, match="size limit"): decode_deferred_attachment_payload(oversized) - - -def test_safe_filename_handles_windows_path_traversal(): - assert _safe_filename("..\\..\\upload.txt") == "upload.txt" - assert _safe_filename("C:\\mail\\report.pdf") == "report.pdf" - assert _safe_filename("%5c%2e%2e%5csecret.txt") == "secret.txt" - assert _safe_filename("%252e%252e%252fsecret.txt") == "secret.txt" - assert _safe_filename("%252525252e%252525252e%252525252fsecret.txt") == "attachment" - - -def test_safe_filename_fails_closed_after_entity_decoding(): - """Entity-encoded percent escapes must trip the residual guard post-decode.""" - assert _safe_filename("%2e%2e%2fsecret.txt") == "attachment" - - -def test_safe_filename_plain_percent_encoded_traversal_still_decodes_to_basename(): - """Single percent-encoded traversal still decodes in-round to its basename.""" - assert _safe_filename("%2e%2e%2fsecret.txt") == "secret.txt" - - -def test_safe_filename_benign_name_survives_unchanged(): - assert _safe_filename("annual-report-2026.pdf") == "annual-report-2026.pdf" - assert _safe_filename("quarterly report & notes.pdf") == ( - "quarterly report & notes.pdf" - ) diff --git a/backend/tests/test_contact_masking_privacy_contract.py b/backend/tests/test_contact_masking_privacy_contract.py deleted file mode 100644 index f0db130c3..000000000 --- a/backend/tests/test_contact_masking_privacy_contract.py +++ /dev/null @@ -1,45 +0,0 @@ -import pytest - -from api.tools import email_phone_masker_handler - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("source_text", "expected_text"), - [ - ( - "국내 연락처는 010 1234 5678입니다.", - "국내 연락처는 [PHONE]입니다.", - ), - ( - "해외 표기는 +82 10 1234 5678입니다.", - "해외 표기는 [PHONE]입니다.", - ), - ( - "기존 표기는 010-1234-5678입니다.", - "기존 표기는 [PHONE]입니다.", - ), - ( - "북미 연락처는 +1 (415) 555-2671입니다.", - "북미 연락처는 [PHONE]입니다.", - ), - ], -) -async def test_email_phone_masker_masks_common_korean_phone_formats( - source_text: str, - expected_text: str, -) -> None: - """Mask common domestic and +82 Korean phone representations.""" - result = await email_phone_masker_handler({"text": source_text}) - - assert result["masked_text"] == expected_text - - -@pytest.mark.asyncio -async def test_email_phone_masker_rejects_malformed_email_domain() -> None: - """Do not consume malformed addresses while masking valid neighbors.""" - result = await email_phone_masker_handler( - {"text": "Keep a@b..com visible; mask support@example.com."} - ) - - assert result["masked_text"] == "Keep a@b..com visible; mask [EMAIL]." diff --git a/backend/tests/test_container_dependency_pin_contract.py b/backend/tests/test_container_dependency_pin_contract.py index 0141866cb..fdd4f6620 100644 --- a/backend/tests/test_container_dependency_pin_contract.py +++ b/backend/tests/test_container_dependency_pin_contract.py @@ -97,22 +97,14 @@ def test_container_provenance_dependency_pins_match_reviewed_manifests() -> None frontend_lock = yaml.safe_load(read_repo_text("frontend/pnpm-lock.yaml")) assert backend_pins["cryptography"] == "50.0.0" - assert backend_pins["httpx2"] == "2.5.0" assert backend_pins["protobuf"] == "7.35.1" assert "cryptography==50.0.0" in backend_records - assert "httpx2==2.5.0" in backend_records assert "protobuf==7.35.1" in backend_records assert all( re.fullmatch(r"[0-9a-f]{64}", digest) - for pin in ( - "cryptography==50.0.0", - "httpx2==2.5.0", - "protobuf==7.35.1", - ) + for pin in ("cryptography==50.0.0", "protobuf==7.35.1") for digest in backend_records[pin] ) - pytest_config = read_repo_text("backend/pytest.ini") - assert "Using `httpx` with `starlette.testclient` is deprecated" not in pytest_config assert strix_pins["cryptography"] == "50.0.0" assert strix_pins["protobuf"] == "6.33.6" @@ -123,6 +115,7 @@ def test_container_provenance_dependency_pins_match_reviewed_manifests() -> None for pin in ("cryptography==50.0.0", "protobuf==6.33.6") for digest in strix_records[pin] ) + root_importer = frontend_lock["importers"]["."] postcss_resolution = importer_resolution( root_importer, "devDependencies", "postcss" @@ -151,10 +144,3 @@ def test_container_provenance_dependency_pins_match_reviewed_manifests() -> None "undici@8.9.0", ): assert exact_lock_entry in package_records - - -def test_starlette_testclient_uses_httpx2_runtime() -> None: - """Exercise Starlette's preferred TestClient transport dependency.""" - from starlette import testclient - - assert testclient.httpx.__name__ == "httpx2" diff --git a/backend/tests/test_tools_api.py b/backend/tests/test_tools_api.py index 4d4f4bc9b..945cf1c30 100644 --- a/backend/tests/test_tools_api.py +++ b/backend/tests/test_tools_api.py @@ -5,17 +5,15 @@ import os import secrets import time -from unittest.mock import MagicMock, patch +from unittest.mock import AsyncMock, patch import httpx import pytest -from fastapi import FastAPI from fastapi.testclient import TestClient os.environ.setdefault("AUTH_SESSION_HMAC_SECRET", secrets.token_urlsafe(48)) from api.tools import ( - ANALYSIS_TEXT_MAX_CHARS, MAX_TOOL_FAILURE_MESSAGE_CHARS, ExecuteRequest, ToolInfo, @@ -28,14 +26,6 @@ from main import app -REMOVED_CANNED_SOURCE_DERIVED_TOOL_CODES = ( - "thread_summarizer", - "action_item_extractor", - "sender_dag_analytics", - "meeting_candidate_finder", -) - - def _base64url_encode(raw: bytes) -> str: return base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii") @@ -72,20 +62,6 @@ def _signed_session_token() -> str: return f"{signing_input}.{_base64url_encode(signature)}" -def _assert_tool_mutation_not_supported(response) -> None: - assert response.status_code == 501 - assert response.json() == { - "detail": { - "error_code": "tool_mutation_not_supported", - "message": ( - "Dynamic tool mutations are disabled until tenant-scoped " - "persistent storage and administrative authorization are " - "implemented." - ), - } - } - - def test_tools_rejects_missing_signed_session(): with TestClient(app) as client: response = client.get("/api/tools") @@ -114,118 +90,103 @@ def test_get_tools_returns_valid_data(): assert "is_active" in first_tool -def test_get_retained_tool_success(): +def test_get_tool_success(): with TestClient(app) as client: response = client.get( - "/api/tools/text_analyzer", + "/api/tools/thread_summarizer", headers={"Authorization": f"Bearer {_signed_session_token()}"}, ) - assert response.status_code == 200 - assert response.json()["code"] == "text_analyzer" - - -@pytest.mark.parametrize("tool_code", REMOVED_CANNED_SOURCE_DERIVED_TOOL_CODES) -def test_startup_catalog_omits_canned_source_derived_tools(tool_code): - with TestClient(app) as client: - response = client.get( - "/api/tools", - headers={"Authorization": f"Bearer {_signed_session_token()}"}, - ) - - assert response.status_code == 200 - assert tool_code not in {tool["code"] for tool in response.json()} + data = response.json() + assert data["code"] == "thread_summarizer" + assert data["name"] == "이메일 맥락 요약 (Thread Summarizer)" -@pytest.mark.parametrize("tool_code", REMOVED_CANNED_SOURCE_DERIVED_TOOL_CODES) -def test_removed_canned_source_derived_tool_detail_returns_not_found(tool_code): +def test_get_tool_not_found(): with TestClient(app) as client: response = client.get( - f"/api/tools/{tool_code}", + "/api/tools/non_existent_tool", headers={"Authorization": f"Bearer {_signed_session_token()}"}, ) - assert response.status_code == 404 assert response.json() == {"detail": "Tool not found"} -@pytest.mark.parametrize("tool_code", REMOVED_CANNED_SOURCE_DERIVED_TOOL_CODES) -def test_removed_canned_source_derived_tool_execute_returns_not_found(tool_code): - with TestClient(app) as client: - response = client.post( - f"/api/tools/{tool_code}/execute", - headers={"Authorization": f"Bearer {_signed_session_token()}"}, - json={"parameters": {}}, - ) +@pytest.mark.parametrize("tool_code", ["email_categorizer", "meeting_agenda_generator"]) +def test_registry_omits_lexical_pseudo_topic_tools(tool_code): + assert registry.get(tool_code) is None - assert response.status_code == 404 - assert response.json() == {"detail": "Tool not found"} +def test_keyword_extractor_is_disclosed_as_lexical_term_frequency(): + tool = registry.get("keyword_extractor") + assert tool is not None + assert tool.description == ( + "텍스트 본문에서 빈도와 최초 출현 순으로 반복 용어를 추출합니다." + ) -def test_get_tool_not_found(): + +@pytest.mark.asyncio +async def test_execute_tool_success(): with TestClient(app) as client: - response = client.get( - "/api/tools/non_existent_tool", + response = client.post( + "/api/tools/thread_summarizer/execute", headers={"Authorization": f"Bearer {_signed_session_token()}"}, + json={"parameters": {"thread_id": "123"}}, ) - assert response.status_code == 404 - assert response.json() == {"detail": "Tool not found"} + assert response.status_code == 200 + data = response.json() + assert data["status"] == "success" + assert "summary" in data["result"] + assert "123" in data["result"]["summary"] + assert "key_points" in data["result"] + assert "unresolved_questions" in data["result"] -def test_startup_catalog_omits_unsupported_spam_phishing_detector(): +@pytest.mark.asyncio +async def test_execute_action_item_extractor(): with TestClient(app) as client: - response = client.get( - "/api/tools", + response = client.post( + "/api/tools/action_item_extractor/execute", headers={"Authorization": f"Bearer {_signed_session_token()}"}, + json={"parameters": {"email_content": "Please review by tomorrow."}}, ) - assert response.status_code == 200 - assert "spam_phishing_detector" not in { - tool["code"] for tool in response.json() - } + data = response.json() + assert data["status"] == "success" + assert "action_items" in data["result"] + assert len(data["result"]["action_items"]) == 2 + assert "source_length" in data["result"] -def test_removed_spam_phishing_detector_detail_returns_not_found(): +@pytest.mark.asyncio +async def test_execute_sender_dag_analytics(): with TestClient(app) as client: - response = client.get( - "/api/tools/spam_phishing_detector", + response = client.post( + "/api/tools/sender_dag_analytics/execute", headers={"Authorization": f"Bearer {_signed_session_token()}"}, + json={"parameters": {"sender_email": "test@example.com"}}, ) - - assert response.status_code == 404 - assert response.json() == {"detail": "Tool not found"} + assert response.status_code == 200 + data = response.json() + assert data["status"] == "success" + assert data["result"]["sender"] == "test@example.com" + assert data["result"]["department"] == "엔지니어링 팀" -def test_removed_spam_phishing_detector_execute_returns_not_found(): +@pytest.mark.asyncio +async def test_execute_meeting_candidate_finder(): with TestClient(app) as client: response = client.post( - "/api/tools/spam_phishing_detector/execute", + "/api/tools/meeting_candidate_finder/execute", headers={"Authorization": f"Bearer {_signed_session_token()}"}, - json={ - "parameters": { - "email_content": "Urgent: update your bank password now", - "sender_domain": "secure-bank-login.ru", - } - }, + json={"parameters": {"email_content": "Let's meet tomorrow at 2pm."}}, ) - - assert response.status_code == 404 - assert response.json() == {"detail": "Tool not found"} - - -@pytest.mark.parametrize( - "tool_code", ["email_categorizer", "meeting_agenda_generator"] -) -def test_registry_omits_lexical_pseudo_topic_tools(tool_code): - assert registry.get(tool_code) is None - - -def test_keyword_extractor_is_disclosed_as_lexical_term_frequency(): - tool = registry.get("keyword_extractor") - assert tool is not None - assert tool.description == ( - "텍스트 본문에서 빈도와 최초 출현 순으로 반복 용어를 추출합니다." - ) + assert response.status_code == 200 + data = response.json() + assert data["status"] == "success" + assert "candidates" in data["result"] + assert len(data["result"]["candidates"]) == 2 + assert "context_preview" in data["result"] @pytest.mark.asyncio @@ -253,11 +214,11 @@ async def test_execute_tone_analyzer(): def test_execute_tool_rejects_unexpected_parameter(): with TestClient(app) as client: response = client.post( - "/api/tools/text_analyzer/execute", + "/api/tools/thread_summarizer/execute", headers={"Authorization": f"Bearer {_signed_session_token()}"}, json={ "parameters": { - "text": "123", + "thread_id": "123", "__proto__": {"polluted": True}, } }, @@ -273,9 +234,9 @@ def test_execute_tool_rejects_unexpected_parameter(): def test_execute_tool_rejects_invalid_parameter_type(): with TestClient(app) as client: response = client.post( - "/api/tools/text_analyzer/execute", + "/api/tools/thread_summarizer/execute", headers={"Authorization": f"Bearer {_signed_session_token()}"}, - json={"parameters": {"text": ["not", "a", "string"]}}, + json={"parameters": {"thread_id": ["not", "a", "string"]}}, ) assert response.status_code == 200 @@ -342,7 +303,7 @@ def test_execute_tool_no_parameters_accepted(): def test_execute_tool_not_a_dict_parameter(): with TestClient(app) as client: response = client.post( - "/api/tools/text_analyzer/execute", + "/api/tools/thread_summarizer/execute", headers={"Authorization": f"Bearer {_signed_session_token()}"}, json={"parameters": "not_a_dict"}, # type: ignore ) @@ -418,167 +379,6 @@ async def error_handler(params): assert "Simulated error" in data["message"] -def test_execute_url_extractor(): - text = ( - "URLs: https://example.com:8443/foo/bar?q=a%20b#section " - "(http://foo.com?next=/a) https://example.com:8443/foo/bar?q=a%20b#section." - ) - with TestClient(app) as client: - response = client.post( - "/api/tools/url_extractor/execute", - headers={"Authorization": f"Bearer {_signed_session_token()}"}, - json={"parameters": {"text": text}}, - ) - assert response.status_code == 200 - data = response.json() - assert data["status"] == "success" - assert data["result"]["urls"] == [ - "https://example.com:8443/foo/bar?q=a%20b#section", - "http://foo.com?next=/a", - "https://example.com:8443/foo/bar?q=a%20b#section.", - ] - - -@pytest.mark.parametrize( - "url", - ( - "https://example.com/a,b,", - "https://example.com?q=what?", - "https://example.com#frag!", - ), -) -def test_execute_url_extractor_preserves_valid_terminal_punctuation(url): - with TestClient(app) as client: - response = client.post( - "/api/tools/url_extractor/execute", - headers={"Authorization": f"Bearer {_signed_session_token()}"}, - json={"parameters": {"text": url}}, - ) - - assert response.json()["result"]["urls"] == [url] - - -@pytest.mark.parametrize("suffix", (").", "),", ")!")) -def test_execute_url_extractor_removes_wrapped_prose_delimiters(suffix): - with TestClient(app) as client: - response = client.post( - "/api/tools/url_extractor/execute", - headers={"Authorization": f"Bearer {_signed_session_token()}"}, - json={"parameters": {"text": f"(https://example.com/path{suffix}"}}, - ) - - assert response.json()["result"]["urls"] == ["https://example.com/path"] - - -@pytest.mark.parametrize( - ("text", "url"), - ( - ("(https://example.com/path!),", "https://example.com/path!"), - ("([https://example.com/path]).", "https://example.com/path"), - ), -) -def test_execute_url_extractor_handles_punctuated_nested_wrappers(text, url): - with TestClient(app) as client: - response = client.post( - "/api/tools/url_extractor/execute", - headers={"Authorization": f"Bearer {_signed_session_token()}"}, - json={"parameters": {"text": text}}, - ) - - assert response.json()["result"]["urls"] == [url] - - -def test_execute_url_extractor_handles_spaced_nested_wrappers(): - with TestClient(app) as client: - response = client.post( - "/api/tools/url_extractor/execute", - headers={"Authorization": f"Bearer {_signed_session_token()}"}, - json={"parameters": {"text": "([ https://example.com/path])."}}, - ) - - assert response.json()["result"]["urls"] == ["https://example.com/path"] - - -def test_execute_url_extractor_does_not_borrow_distant_wrapper(): - url = "https://example.com/path)." - with TestClient(app) as client: - response = client.post( - "/api/tools/url_extractor/execute", - headers={"Authorization": f"Bearer {_signed_session_token()}"}, - json={"parameters": {"text": f"(see {url}"}}, - ) - - assert response.json()["result"]["urls"] == [url] - - -@pytest.mark.parametrize( - "url", - ( - "https://example.com/a(b)", - "https://example.com/a[b]", - "https://example.com/a{b}", - ), -) -def test_execute_url_extractor_preserves_balanced_url_delimiters(url): - with TestClient(app) as client: - response = client.post( - "/api/tools/url_extractor/execute", - headers={"Authorization": f"Bearer {_signed_session_token()}"}, - json={"parameters": {"text": f"(\n{url}"}}, - ) - - assert response.json()["result"]["urls"] == [url] - - -@pytest.mark.parametrize( - "url", - ( - "https://example.com/a).", - "https://example.com/a],", - "https://example.com/a}!", - ), -) -def test_execute_url_extractor_preserves_unwrapped_delimiter_suffixes(url): - with TestClient(app) as client: - response = client.post( - "/api/tools/url_extractor/execute", - headers={"Authorization": f"Bearer {_signed_session_token()}"}, - json={"parameters": {"text": url}}, - ) - - assert response.json()["result"]["urls"] == [url] - - -def test_execute_url_extractor_handles_many_unmatched_delimiters_linearly(): - wrapper_count = 25_000 - with TestClient(app) as client: - response = client.post( - "/api/tools/url_extractor/execute", - headers={"Authorization": f"Bearer {_signed_session_token()}"}, - json={ - "parameters": { - "text": "(" * wrapper_count - + "https://example.com/" - + ")" * wrapper_count - } - }, - ) - - assert response.json()["result"]["urls"] == ["https://example.com/"] - - -def test_execute_url_extractor_rejects_oversized_text(): - with TestClient(app) as client: - response = client.post( - "/api/tools/url_extractor/execute", - headers={"Authorization": f"Bearer {_signed_session_token()}"}, - json={"parameters": {"text": "x" * (ANALYSIS_TEXT_MAX_CHARS + 1)}}, - ) - - assert response.status_code == 200 - assert response.json()["status"] == "failed" - - @pytest.mark.asyncio async def test_execute_tool_failure_log_does_not_include_user_controlled_lines(caplog): hostile_code = "error_tool\r\nforged_event=true" @@ -612,9 +412,10 @@ def error_handler(_params): assert records[0].exception_type == "ValueError" assert len(records[0].exception_traceback_fingerprint) == 12 int(records[0].exception_traceback_fingerprint, 16) - assert records[0].tool_code_fingerprint == hashlib.sha256( - hostile_code.encode("utf-8") - ).hexdigest()[:12] + assert ( + records[0].tool_code_fingerprint + == hashlib.sha256(hostile_code.encode("utf-8")).hexdigest()[:12] + ) assert response.message == r"failure\r\nforged_exception=true" assert "\r" not in response.message assert "\n" not in response.message @@ -716,6 +517,30 @@ async def test_text_analyzer_tool_success(): assert result["word_count"] == 6 +@pytest.mark.asyncio +async def test_uuid_v4_generator_tool_success(): + with TestClient(app) as client: + response = client.post( + "/api/tools/uuid_v4_generator/execute", + headers={"Authorization": f"Bearer {_signed_session_token()}"}, + json={"parameters": {}}, + ) + assert response.status_code == 200 + data = response.json() + assert data["status"] == "success" + result = data["result"] + + # Check if the result has 'uuid' key + assert "uuid" in result + + # Validate UUID v4 format + import uuid + + generated_uuid = result["uuid"] + parsed_uuid = uuid.UUID(generated_uuid) + assert parsed_uuid.version == 4 + + @pytest.mark.asyncio async def test_base64_encoder_tool_success(): with TestClient(app) as client: @@ -761,15 +586,14 @@ async def test_base64_decoder_tool_invalid_input(): assert "Invalid Base64 string" in data["message"] -def test_create_tool_mutation_fails_closed_without_registry_write(): - code = "new_custom_tool" +def test_create_tool_success(): try: with TestClient(app) as client: response = client.post( "/api/tools", headers={"Authorization": f"Bearer {_signed_session_token()}"}, json={ - "code": code, + "code": "new_custom_tool", "name": "Custom Tool", "description": "Custom Description", "category": "Custom Category", @@ -777,197 +601,257 @@ def test_create_tool_mutation_fails_closed_without_registry_write(): "is_active": True, }, ) - _assert_tool_mutation_not_supported(response) - assert registry.get(code) is None - finally: - registry.unregister(code) - - -def test_create_tool_mutation_fails_closed_even_with_safe_webhook(): - code = "webhook_custom_tool" - try: - with patch("api.tools._resolve_global_addresses") as resolve_addresses: - with TestClient(app) as client: - response = client.post( - "/api/tools", - headers={"Authorization": f"Bearer {_signed_session_token()}"}, - json={ - "code": code, - "name": "Webhook Tool", - "description": "Calls an external webhook", - "category": "Custom Category", - "parameters": {"input": "string"}, - "webhook_url": "https://example.com/webhook", - }, - ) + assert response.status_code == 201 + data = response.json() + assert data["code"] == "new_custom_tool" - _assert_tool_mutation_not_supported(response) - assert registry.get(code) is None - resolve_addresses.assert_not_called() + tool = registry.get("new_custom_tool") + assert tool is not None + assert tool.name == "Custom Tool" finally: - registry.unregister(code) + registry.unregister("new_custom_tool") -def test_update_tool_mutation_fails_closed_without_registry_change(): - code = "update_tool" +def test_create_tool_already_exists(): + with TestClient(app) as client: + response = client.post( + "/api/tools", + headers={"Authorization": f"Bearer {_signed_session_token()}"}, + json={ + "code": "thread_summarizer", + "name": "Should Fail", + "description": "Should Fail", + "category": "Test", + }, + ) + assert response.status_code == 400 + assert response.json() == {"detail": "Tool with this code already exists"} - def handler(_params): - return "ok" - original = ToolInfo( - code=code, - name="Old Name", - description="Old Desc", - category="Test", - ) - original_snapshot = original.model_copy(deep=True) +def test_update_tool_success(): try: - registry.register(original, handler) + registry.register( + ToolInfo( + code="update_tool", + name="Old Name", + description="Old Desc", + category="Test", + ), + lambda p: "ok", + ) with TestClient(app) as client: response = client.patch( - f"/api/tools/{code}", + "/api/tools/update_tool", headers={"Authorization": f"Bearer {_signed_session_token()}"}, json={"name": "New Name", "is_active": False}, ) - _assert_tool_mutation_not_supported(response) - assert registry.get(code) == original_snapshot - assert registry._handlers[code] is handler + assert response.status_code == 200 + data = response.json() + assert data["name"] == "New Name" + assert data["is_active"] is False + + tool = registry.get("update_tool") + assert tool.name == "New Name" + assert tool.is_active is False finally: - registry.unregister(code) + registry.unregister("update_tool") -def test_delete_tool_mutation_fails_closed_without_registry_change(): - code = "delete_tool" +def test_update_tool_with_webhook(): + try: + registry.register( + ToolInfo( + code="webhook_update_tool", + name="Old", + description="Old", + category="Test", + ), + lambda p: "ok", + ) - def handler(_params): - return "ok" + with patch( + "api.tools._resolve_global_addresses", + return_value=("93.184.216.34",), + ): + with TestClient(app) as client: + response = client.patch( + "/api/tools/webhook_update_tool", + headers={"Authorization": f"Bearer {_signed_session_token()}"}, + json={"webhook_url": "https://example.com/webhook"}, + ) - original = ToolInfo( - code=code, - name="Do Not Delete", - description="Do Not Delete", - category="Test", - ) + assert response.status_code == 200 + tool = registry.get("webhook_update_tool") + assert tool.webhook_url == "https://example.com/webhook" + finally: + registry.unregister("webhook_update_tool") + + +def test_update_tool_remove_webhook(): try: - registry.register(original, handler) + registry.register( + ToolInfo( + code="webhook_remove_tool", + name="Old", + description="Old", + category="Test", + webhook_url="https://example.com/webhook", + ), + lambda p: "ok", + ) with TestClient(app) as client: - response = client.delete( - f"/api/tools/{code}", + response = client.patch( + "/api/tools/webhook_remove_tool", headers={"Authorization": f"Bearer {_signed_session_token()}"}, + json={"webhook_url": None}, ) - _assert_tool_mutation_not_supported(response) - assert registry.get(code) == original - assert registry._handlers[code] is handler + assert response.status_code == 200 + tool = registry.get("webhook_remove_tool") + assert tool.webhook_url is None finally: - registry.unregister(code) + registry.unregister("webhook_remove_tool") -@pytest.mark.parametrize( - ("method", "path", "payload"), - [ - ( - "POST", - "/api/tools", - { - "code": "unauthorized_tool", - "name": "Unauthorized Tool", - "description": "Must not be registered", - "category": "Test", - }, - ), - ("PATCH", "/api/tools/text_analyzer", {"name": "Unauthorized"}), - ("DELETE", "/api/tools/text_analyzer", None), - ], -) -def test_tool_mutation_routes_require_signed_session(method, path, payload): +def test_update_tool_not_found(): with TestClient(app) as client: - response = client.request(method, path, json=payload) - - assert response.status_code == 401 - assert response.json() == {"detail": "Authentication required"} + response = client.patch( + "/api/tools/non_existent_tool", + headers={"Authorization": f"Bearer {_signed_session_token()}"}, + json={"name": "New Name"}, + ) + assert response.status_code == 404 -@pytest.mark.parametrize( - ("method", "path"), - [ - ("POST", "/api/tools"), - ("PATCH", "/api/tools/non_existent_tool"), - ], -) -def test_tool_mutation_tombstones_do_not_validate_request_models(method, path): - with TestClient(app) as client: - response = client.request( - method, - path, - headers={ - "Authorization": f"Bearer {_signed_session_token()}", - "Content-Type": "application/json", - }, - content="{not-json", +def test_delete_tool_success(): + try: + registry.register( + ToolInfo( + code="delete_tool", + name="To Delete", + description="To Delete", + category="Test", + ), + lambda p: "ok", ) - _assert_tool_mutation_not_supported(response) - + with TestClient(app) as client: + response = client.delete( + "/api/tools/delete_tool", + headers={"Authorization": f"Bearer {_signed_session_token()}"}, + ) -def test_tool_mutation_routes_are_hidden_from_openapi(): - from api.tools import router as tools_router + assert response.status_code == 204 + assert registry.get("delete_tool") is None + finally: + registry.unregister("delete_tool") - schema_app = FastAPI() - schema_app.include_router(tools_router) - paths = schema_app.openapi()["paths"] - assert "post" not in paths["/api/tools"] - assert "patch" not in paths["/api/tools/{code}"] - assert "delete" not in paths["/api/tools/{code}"] +def test_delete_tool_not_found(): + with TestClient(app) as client: + response = client.delete( + "/api/tools/non_existent_tool", + headers={"Authorization": f"Bearer {_signed_session_token()}"}, + ) + assert response.status_code == 404 @pytest.mark.asyncio async def test_webhook_handler_success(): - from api.tools import make_webhook_handler + try: + with patch( + "api.tools._resolve_global_addresses", + return_value=("93.184.216.34",), + ): + with TestClient(app) as client: + client.post( + "/api/tools", + headers={"Authorization": f"Bearer {_signed_session_token()}"}, + json={ + "code": "webhook_tool", + "name": "Webhook Tool", + "description": "Calls external webhook", + "category": "Test", + "parameters": {"input": "string"}, + "webhook_url": "https://example.com/webhook", + }, + ) - with patch( - "api.tools._resolve_global_addresses", - return_value=("93.184.216.34",), - ): - handler = make_webhook_handler("https://example.com/webhook") - with patch("httpx.AsyncClient.post") as mock_post: - mock_response = MagicMock() - mock_response.json.return_value = {"webhook_success": True} - mock_response.raise_for_status.return_value = None - mock_post.return_value = mock_response - - result = await handler({"input": "hello"}) - - assert result == {"webhook_success": True} - mock_post.assert_awaited_once_with( - "https://example.com/webhook", - json={"parameters": {"input": "hello"}}, - timeout=10.0, - ) + with patch("httpx.AsyncClient.post") as mock_post: + mock_response = AsyncMock() + mock_response.json.return_value = { + "webhook_success": True + } # json() is sync, return_value returns coroutine from AsyncMock, wait... + from unittest.mock import MagicMock + + mock_response.json = MagicMock(return_value={"webhook_success": True}) + mock_response.raise_for_status = lambda: None + mock_post.return_value = mock_response + + with TestClient(app) as client: + response = client.post( + "/api/tools/webhook_tool/execute", + headers={"Authorization": f"Bearer {_signed_session_token()}"}, + json={"parameters": {"input": "hello"}}, + ) + + assert response.status_code == 200 + data = response.json() + assert data["status"] == "success" + assert data["result"] == {"webhook_success": True} + + mock_post.assert_called_once() + args, kwargs = mock_post.call_args + assert args[0] == "https://example.com/webhook" + assert kwargs["json"] == {"parameters": {"input": "hello"}} + + finally: + registry.unregister("webhook_tool") @pytest.mark.asyncio async def test_webhook_handler_http_error(): - from api.tools import make_webhook_handler - - with patch( - "api.tools._resolve_global_addresses", - return_value=("93.184.216.34",), - ): - handler = make_webhook_handler("https://example.com/webhook") + try: with patch( - "httpx.AsyncClient.post", - side_effect=httpx.HTTPError("Simulated HTTP Error"), + "api.tools._resolve_global_addresses", + return_value=("93.184.216.34",), ): - with pytest.raises( - ValueError, - match="Webhook execution failed: Simulated HTTP Error", - ): - await handler({"input": "hello"}) + with TestClient(app) as client: + client.post( + "/api/tools", + headers={"Authorization": f"Bearer {_signed_session_token()}"}, + json={ + "code": "webhook_fail_tool", + "name": "Webhook Fail Tool", + "description": "Calls external webhook", + "category": "Test", + "parameters": {"input": "string"}, + "webhook_url": "https://example.com/webhook", + }, + ) + + with patch("httpx.AsyncClient.post") as mock_post: + mock_post.side_effect = httpx.HTTPError("Simulated HTTP Error") + + with TestClient(app) as client: + response = client.post( + "/api/tools/webhook_fail_tool/execute", + headers={"Authorization": f"Bearer {_signed_session_token()}"}, + json={"parameters": {"input": "hello"}}, + ) + + assert response.status_code == 200 + data = response.json() + assert data["status"] == "failed" + assert ( + "Webhook execution failed: Simulated HTTP Error" in data["message"] + ) + + finally: + registry.unregister("webhook_fail_tool") def test_tool_registry_execute_no_handler(): @@ -1038,6 +922,46 @@ def test_validate_parameters_not_dict(): registry._validate_parameters("some_code", "not a dict") # type: ignore +def test_create_tool_unsafe_webhook(): + with TestClient(app) as client: + response = client.post( + "/api/tools", + headers={"Authorization": f"Bearer {_signed_session_token()}"}, + json={ + "code": "unsafe_tool", + "name": "Unsafe", + "description": "Unsafe", + "category": "Test", + "webhook_url": "http://localhost:8080/admin", + }, + ) + assert response.status_code == 400 + assert "Invalid or unsafe webhook URL" in response.json()["detail"] + + +def test_update_tool_unsafe_webhook(): + try: + registry.register( + ToolInfo( + code="unsafe_update_tool", + name="Safe", + description="Safe", + category="Test", + ), + lambda p: "ok", + ) + with TestClient(app) as client: + response = client.patch( + "/api/tools/unsafe_update_tool", + headers={"Authorization": f"Bearer {_signed_session_token()}"}, + json={"webhook_url": "http://169.254.169.254/latest/meta-data/"}, + ) + assert response.status_code == 400 + assert "Invalid or unsafe webhook URL" in response.json()["detail"] + finally: + registry.unregister("unsafe_update_tool") + + def test_is_safe_webhook_url_coverage(): from api.tools import is_safe_webhook_url @@ -1077,6 +1001,27 @@ def test_execute_email_translator(): assert data["result"]["source_language_detected"] == "en" +def test_execute_spam_phishing_detector(): + with TestClient(app) as client: + response = client.post( + "/api/tools/spam_phishing_detector/execute", + headers={"Authorization": f"Bearer {_signed_session_token()}"}, + json={ + "parameters": { + "email_content": "Urgent: update your bank password now", + "sender_domain": "secure-bank-login.ru", + } + }, + ) + assert response.status_code == 200 + data = response.json() + assert data["status"] == "success" + assert data["result"]["is_phishing"] is True + assert data["result"]["is_spam"] is True + assert data["result"]["risk_score"] >= 90 + assert any("sender domain" in warning for warning in data["result"]["warnings"]) + + def test_execute_reply_drafter(): with TestClient(app) as client: response = client.post( @@ -1119,7 +1064,7 @@ def test_execute_data_anonymizer(): headers={"Authorization": f"Bearer {_signed_session_token()}"}, json={ "parameters": { - "text": "제 이메일은 test.user-1@gmail.com 이고, 폰 번호는 010-1234-5678, 주민번호는 900101-1234567 입니다. 011-123-4567도 됩니다." + "text": "제 이메일은 test.user-1@gmail.com 이고, 폰 번호는 010-1234-5678, 프랑스 폰 번호는 +33 6 12 34 56 78, 주민번호는 900101 1234567 입니다. 011-123-4567도 됩니다." } }, ) @@ -1133,6 +1078,8 @@ def test_execute_data_anonymizer(): assert "test.user-1@gmail.com" not in anonymized assert "010-1234-5678" not in anonymized assert "900101-1234567" not in anonymized + assert "900101 1234567" not in anonymized + assert "+33 6 12 34 56 78" not in anonymized # 빈 텍스트 케이스 response_empty = client.post( @@ -1149,29 +1096,6 @@ def test_execute_data_anonymizer(): # fallback 커버리지를 위해 직접 handler를 호출하는 비동기 테스트를 아래에 추가합니다. -def test_execute_data_anonymizer_masks_separator_free_and_international_formats(): - source_values = ( - "01012345678", - "9001011234567", - "01 42 68 53 00", - "사용자@예시.한국", - ) - with TestClient(app) as client: - response = client.post( - "/api/tools/data_anonymizer/execute", - headers={"Authorization": f"Bearer {_signed_session_token()}"}, - json={"parameters": {"text": " / ".join(source_values) + "."}}, - ) - - assert response.status_code == 200 - anonymized = response.json()["result"]["anonymized_text"] - assert all(source_value not in anonymized for source_value in source_values) - assert anonymized.endswith(".") - assert anonymized.count("***-****-****") == 2 - assert "******-*******" in anonymized - assert "***@***" in anonymized - - @pytest.mark.asyncio async def test_data_anonymizer_handler_none(): from api.tools import data_anonymizer_handler @@ -1182,11 +1106,6 @@ async def test_data_anonymizer_handler_none(): result_missing = await data_anonymizer_handler({}) assert result_missing["anonymized_text"] == "" - with pytest.raises(ValueError, match="Analysis text must not exceed"): - await data_anonymizer_handler( - {"text": "x" * (ANALYSIS_TEXT_MAX_CHARS + 1)} - ) - def test_execute_grammar_checker(): with TestClient(app) as client: @@ -1208,6 +1127,14 @@ def test_execute_grammar_checker(): assert data["result"]["errors_found"] == 3 +@pytest.mark.asyncio +async def test_mock_handler(): + from api.tools import mock_handler + + res = await mock_handler({"test": 123}) + assert "123" in res + + def test_validate_webhook_url_no_host(): from api.tools import validate_webhook_url @@ -1251,6 +1178,7 @@ async def test_analysis_handlers_safe_and_fallthrough_paths(): email_translator_handler, grammar_checker_handler, sentiment_analyzer_handler, + spam_phishing_detector_handler, ) untranslated = await email_translator_handler( @@ -1259,6 +1187,19 @@ async def test_analysis_handlers_safe_and_fallthrough_paths(): assert untranslated["translated_text"] == "Hello, thank you for the meeting." assert untranslated["source_language_detected"] == "en" + safe_email = await spam_phishing_detector_handler( + { + "email_content": "Here are the approved meeting notes.", + "sender_domain": "example.com", + } + ) + assert safe_email == { + "is_spam": False, + "is_phishing": False, + "risk_score": 10, + "warnings": [], + } + nonurgent_negative = await sentiment_analyzer_handler( {"text": "I am disappointed."} ) @@ -1301,70 +1242,6 @@ async def test_keyword_extractor_handler(): assert empty == {"keywords": [], "keyword_count": 0} -@pytest.mark.asyncio -async def test_email_address_extractor_handler(): - from api.tools import email_address_extractor_handler - - text = ( - "Please contact John.Doe@example.com or support@example.com. " - "Then john.doe@EXAMPLE.COM or user@mail.example.com" - ) - first = await email_address_extractor_handler({"text": text}) - second = await email_address_extractor_handler({"text": text}) - - assert first == second - assert first == { - "emails": [ - "John.Doe@example.com", - "support@example.com", - "user@mail.example.com", - ], - "count": 3, - } - assert await email_address_extractor_handler({"text": "No emails here."}) == { - "emails": [], - "count": 0, - } - assert await email_address_extractor_handler( - {"text": "Reject a@b..com but keep support@example.com..."} - ) == {"emails": ["support@example.com"], "count": 1} - - -def test_execute_email_address_extractor_envelope(): - with TestClient(app) as client: - response = client.post( - "/api/tools/email_address_extractor/execute", - headers={"Authorization": f"Bearer {_signed_session_token()}"}, - json={"parameters": {"text": "Contact test@example.com."}}, - ) - - assert response.status_code == 200 - assert response.json()["result"] == { - "emails": ["test@example.com"], - "count": 1, - } - - -def test_email_address_extractor_rejects_oversized_text(): - from api.tools import ANALYSIS_TEXT_MAX_CHARS - - with TestClient(app) as client: - response = client.post( - "/api/tools/email_address_extractor/execute", - headers={"Authorization": f"Bearer {_signed_session_token()}"}, - json={"parameters": {"text": "x" * (ANALYSIS_TEXT_MAX_CHARS + 1)}}, - ) - - assert response.status_code == 200 - assert response.json() == { - "status": "failed", - "result": None, - "message": ( - f"Analysis text must not exceed {ANALYSIS_TEXT_MAX_CHARS} characters" - ), - } - - def test_execute_analysis_tool_rejects_oversized_text(): from api.tools import ANALYSIS_TEXT_MAX_CHARS @@ -1383,125 +1260,3 @@ def test_execute_analysis_tool_rejects_oversized_text(): f"Analysis text must not exceed {ANALYSIS_TEXT_MAX_CHARS} characters" ), } - - -@pytest.mark.asyncio -async def test_first_last_sentence_handler_success(): - params = {"text": "Hello world. This is a test. How are you?"} - result = await registry.invoke_tool("first_last_sentence", params) - assert result == {"excerpt": "Hello world. How are you?"} - - -@pytest.mark.asyncio -async def test_first_last_sentence_handler_cjk_punctuation(): - params = {"text": "첫 번째 문장입니다． 두 번째 문장입니다！ 세 번째 문장입니다？"} - result = await registry.invoke_tool("first_last_sentence", params) - assert result == {"excerpt": "첫 번째 문장입니다． 세 번째 문장입니다？"} - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("text", "excerpt"), - [ - ( - "Dr. Smith approved it. Please proceed.", - "Dr. Smith approved it. Please proceed.", - ), - ("Version 1.2 works. Please deploy.", "Version 1.2 works. Please deploy."), - ( - "Contact alice@example.com for help. Thanks.", - "Contact alice@example.com for help. Thanks.", - ), - ( - "Read https://example.com/docs.html first. Done.", - "Read https://example.com/docs.html first. Done.", - ), - ('He said "First." She said "Last."', 'He said "First." She said "Last."'), - ], -) -async def test_first_last_sentence_handler_internal_periods_and_closers(text, excerpt): - result = await registry.invoke_tool("first_last_sentence", {"text": text}) - assert result == {"excerpt": excerpt} - - -@pytest.mark.asyncio -async def test_first_last_sentence_handler_empty_text(): - params = {"text": ""} - result = await registry.invoke_tool("first_last_sentence", params) - assert result == {"excerpt": ""} - - -@pytest.mark.asyncio -async def test_first_last_sentence_handler_no_sentences(): - params = {"text": "..."} - result = await registry.invoke_tool("first_last_sentence", params) - assert result == {"excerpt": "..."} - - -@pytest.mark.asyncio -async def test_first_last_sentence_handler_one_sentence(): - params = {"text": "Just one sentence."} - result = await registry.invoke_tool("first_last_sentence", params) - assert result == {"excerpt": "Just one sentence."} - - -@pytest.mark.asyncio -async def test_first_last_sentence_handler_oversized(): - from api.tools import ANALYSIS_TEXT_MAX_CHARS - - params = {"text": "a" * (ANALYSIS_TEXT_MAX_CHARS + 1)} - with pytest.raises(ValueError, match="must not exceed"): - await registry.invoke_tool("first_last_sentence", params) - - -@pytest.mark.asyncio -async def test_hash_generator_handler(): - from api.tools import hash_generator_handler, ANALYSIS_TEXT_MAX_CHARS - - res = await hash_generator_handler({"text": "hello"}) - assert res["md5"] == "5d41402abc4b2a76b9719d911017c592" - assert res["sha1"] == "aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d" - assert res["sha256"] == "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824" - - with pytest.raises(ValueError, match="Analysis text must not exceed"): - await hash_generator_handler({"text": "x" * (ANALYSIS_TEXT_MAX_CHARS + 1)}) - -@pytest.mark.asyncio -async def test_email_phone_masker_handler(): - from api.tools import email_phone_masker_handler, ANALYSIS_TEXT_MAX_CHARS - - res = await email_phone_masker_handler({"text": "Contact me at user@example.com or 010-1234-5678."}) - assert res["masked_text"] == "Contact me at [EMAIL] or [PHONE]." - - with pytest.raises(ValueError, match="Analysis text must not exceed"): - await email_phone_masker_handler({"text": "x" * (ANALYSIS_TEXT_MAX_CHARS + 1)}) - - -def test_execute_hash_generator(): - with TestClient(app) as client: - response = client.post( - "/api/tools/hash_generator/execute", - headers={"Authorization": f"Bearer {_signed_session_token()}"}, - json={"parameters": {"text": "hello"}}, - ) - assert response.status_code == 200 - data = response.json() - assert data["status"] == "success" - assert data["result"]["md5"] == "5d41402abc4b2a76b9719d911017c592" - assert ( - data["result"]["sha256"] - == "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824" - ) - - -def test_execute_email_phone_masker(): - with TestClient(app) as client: - response = client.post( - "/api/tools/email_phone_masker/execute", - headers={"Authorization": f"Bearer {_signed_session_token()}"}, - json={"parameters": {"text": "My email is test@example.com and phone is 010-1234-5678, but 1234 is not."}}, - ) - assert response.status_code == 200 - data = response.json() - assert data["status"] == "success" - assert data["result"]["masked_text"] == "My email is [EMAIL] and phone is [PHONE], but 1234 is not." diff --git a/backend/tests/test_tools_uuid_generator_contract.py b/backend/tests/test_tools_uuid_generator_contract.py deleted file mode 100644 index f15ca254d..000000000 --- a/backend/tests/test_tools_uuid_generator_contract.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Regression contract for the retained UUID v4 built-in tool.""" - -from __future__ import annotations - -import uuid - -import pytest - -from api.tools import registry - - -@pytest.mark.asyncio -async def test_uuid_v4_generator_remains_available_after_mutation_freeze() -> None: - """Keep the safe built-in utility while disabling only dynamic mutations.""" - tool = registry.get("uuid_v4_generator") - - assert tool is not None - assert tool.parameters == {} - result = await registry.invoke_tool("uuid_v4_generator", {}) - generated_uuid = uuid.UUID(result["uuid"]) - assert generated_uuid.version == 4 diff --git a/backend/uv.lock b/backend/uv.lock index d455f2a61..00a57a0c5 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -683,19 +683,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, ] -[[package]] -name = "httpcore2" -version = "2.5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "h11" }, - { name = "truststore" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/47/06/5c12df521b5322fb1114a83d46911b2fbcb8855ddb3a635f11c01a214af5/httpcore2-2.5.0.tar.gz", hash = "sha256:88aa170137c17328d5ac44234f9fd10706466d5fb347f3edac4d39b91137b09d", size = 64808, upload-time = "2026-06-25T14:16:56.472Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c9/a1/7564199d1a8728fe737b0a72e5b3f8d92dfe085a74ddf7cdd83bce5f206d/httpcore2-2.5.0-py3-none-any.whl", hash = "sha256:5ce35188de461d31e8d000bfb8ef8bf22c6c16587a211e5571deaa5e9bdf842a", size = 80330, upload-time = "2026-06-25T14:16:53.634Z" }, -] - [[package]] name = "httplib2" version = "0.32.0" @@ -723,22 +710,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] -[[package]] -name = "httpx2" -version = "2.5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "httpcore2" }, - { name = "idna" }, - { name = "truststore" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d0/e2/b5dedc0cf35aa65de5f541ccd30d2bc1fd7f1d43c9ab09f8ed9a7342317b/httpx2-2.5.0.tar.gz", hash = "sha256:e2df9cb4611021527ff8a675b1c320b610a2ec397acc8d6fe6e91df2d9b33c29", size = 83121, upload-time = "2026-06-25T14:16:57.491Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/31/22/859d8252dad9bc9adee34b52e62cde621ece07b042ccb2ab4da1be46695f/httpx2-2.5.0-py3-none-any.whl", hash = "sha256:3d2d4d9cf4b61f1a1f46a95947cfdb47e80cb56a2f91c6256ac8f58e4891df41", size = 76652, upload-time = "2026-06-25T14:16:55.23Z" }, -] - [[package]] name = "icalendar" version = "7.2.0" @@ -1048,9 +1019,9 @@ dependencies = [ [package.dev-dependencies] dev = [ { name = "coverage" }, - { name = "httpx2" }, { name = "pytest" }, { name = "pytest-asyncio" }, + { name = "pytest-cov" }, { name = "ruff" }, ] @@ -1095,9 +1066,9 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ { name = "coverage", specifier = "==7.15.1" }, - { name = "httpx2", specifier = "==2.5.0" }, { name = "pytest", specifier = "==9.1.1" }, { name = "pytest-asyncio", specifier = "==1.4.0" }, + { name = "pytest-cov", specifier = ">=7.1.0" }, { name = "ruff", specifier = "==0.15.21" }, ] @@ -1616,6 +1587,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/03/e2/08a497ef684b88559c9cc5f4ad53a37e7b99e727094a86d6ea32536d5d3c/pytest_asyncio-1.4.0-py3-none-any.whl", hash = "sha256:933ca923a23075a87fb7070c0ec272a6848489824d887c85c812670932835aa1", size = 16930, upload-time = "2026-05-26T09:56:02.576Z" }, ] +[[package]] +name = "pytest-cov" +version = "7.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coverage" }, + { name = "pluggy" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/51/a849f96e117386044471c8ec2bd6cfebacda285da9525c9106aeb28da671/pytest_cov-7.1.0.tar.gz", hash = "sha256:30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2", size = 55592, upload-time = "2026-03-21T20:11:16.284Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -2003,15 +1988,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/22/2a/5e5e750890ada51017d18d0d4c30da696e5b5bd3180947729927628fc3cb/tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2", size = 676612, upload-time = "2026-07-07T09:58:16.256Z" }, ] -[[package]] -name = "truststore" -version = "0.10.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/53/a3/1585216310e344e8102c22482f6060c7a6ea0322b63e026372e6dcefcfd6/truststore-0.10.4.tar.gz", hash = "sha256:9d91bd436463ad5e4ee4aba766628dd6cd7010cf3e2461756b3303710eebc301", size = 26169, upload-time = "2025-08-12T18:49:02.73Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/19/97/56608b2249fe206a67cd573bc93cd9896e1efb9e98bce9c163bcdc704b88/truststore-0.10.4-py3-none-any.whl", hash = "sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981", size = 18660, upload-time = "2025-08-12T18:49:01.46Z" }, -] - [[package]] name = "typing-extensions" version = "4.16.0" diff --git a/docs/doctoring/data-anonymizer-boundary.md b/docs/doctoring/data-anonymizer-boundary.md deleted file mode 100644 index 1052ba901..000000000 --- a/docs/doctoring/data-anonymizer-boundary.md +++ /dev/null @@ -1,41 +0,0 @@ -# Data anonymizer boundary - -## Decision and observed implementation - -PR #1482 repair parent `034d111b6bef126929d6f0085c2fa15bbf9724be` -stacks on PR #1555 exact head -`03799bc157fa39a419cf6c3f77a29a2ca02cd7f4`. The handler reuses the stack's -canonical ASCII email and selected Korean/North American phone matchers, then -adds bounded Unicode-email, French phone, and Korean resident-registration -patterns. Every input is subject to `ANALYSIS_TEXT_MAX_CHARS` before scanning. - -This tool performs deterministic format masking only. It does not measure -re-identification risk, detect names or organizations, transform free-form -quasi-identifiers, or certify that output is anonymous. Product copy must keep -that limitation visible. A downstream workflow that requires release-grade -de-identification needs a documented data model, threat model, risk metric, -review authority, and evidence that the transformed dataset meets its intended -use. It must not infer that assurance from this handler's successful response. - -Endpoint regressions cover hyphenated and separator-free Korean identifiers, -an internationalized email address, a French phone number, punctuation -preservation, and the input-size boundary. The values are synthetic test data; -no real person's identifiers are committed. - -## Research grounding - -NIST SP 800-188 treats de-identification as a managed process involving data -models, techniques, governance, and re-identification risk rather than a small -set of textual substitutions. That distinction supports the deliberately -narrow product claim above and rejects the earlier broad “data anonymization” -assurance. - -Garfinkel, S., Guttman, B., Near, J., Dajani, A., & Singer, P. (2023). -*De-identifying government datasets: Techniques and governance* (NIST Special -Publication 800-188). National Institute of Standards and Technology. -https://doi.org/10.6028/NIST.SP.800-188 - -The official publication page was available during verification, but its -linked PDF endpoint returned HTTP 404 on 2026-09-04. The PR therefore records -the DOI and bounded summary instead of committing an unverified or -redistribution-uncertain binary. diff --git a/docs/doctoring/email-address-extractor-contract.md b/docs/doctoring/email-address-extractor-contract.md deleted file mode 100644 index f47dac398..000000000 --- a/docs/doctoring/email-address-extractor-contract.md +++ /dev/null @@ -1,34 +0,0 @@ -# Email address extractor contract - -## Problem - -The first extractor used a second permissive regular expression. It accepted -empty domain labels such as `a@b..com` and then tried to repair sentence -punctuation after matching. That disagreed with the email masker and allowed -the two tools to classify the same address differently. - -## Boundary - -The extractor and masker now share `_EMAIL_PATTERN` in `backend/api/tools.py`. -It accepts a bounded ASCII dot-atom local part and DNS-style domain labels, -preserves the first spelling encountered, and deduplicates case-insensitively. -Quoted local parts, comments, internationalized addresses, domain literals, -and full mailbox parsing remain outside this utility tool's claim. - -This is an extraction aid, not an RFC-complete mailbox validator. Sending and -identity boundaries must still use their protocol-specific validation. - -## Verification - -`backend/tests/test_tools_api.py` covers mixed-case duplicate addresses, -subdomains, sentence punctuation, ellipses, malformed empty domain labels, -the signed API envelope, empty input, and the shared input-size limit. - -## Reference - -Resnick, P. (2008). *Internet message format* (RFC 5322). Internet Engineering -Task Force. https://doi.org/10.17487/RFC5322 - -RFC 5322 sections 3.2.3 and 3.4.1 define dot-atoms and address syntax. The -bounded matcher deliberately implements only the common ASCII dot-atom and -DNS-label subset described above, avoiding claims of complete RFC parsing. diff --git a/docs/doctoring/starlette-httpx2-testclient-dependency.md b/docs/doctoring/starlette-httpx2-testclient-dependency.md deleted file mode 100644 index c66b73990..000000000 --- a/docs/doctoring/starlette-httpx2-testclient-dependency.md +++ /dev/null @@ -1,48 +0,0 @@ -# Starlette TestClient `httpx2` dependency - -## Observed failure - -Protected `develop@042b0c70531b229af3acbd0421a2f23098d848b3` pins Starlette -1.3.1 but did not install `httpx2`. Importing `starlette.testclient` therefore -fell back to deprecated `httpx`; warning-as-error test runs stopped during -collection. Removing the warning filter without installing the preferred -transport would expose the defect without repairing it. - -## Decision and boundary - -Pin `httpx2==2.5.0` in the repository's existing combined backend -development/direct-test manifests and immutable locks. Keep application HTTP -clients on their existing `httpx` path. A runtime regression test imports -Starlette's TestClient module and verifies that its selected transport module is -`httpx2`; manifest and digest checks alone are insufficient evidence. - -Starlette 1.2.0 introduced TestClient support for `httpx2`, and 1.3.0 added it -to the `full` extra. The 2.5.0 wheel in this change matches PyPI's published -SHA-256 digest `3d2d4d9cf4b61f1a1f46a95947cfdb47e80cb56a2f91c6256ac8f58e4891df41`. -PyPI records a trusted-publishing attestation from the `pydantic/httpx2` -repository at tag `v2.5.0`. These facts establish origin and integrity; they do -not transfer current-head CI or protected-merge authority. - -## Verification and rollback - -Run from `backend/`: - -```bash -uv run --frozen pytest -q -W error tests/test_container_dependency_pin_contract.py -uv run --frozen ruff check tests/test_container_dependency_pin_contract.py -``` - -Rollback removes the direct pin, regenerated lock records, runtime assertion, -and obsolete-warning-filter removal together. Do not restore only the warning -suppression. - -## References - -Kludex. (2026). *Starlette release notes*. GitHub. -https://github.com/Kludex/starlette/blob/main/docs/release-notes.md - -Python Packaging Authority. (2026). *httpx2 2.5.0 file details and provenance*. -PyPI. https://pypi.org/project/httpx2/2.5.0/ - -Pydantic. (2026). *HTTPX2 v2.5.0* [Source code]. GitHub. -https://github.com/pydantic/httpx2/tree/v2.5.0