From acb374e9b68faf1911a4298155aead3e4144b22e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 15:25:57 +0900 Subject: [PATCH 01/16] fix(ci): coalesce Bandit pull request scans Keep push and manual scans independent while cancelling only superseded scans for the same pull request and repository. Signed-off-by: Seongho Bae Co-authored-by: Codex --- .github/workflows/bandit.yml | 4 ++++ backend/tests/test_release_governance.py | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/.github/workflows/bandit.yml b/.github/workflows/bandit.yml index c5c613c08..1026da4f5 100644 --- a/.github/workflows/bandit.yml +++ b/.github/workflows/bandit.yml @@ -10,6 +10,10 @@ on: permissions: contents: read +concurrency: + group: bandit-${{ github.repository }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: security: runs-on: ubuntu-latest diff --git a/backend/tests/test_release_governance.py b/backend/tests/test_release_governance.py index efe0acd0e..16ae7d742 100644 --- a/backend/tests/test_release_governance.py +++ b/backend/tests/test_release_governance.py @@ -138,6 +138,15 @@ def test_backend_images_use_python_314_runtime() -> None: assert 'python-version: "3.12"' not in bandit_workflow +def test_bandit_pr_runs_share_only_the_same_repository_pr_group() -> None: + workflow = read_repo_text(".github/workflows/bandit.yml") + assert ( + "group: bandit-${{ github.repository }}-${{ github.event_name == 'pull_request' " + "&& github.event.pull_request.number || github.ref }}" + ) in workflow + assert "cancel-in-progress: ${{ github.event_name == 'pull_request' }}" in workflow + + def test_python_314_backend_image_uses_binary_wheel_dependencies() -> None: dockerfile = read_repo_text("Dockerfile") requirements = read_repo_text("backend/requirements.txt") From 55d601cdc494921cb2e98e04612709e43e4f6a69 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 15:43:18 +0900 Subject: [PATCH 02/16] chore(ci): defer Bandit concurrency to canonical owner --- .github/workflows/bandit.yml | 4 - .github/workflows/docker-publish.yml | 60 +- .jules/sentinel.md | 5 + AGENTS.md | 31 + ARCHITECTURE.md | 15 + CHANGELOG.md | 32 + CLAUDE.md | 7 + Dockerfile | 13 +- Dockerfile.ollama | 2 +- README.md | 8 + backend/api/calendar_conflicts.py | 204 +++ backend/api/tools.py | 93 +- backend/core/local_http.py | 6 +- backend/main.py | 2 + backend/services/attachment_parser.py | 17 +- backend/services/batch_embedding_service.py | 148 +- backend/services/calendar_conflict_ics.py | 220 +++ backend/services/calendar_conflict_policy.py | 223 +++ backend/services/email_client.py | 4 + backend/services/email_import_service.py | 100 +- backend/services/embedding.py | 14 +- backend/services/text_safety.py | 8 +- .../calendar/existing-cancelled-1000z.ics | 12 + .../calendar/existing-confirmed-1000z.ics | 12 + .../existing-confirmed-adjacent-1100z.ics | 12 + .../calendar/existing-tentative-1030z.ics | 12 + .../calendar/proposed-confirmed-1000z.ics | 12 + .../calendar/proposed-tentative-1000z.ics | 12 + backend/tests/runner/utils/test_dispatch.py | 15 + backend/tests/test_attachment_parser.py | 26 + backend/tests/test_batch_embedding_service.py | 136 +- backend/tests/test_calendar_conflict_api.py | 226 +++ backend/tests/test_calendar_conflict_ics.py | 194 +++ .../tests/test_calendar_conflict_policy.py | 240 ++++ .../test_container_dependency_pin_contract.py | 146 ++ backend/tests/test_email_client.py | 15 + backend/tests/test_email_import_service.py | 219 ++- backend/tests/test_embedding.py | 32 + backend/tests/test_local_http.py | 70 + backend/tests/test_oidc_jwks_preload.py | 38 + backend/tests/test_release_governance.py | 165 +-- backend/tests/test_repo_hygiene.py | 2 +- backend/tests/test_text_safety.py | 4 + backend/tests/test_tools_api.py | 96 +- .../test_topic_intelligence_documentation.py | 256 ++++ backend/tests/test_url_validation.py | 137 +- connector/Dockerfile | 2 +- docs/adr/0001-topic-measurement-authority.md | 77 ++ .../0002-fitted-topic-artifact-consumption.md | 81 ++ ...opic-measurement-from-agenda-generation.md | 63 + ...0004-status-weighted-calendar-conflicts.md | 79 ++ docs/adr/README.md | 30 + docs/doctoring/kanban-task-keyboard-focus.md | 37 + .../local-http-origin-port-validation.md | 36 + .../status-weighted-calendar-conflicts.md | 39 + .../structural-topic-model-boundary.md | 94 ++ .../container-provenance-contract.md | 41 + docs/planning/naruon-platform-plan.md | 13 +- docs/product-technical-gap-baseline.md | 1016 ++++++++++++++ .../email-authentication-xoauth2/README.md | 54 + .../2026-08-09-structural-topic-boundary.md | 228 +++ ...-08-09-structural-topic-boundary-design.md | 127 ++ docs/topic-intelligence/API_CONTRACT.md | 361 +++++ docs/topic-intelligence/ARCHITECTURE.md | 249 ++++ docs/topic-intelligence/DATA_MODEL.md | 302 ++++ .../DOCUMENTATION_FITNESS.md | 105 ++ docs/topic-intelligence/OPERABILITY.md | 122 ++ docs/topic-intelligence/PRD.md | 129 ++ docs/topic-intelligence/README.md | 159 +++ docs/topic-intelligence/REFERENCES.md | 100 ++ docs/topic-intelligence/SECURITY.md | 115 ++ docs/topic-intelligence/TEST_STRATEGY.md | 156 +++ docs/topic-intelligence/THREAT_MODEL.md | 113 ++ docs/topic-intelligence/TRACEABILITY.md | 116 ++ docs/topic-intelligence/TRD.md | 206 +++ docs/topic-intelligence/UML.md | 253 ++++ .../topic-inference-result-v1.schema.json | 1228 +++++++++++++++++ frontend/.Jules/palette.md | 4 + frontend/Dockerfile | 11 +- frontend/src/app/calendar/page.test.tsx | 56 + frontend/src/components/CalendarLayout.tsx | 9 +- frontend/src/components/EmailDetail.test.tsx | 16 + frontend/src/components/EmailDetail.tsx | 8 - .../NetworkGraph.map-lookup.test.ts | 66 + frontend/src/components/NetworkGraph.test.tsx | 200 +++ frontend/src/components/NetworkGraph.tsx | 103 +- .../TasksLayout.focus-visible.test.ts | 38 + frontend/src/components/TasksLayout.tsx | 2 +- .../calendar/CalendarCoordinationView.tsx | 86 +- frontend/src/components/calendar/constants.ts | 7 +- frontend/src/components/calendar/helpers.ts | 36 +- frontend/src/components/calendar/types.ts | 17 + frontend/tests/e2e/helpers.ts | 34 + plan.md | 21 + 94 files changed, 9306 insertions(+), 444 deletions(-) create mode 100644 backend/api/calendar_conflicts.py create mode 100644 backend/services/calendar_conflict_ics.py create mode 100644 backend/services/calendar_conflict_policy.py create mode 100644 backend/tests/fixtures/calendar/existing-cancelled-1000z.ics create mode 100644 backend/tests/fixtures/calendar/existing-confirmed-1000z.ics create mode 100644 backend/tests/fixtures/calendar/existing-confirmed-adjacent-1100z.ics create mode 100644 backend/tests/fixtures/calendar/existing-tentative-1030z.ics create mode 100644 backend/tests/fixtures/calendar/proposed-confirmed-1000z.ics create mode 100644 backend/tests/fixtures/calendar/proposed-tentative-1000z.ics create mode 100644 backend/tests/runner/utils/test_dispatch.py create mode 100644 backend/tests/test_calendar_conflict_api.py create mode 100644 backend/tests/test_calendar_conflict_ics.py create mode 100644 backend/tests/test_calendar_conflict_policy.py create mode 100644 backend/tests/test_container_dependency_pin_contract.py create mode 100644 backend/tests/test_oidc_jwks_preload.py create mode 100644 backend/tests/test_topic_intelligence_documentation.py create mode 100644 docs/adr/0001-topic-measurement-authority.md create mode 100644 docs/adr/0002-fitted-topic-artifact-consumption.md create mode 100644 docs/adr/0003-separate-topic-measurement-from-agenda-generation.md create mode 100644 docs/adr/0004-status-weighted-calendar-conflicts.md create mode 100644 docs/adr/README.md create mode 100644 docs/doctoring/kanban-task-keyboard-focus.md create mode 100644 docs/doctoring/local-http-origin-port-validation.md create mode 100644 docs/doctoring/status-weighted-calendar-conflicts.md create mode 100644 docs/doctoring/structural-topic-model-boundary.md create mode 100644 docs/operations/container-provenance-contract.md create mode 100644 docs/product-technical-gap-baseline.md create mode 100644 docs/research/email-authentication-xoauth2/README.md create mode 100644 docs/superpowers/plans/2026-08-09-structural-topic-boundary.md create mode 100644 docs/superpowers/specs/2026-08-09-structural-topic-boundary-design.md create mode 100644 docs/topic-intelligence/API_CONTRACT.md create mode 100644 docs/topic-intelligence/ARCHITECTURE.md create mode 100644 docs/topic-intelligence/DATA_MODEL.md create mode 100644 docs/topic-intelligence/DOCUMENTATION_FITNESS.md create mode 100644 docs/topic-intelligence/OPERABILITY.md create mode 100644 docs/topic-intelligence/PRD.md create mode 100644 docs/topic-intelligence/README.md create mode 100644 docs/topic-intelligence/REFERENCES.md create mode 100644 docs/topic-intelligence/SECURITY.md create mode 100644 docs/topic-intelligence/TEST_STRATEGY.md create mode 100644 docs/topic-intelligence/THREAT_MODEL.md create mode 100644 docs/topic-intelligence/TRACEABILITY.md create mode 100644 docs/topic-intelligence/TRD.md create mode 100644 docs/topic-intelligence/UML.md create mode 100644 docs/topic-intelligence/schema/topic-inference-result-v1.schema.json create mode 100644 frontend/src/components/NetworkGraph.map-lookup.test.ts create mode 100644 frontend/src/components/TasksLayout.focus-visible.test.ts create mode 100644 plan.md diff --git a/.github/workflows/bandit.yml b/.github/workflows/bandit.yml index 1026da4f5..c5c613c08 100644 --- a/.github/workflows/bandit.yml +++ b/.github/workflows/bandit.yml @@ -10,10 +10,6 @@ on: permissions: contents: read -concurrency: - group: bandit-${{ github.repository }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - jobs: security: runs-on: ubuntu-latest diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 879b906ec..fc7058413 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -32,18 +32,21 @@ jobs: - component: backend image: ai_email_client-backend dockerfile: Dockerfile + base_dockerfile: Dockerfile context: . build_args: | BUILDKIT_INLINE_CACHE=1 - component: naruon image: naruon dockerfile: Dockerfile + base_dockerfile: Dockerfile context: . build_args: | BUILDKIT_INLINE_CACHE=1 - component: frontend image: ai_email_client-frontend dockerfile: frontend/Dockerfile + base_dockerfile: frontend/Dockerfile context: . build_args: | BUILDKIT_INLINE_CACHE=1 @@ -64,9 +67,28 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + - name: Resolve pinned Ollama base manifest + if: matrix.component == 'naruon' + run: | + base_image="$(awk 'toupper($1) == "FROM" { print $2; exit }' Dockerfile.ollama)" + if ! printf '%s\n' "$base_image" | grep -Eq '^ollama/ollama@sha256:[0-9a-f]{64}$'; then + printf '::error file=Dockerfile.ollama,line=1::Expected an exact ollama/ollama sha256 base pin; found %s\n' "$base_image" + exit 1 + fi + printf 'Resolving pinned Ollama base manifest: %s\n' "$base_image" + manifest_output="$(docker buildx imagetools inspect "$base_image")" + printf '%s\n' "$manifest_output" + for platform in linux/amd64 linux/arm64; do + if ! printf '%s\n' "$manifest_output" | grep -Eq "^[[:space:]]*Platform:[[:space:]]+${platform}[[:space:]]*$"; then + printf '::error file=Dockerfile.ollama,line=1::Pinned Ollama manifest is missing %s\n' "$platform" + exit 1 + fi + done + - name: Prepare OCI annotation values id: oci env: + BASE_DOCKERFILE: ${{ matrix.base_dockerfile }} GIT_REF_NAME: ${{ github.ref_name }} IMAGE_COMPONENT: ${{ matrix.component }} IMAGE_NAME: ${{ matrix.image }} @@ -76,24 +98,29 @@ jobs: version="$(cat VERSION)" created="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" vendor="${REPOSITORY%%/*}" + base_reference="$(awk 'toupper($1) == "FROM" { print $2; exit }' "$BASE_DOCKERFILE")" + if ! printf '%s\n' "$base_reference" | grep -Eq '^[A-Za-z0-9._/-]+:[A-Za-z0-9._-]+@sha256:[0-9a-f]{64}$'; then + printf '::error file=%s,line=1::Expected an exact tagged sha256 base pin; found %s\n' "$BASE_DOCKERFILE" "$base_reference" + exit 1 + fi + base_digest="${base_reference##*@}" + base_repository="${base_reference%@*}" + case "$base_repository" in + */*) base_name="$base_reference" ;; + *) base_name="docker.io/library/$base_reference" ;; + esac case "$IMAGE_COMPONENT" in frontend) title="naruon frontend" description="Naruon Next.js frontend runtime image" - base_digest="sha256:191ef878ecb351d68b78219593de18bd8942afd59af59f29960dc4b24805a3f1" - base_name="docker.io/library/node:26-slim@${base_digest}" ;; backend) title="naruon backend" description="Naruon FastAPI backend runtime image" - base_digest="sha256:44dd04494ee8f3b538294360e7c4b3acb87c8268e4d0a4828a6500b1eff50061" - base_name="docker.io/library/python:3.14-slim@${base_digest}" ;; *) title="naruon" description="Naruon combined FastAPI and Next.js runtime image" - base_digest="sha256:44dd04494ee8f3b538294360e7c4b3acb87c8268e4d0a4828a6500b1eff50061" - base_name="docker.io/library/python:3.14-slim@${base_digest}" ;; esac { @@ -158,18 +185,21 @@ jobs: - component: backend image: ai_email_client-backend dockerfile: Dockerfile + base_dockerfile: Dockerfile context: . build_args: | BUILDKIT_INLINE_CACHE=1 - component: naruon image: naruon dockerfile: Dockerfile + base_dockerfile: Dockerfile context: . build_args: | BUILDKIT_INLINE_CACHE=1 - component: frontend image: ai_email_client-frontend dockerfile: frontend/Dockerfile + base_dockerfile: frontend/Dockerfile context: . build_args: | BUILDKIT_INLINE_CACHE=1 @@ -204,6 +234,7 @@ jobs: - name: Prepare OCI annotation values id: oci env: + BASE_DOCKERFILE: ${{ matrix.base_dockerfile }} GIT_REF_NAME: ${{ github.ref_name }} IMAGE_COMPONENT: ${{ matrix.component }} IMAGE_NAME: ${{ matrix.image }} @@ -214,24 +245,29 @@ jobs: version="${VERSION_VALUE:-$(cat VERSION)}" created="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" vendor="${REPOSITORY%%/*}" + base_reference="$(awk 'toupper($1) == "FROM" { print $2; exit }' "$BASE_DOCKERFILE")" + if ! printf '%s\n' "$base_reference" | grep -Eq '^[A-Za-z0-9._/-]+:[A-Za-z0-9._-]+@sha256:[0-9a-f]{64}$'; then + printf '::error file=%s,line=1::Expected an exact tagged sha256 base pin; found %s\n' "$BASE_DOCKERFILE" "$base_reference" + exit 1 + fi + base_digest="${base_reference##*@}" + base_repository="${base_reference%@*}" + case "$base_repository" in + */*) base_name="$base_reference" ;; + *) base_name="docker.io/library/$base_reference" ;; + esac case "$IMAGE_COMPONENT" in frontend) title="naruon frontend" description="Naruon Next.js frontend runtime image" - base_digest="sha256:191ef878ecb351d68b78219593de18bd8942afd59af59f29960dc4b24805a3f1" - base_name="docker.io/library/node:26-slim@${base_digest}" ;; backend) title="naruon backend" description="Naruon FastAPI backend runtime image" - base_digest="sha256:44dd04494ee8f3b538294360e7c4b3acb87c8268e4d0a4828a6500b1eff50061" - base_name="docker.io/library/python:3.14-slim@${base_digest}" ;; *) title="naruon" description="Naruon combined FastAPI and Next.js runtime image" - base_digest="sha256:44dd04494ee8f3b538294360e7c4b3acb87c8268e4d0a4828a6500b1eff50061" - base_name="docker.io/library/python:3.14-slim@${base_digest}" ;; esac { diff --git a/.jules/sentinel.md b/.jules/sentinel.md index 6f502e1c7..9208f58b1 100644 --- a/.jules/sentinel.md +++ b/.jules/sentinel.md @@ -133,3 +133,8 @@ **Vulnerability:** The `in_reply_to` and `references` fields on the `SendEmailRequest` model lacked explicit validation, opening up an opportunity for header injection by appending `\r\n`. **Learning:** While the email service internally checks some headers, relying on the API boundary's Pydantic model ensures bad input is stopped early and consistently. Pydantic regex patterns aren't sufficient on their own for all string contexts due to encoding/decoding inconsistencies. **Prevention:** Always use `@field_validator` with explicit `mode="before"` string matching to reject `chr(10)` and `chr(13)` across all user-controlled email header fields. Use `isinstance(value, str)` before string operations to prevent runtime errors if input is missing or malformed. + +## 2026-08-05 - [Prevent Path Traversal via Backslashes in Attachment Parser] +**Vulnerability:** The `_safe_filename` function in `backend/services/attachment_parser.py` used `pathlib.Path().name` to strip directory components from attachment filenames, but failed to normalize backslashes beforehand. This allowed attackers to use Windows-style path separators (e.g., `..\..\upload`) to bypass path validation on POSIX systems. +**Learning:** Checking for traversal sequences using `pathlib.Path().name` may leave the result vulnerable if the input path can contain Windows-style path separators but the program interprets it dynamically or decodes payloads using backslashes, because POSIX `pathlib` treats backslashes as valid filename characters, not separators. +**Prevention:** Always convert backslashes to forward slashes before parsing filenames using `pathlib.Path().name`. diff --git a/AGENTS.md b/AGENTS.md index 35a593547..9104dd1f4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -95,6 +95,17 @@ in this repo. knowledge-graph pipeline (DOM decomposition, entity/relation extraction, grounded graph retrieval) should ground itself in the relevant layout-analysis and knowledge-graph / grounded-retrieval literature. + +### Structural topic-model boundary + +- Do not implement or describe hard-coded term lists, term frequency, + embeddings, or LLM-assigned labels as structural topic modeling (STM). + Fixed business labels are not topic-posterior estimates, and the explicitly + lexical `keyword_extractor` must not be used as topic evidence. +- Topic inference requires a versioned fitted TEPP model and its frozen + preprocessing and vocabulary contract. If that fitted model is unavailable, + fail closed; do not return a default label, template agenda, or substitute + keyword/embedding/LLM result presented as STM. ## Release governance defaults @@ -425,7 +436,23 @@ in this repo. - Public audit/event identifiers that may use human-readable prefixes must not be stored in artificially short `varchar(n)` columns; use opaque source UIDs that fit seeded smoke data and provider evidence without truncation. +- Conceptual ERDs, API schemas, persistence models, and fixtures must not mark a + reusable business identifier such as `document_ref`, `model_id`, `topic_id`, + or `label_id` as an unscoped primary or foreign key. Use an opaque immutable + reference that binds the full scope or an explicit composite identity with the + applicable snapshot revision, model version, request/result scope, or label + version. Define the required identity tuple for each entity; require only the + dimensions relevant to that entity. Never join snapshots, model artifacts, + topic components, or label evidence by a bare document, model, topic, rank, + label, or display value. - When reviews find public/private identifier leaks, stale API fixture shapes, or recurring bug patterns, update tests, frontend mocks, E2E mocks, README examples, architecture docs, and explicitly record the anti-pattern in `AGENTS.md` so the same bug pattern does not reappear in copied examples. +- Memoized id-to-record Maps must be first-wins (`if (!map.has(key)) map.set(...)`). + `new Map(items.map((item) => [String(item.id), item]))` is last-wins and + desynchronizes first-wins label maps from the selected node or edge when + ids collide. Keep a rendered selection test that repeats an id and asserts + the first instance is the one opened. Do not treat a source-substring scan + as the only selection-path contract; fire the vis-network `selectNode` / + `selectEdge` callbacks with mixed numeric and string ids. - When reviews find missing browser security headers or tabnabbing hardening, update both backend header tests and frontend link tests. Global backend responses must include `Referrer-Policy`, and `target="_blank"` links must @@ -490,6 +517,10 @@ in this repo. - Calendar writeback UI must fail closed while the signed source registry is loading or errored; do not emit intent POSTs without a confirmed opaque `target_source_id`, and keep tests covering the loading/error boundary. +- Calendar coordination must not present canned ICS documents or fixed + conflict outcomes as production evidence. Use selectable sources from the + signed `/api/calendar/writeback-sources` registry, or omit the evaluate call + until source-backed VEVENT evidence exists. Known `.ics` pairs stay in tests. - Calendar and WebDAV workspaces must expose the current opaque writeback source as a deliberate user selection with capability and ETag/If-Match state. Automatic first-source fallback may initialize the control, but intent POSTs diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 2139d7984..9d2cbba18 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -20,6 +20,21 @@ Runtime database connectivity is secret-injected: `backend/core/config.py` has no fallback `DATABASE_URL`, so missing database configuration fails at startup rather than silently using shared development credentials. +## Topic-intelligence boundary + +Naruon has no live Structural Topic Modeling endpoint, fitted topic artifact, +or topic-result persistence. The retained `keyword_extractor` is deterministic +lexical metadata and must not feed topic, agenda, search, or norm-group +inference. A future adapter may consume a separately accepted, versioned TEPP +artifact/API only when frozen preprocessing and vocabulary, covariate design, +mixed-membership posterior uncertainty, diagnostics, provenance, and explicit +abstention are all available. Missing or incompatible scientific authority +fails closed. Naruon owns authentication, authorization, request validation, +the adapter envelope, and disclosure policy; TEPP would own the scientific +payload. See the canonical documentation graph in +[`docs/topic-intelligence/README.md`](docs/topic-intelligence/README.md) and +[`ADR-0001`](docs/adr/0001-topic-measurement-authority.md). + ## Workspace navigation boundary The Next.js shell opens the Today execution dashboard for first-run sessions and diff --git a/CHANGELOG.md b/CHANGELOG.md index 778b891e0..7ec84c36f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,37 @@ ## [Unreleased] +- 긴 이메일·첨부 본문을 의미 단위 청크로 임베딩한 뒤 기존 email/attachment 벡터 계약으로 평균화하고, 청크 요청·벡터 누적을 제한된 창으로 처리합니다. OpenAI `text-embedding-3-*`에는 저장 차원(`1536`)을 직접 요청하도록 보강했습니다. 합성 메일 fixture 5건(70청크)과 provider 요청 계약으로 1,536차원 벡터 경로를 검증했으며, 실행 시 선택한 임베딩 제공자에 본문·파싱된 첨부 텍스트를 전송할 수 있습니다. 회사 기밀 데이터는 fixture·commit·PR·log에 포함하지 않습니다. +- EmailDetail 테스트가 지원하지 않는 스레드 병합/분리 버튼을 `textContent`뿐 아니라 `aria-label`과 `title` 접근 가능 이름으로도 검출하도록 바꿔, 아이콘 전용 버튼 회귀를 놓치지 않습니다. + +### 캘린더 충돌 (Status-weighted conflicts) + +- 상태 가중 일정 충돌 평가가 RFC 5545 `VEVENT` 증거를 직접 받습니다. + `POST /api/calendar/conflicts/evaluate`는 구조화 commitment 또는 + `proposed_ics`/`existing_ics`를 받아 `available` / `review_required` / + `blocked`와 다음 행동을 반환합니다. `STATUS:CANCELLED`는 유효한 증거라 + 시간을 차지하지 않으므로, 취소된 기존 일정과 겹치는 확정 제안은 진행할 수 + 있습니다. 잠정 겹침은 검토를, 확정 겹침은 이중 예약을 차단합니다. + Calendar 회의 조율 화면은 서명된 writeback 원본만 선택하고, 알려진 `.ics` + 쌍은 테스트 고정값으로만 유지합니다. 요청 검증 실패는 + `calendar_proposed_source_missing` 또는 `calendar_request_invalid` 봉투를 + 반환합니다. 반복 VEVENT와 과도한 ICS 바이트는 fail-closed 합니다. 공급자 + CalDAV 쓰기는 하지 않습니다. +- 검증: `python -m pytest backend/tests/test_calendar_conflict_policy.py backend/tests/test_calendar_conflict_ics.py backend/tests/test_calendar_conflict_api.py -q`, + `corepack pnpm@11.5.3 --dir frontend exec vitest run src/app/calendar/page.test.tsx`. +### 주제 측정 경계 (Topic Measurement) + +- STM 결과로 오인될 수 있었던 하드코딩 용어표 기반 + `email_categorizer`와 `meeting_agenda_generator`를 도구 레지스트리에서 + 제거했습니다. `keyword_extractor`는 결정론적 단어 빈도 유틸리티로 유지하되 + 주제 posterior 근거로 사용하지 않는 경계를 문서화했습니다. 현재 Naruon에는 + fitted TEPP 모델 기반 production 주제 측정 API가 없으므로, 모델 부재 시 + 기본 라벨이나 템플릿으로 대체하지 않고 fail closed 합니다. +- 이 경계의 PRD, TRD, ADR, Architecture, API 계약, JSON Schema, UML, + 개념 ERD, 보안·위협 모델, 테스트·운영 전략, 추적성 및 문서 적합성 평가를 + `docs/topic-intelligence/`에 하나의 상태 표시 문서 그래프로 정리했습니다. + 이는 미래 계약의 설계 근거이며, 현재 runtime 구현이나 물리 DB 엔터티가 + 존재한다는 주장이 아닙니다. - UUID V4 제너레이터(`uuid_v4_generator`) 도구를 추가하여 런타임에서 범용 고유 식별자 버전 4를 랜덤으로 생성할 수 있게 하였습니다. 테스트 커버리지 100%를 보장합니다. + ### 보안 패치 (CodeQL extended current-head) - `cryptography`를 `50.0.0`으로 갱신해 공격자 제공 PKCS#7 EnvelopedData 복호화 결과의 오류·타이밍 차이로 발생하는 Bleichenbacher oracle(`CVE-2026-69247`, `GHSA-g6cj-pr64-35w5`)을 제거하고, backend·uv lock·hash lock·Strix CI 의존성 증거를 같은 버전으로 동기화했습니다. Strix 잠금은 `google-cloud-aiplatform==1.160.0`의 `<7` 제약을 위반하던 `protobuf==7.35.1`을 이미 검증된 `6.33.6`으로 복구해 다시 해석·설치 가능하게 했습니다. diff --git a/CLAUDE.md b/CLAUDE.md index 896963575..be67bc80c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -146,6 +146,13 @@ Next.js frontend ──> FastAPI backend (control plane) ──> Postgres + pgve test/lint/build), plus `bandit`, `codeql`, `trivy`, `scorecard`, `pr-governance`, `docker-publish` (GHCR on `v*` tags matching `VERSION`), and `mail-smoke`. Actions are pinned to full commit SHAs. +- Topic intelligence is **not implemented**. Never use lexical frequencies, + embeddings, zero-shot labels, or request-time LLM labels as an STM result. + The retained `keyword_extractor` is lexical metadata only. Any future adapter + is blocked on a versioned fitted TEPP artifact/API with frozen preprocessing, + mixed-membership uncertainty and diagnostics; absence or incompatibility + fails closed. Start at `docs/topic-intelligence/README.md` and + `docs/adr/0001-topic-measurement-authority.md`. ## Key conventions diff --git a/Dockerfile b/Dockerfile index d51e6dafc..68c5d2e91 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Stage 1: Backend runtime for local Compose and backend-only deployments -FROM python:3.14-slim@sha256:b877e50bd90de10af8d82c57a022fc2e0dc731c5320d762a27986facfc3355c1 AS backend-runtime +FROM python:3.14-slim@sha256:a7fb1e634c4a578f9e0bd6327f11a3cde11b7a9395f48e24360c0988bcc5c2bc AS backend-runtime WORKDIR /app ENV PYTHONDONTWRITEBYTECODE=1 @@ -25,7 +25,7 @@ EXPOSE 8000 CMD ["python", "scripts/start_backend.py", "--host", "0.0.0.0", "--port", "8000"] # Stage 2: Build Frontend -FROM node:26-slim@sha256:ffc78385a788964bb3cbab5e434ff79a10bdc25b8ae6db03fe5fe6cb14053c09 AS frontend-builder +FROM node:26-slim@sha256:4ebb5ace66f15a24c14c492e01a8beeed4fddf970a856109f5126e703e5fe503 AS frontend-builder WORKDIR /app ENV NPM_CONFIG_UPDATE_NOTIFIER=false ENV PNPM_VERSION=11.5.3 @@ -63,8 +63,13 @@ ARG OCI_IMAGE_LICENSES="LicenseRef-Naruon-Proprietary" ARG OCI_IMAGE_REF_NAME="" ARG OCI_IMAGE_TITLE="naruon" ARG OCI_IMAGE_DESCRIPTION="Naruon combined FastAPI and Next.js runtime image" -ARG OCI_IMAGE_BASE_DIGEST="sha256:44dd04494ee8f3b538294360e7c4b3acb87c8268e4d0a4828a6500b1eff50061" -ARG OCI_IMAGE_BASE_NAME="docker.io/library/python:3.14-slim@sha256:44dd04494ee8f3b538294360e7c4b3acb87c8268e4d0a4828a6500b1eff50061" +ARG OCI_IMAGE_BASE_DIGEST="sha256:a7fb1e634c4a578f9e0bd6327f11a3cde11b7a9395f48e24360c0988bcc5c2bc" +ARG OCI_IMAGE_BASE_NAME="docker.io/library/python:3.14-slim@sha256:a7fb1e634c4a578f9e0bd6327f11a3cde11b7a9395f48e24360c0988bcc5c2bc" + +# Defaults keep local builds provenance-complete. The publishing workflow derives +# and overrides both values from the exact first FROM instruction, while +# repository governance tests prevent the reviewed defaults from drifting. +RUN test -n "$OCI_IMAGE_BASE_DIGEST" && test -n "$OCI_IMAGE_BASE_NAME" LABEL org.opencontainers.image.created="${OCI_IMAGE_CREATED}" \ org.opencontainers.image.authors="${OCI_IMAGE_AUTHORS}" \ diff --git a/Dockerfile.ollama b/Dockerfile.ollama index d4b369689..c4afd9598 100644 --- a/Dockerfile.ollama +++ b/Dockerfile.ollama @@ -1,4 +1,4 @@ -FROM ollama/ollama@sha256:509fdf54e23bd50d87af646cb51c0a7a203d6a83cc4d6695b3b08c5be1c62c0a +FROM ollama/ollama@sha256:b88c73ace3e115f8ec53dc8761ae1c0aabfa675406e3681786b98757ce050f42 ENV OLLAMA_MODELS=/usr/share/ollama/.ollama/models diff --git a/README.md b/README.md index a5cc6252f..ff64840e9 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,8 @@ mail/calendar/file systems. ## Quick Links - [Installation & Setup](#five-minute-local-path) - [Architecture](docs/architecture/) +- [Topic-intelligence documentation set](docs/topic-intelligence/README.md) +- [Architecture decisions](docs/adr/README.md) - [Contributing](CONTRIBUTING.md) - [Code of Conduct](CODE_OF_CONDUCT.md) - [Security Policy](SECURITY.md) @@ -48,6 +50,12 @@ mail/calendar/file systems. auto-merge, and mechanical merge actions run as the target repository's `github-actions[bot]` through the central workflow. Pending CodeRabbit or required-check evidence is a wait state, not a hard blocker. +- Topic intelligence is not currently a live Naruon capability. The lexical + `keyword_extractor` is metadata only; Naruon fails closed rather than present + keyword, embedding, or LLM labels as Structural Topic Modeling. The product, + technical, architecture, contract, security, UML, conceptual ERD, test, and + operability records are indexed in + [`docs/topic-intelligence/`](docs/topic-intelligence/README.md). - Security governance is source-backed through signed `/api/security/access-surface`. The endpoint reads scoped WebDAV, CalDAV, and connector evidence plus durable `security_audit_events`, reuses the deny-first diff --git a/backend/api/calendar_conflicts.py b/backend/api/calendar_conflicts.py new file mode 100644 index 000000000..d61293f82 --- /dev/null +++ b/backend/api/calendar_conflicts.py @@ -0,0 +1,204 @@ +"""Authenticated API surface for deterministic calendar conflict decisions.""" + +from __future__ import annotations + +from typing import Literal, Self + +from fastapi import APIRouter +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from fastapi.routing import APIRoute +from pydantic import AwareDatetime, BaseModel, ConfigDict, Field, model_validator +from starlette.requests import Request +from starlette.responses import Response + +from services.calendar_conflict_ics import ( + parse_existing_calendar_commitments_from_ics, + parse_proposed_calendar_commitment_from_ics, +) +from services.calendar_conflict_policy import ( + CalendarCommitment, + CalendarConflictDecision, + CalendarPolicyValidationError, + CommitmentStatus, + evaluate_calendar_conflicts, +) + +MAX_EXISTING_COMMITMENTS = 500 +MAX_PROPOSED_ICS_CHARS = 65_536 +MAX_EXISTING_ICS_CHARS = 262_144 +POLICY_VALIDATION_HTTP_STATUS = 422 +REQUEST_INVALID_ERROR_CODE = "calendar_request_invalid" +PROPOSED_SOURCE_REQUIRED_DETAIL = "Provide exactly one of proposed or proposed_ics" + + +class CalendarConflictAPIRoute(APIRoute): + """Keep request-model failures on the stable calendar conflict error envelope.""" + + def get_route_handler(self): + """Wrap the FastAPI handler so validation uses CalendarConflictErrorResponse.""" + original_route_handler = super().get_route_handler() + + async def calendar_conflict_route_handler(request: Request) -> Response: + try: + return await original_route_handler(request) + except RequestValidationError as exc: + return _request_validation_error_response(exc) + + return calendar_conflict_route_handler + + +router = APIRouter( + prefix="/api/calendar/conflicts", + tags=["calendar"], + route_class=CalendarConflictAPIRoute, +) + + +class CalendarCommitmentPayload(BaseModel): + """One bounded calendar commitment accepted by the decision endpoint.""" + + model_config = ConfigDict(extra="forbid") + + commitment_id: str = Field(min_length=1, max_length=256) + start_at: AwareDatetime + end_at: AwareDatetime + status: CommitmentStatus + + +class CalendarConflictRequest(BaseModel): + """Candidate commitment plus existing evidence used for one decision.""" + + model_config = ConfigDict(extra="forbid") + + proposed: CalendarCommitmentPayload | None = None + existing: list[CalendarCommitmentPayload] = Field( + default_factory=list, + max_length=MAX_EXISTING_COMMITMENTS, + ) + proposed_ics: str | None = Field(default=None, min_length=1, max_length=MAX_PROPOSED_ICS_CHARS) + existing_ics: str | None = Field(default=None, min_length=1, max_length=MAX_EXISTING_ICS_CHARS) + + @model_validator(mode="after") + def require_exactly_one_proposed_source(self) -> Self: + """Accept either a structured proposal or exactly one proposed VEVENT.""" + has_proposed = self.proposed is not None + has_proposed_ics = self.proposed_ics is not None + if has_proposed == has_proposed_ics: + raise ValueError(PROPOSED_SOURCE_REQUIRED_DETAIL) + return self + + +class CalendarConflictEvidence(BaseModel): + """Conflict evidence returned to the customer for explicit resolution.""" + + commitment_id: str + start_at: AwareDatetime + end_at: AwareDatetime + status: CommitmentStatus + + +class CalendarConflictResponse(BaseModel): + """Buyer-visible decision, evidence, policy version, and next action.""" + + decision_code: Literal["available", "blocked", "review_required"] + reason_code: str + conflicts: list[CalendarConflictEvidence] + recommended_action: str + policy_version: str + + +class CalendarConflictErrorResponse(BaseModel): + """Stable machine code plus safe explanation for policy validation failures.""" + + error_code: str + detail: str + + +def _request_validation_error_response(exc: RequestValidationError) -> JSONResponse: + """Map FastAPI request validation onto the existing error_code envelope.""" + messages = [str(error.get("msg", "")) for error in exc.errors()] + if any(PROPOSED_SOURCE_REQUIRED_DETAIL in message for message in messages): + error = CalendarConflictErrorResponse( + error_code="calendar_proposed_source_missing", + detail=PROPOSED_SOURCE_REQUIRED_DETAIL, + ) + else: + error = CalendarConflictErrorResponse( + error_code=REQUEST_INVALID_ERROR_CODE, + detail="Calendar conflict request fields are malformed", + ) + return JSONResponse( + status_code=POLICY_VALIDATION_HTTP_STATUS, + content=error.model_dump(), + ) + + +def _to_commitment(payload: CalendarCommitmentPayload) -> CalendarCommitment: + """Convert a validated transport payload into deterministic policy evidence.""" + return CalendarCommitment( + commitment_id=payload.commitment_id, + start_at=payload.start_at, + end_at=payload.end_at, + status=payload.status, + ) + + +def _to_response(decision: CalendarConflictDecision) -> CalendarConflictResponse: + """Convert the policy decision into the stable public response envelope.""" + return CalendarConflictResponse( + decision_code=decision.decision_code, + reason_code=decision.reason_code, + conflicts=[ + CalendarConflictEvidence( + commitment_id=conflict.commitment_id, + start_at=conflict.start_at, + end_at=conflict.end_at, + status=conflict.status, + ) + for conflict in decision.conflicts + ], + recommended_action=decision.recommended_action, + policy_version=decision.policy_version, + ) + + +@router.post( + "/evaluate", + response_model=CalendarConflictResponse, + responses={POLICY_VALIDATION_HTTP_STATUS: {"model": CalendarConflictErrorResponse}}, +) +def evaluate_calendar_conflict_request( + request: CalendarConflictRequest, +) -> CalendarConflictResponse | JSONResponse: + """Evaluate double-booking risk without mutating any provider calendar.""" + try: + proposed_payload = request.proposed + if request.proposed_ics is not None: + proposed = parse_proposed_calendar_commitment_from_ics(request.proposed_ics) + elif proposed_payload is not None: + proposed = _to_commitment(proposed_payload) + else: + raise CalendarPolicyValidationError( + "calendar_proposed_source_missing", + PROPOSED_SOURCE_REQUIRED_DETAIL, + ) + existing = [_to_commitment(item) for item in request.existing] + if request.existing_ics is not None: + existing.extend(parse_existing_calendar_commitments_from_ics(request.existing_ics)) + if len(existing) > MAX_EXISTING_COMMITMENTS: + raise CalendarPolicyValidationError( + "calendar_existing_batch_exceeded", + "existing evidence exceeds the bounded commitment batch", + ) + except CalendarPolicyValidationError as exc: + error = CalendarConflictErrorResponse( + error_code=exc.error_code, + detail=str(exc), + ) + return JSONResponse( + status_code=POLICY_VALIDATION_HTTP_STATUS, + content=error.model_dump(), + ) + + return _to_response(evaluate_calendar_conflicts(proposed, existing)) diff --git a/backend/api/tools.py b/backend/api/tools.py index 248996af7..bd15abfac 100644 --- a/backend/api/tools.py +++ b/backend/api/tools.py @@ -706,26 +706,6 @@ async def base64_decoder_handler(params: Dict[str, Any]) -> Dict[str, str]: "합니다", } ) -_CATEGORY_TERMS = ( - ("Urgent", ("urgent", "asap", "immediate", "긴급", "시급", "빨리")), - ("Finance", ("invoice", "billing", "payment", "결제", "청구", "송금")), - ("Scheduling", ("meeting", "schedule", "appointment", "회의", "일정", "약속")), -) -_AGENDA_TOPICS = ( - ("Project Status Update", ("project", "프로젝트", "과제")), - ("Discuss Pending Issues", ("issue", "bug", "blocker", "문제", "오류", "장애")), - ("Decisions Required", ("decision", "approve", "결정", "승인")), - ( - "Timeline and Milestones", - ("deadline", "milestone", "timeline", "마감", "기한", "일정"), - ), - ( - "Budget and Resource Review", - ("budget", "cost", "resource", "예산", "비용", "자원"), - ), -) - - def _normalize_analysis_text(value: str) -> str: """Normalize user text for deterministic, multilingual rule matching.""" if len(value) > ANALYSIS_TEXT_MAX_CHARS: @@ -740,44 +720,8 @@ def _analysis_tokens(value: str) -> list[str]: return _ANALYSIS_TOKEN_PATTERN.findall(_normalize_analysis_text(value)) -def _contains_analysis_term(normalized_text: str, term: str) -> bool: - """Match ASCII terms on word boundaries and Korean terms as morpheme stems.""" - normalized_term = _normalize_analysis_text(term) - if normalized_term.isascii(): - pattern = rf"(? Any: - """Categorize email text with deterministic Korean and English rules.""" - content = _normalize_analysis_text(params.get("email_content", "")) - categories = [ - category - for category, terms in _CATEGORY_TERMS - if any(_contains_analysis_term(content, term) for term in terms) - ] - - if not categories: - categories = ["General"] - - return {"categories": categories, "primary_category": categories[0]} - - -registry.register( - ToolInfo( - code="email_categorizer", - name="이메일 자동 분류기 (Email Categorizer)", - description="이메일 내용을 분석하여 알맞은 카테고리로 자동 분류합니다.", - category="이메일 분석", - parameters={"email_content": "string"}, - ), - email_categorizer_handler, -) - - async def keyword_extractor_handler(params: Dict[str, Any]) -> Any: - """Extract stable keywords ranked by frequency and first occurrence.""" + """Extract deterministic lexical terms by frequency and first occurrence.""" candidates = [ token for token in _analysis_tokens(params.get("text", "")) @@ -801,7 +745,7 @@ async def keyword_extractor_handler(params: Dict[str, Any]) -> Any: ToolInfo( code="keyword_extractor", name="주요 키워드 추출기 (Keyword Extractor)", - description="텍스트 본문에서 가장 중요한 키워드를 추출합니다.", + description="텍스트 본문에서 빈도와 최초 출현 순으로 반복 용어를 추출합니다.", category="이메일 분석", parameters={"text": "string"}, ), @@ -809,38 +753,6 @@ async def keyword_extractor_handler(params: Dict[str, Any]) -> Any: ) -async def meeting_agenda_generator_handler(params: Dict[str, Any]) -> Any: - """Generate a deterministic agenda from Korean or English discussion topics.""" - context = _normalize_analysis_text(params.get("discussion_context", "")) - if len(_analysis_tokens(context)) < 2: - return { - "agenda_items": ["Introductions", "Open Discussion"], - "estimated_duration_minutes": 30, - } - - items = ["Review previous action items"] - items.extend( - agenda_item - for agenda_item, terms in _AGENDA_TOPICS - if any(_contains_analysis_term(context, term) for term in terms) - ) - items.append("Next Steps and Action Items") - - return {"agenda_items": items, "estimated_duration_minutes": len(items) * 15} - - -registry.register( - ToolInfo( - code="meeting_agenda_generator", - name="회의 아젠다 생성기 (Meeting Agenda Generator)", - description="논의 컨텍스트를 바탕으로 적절한 회의 아젠다를 자동으로 생성합니다.", - category="일정 관리", - parameters={"discussion_context": "string"}, - ), - meeting_agenda_generator_handler, -) - - async def uuid_v4_generator_handler(params: Dict[str, Any]) -> Dict[str, str]: return {"uuid": str(uuid.uuid4())} @@ -857,6 +769,7 @@ async def uuid_v4_generator_handler(params: Dict[str, Any]) -> Dict[str, str]: ) + @router.get("/tools", response_model=list[ToolInfo]) def get_tools() -> list[ToolInfo]: """ diff --git a/backend/core/local_http.py b/backend/core/local_http.py index a0e7e6691..97aa25575 100644 --- a/backend/core/local_http.py +++ b/backend/core/local_http.py @@ -70,7 +70,11 @@ def validate_loopback_http_origin(value: str) -> LocalHTTPOrigin: safe_hostname = address.compressed try: - port = parsed.port or (443 if parsed.scheme == "https" else 80) + port = ( + parsed.port + if parsed.port is not None + else (443 if parsed.scheme == "https" else 80) + ) except ValueError as exc: raise LocalHTTPValidationError("local HTTP origin port is invalid") from exc if not 1 <= port <= 65535: diff --git a/backend/main.py b/backend/main.py index 0ad7762a8..51b054dbf 100644 --- a/backend/main.py +++ b/backend/main.py @@ -10,6 +10,7 @@ from api.search import router as search_router from api.llm import router as llm_router from api.calendar import router as calendar_router +from api.calendar_conflicts import router as calendar_conflicts_router from api.network import router as network_router from api.emails import router as emails_router from api.runner_config import router as runner_config_router @@ -217,6 +218,7 @@ async def add_security_headers(request: Request, call_next): app.include_router(search_router, dependencies=PRIVATE_API_DEPENDENCIES) app.include_router(llm_router, dependencies=PRIVATE_API_DEPENDENCIES) app.include_router(calendar_router, dependencies=PRIVATE_API_DEPENDENCIES) +app.include_router(calendar_conflicts_router, dependencies=PRIVATE_API_DEPENDENCIES) app.include_router(network_router, dependencies=PRIVATE_API_DEPENDENCIES) app.include_router(emails_router, dependencies=PRIVATE_API_DEPENDENCIES) app.include_router(runner_config_router, dependencies=PRIVATE_API_DEPENDENCIES) diff --git a/backend/services/attachment_parser.py b/backend/services/attachment_parser.py index 868b9b183..7359d6b2f 100644 --- a/backend/services/attachment_parser.py +++ b/backend/services/attachment_parser.py @@ -5,6 +5,7 @@ from dataclasses import dataclass from pathlib import Path from typing import Any +from urllib.parse import unquote from .text_safety import strip_html_markup @@ -16,6 +17,7 @@ } MAX_ATTACHMENT_PARSE_SOURCE_CHARS = 1_000_000 MAX_ATTACHMENT_PARSE_SOURCE_BYTES = 20 * 1024 * 1024 +MAX_ATTACHMENT_FILENAME_DECODE_ROUNDS = 3 @dataclass(frozen=True) @@ -266,8 +268,19 @@ def _parser_key_for(parse_content_type: str, parse_status: str) -> str: def _safe_filename(filename: str | None) -> str: """Return a basename-only attachment display filename.""" - display_filename = strip_html_markup(_sanitize_nul(filename or "attachment")) - display_filename = Path(display_filename).name.strip() + display_filename = filename or "attachment" + for _ in range(MAX_ATTACHMENT_FILENAME_DECODE_ROUNDS): + decoded_filename = unquote(display_filename) + if decoded_filename == display_filename: + break + display_filename = decoded_filename + # Entity-encoded percent escapes (for example ``%2e``) only become + # literal ``%`` sequences during markup decoding, so the residual-encoding + # guard must run after ``strip_html_markup`` to stay fail-closed. + display_filename = strip_html_markup(_sanitize_nul(display_filename)) + if unquote(display_filename) != display_filename: + return "attachment" + display_filename = Path(display_filename.replace("\\", "/")).name.strip() if display_filename in {"", ".", ".."}: return "attachment" return display_filename diff --git a/backend/services/batch_embedding_service.py b/backend/services/batch_embedding_service.py index b9fe02b76..0bd6cc4bb 100644 --- a/backend/services/batch_embedding_service.py +++ b/backend/services/batch_embedding_service.py @@ -32,6 +32,7 @@ from __future__ import annotations import asyncio +import json import logging import uuid from dataclasses import dataclass @@ -61,6 +62,11 @@ # Poll budget while the orchestrator drains the batch through pg-llm-batch. _ORCHESTRATOR_POLL_INTERVAL_SECONDS = 1.0 _ORCHESTRATOR_MAX_POLLS = 30 +# Keep each JSON request below the orchestrator's body budget with envelope +# headroom. Import chunks are normally <= 1,000 characters, but the byte check +# also protects direct callers that provide longer or multibyte inputs. +_ORCHESTRATOR_MAX_INPUTS_PER_REQUEST = 32 +_ORCHESTRATOR_MAX_INPUT_BYTES = 48 * 1024 _SUCCESS_STATUSES = frozenset({"completed", "succeeded"}) # Cache the (im)port result so repeated imports don't re-probe sys.path. @@ -114,6 +120,14 @@ def has_local_fallback(self) -> bool: return bool(self.local_dsn) +@dataclass(frozen=True) +class BatchEmbeddingPartial: + """Completed prefix plus inputs that still need the normal fallback path.""" + + completed_vectors: list[list[float]] + pending_texts: list[str] + + async def resolve_batch_embedding_settings( session: AsyncSession, *, @@ -145,9 +159,7 @@ async def resolve_batch_embedding_settings( attribution_service=_clean( getattr(tenant_config, "batch_attribution_service", None) ), - attribution_team=_clean( - getattr(tenant_config, "batch_attribution_team", None) - ), + attribution_team=_clean(getattr(tenant_config, "batch_attribution_team", None)), attribution_group=_clean( getattr(tenant_config, "batch_attribution_group", None) ), @@ -176,15 +188,16 @@ async def try_batch_import_embeddings( user_id: str, organization_id: str | None, dimension: int = STORAGE_EMBEDDING_DIMENSION, -) -> list[list[float]] | None: +) -> list[list[float]] | BatchEmbeddingPartial | None: """Route bulk embeddings through the batch path, or ``None`` to fall back. On success returns one fitted vector per input text (original order). The primary path submits to contextual-orchestrator; only if the orchestrator is unconfigured or unavailable does it consider the local ``pg-llm-batch`` - package fallback. Any failure returns ``None`` so the caller uses its - per-item path. The run is recorded in ``llm_batch_jobs`` / ``llm_batch_items`` - for observability. + package fallback. A later partition failure returns a completed prefix plus + only the unfinished inputs so the caller does not resend successful work. + The run is recorded in ``llm_batch_jobs`` / ``llm_batch_items`` for + observability. """ if not texts: return None @@ -198,7 +211,7 @@ async def try_batch_import_embeddings( model = settings.model or embedding_provider.embedding_model if settings.has_orchestrator: - result = await _run_orchestrator_batch( + result = await _run_orchestrator_batches( session, texts, settings=settings, @@ -229,6 +242,121 @@ async def try_batch_import_embeddings( # --- Primary path: contextual-orchestrator batch API ------------------------ +def _serialized_orchestrator_payload_bytes( + inputs: list[str], + *, + model: str, + endpoint_alias: str | None, + metadata: dict[str, str], +) -> int: + """Return the UTF-8 size of the request envelope sent to the orchestrator.""" + payload = { + "model": model, + "endpoint": endpoint_alias, + "inputs": inputs, + "metadata": metadata, + } + return len( + json.dumps(payload, ensure_ascii=True, separators=(",", ":")).encode("utf-8") + ) + + +def _partition_orchestrator_inputs( + texts: list[str], + *, + model: str = "", + endpoint_alias: str | None = None, + metadata: dict[str, str] | None = None, +) -> list[list[str]] | None: + """Partition inputs by count and serialized JSON request bytes.""" + request_metadata = metadata or {} + partitions: list[list[str]] = [] + current: list[str] = [] + for text in texts: + candidate = [*current, text] + if current and ( + len(candidate) > _ORCHESTRATOR_MAX_INPUTS_PER_REQUEST + or _serialized_orchestrator_payload_bytes( + candidate, + model=model, + endpoint_alias=endpoint_alias, + metadata=request_metadata, + ) + > _ORCHESTRATOR_MAX_INPUT_BYTES + ): + partitions.append(current) + candidate = [text] + if ( + _serialized_orchestrator_payload_bytes( + candidate, + model=model, + endpoint_alias=endpoint_alias, + metadata=request_metadata, + ) + > _ORCHESTRATOR_MAX_INPUT_BYTES + ): + return None + current = candidate + if current: + partitions.append(current) + return partitions + + +async def _run_orchestrator_batches( + session: AsyncSession, + texts: list[str], + *, + settings: BatchEmbeddingSettings, + model: str, + user_id: str, + organization_id: str | None, + dimension: int, +) -> list[list[float]] | BatchEmbeddingPartial | None: + """Submit bounded requests and concatenate vectors in original order.""" + metadata = _attribution_metadata( + settings=settings, + user_id=user_id, + organization_id=organization_id, + ) + partitions = _partition_orchestrator_inputs( + texts, + model=model, + endpoint_alias=settings.endpoint_alias, + metadata=metadata, + ) + if partitions is None: + logger.warning( + "Orchestrator batch input exceeded one-request byte budget; falling back: " + "text_count=%s", + len(texts), + ) + return None + + vectors: list[list[float]] = [] + for partition_index, partition in enumerate(partitions): + partition_vectors = await _run_orchestrator_batch( + session, + partition, + settings=settings, + model=model, + user_id=user_id, + organization_id=organization_id, + dimension=dimension, + ) + if partition_vectors is None: + if not vectors: + return None + pending_texts = [ + text for remaining in partitions[partition_index:] for text in remaining + ] + return BatchEmbeddingPartial( + completed_vectors=vectors, + pending_texts=pending_texts, + ) + vectors.extend(partition_vectors) + return vectors + + async def _run_orchestrator_batch( session: AsyncSession, texts: list[str], @@ -372,9 +500,7 @@ async def _submit_and_await( if status in _SUCCESS_STATUSES and document.get("embeddings") is not None: return document if status in ("failed", "error", "canceled"): - raise EmbeddingGenerationError( - f"orchestrator batch rejected: status={status}" - ) + raise EmbeddingGenerationError(f"orchestrator batch rejected: status={status}") batch_id = document.get("batch_id") or document.get("id") if not batch_id: diff --git a/backend/services/calendar_conflict_ics.py b/backend/services/calendar_conflict_ics.py new file mode 100644 index 000000000..4f21a5cc3 --- /dev/null +++ b/backend/services/calendar_conflict_ics.py @@ -0,0 +1,220 @@ +"""Parse RFC 5545 VEVENT evidence into status-weighted calendar commitments.""" + +from __future__ import annotations + +import datetime +from typing import Any + +from icalendar import Calendar + +from services.calendar_conflict_policy import ( + CalendarCommitment, + CalendarConflictDecision, + CalendarPolicyValidationError, + CommitmentStatus, + PolicyValidationCode, + evaluate_calendar_conflicts, +) + +_ICS_STATUS_MAP: dict[str, CommitmentStatus] = { + "CONFIRMED": "confirmed", + "TENTATIVE": "tentative", + "CANCELLED": "cancelled", +} +_MAX_EXISTING_ICS_COMMITMENTS = 500 +_MAX_CONVERTED_VEVENTS = _MAX_EXISTING_ICS_COMMITMENTS + 1 +_MAX_ICS_DOCUMENT_BYTES = 262_144 +_RECURRENCE_PROPERTY_NAMES = ("RRULE", "RDATE", "EXDATE") + + +def parse_calendar_commitments_from_ics(ics_text: str) -> tuple[CalendarCommitment, ...]: + """Extract VEVENT commitments from one iCalendar/ICS document. + + RFC 5545 VEVENT ``STATUS`` defaults to ``CONFIRMED`` when omitted. Date-only + and floating date-times are rejected because their absolute instant is + ambiguous. ``DURATION`` is accepted in place of ``DTEND``. + """ + calendar = _parse_calendar(ics_text) + commitments = _commitments_from_calendar(calendar) + if not commitments: + raise CalendarPolicyValidationError( + "calendar_ics_vevent_required", + "iCalendar evidence must include at least one VEVENT", + ) + return commitments + + +def parse_existing_calendar_commitments_from_ics( + ics_text: str, +) -> tuple[CalendarCommitment, ...]: + """Extract zero or more existing VEVENT commitments from one document.""" + return _commitments_from_calendar(_parse_calendar(ics_text)) + + +def parse_proposed_calendar_commitment_from_ics(ics_text: str) -> CalendarCommitment: + """Extract exactly one proposed VEVENT commitment from iCalendar text.""" + proposed_commitments = parse_calendar_commitments_from_ics(ics_text) + if len(proposed_commitments) != 1: + raise CalendarPolicyValidationError( + "calendar_ics_single_vevent_required", + "proposed iCalendar evidence must contain exactly one VEVENT", + ) + return proposed_commitments[0] + + +def evaluate_calendar_conflicts_from_ics( + proposed_ics: str, + existing_ics: str, +) -> CalendarConflictDecision: + """Evaluate one proposed VEVENT against existing VEVENT evidence.""" + proposed_commitment = parse_proposed_calendar_commitment_from_ics(proposed_ics) + existing_commitments = parse_existing_calendar_commitments_from_ics(existing_ics) + if len(existing_commitments) > _MAX_EXISTING_ICS_COMMITMENTS: + raise CalendarPolicyValidationError( + "calendar_existing_batch_exceeded", + "existing iCalendar evidence exceeds the bounded commitment batch", + ) + return evaluate_calendar_conflicts(proposed_commitment, existing_commitments) + + +def _parse_calendar(ics_text: str) -> Calendar: + """Parse iCalendar text without leaking parser internals.""" + if len(ics_text.encode("utf-8")) > _MAX_ICS_DOCUMENT_BYTES: + raise CalendarPolicyValidationError( + "calendar_ics_byte_limit_exceeded", + "iCalendar evidence exceeds the bounded document size", + ) + try: + calendar = Calendar.from_ical(ics_text) + except (ValueError, TypeError, KeyError) as exc: + raise CalendarPolicyValidationError( + "calendar_ics_invalid", + "iCalendar evidence is not a valid VCALENDAR document", + ) from exc + if not isinstance(calendar, Calendar): + raise CalendarPolicyValidationError( + "calendar_ics_invalid", + "iCalendar evidence is not a valid VCALENDAR document", + ) + return calendar + + +def _commitments_from_calendar(calendar: Calendar) -> tuple[CalendarCommitment, ...]: + """Convert VEVENTs until the bounded batch plus one overflow item.""" + commitments: list[CalendarCommitment] = [] + for component in calendar.walk("VEVENT"): + if len(commitments) >= _MAX_CONVERTED_VEVENTS: + break + commitments.append(_commitment_from_vevent(component)) + return tuple(commitments) + + +def _reject_recurrence_properties(component: Any) -> None: + """Fail closed when RRULE, RDATE, or EXDATE would hide later instances.""" + if any(property_name in component for property_name in _RECURRENCE_PROPERTY_NAMES): + raise CalendarPolicyValidationError( + "calendar_ics_recurrence_unsupported", + "iCalendar evidence must not include RRULE, RDATE, or EXDATE", + ) + + +def _commitment_from_vevent(component: Any) -> CalendarCommitment: + """Convert one VEVENT into a timezone-aware policy commitment.""" + _reject_recurrence_properties(component) + commitment_id = _text_property(component, "UID") + if commitment_id is None or not commitment_id.strip(): + raise CalendarPolicyValidationError( + "calendar_ics_uid_required", + "VEVENT evidence must include a non-blank UID", + ) + start_at = _aware_datetime_property(component, "DTSTART", "calendar_ics_dtstart_required") + end_at = _vevent_end_at(component, start_at) + return CalendarCommitment( + commitment_id=commitment_id.strip(), + start_at=start_at, + end_at=end_at, + status=_vevent_status(component), + ) + + +def _vevent_status(component: Any) -> CommitmentStatus: + """Map RFC 5545 VEVENT STATUS, defaulting to confirmed when omitted.""" + raw_status = _text_property(component, "STATUS") + if raw_status is None or not raw_status.strip(): + return "confirmed" + mapped = _ICS_STATUS_MAP.get(raw_status.strip().upper()) + if mapped is None: + raise CalendarPolicyValidationError( + "calendar_status_unsupported", + f"Unsupported commitment status: {raw_status}", + ) + return mapped + + +def _vevent_end_at( + component: Any, + start_at: datetime.datetime, +) -> datetime.datetime: + """Resolve exclusive end from DTEND or DURATION, never both.""" + has_end = "DTEND" in component + has_duration = "DURATION" in component + if has_end and has_duration: + raise CalendarPolicyValidationError( + "calendar_ics_interval_required", + "VEVENT evidence must not include both DTEND and DURATION", + ) + if has_end: + return _aware_datetime_property( + component, + "DTEND", + "calendar_ics_interval_required", + ) + if has_duration: + duration = component.decoded("DURATION") + if not isinstance(duration, datetime.timedelta) or duration <= datetime.timedelta(0): + raise CalendarPolicyValidationError( + "calendar_ics_interval_required", + "VEVENT DURATION must be a positive interval", + ) + return start_at + duration + raise CalendarPolicyValidationError( + "calendar_ics_interval_required", + "VEVENT evidence must include DTEND or DURATION", + ) + + +def _aware_datetime_property( + component: Any, + property_name: str, + missing_error_code: PolicyValidationCode, +) -> datetime.datetime: + """Read a timezone-aware date-time property or fail closed.""" + if property_name not in component: + raise CalendarPolicyValidationError( + missing_error_code, + f"VEVENT evidence must include {property_name}", + ) + value = component.decoded(property_name) + if isinstance(value, datetime.date) and not isinstance(value, datetime.datetime): + raise CalendarPolicyValidationError( + "calendar_ics_datetime_required", + "VEVENT date-times must be DATE-TIME values, not DATE", + ) + if not isinstance(value, datetime.datetime): + raise CalendarPolicyValidationError( + "calendar_ics_datetime_required", + "VEVENT date-times must be DATE-TIME values, not DATE", + ) + return value + + +def _text_property(component: Any, property_name: str) -> str | None: + """Return a decoded iCalendar text property, or None when absent.""" + if property_name not in component: + return None + value = component.decoded(property_name) + if isinstance(value, bytes): + return value.decode("utf-8") + if isinstance(value, str): + return value + return str(value) diff --git a/backend/services/calendar_conflict_policy.py b/backend/services/calendar_conflict_policy.py new file mode 100644 index 000000000..6e1a58546 --- /dev/null +++ b/backend/services/calendar_conflict_policy.py @@ -0,0 +1,223 @@ +"""Deterministic policy for preventing silent calendar double-booking. + +The policy treats event time ranges as half-open intervals (inclusive start, +exclusive end) and ranks occupying Naruon commitment statuses as confirmed > +tentative > desired. RFC 5545 STATUS:CANCELLED is valid evidence and does not +occupy the interval. The occupying rank is a product policy, not an iCalendar +standard requirement. No lower-priority event is mutated or displaced +automatically. +""" + +from __future__ import annotations + +import datetime +from dataclasses import dataclass +from typing import Literal + +CommitmentStatus = Literal["confirmed", "tentative", "desired", "cancelled"] +DecisionCode = Literal["available", "blocked", "review_required"] +PolicyValidationCode = Literal[ + "calendar_commitment_id_required", + "calendar_timestamp_timezone_required", + "calendar_interval_invalid", + "calendar_status_unsupported", + "calendar_ics_invalid", + "calendar_ics_vevent_required", + "calendar_ics_uid_required", + "calendar_ics_dtstart_required", + "calendar_ics_interval_required", + "calendar_ics_datetime_required", + "calendar_ics_single_vevent_required", + "calendar_ics_byte_limit_exceeded", + "calendar_ics_recurrence_unsupported", + "calendar_existing_batch_exceeded", + "calendar_proposed_source_missing", +] + +_STATUS_PRIORITY: dict[str, int] = { + "desired": 1, + "tentative": 2, + "confirmed": 3, +} +_OCCUPYING_STATUSES = frozenset(_STATUS_PRIORITY) +_KNOWN_STATUSES = frozenset((*_STATUS_PRIORITY, "cancelled")) +UTC = datetime.timezone.utc + + +class CalendarPolicyValidationError(ValueError): + """Stable typed validation failure emitted by the calendar policy boundary. + + Attributes: + error_code: Machine-readable code that remains stable when explanatory + wording changes. + """ + + def __init__(self, error_code: PolicyValidationCode, message: str) -> None: + """Create a validation failure with a stable public-facing code.""" + super().__init__(message) + self.error_code = error_code + + +@dataclass(frozen=True, slots=True) +class CalendarCommitment: + """One auditable scheduling commitment considered by the conflict policy. + + Attributes: + commitment_id: Opaque non-blank identifier used to correlate evidence. + start_at: Inclusive timezone-aware start instant. + end_at: Exclusive timezone-aware end instant, strictly after ``start_at``. + status: Naruon commitment priority or RFC 5545 cancelled (non-occupying). + """ + + commitment_id: str + start_at: datetime.datetime + end_at: datetime.datetime + status: CommitmentStatus + + def __post_init__(self) -> None: + """Fail closed when scheduling evidence is ambiguous or unsupported.""" + if not self.commitment_id.strip(): + raise CalendarPolicyValidationError( + "calendar_commitment_id_required", + "commitment_id must be non-blank", + ) + _require_timezone_aware(self.start_at) + _require_timezone_aware(self.end_at) + if _as_utc(self.end_at) <= _as_utc(self.start_at): + raise CalendarPolicyValidationError( + "calendar_interval_invalid", + "end_at must be later than start_at", + ) + if self.status not in _KNOWN_STATUSES: + raise CalendarPolicyValidationError( + "calendar_status_unsupported", + f"Unsupported commitment status: {self.status}", + ) + + +@dataclass(frozen=True, slots=True) +class CalendarConflictDecision: + """Deterministic conflict evidence and the customer's required next action.""" + + decision_code: DecisionCode + reason_code: str + conflicts: tuple[CalendarCommitment, ...] + recommended_action: str + policy_version: str = "status-weighted-v1" + + +def _require_timezone_aware(value: datetime.datetime) -> None: + """Reject local/naive timestamps whose absolute instant is ambiguous.""" + if value.tzinfo is None or value.utcoffset() is None: + raise CalendarPolicyValidationError( + "calendar_timestamp_timezone_required", + "calendar commitment timestamps must be timezone-aware", + ) + + +def _as_utc(value: datetime.datetime) -> datetime.datetime: + """Return an already-validated aware timestamp in absolute UTC time.""" + return value.astimezone(UTC) + + +def occupies_interval(commitment: CalendarCommitment) -> bool: + """Return whether the commitment claims its half-open interval. + + RFC 5545 ``STATUS:CANCELLED`` remains valid scheduling evidence, but the + cancelled VEVENT no longer occupies the slot. Naruon ``desired``, + ``tentative``, and ``confirmed`` commitments do occupy the interval. + """ + return commitment.status in _OCCUPYING_STATUSES + + +def _overlaps( + left: CalendarCommitment, + right: CalendarCommitment, +) -> bool: + """Return whether two half-open event intervals overlap in absolute time.""" + left_start = _as_utc(left.start_at) + left_end = _as_utc(left.end_at) + right_start = _as_utc(right.start_at) + right_end = _as_utc(right.end_at) + return left_start < right_end and right_start < left_end + + +def _conflict_sort_key( + commitment: CalendarCommitment, +) -> tuple[datetime.datetime, str]: + """Sort provider evidence deterministically by UTC instant then opaque ID.""" + return _as_utc(commitment.start_at), commitment.commitment_id + + +def evaluate_calendar_conflicts( + proposed: CalendarCommitment, + existing: list[CalendarCommitment] | tuple[CalendarCommitment, ...], +) -> CalendarConflictDecision: + """Classify a proposed commitment without silently mutating existing events. + + Existing commitments with the same opaque identifier are treated as the + current representation of the proposal rather than as a self-conflict. + Cancelled commitments do not occupy an interval. Equal or higher-priority + occupying overlaps block scheduling. Lower-priority occupying overlaps + require explicit human review instead of automatic displacement. + + Args: + proposed: Candidate commitment being considered for scheduling. + existing: Provider- or database-derived commitments in any order. + + Returns: + A deterministic decision with sorted conflict evidence and a concrete + next action for the customer. + """ + if not occupies_interval(proposed): + return CalendarConflictDecision( + decision_code="available", + reason_code="no_overlapping_commitment", + conflicts=(), + recommended_action="Proceed with scheduling.", + ) + + conflicts = tuple( + sorted( + ( + commitment + for commitment in existing + if commitment.commitment_id != proposed.commitment_id + and occupies_interval(commitment) + and _overlaps(proposed, commitment) + ), + key=_conflict_sort_key, + ) + ) + if not conflicts: + return CalendarConflictDecision( + decision_code="available", + reason_code="no_overlapping_commitment", + conflicts=(), + recommended_action="Proceed with scheduling.", + ) + + proposed_priority = _STATUS_PRIORITY[proposed.status] + if any( + _STATUS_PRIORITY[commitment.status] >= proposed_priority + for commitment in conflicts + ): + return CalendarConflictDecision( + decision_code="blocked", + reason_code="equal_or_higher_priority_conflict", + conflicts=conflicts, + recommended_action=( + "Choose another time or explicitly resolve the equal/higher-priority " + "conflict first." + ), + ) + + return CalendarConflictDecision( + decision_code="review_required", + reason_code="lower_priority_conflict_requires_explicit_resolution", + conflicts=conflicts, + recommended_action=( + "Review and explicitly reschedule or accept the lower-priority conflict " + "before proceeding." + ), + ) diff --git a/backend/services/email_client.py b/backend/services/email_client.py index db17eb77a..8763a41aa 100644 --- a/backend/services/email_client.py +++ b/backend/services/email_client.py @@ -64,6 +64,10 @@ class SmtpConfig: def generate_oauth2_string(user: str, access_token: str) -> bytes: """Generates an OAuth2 string for IMAP/SMTP authentication.""" + if "\x01" in user or "\x01" in access_token: + raise ValueError( + "OAuth2 authentication fields must not contain SASL delimiters" + ) auth_string = f"user={user}\x01auth=Bearer {access_token}\x01\x01" return base64.b64encode(auth_string.encode("utf-8")) diff --git a/backend/services/email_import_service.py b/backend/services/email_import_service.py index 1ff9a2bb3..ddfa350fd 100644 --- a/backend/services/email_import_service.py +++ b/backend/services/email_import_service.py @@ -26,12 +26,16 @@ KnowledgeGraphEdgeRecord, ) from services.archive import extract_backup_async -from services.batch_embedding_service import try_batch_import_embeddings +from services.batch_embedding_service import ( + BatchEmbeddingPartial, + try_batch_import_embeddings, +) from services.content_graph import ParseResult, parse_content from services.email_dedupe_service import strong_email_fingerprint from services.email_parser import EmailData, parse_eml_bytes from services.embedding import ( STORAGE_EMBEDDING_DIMENSION, + chunk_text, fit_embedding_vector, generate_embeddings, ) @@ -52,10 +56,14 @@ EMBEDDING_DIMENSION = STORAGE_EMBEDDING_DIMENSION MAX_IMPORT_UPLOADS = 10 -MAX_IMPORT_UPLOAD_BYTES = 20 * 1024 * 1024 +# Transport safety ceiling only; parser and embedding chunking must accept +# sources larger than 20 MiB without confusing the request guard for a parser +# limit. +MAX_IMPORT_UPLOAD_BYTES = 64 * 1024 * 1024 MAX_IMPORT_EML_FILES = 100 MAX_IMPORT_EMAILS_PER_OWNER = 1000 MAX_UPLOAD_FILENAME_DECODE_ROUNDS = 8 +MAX_EMBEDDING_CHUNKS_PER_WINDOW = 32 SUPPORTED_EMAIL_IMPORT_SUFFIXES = frozenset({".eml", ".mbox", ".zip"}) EMAIL_IMPORT_QUOTA_LOCK_NAMESPACE = "naruon-email-import-quota" logger = logging.getLogger(__name__) @@ -288,15 +296,52 @@ async def _extract_and_generate_embeddings( batch_context: "EmailImportBatchContext | None" = None, ) -> tuple[list[dict], list[list[float]]]: attachment_payloads = list(parsed.get("attachments", [])) - embedding_texts = [str(parsed.get("body") or "")] - embedding_texts.extend( - str(attachment.get("content") or "") for attachment in attachment_payloads - ) - fitted_embeddings = await _generate_import_embeddings( - embedding_texts, - embedding_provider=embedding_provider, - batch_context=batch_context, + body_parse_content = parsed.get("body_parse_content") + source_texts = [ + str( + body_parse_content + if body_parse_content is not None + else parsed.get("body") or "" + ) + ] + source_texts.extend( + str( + "" + if (attachment.get("parse_status") or "parsed") != "parsed" + else ( + attachment.get("parse_content") + if attachment.get("parse_content") is not None + else attachment.get("content") or "" + ) + ) + for attachment in attachment_payloads ) + fitted_embeddings: list[list[float]] = [] + for source_text in source_texts: + source_chunks = chunk_text(source_text) + if not source_chunks: + fitted_embeddings.append(_zero_embedding()) + continue + + vector_sum: list[float] | None = None + vector_count = 0 + for start in range(0, len(source_chunks), MAX_EMBEDDING_CHUNKS_PER_WINDOW): + chunk_embeddings = await _generate_import_embeddings( + source_chunks[start : start + MAX_EMBEDDING_CHUNKS_PER_WINDOW], + embedding_provider=embedding_provider, + batch_context=batch_context, + ) + for embedding in chunk_embeddings: + if vector_sum is None: + vector_sum = [0.0] * len(embedding) + for index, value in enumerate(embedding): + vector_sum[index] += value + vector_count += 1 + fitted_embeddings.append( + [value / vector_count for value in vector_sum] + if vector_sum and vector_count + else _zero_embedding() + ) return attachment_payloads, fitted_embeddings @@ -394,7 +439,11 @@ def _fallback_attachment_parser_key( return "calendar" if parse_content_type == "text/html": return "html" - if parse_content_type in {"text/markdown", "text/x-markdown", "application/markdown"}: + if parse_content_type in { + "text/markdown", + "text/x-markdown", + "application/markdown", + }: return "markdown" if parse_content_type == "text/plain": return "plain_text" @@ -586,9 +635,9 @@ def add_edge( item.segment_path, ), ): - segments_by_source[ - (segment.source_kind, segment.source_record_uid) - ].append(segment) + segments_by_source[(segment.source_kind, segment.source_record_uid)].append( + segment + ) add_edge( edge_kind="node_has_segment", edge_path=f"{segment.content_node.node_path}/has/{segment.segment_path}", @@ -603,8 +652,7 @@ def add_edge( add_edge( edge_kind="segment_next", edge_path=( - f"{source_segment.segment_path}/next/" - f"{target_segment.segment_path}" + f"{source_segment.segment_path}/next/{target_segment.segment_path}" ), source_kind=source_segment.source_kind, source_record_uid=source_segment.source_record_uid, @@ -628,8 +676,7 @@ def add_edge( add_edge( edge_kind="heading_contains_segment", edge_path=( - f"{heading_segment.segment_path}/contains/" - f"{segment.segment_path}" + f"{heading_segment.segment_path}/contains/{segment.segment_path}" ), source_kind=segment.source_kind, source_record_uid=segment.source_record_uid, @@ -905,6 +952,8 @@ async def _generate_import_embeddings( embedding_provider: EmailImportEmbeddingProvider | None, batch_context: "EmailImportBatchContext | None" = None, ) -> list[list[float]]: + if not texts: + return [] if embedding_provider is None: return [_zero_embedding() for _ in texts] if batch_context is not None and texts: @@ -921,6 +970,21 @@ async def _generate_import_embeddings( dimension=EMBEDDING_DIMENSION, ) if batched is not None: + if isinstance(batched, BatchEmbeddingPartial): + remainder: list[list[float]] = [] + for start in range( + 0, len(batched.pending_texts), MAX_EMBEDDING_CHUNKS_PER_WINDOW + ): + remainder.extend( + await _generate_import_embeddings( + batched.pending_texts[ + start : start + MAX_EMBEDDING_CHUNKS_PER_WINDOW + ], + embedding_provider=embedding_provider, + batch_context=None, + ) + ) + return [*batched.completed_vectors, *remainder] return batched try: provider_embeddings = await generate_embeddings( diff --git a/backend/services/embedding.py b/backend/services/embedding.py index 626a44f2e..404e3c49f 100644 --- a/backend/services/embedding.py +++ b/backend/services/embedding.py @@ -36,6 +36,11 @@ def fit_embedding_vector( return embedding[:target_dimension] +def _supports_native_dimensions(model: str) -> bool: + """Return whether the selected OpenAI embedding family accepts dimensions.""" + return model.rsplit("/", 1)[-1].startswith("text-embedding-3-") + + async def generate_embeddings( texts: list[str], openai_api_key: str, @@ -56,13 +61,16 @@ async def generate_embeddings( http_client=http_client, ) + selected_model = model or settings.OPENAI_EMBEDDING_MODEL + request = {"model": selected_model, "input": texts} + if _supports_native_dimensions(selected_model): + request["dimensions"] = STORAGE_EMBEDDING_DIMENSION + try: response = await provider_circuit_breaker.call( validated_base_url or "openai-default", lambda: retry_transient( - lambda: client.embeddings.create( - model=model or settings.OPENAI_EMBEDDING_MODEL, input=texts - ), + lambda: client.embeddings.create(**request), operation_name="embedding generation", ), ) diff --git a/backend/services/text_safety.py b/backend/services/text_safety.py index d468a7d2f..451b8ca29 100644 --- a/backend/services/text_safety.py +++ b/backend/services/text_safety.py @@ -452,16 +452,20 @@ def strip_html_markup(value: str) -> str: decoded = _decode_entities(value) masked, placeholders = _mask_angle_emails(decoded) + # HTMLParser can expose the tail of the malformed ```` opener as + # literal data. Normalize that opener into an ignored comment boundary + # without deleting legitimate ``-->`` text elsewhere in user content. + masked = masked.replace("", " as text" + + @pytest.mark.parametrize( "safe_text", [ diff --git a/backend/tests/test_tools_api.py b/backend/tests/test_tools_api.py index 8af3435e3..8e537cef7 100644 --- a/backend/tests/test_tools_api.py +++ b/backend/tests/test_tools_api.py @@ -112,6 +112,21 @@ def test_get_tool_not_found(): assert response.json() == {"detail": "Tool not found"} +@pytest.mark.parametrize( + "tool_code", ["email_categorizer", "meeting_agenda_generator"] +) +def test_registry_omits_lexical_pseudo_topic_tools(tool_code): + assert registry.get(tool_code) is None + + +def test_keyword_extractor_is_disclosed_as_lexical_term_frequency(): + tool = registry.get("keyword_extractor") + assert tool is not None + assert tool.description == ( + "텍스트 본문에서 빈도와 최초 출현 순으로 반복 용어를 추출합니다." + ) + + @pytest.mark.asyncio async def test_execute_tool_success(): with TestClient(app) as client: @@ -1155,52 +1170,6 @@ def test_detect_text_language_ko(): assert _detect_text_language("안녕하세요") == "ko" -@pytest.mark.asyncio -async def test_email_categorizer_handler(): - from api.tools import email_categorizer_handler - - # Test Finance category - result = await email_categorizer_handler( - {"email_content": "Please pay this invoice soon."} - ) - assert "Finance" in result["categories"] - - # Test Scheduling category - result = await email_categorizer_handler( - {"email_content": "Let's schedule a meeting."} - ) - assert "Scheduling" in result["categories"] - - # Test Urgent category - result = await email_categorizer_handler({"email_content": "This is urgent!"}) - assert "Urgent" in result["categories"] - - # Test General category (fallback) - result = await email_categorizer_handler({"email_content": "Hello, how are you?"}) - assert "General" in result["categories"] - - # Test multiple categories - result = await email_categorizer_handler( - {"email_content": "URGENT: Meeting to discuss invoice payment"} - ) - assert result == { - "categories": ["Urgent", "Finance", "Scheduling"], - "primary_category": "Urgent", - } - - # ASCII category rules use token boundaries instead of substring matching. - result = await email_categorizer_handler( - {"email_content": "The prepayment plan is documented."} - ) - assert result["categories"] == ["General"] - - # Unicode compatibility forms and Korean stems remain matchable. - result = await email_categorizer_handler( - {"email_content": "긴급 회의에서 청구 금액을 검토합니다."} - ) - assert result["categories"] == ["Urgent", "Finance", "Scheduling"] - - @pytest.mark.asyncio async def test_keyword_extractor_handler(): from api.tools import keyword_extractor_handler @@ -1224,41 +1193,6 @@ async def test_keyword_extractor_handler(): assert empty == {"keywords": [], "keyword_count": 0} -@pytest.mark.asyncio -async def test_meeting_agenda_generator_handler(): - from api.tools import meeting_agenda_generator_handler - - # Test with short context - result = await meeting_agenda_generator_handler({"discussion_context": "short"}) - assert result["agenda_items"] == ["Introductions", "Open Discussion"] - assert result["estimated_duration_minutes"] == 30 - - # Test with project and issue context - result = await meeting_agenda_generator_handler( - {"discussion_context": "The project has an issue that needs fixing."} - ) - assert "Review previous action items" in result["agenda_items"] - assert "Project Status Update" in result["agenda_items"] - assert "Discuss Pending Issues" in result["agenda_items"] - assert "Next Steps and Action Items" in result["agenda_items"] - assert result["estimated_duration_minutes"] == len(result["agenda_items"]) * 15 - - # Korean context covers decision, timeline, and resource agenda paths. - result = await meeting_agenda_generator_handler( - {"discussion_context": "프로젝트 예산 승인과 마감 일정 문제를 결정합니다."} - ) - assert result["agenda_items"] == [ - "Review previous action items", - "Project Status Update", - "Discuss Pending Issues", - "Decisions Required", - "Timeline and Milestones", - "Budget and Resource Review", - "Next Steps and Action Items", - ] - assert result["estimated_duration_minutes"] == 105 - - def test_execute_analysis_tool_rejects_oversized_text(): from api.tools import ANALYSIS_TEXT_MAX_CHARS diff --git a/backend/tests/test_topic_intelligence_documentation.py b/backend/tests/test_topic_intelligence_documentation.py new file mode 100644 index 000000000..f1f930a87 --- /dev/null +++ b/backend/tests/test_topic_intelligence_documentation.py @@ -0,0 +1,256 @@ +"""Machine-check the topic-intelligence documentation authority graph.""" + +from __future__ import annotations + +import json +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +DOC_ROOT = REPO_ROOT / "docs" / "topic-intelligence" +SCHEMA_PATH = DOC_ROOT / "schema" / "topic-inference-result-v1.schema.json" + +REQUIRED_DOCUMENTS = ( + "README.md", + "PRD.md", + "TRD.md", + "ARCHITECTURE.md", + "UML.md", + "DATA_MODEL.md", + "API_CONTRACT.md", + "SECURITY.md", + "THREAT_MODEL.md", + "TEST_STRATEGY.md", + "OPERABILITY.md", + "TRACEABILITY.md", + "DOCUMENTATION_FITNESS.md", + "REFERENCES.md", +) + + +def _read(path: Path) -> str: + return path.read_text(encoding="utf-8") + + +def test_topic_intelligence_package_is_complete_and_indexed() -> None: + index = _read(DOC_ROOT / "README.md") + + for filename in REQUIRED_DOCUMENTS: + assert (DOC_ROOT / filename).is_file() + if filename != "README.md": + assert f"({filename})" in index + + assert SCHEMA_PATH.is_file() + assert "(schema/topic-inference-result-v1.schema.json)" in _read( + DOC_ROOT / "API_CONTRACT.md" + ) + + +def test_topic_intelligence_package_is_discoverable_from_root_docs() -> None: + for path in ( + REPO_ROOT / "README.md", + REPO_ROOT / "ARCHITECTURE.md", + REPO_ROOT / "CLAUDE.md", + ): + assert "docs/topic-intelligence/" in _read(path) + + +def test_maturity_vocabulary_separates_runtime_truth_from_design() -> None: + index = _read(DOC_ROOT / "README.md") + + for status in ( + "IMPLEMENTED-ON-PROTECTED-DEVELOP", + "ACTIVE-PR", + "ACCEPTED-NARUON-POLICY", + "PLANNED", + "BLOCKED-UPSTREAM", + ): + assert status in index + + assert "not evidence that STM is available in Naruon" in " ".join(index.split()) + + +def test_platform_plan_does_not_claim_live_stm_signals() -> None: + plan = " ".join( + _read(REPO_ROOT / "docs" / "planning" / "naruon-platform-plan.md").split() + ) + + assert "structured topic modeling (STM) feeds search" not in plan + assert "account, STM topic, past patterns" not in plan + assert "PLANNED, not LIVE" in plan + assert "keyword_extractor` is never topic evidence" in plan + + +def test_contract_separates_errors_from_scientific_abstention() -> None: + contract = _read(DOC_ROOT / "API_CONTRACT.md") + normalized_contract = " ".join(contract.split()) + + for status_code in ("`409`", "`422`", "`502`", "`503`"): + assert status_code in contract + assert "`error_code` is a required Naruon extension" in contract + assert "`status=abstained`" in contract + assert "must never return HTTP `200` or `status=abstained`" in normalized_contract + + +def test_uml_and_erd_are_conceptual_and_fail_closed() -> None: + uml = _read(DOC_ROOT / "UML.md") + data_model = _read(DOC_ROOT / "DATA_MODEL.md") + + assert uml.count("```mermaid") >= 4 + assert "no fallback transition" in uml + assert "**Persistence status:** `NOT-APPLICABLE`" in data_model + assert "no Alembic migration is authorized" in data_model + assert data_model.count("```mermaid") >= 3 + + +def test_conceptual_erd_uses_scoped_immutable_identities() -> None: + data_model = _read(DOC_ROOT / "DATA_MODEL.md") + agents = " ".join(_read(REPO_ROOT / "AGENTS.md").split()) + + for scoped_reference in ( + "snapshot_ref PK", + "model_artifact_ref PK", + "component_ref PK", + "label_evidence_ref PK", + ): + assert scoped_reference in data_model + + for unscoped_identity in ( + "string document_ref PK", + "string model_id PK", + "int topic_id PK", + "int topic_id FK", + ): + assert unscoped_identity not in data_model + + assert "must not mark a reusable business identifier" in agents + assert "as an unscoped primary or foreign key" in agents + + +def test_digest_contract_defines_one_schema_digest_and_raw_byte_boundary() -> None: + contract = " ".join(_read(DOC_ROOT / "API_CONTRACT.md").split()) + index = " ".join(_read(DOC_ROOT / "README.md").split()) + + assert "naruon.topic-inference.schema.v1" in contract + assert ( + "the complete parsed JSON value of the immutable schema resource " + "identified by the pinned `$id`" + ) in contract + assert "exactly 14 canonical digest fields" in contract + assert "artifact_digest` binds the fitted-artifact **descriptor**" in contract + assert "do not by themselves verify descriptor truth or completeness" in index + assert "does not add a canonical digest field to this inventory" in index + + +def test_security_treats_every_derived_digest_as_sensitive() -> None: + security = " ".join(_read(DOC_ROOT / "SECURITY.md").split()) + + assert ( + "every content-, evidence-, covariate-, membership-, temporal-, design-, " + "or label-derived digest. Such digests are sensitive pseudonymous linkage " + "values" + ) in security + assert "sensitive pseudonymous linkage values" in security + assert "ecological-fallacy" in security + + +def test_planned_schema_has_closed_revision_and_ownership_metadata() -> None: + schema = json.loads(_read(SCHEMA_PATH)) + + assert schema["$schema"] == "https://json-schema.org/draft/2020-12/schema" + assert "2026-08-09.1" in schema["$id"] + assert schema["x-owner"] == "NARUON" + assert "x-upstream-owner" not in schema + assert schema["x-expected-upstream-producer"] == "TEPP" + assert schema["x-runtime-status"] == "NOT_IMPLEMENTED" + assert schema["x-schema-digest-required"] is True + assert schema["additionalProperties"] is False + assert schema["properties"]["status"]["enum"] == ["inferred", "abstained"] + + +def test_every_typed_schema_object_is_closed() -> None: + schema = json.loads(_read(SCHEMA_PATH)) + + def visit(value: object, location: str) -> None: + if isinstance(value, dict): + if value.get("type") == "object": + assert value.get("additionalProperties") is False, location + for key, child in value.items(): + visit(child, f"{location}/{key}") + elif isinstance(value, list): + for index, child in enumerate(value): + visit(child, f"{location}/{index}") + + visit(schema, "#") + + +def test_schema_requires_input_and_numerical_diagnostics() -> None: + schema = json.loads(_read(SCHEMA_PATH)) + definitions = schema["$defs"] + + input_required = set(definitions["inputDiagnostics"]["required"]) + assert {"retained_token_count", "out_of_vocabulary_ratio"} <= input_required + + posterior_required = set(definitions["posteriorDiagnostics"]["required"]) + assert { + "convergence_code", + "numerical_status", + "quality_codes", + } <= posterior_required + + +def test_schema_declares_required_runtime_cross_field_validation() -> None: + schema = json.loads(_read(SCHEMA_PATH)) + invariants = " ".join(schema["x-runtime-invariants"]) + + for requirement in ( + "fitted_topic_count", + "observed_topic_count", + "number of topic_components", + "snapshot_revision", + "scope_binding_ref", + "availability_time is at or before knowledge_cutoff_time", + "unknown registry version or code is an upstream protocol error", + ): + assert requirement in invariants + + +def test_public_contract_preserves_semantics_and_fail_closed_errors() -> None: + contract = _read(DOC_ROOT / "API_CONTRACT.md") + + for semantic_field in ( + '"model_id"', + '"model_version"', + '"analysis_unit"', + '"estimand_id"', + '"causal_design"', + '"covariate_level"', + ): + assert semantic_field in contract + + for error_code in ( + "topic_authentication_required", + "topic_evidence_forbidden", + "topic_rate_limited", + "topic_upstream_timeout", + "topic_upstream_protocol_error", + ): + assert error_code in contract + + +def test_traceability_covers_every_product_requirement() -> None: + prd = _read(DOC_ROOT / "PRD.md") + traceability = _read(DOC_ROOT / "TRACEABILITY.md") + + for number in range(1, 11): + requirement_id = f"TI-REQ-{number:03d}" + assert requirement_id in prd + assert requirement_id in traceability + + +def test_references_pin_the_inspected_tepp_evidence() -> None: + references = _read(DOC_ROOT / "REFERENCES.md") + + assert "b8e26aae334397daa1974d4a24c9015cfd682600" in references + assert "2026-08-06T11:33:18+09:00" in references + assert "There is no corresponding" in references + assert "production topic-measurement crate or endpoint" in references diff --git a/backend/tests/test_url_validation.py b/backend/tests/test_url_validation.py index 2857f61a2..05c75ed43 100644 --- a/backend/tests/test_url_validation.py +++ b/backend/tests/test_url_validation.py @@ -5,6 +5,7 @@ from core.url_validation import ( parse_allowed_hosts, validate_https_url_host, + validate_same_or_subdomain_host, validate_https_url_host_details, _normalize_host, _reject_unsafe_ip_literal, @@ -12,6 +13,7 @@ _resolve_global_addresses, ) + def test_parse_allowed_hosts(): assert parse_allowed_hosts("example.com, TEST.COM. , [2001:db8::1]") == frozenset( {"example.com", "test.com", "2001:db8::1"} @@ -22,11 +24,13 @@ def test_parse_allowed_hosts(): {"example.com", "example.net"} ) + def test_normalize_host(): assert _normalize_host(" Example.COM. ") == "example.com" assert _normalize_host("[2001:db8::1]") == "2001:db8::1" assert _normalize_host("test") == "test" + def test_reject_unsafe_ip_literal(): # Safe global IP _reject_unsafe_ip_literal("setting", "8.8.8.8") @@ -38,69 +42,109 @@ def test_reject_unsafe_ip_literal(): with pytest.raises(ValueError, match="setting IP host must be globally routable"): _reject_unsafe_ip_literal("setting", "::1") - with pytest.raises(ValueError, match="setting host must not be a local or internal domain"): + with pytest.raises( + ValueError, match="setting host must not be a local or internal domain" + ): _reject_unsafe_ip_literal("setting", "localhost") - with pytest.raises(ValueError, match="setting host must not be a local or internal domain"): + with pytest.raises( + ValueError, match="setting host must not be a local or internal domain" + ): _reject_unsafe_ip_literal("setting", "test.localhost") - with pytest.raises(ValueError, match="setting host must not be a local or internal domain"): + with pytest.raises( + ValueError, match="setting host must not be a local or internal domain" + ): _reject_unsafe_ip_literal("setting", "internal") - with pytest.raises(ValueError, match="setting host must not be a local or internal domain"): + with pytest.raises( + ValueError, match="setting host must not be a local or internal domain" + ): _reject_unsafe_ip_literal("setting", "test.internal") - with pytest.raises(ValueError, match="setting host must not be a local or internal domain"): + with pytest.raises( + ValueError, match="setting host must not be a local or internal domain" + ): _reject_unsafe_ip_literal("setting", "test.local") # Standard domain name _reject_unsafe_ip_literal("setting", "example.com") + def test_validate_global_address(): assert _validate_global_address("setting", "8.8.8.8") == "8.8.8.8" - assert _validate_global_address("setting", "2001:4860:4860::8888") == "2001:4860:4860::8888" + assert ( + _validate_global_address("setting", "2001:4860:4860::8888") + == "2001:4860:4860::8888" + ) - with pytest.raises(ValueError, match="setting resolved IP host must be globally routable"): + with pytest.raises( + ValueError, match="setting resolved IP host must be globally routable" + ): _validate_global_address("setting", "127.0.0.1") - with pytest.raises(ValueError, match="setting resolved IP host must be globally routable"): + with pytest.raises( + ValueError, match="setting resolved IP host must be globally routable" + ): _validate_global_address("setting", "invalid-ip") + @patch("socket.getaddrinfo") def test_resolve_global_addresses(mock_getaddrinfo): mock_getaddrinfo.return_value = [ (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("8.8.8.8", 443)), (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("8.8.4.4", 443)), - (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("8.8.8.8", 443)), # duplicate - (socket.AF_INET6, socket.SOCK_STREAM, 6, "", ("2001:4860:4860::8888", 443, 0, 0)), + (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("8.8.8.8", 443)), # duplicate + ( + socket.AF_INET6, + socket.SOCK_STREAM, + 6, + "", + ("2001:4860:4860::8888", 443, 0, 0), + ), ] addresses = _resolve_global_addresses("setting", "example.com", 443) assert addresses == ("8.8.8.8", "8.8.4.4", "2001:4860:4860::8888") - mock_getaddrinfo.assert_called_once_with("example.com", 443, type=socket.SOCK_STREAM) + mock_getaddrinfo.assert_called_once_with( + "example.com", 443, type=socket.SOCK_STREAM + ) + @patch("socket.getaddrinfo") def test_resolve_global_addresses_gaierror(mock_getaddrinfo): mock_getaddrinfo.side_effect = socket.gaierror("Name or service not known") - with pytest.raises(ValueError, match="setting host must resolve to a global address"): + with pytest.raises( + ValueError, match="setting host must resolve to a global address" + ): _resolve_global_addresses("setting", "example.com", 443) + @patch("socket.getaddrinfo") def test_resolve_global_addresses_no_global(mock_getaddrinfo): mock_getaddrinfo.return_value = [ (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", 443)), ] - with pytest.raises(ValueError, match="setting resolved IP host must be globally routable"): + with pytest.raises( + ValueError, match="setting resolved IP host must be globally routable" + ): _resolve_global_addresses("setting", "example.com", 443) + @patch("socket.getaddrinfo") def test_resolve_global_addresses_empty(mock_getaddrinfo): mock_getaddrinfo.return_value = [] - with pytest.raises(ValueError, match="setting host must resolve to a global address"): + with pytest.raises( + ValueError, match="setting host must resolve to a global address" + ): _resolve_global_addresses("setting", "example.com", 443) + @patch("core.url_validation._resolve_global_addresses") def test_validate_https_url_host_details(mock_resolve): mock_resolve.return_value = ("8.8.8.8",) # Success res = validate_https_url_host_details( - "setting", "https://example.com/path", frozenset({"example.com"}), "ALLOWED_HOSTS" + "setting", + "https://example.com/path", + frozenset({"example.com"}), + "ALLOWED_HOSTS", ) assert res.normalized_url == "https://example.com/path" assert res.hostname == "example.com" @@ -109,7 +153,10 @@ def test_validate_https_url_host_details(mock_resolve): # Success with port res2 = validate_https_url_host_details( - "setting", "https://example.com:8443/path", frozenset({"example.com"}), "ALLOWED_HOSTS" + "setting", + "https://example.com:8443/path", + frozenset({"example.com"}), + "ALLOWED_HOSTS", ) assert res2.normalized_url == "https://example.com:8443/path" assert res2.hostname == "example.com" @@ -119,19 +166,28 @@ def test_validate_https_url_host_details(mock_resolve): # Not https with pytest.raises(ValueError, match="setting must use https"): validate_https_url_host_details( - "setting", "http://example.com/path", frozenset({"example.com"}), "ALLOWED_HOSTS" + "setting", + "http://example.com/path", + frozenset({"example.com"}), + "ALLOWED_HOSTS", ) # Userinfo with pytest.raises(ValueError, match="setting must not include userinfo"): validate_https_url_host_details( - "setting", "https://user:pass@example.com/path", frozenset({"example.com"}), "ALLOWED_HOSTS" + "setting", + "https://user:pass@example.com/path", + frozenset({"example.com"}), + "ALLOWED_HOSTS", ) # Fragment with pytest.raises(ValueError, match="setting must not include a fragment"): validate_https_url_host_details( - "setting", "https://example.com/path#frag", frozenset({"example.com"}), "ALLOWED_HOSTS" + "setting", + "https://example.com/path#frag", + frozenset({"example.com"}), + "ALLOWED_HOSTS", ) # No host @@ -141,12 +197,47 @@ def test_validate_https_url_host_details(mock_resolve): ) # Host not in allowed - with pytest.raises(ValueError, match="setting host must be listed in ALLOWED_HOSTS"): + with pytest.raises( + ValueError, match="setting host must be listed in ALLOWED_HOSTS" + ): validate_https_url_host_details( - "setting", "https://bad.com/path", frozenset({"example.com"}), "ALLOWED_HOSTS" + "setting", + "https://bad.com/path", + frozenset({"example.com"}), + "ALLOWED_HOSTS", ) + @patch("core.url_validation.validate_https_url_host_details") def test_validate_https_url_host(mock_details): - validate_https_url_host("setting", "https://example.com", frozenset({"example.com"}), "ALLOWED_HOSTS") - mock_details.assert_called_once_with("setting", "https://example.com", frozenset({"example.com"}), "ALLOWED_HOSTS") + validate_https_url_host( + "setting", "https://example.com", frozenset({"example.com"}), "ALLOWED_HOSTS" + ) + mock_details.assert_called_once_with( + "setting", "https://example.com", frozenset({"example.com"}), "ALLOWED_HOSTS" + ) + + +def test_validate_same_or_subdomain_host_rejects_suffix_confusion(): + for valid_host in ( + "issuer.example.com", + "jwks.issuer.example.com", + "a.b.c.issuer.example.com", + ): + validate_same_or_subdomain_host( + "OIDC_JWKS_URL", valid_host, "OIDC_ISSUER_URL", "issuer.example.com" + ) + + for invalid_host in ( + "other.com", + "fakeissuer.example.com", + "issuer.example.com.attacker.com", + "notexample.com", + ): + with pytest.raises( + ValueError, + match="OIDC_JWKS_URL host must match or be a subdomain of OIDC_ISSUER_URL host", + ): + validate_same_or_subdomain_host( + "OIDC_JWKS_URL", invalid_host, "OIDC_ISSUER_URL", "issuer.example.com" + ) diff --git a/connector/Dockerfile b/connector/Dockerfile index db7e95e7e..fa45883d0 100644 --- a/connector/Dockerfile +++ b/connector/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.14-slim@sha256:cea0e6040540fb2b965b6e7fb5ffa00871e632eef63719f0ea54bca189ce14a6 +FROM python:3.14-slim@sha256:a7fb1e634c4a578f9e0bd6327f11a3cde11b7a9395f48e24360c0988bcc5c2bc WORKDIR /app ENV PYTHONDONTWRITEBYTECODE=1 diff --git a/docs/adr/0001-topic-measurement-authority.md b/docs/adr/0001-topic-measurement-authority.md new file mode 100644 index 000000000..e076ebe72 --- /dev/null +++ b/docs/adr/0001-topic-measurement-authority.md @@ -0,0 +1,77 @@ +# ADR-0001: Naruon-local policy for consuming structural topic measurement + +**Status:** Accepted (Naruon-local consumption policy) +**Date:** 2026-08-09 +**Decision owner:** Naruon maintainers +**Scope:** Naruon's product behavior and any future Naruon adapter. This ADR does not transfer product or scientific authority to TEPP, govern TEPP, or record TEPP's acceptance of a Naruon contract. + +**Related records:** the complete documentation graph is indexed in +[`docs/topic-intelligence/README.md`](../topic-intelligence/README.md). Proposed +implementation decisions are split into [ADR-0002](0002-fitted-topic-artifact-consumption.md) +and [ADR-0003](0003-separate-topic-measurement-from-agenda-generation.md). + +## Upstream direction evidence + +[TEPP's protected-`main` architecture at commit `b8e26aae334397daa1974d4a24c9015cfd682600`](https://github.com/ContextualWisdomLab/TEPP/blob/b8e26aae334397daa1974d4a24c9015cfd682600/ARCHITECTURE.md#bounded-services-and-rust-crates) lists `topic_measurement` and states that its boundaries expose versioned integration contracts. This is direction evidence for Naruon's future-consumption policy only. It is not TEPP's acceptance of this ADR, not a transfer of authority, and not evidence of a production API or contract. + +## Context + +Naruon historically exposed `email_categorizer` and `meeting_agenda_generator` from small hard-coded Korean/English term tables. Those outputs were deterministic, but they were lexical rules presented through product names that implied semantic topic inference. That is not a Structural Topic Model and provides no fitted corpus-level topic identity, mixed-membership posterior, uncertainty, prevalence/content covariate effect, multilingual measurement evidence, or model-artifact provenance. + +The upstream architecture is compatible with a future fitted-model integration, but Naruon has no independently published TEPP production artifact/API/contract to consume today. This ADR therefore makes a local product-truth decision: Naruon will not present lexical, embedding, zero-shot, or LLM output as Structural Topic Modeling (STM), and it will fail closed until a separately accepted upstream contract is available. + +## Decision + +1. Naruon's retained `keyword_extractor` remains explicitly lexical metadata only. It must never be described as a topic model or semantic classifier. +2. Naruon will not replace removed pseudo-topic tools with a larger keyword table, embedding cluster, zero-shot labeler, or LLM prompt while naming the result Structural Topic Modeling. +3. A Naruon adapter remains blocked until TEPP independently publishes a versioned production fitted-model artifact, API, or contract and its own acceptance evidence. If Naruon later chooses to consume that published contract, it must use a stable typed integration boundary and must not refit an STM per request. +4. Naruon's acceptance criteria for any future consumed inference contract include: model artifact/version and digest; immutable source/document identity; frozen preprocessing and vocabulary; OOV/retained-token diagnostics; language profile/support status; relevant prevalence/content and multilevel/cross-classified/multiple-membership covariates; event/document/availability/knowledge-cutoff time semantics when the model uses them; mixed-membership topic proportions; posterior uncertainty/diagnostics; and explicit abstention/failure status. +5. Human-readable topic labels and generated agenda/action summaries are presentation/generation artifacts. They are never the numeric topic identity and cannot change the fitted posterior. +6. If a required published model/API/artifact is unavailable, incompatible, under-supported for the document language, or cannot produce an evidence-valid posterior, Naruon fails closed. It does not fabricate `General`, empty agenda semantics, or an embedding/LLM substitute under the same contract. +7. Naruon remains useful without topic inference. Any future integration is optional and versioned; Naruon must not read an upstream service's private database directly. This ADR imposes no obligations on TEPP. + +## Alternatives rejected + +### Keep deterministic keyword categories + +Rejected because deterministic lexical matching is not mixed-membership topic measurement and would preserve the original product-truth defect. + +### Use embeddings or clustering as a drop-in STM replacement + +Rejected as a semantic product substitution. Such methods may be useful in separate features, but equal semantic usefulness does not make them an STM posterior or preserve the same prevalence/content/uncertainty contract. + +### Ask an LLM for topic labels at request time + +Rejected as the statistical authority. LLMs may interpret or label fitted evidence behind a separate bounded contract, but request-time labels do not replace a fitted corpus-level model and its uncertainty. + +### Fit a fresh topic model for every Naruon request + +Rejected because new-document inference must be comparable against a stable fitted model. Per-request refits destroy topic identity, reproducibility, governance, and longitudinal comparability. + +## Consequences + +- PR #1297 removes the misleading pseudo-topic tools rather than shipping an unvalidated replacement. +- A Naruon adapter cannot be proposed until TEPP independently publishes a versioned production artifact/API/contract and its own acceptance evidence. +- Naruon tests must keep lexical utilities labelled lexical and must fail if removed pseudo-topic registry entries reappear without a locally accepted replacement contract. +- Any future adapter must carry model/provenance/uncertainty/diagnostic fields rather than only a label string. +- Product documentation must distinguish `implemented on protected develop`, + `active PR`, `accepted Naruon-local policy`, `proposed target`, and + `blocked-upstream`; this ADR neither claims that TEPP topic inference exists + today nor that TEPP accepted Naruon's consumption policy. + +## Naruon adapter acceptance criteria + +A future Naruon topic-measurement adapter remains blocked unless TEPP independently publishes a versioned production artifact/API/contract and its own acceptance evidence. Once that upstream precondition exists, Naruon may evaluate an adapter against these local criteria before promoting it to protected `develop`: + +- a published TEPP production artifact/inference API at a versioned contract, plus TEPP's own acceptance evidence; +- fitted-model and preprocessing/vocabulary identity validation; +- positive, negative, OOV/insufficient-text, unsupported-language and model-unavailable tests; +- posterior normalization and uncertainty/diagnostic tests; +- multilevel/multiple-membership and temporal-covariate contract tests when those inputs are part of the fitted model; +- tenant/source authorization at the Naruon boundary; +- exact-head CI/security/coverage and independent review; +- no claim that topic labels or LLM interpretations are the numeric topic identity. + +## Supersession rule + +Changing this Naruon-local consumption policy, changing new-document topic identity semantics, or authorizing Naruon to fit its own production topic models requires a superseding Naruon ADR plus synchronized product/technical/architecture/test/operability documentation and scientific validation evidence. diff --git a/docs/adr/0002-fitted-topic-artifact-consumption.md b/docs/adr/0002-fitted-topic-artifact-consumption.md new file mode 100644 index 000000000..76460ede9 --- /dev/null +++ b/docs/adr/0002-fitted-topic-artifact-consumption.md @@ -0,0 +1,81 @@ +# ADR-0002: Consume only a versioned fitted topic artifact + +**Status:** Proposed + +**Date:** 2026-08-09 + +**Decision owner:** Naruon maintainers + +**Capability maturity:** target `PLANNED`; runtime `BLOCKED-UPSTREAM` + +**Scope:** a possible future Naruon consumption decision only. This ADR does not +assign an external scientific owner, impose obligations on TEPP or another +publisher, or record upstream acceptance. + +**Trigger for acceptance:** an upstream publisher independently releases a +versioned production inference contract and its own acceptance evidence, and +Naruon approves that exact contract in an implementing PR. + +**Related requirements:** [TI-REQ-003, TI-REQ-004, and +TI-REQ-006](../topic-intelligence/PRD.md#product-requirements) + +## Context + +New-document structural topic inference is meaningful only relative to a stable +fitted corpus-level model. A request-time refit, an embedding cluster, a keyword +table, or an LLM label cannot preserve topic identity, covariate design, +uncertainty, or longitudinal comparability. Naruon currently has no production +topic endpoint and no fitted topic artifact to consume. + +## Proposed decision + +Naruon will add no topic adapter until an independently published contract can +bind all of the following in one result: + +- every exact field in the [canonical 14-field digest + inventory](../topic-intelligence/README.md#canonical-digest-inventory), including + the model-card, validation-report, evidence-time-manifest, + covariate-snapshot, and design-row digests; +- immutable source/snapshot, model, artifact, contract, preprocessing, + vocabulary, design, lineage, model-card, and validation-report identity; +- explicit language support, retained-token count, OOV rate, covariate design, + temporal semantics, multilevel and multiple-membership inputs when fitted; +- a mixed-membership topic vector, conditional posterior uncertainty, + convergence/numerical diagnostics, and stable quality codes; and +- an explicit `inferred` or scientifically `abstained` outcome, never a + fabricated default topic. + +This proposed decision assigns only Naruon responsibilities: tenant +authorization, input bounds, disclosure policy, request and response envelopes, +transport resilience, compatibility validation, activation, and error mapping. +Naruon would consume only scientific evidence accompanied by the publisher's +independently issued acceptance evidence; this ADR neither determines who holds +external scientific authority nor delegates Naruon's compatibility decision. +Naruon must not read an upstream private database or refit the model per request. + +Preflight incompatibility is an error: invalid language, insufficient retained +tokens, excessive OOV, missing fitted covariates, or an incompatible design row +returns a stable `422` problem. No active compatible deployment is `503`. +Request/version conflicts are `409`. Only a compatible active model's posterior, +diagnostic, or policy rejection may return HTTP `200` with `status=abstained`. + +## Consequences + +- The proposed schema and HTTP contract are design artifacts, not live API + claims. +- The 14-field digest inventory is a Naruon acceptance profile, not evidence of + upstream adoption, scientific validity, retained objects, or replayability. +- Any implementation requires a superseding or acceptance edit to this ADR, + an upstream compatibility fixture, tenant-boundary tests, scientific + calibration evidence, and exact-head CI/security review. +- Absence, incompatibility, malformed results, or upstream failure remains a + visible fail-closed condition. + +## Alternatives rejected + +- **Per-request fitting:** destroys stable topic identity and is operationally + unbounded. +- **Keyword/embedding/LLM substitution:** may support separately named product + features but is not the same estimand. +- **Best-effort fallback:** converts missing scientific evidence into false + certainty. diff --git a/docs/adr/0003-separate-topic-measurement-from-agenda-generation.md b/docs/adr/0003-separate-topic-measurement-from-agenda-generation.md new file mode 100644 index 000000000..ec2bcda7d --- /dev/null +++ b/docs/adr/0003-separate-topic-measurement-from-agenda-generation.md @@ -0,0 +1,63 @@ +# ADR-0003: Separate topic measurement from agenda generation + +**Status:** Proposed + +**Date:** 2026-08-09 + +**Decision owner:** Naruon maintainers + +**Capability maturity:** target and future agenda capability `PLANNED`; no +implementation is authorized + +**Scope:** a possible future Naruon agenda-generation decision only. This ADR +does not govern a model or generation provider, assign external ownership, or +record provider acceptance. + +**Trigger for acceptance:** a separately reviewed agenda-generation product +contract and implementation PR. + +**Related requirement:** +[TI-REQ-009](../topic-intelligence/PRD.md#product-requirements) + +## Context + +The removed `meeting_agenda_generator` mapped words directly to a fixed agenda +template. That coupled a lexical trigger, an implied topic assertion, and a +generated action artifact. Even a valid fitted topic posterior would be +descriptive evidence, not authorization to create or execute an agenda. + +## Proposed decision + +ADR-0001 already supplies the accepted Naruon-local separation policy. This ADR +is the proposed implementation decision for a future bounded agenda capability; +it remains a proposed target rather than accepted architecture. If Naruon +reintroduces agenda generation, the Naruon capability must: + +1. consume tenant-authorized source evidence and, optionally, a versioned topic + posterior by reference; +2. preserve every cited source and model provenance field without converting a + display label into numeric topic identity; +3. declare the generation provider/model and return `review_required=true`; +4. treat source text, labels, and posterior metadata as untrusted data rather + than instructions; and +5. create no calendar/task/provider write unless a separate explicit intent, + capability, consent, and conflict check succeeds. + +The generator may abstain or fail, but it must not fall back to a template and +describe that output as source-backed topic inference. + +## Consequences + +- A statistical posterior can inform a draft but never authorizes a write. +- Human-readable labels are presentation metadata and can be revised without + changing the fitted topic identity. +- Agenda quality, grounding, prompt-injection resistance, and provider-write + safety require tests independent of topic-model validation. + +## Alternatives rejected + +- **One endpoint for measurement and generation:** obscures error ownership and + makes a generative failure look like scientific inference. +- **Template fallback:** recreates the misleading behavior removed by PR #1297. +- **Direct provider write:** bypasses Naruon's source, consent, capability, and + conflict boundaries. diff --git a/docs/adr/0004-status-weighted-calendar-conflicts.md b/docs/adr/0004-status-weighted-calendar-conflicts.md new file mode 100644 index 000000000..b2a4603a6 --- /dev/null +++ b/docs/adr/0004-status-weighted-calendar-conflicts.md @@ -0,0 +1,79 @@ +# ADR-0004: Status-weighted calendar conflicts from iCalendar evidence + +**Status:** Accepted (Naruon-local scheduling policy) +**Date:** 2026-08-17 +**Decision owner:** Naruon maintainers +**Scope:** Naruon's conflict-decision product behavior for customer-owned CalDAV +evidence. This ADR does not make Naruon a calendar host and does not authorize +provider mutation, RSVP send, or automatic reschedule. + +## Context + +Naruon is a web client over customer-owned CalDAV. Buyers cannot trust the +calendar unless overlapping VEVENTs are classified by their RFC 5545 `STATUS` +instead of treating every interval as equally busy. The product statuses +`confirmed`, `tentative`, and `desired` remain a Naruon priority axis. RFC 5545 +also defines `STATUS:CANCELLED`, which must not occupy a slot after the event +is withdrawn. + +## Decision + +1. Occupying commitments rank `confirmed > tentative > desired`. Equal or + higher-priority overlap is `blocked`. Lower-priority-only overlap is + `review_required`. No occupying overlap is `available`. +2. `STATUS:CANCELLED` is valid evidence and does not occupy `[DTSTART, DTEND)`. + A cancelled existing event therefore allows a new booking; a cancelled + proposal does not claim the interval. +3. iCalendar/ICS evidence is accepted as text. The evaluator parses + `VEVENT` `UID`, timezone-aware `DTSTART`, `DTEND` or `DURATION`, and + `STATUS`. Missing `STATUS` defaults to `CONFIRMED`. Date-only and floating + date-times fail closed. +4. The decision is advisory. It does not write CalDAV, change ETags, or + displace an existing event. Customer copy names the next action. +5. Unknown statuses fail closed. The same opaque `UID` is excluded as a + self-update, not as a conflict. + +## Alternatives rejected + +### Treat cancelled as an unsupported status + +Rejected because RFC 5545 already names `CANCELLED`. Rejecting it prevents +buyers from booking a freed slot and forces a false double-booking. + +### Rank cancelled as the lowest occupying priority + +Rejected because a cancelled VEVENT no longer claims the interval. Ranking it +below `desired` would still emit `review_required` and block silent reuse of a +freed hour. + +### Infer conflicts only from JSON commitments + +Rejected as the product path. Customer calendars arrive as `.ics`. The JSON +commitment envelope remains for tests and later structured sources; it is not +a substitute for VEVENT evidence. + +## Consequences + +- `POST /api/calendar/conflicts/evaluate` accepts either structured commitments + or `proposed_ics` / `existing_ics`. +- Calendar coordination selects a signed writeback source. Known VEVENT pairs + remain test fixtures, not production coordination evidence. Writeback remains + a separate ETag/If-Match intent path. +- Tests must keep known `.ics` pairs as the source of conflict-versus-allow + evidence. + +## References (APA 7th) + +Daboo, C. (Ed.). (2009). *iCalendar transport-independent interoperability +protocol (iTIP)* (RFC 5546). RFC Editor. https://doi.org/10.17487/RFC5546 + +Allen, J. F. (1983). Maintaining knowledge about temporal intervals. +*Communications of the ACM, 26*(11), 832–843. +https://doi.org/10.1145/182.358434 +Allen’s interval algebra names the qualitative relations between time +intervals, including overlap, which is the comparison this policy applies to +half-open calendar commitments. The ACM publication is not redistributed here. + +Desruisseaux, B. (Ed.). (2009). *Internet calendaring and scheduling core +object specification (iCalendar)* (RFC 5545). RFC Editor. +https://doi.org/10.17487/RFC5545 diff --git a/docs/adr/README.md b/docs/adr/README.md new file mode 100644 index 000000000..4d461fff6 --- /dev/null +++ b/docs/adr/README.md @@ -0,0 +1,30 @@ +# Naruon Architecture Decision Records + +This index records cross-cutting Naruon decisions that must survive beyond an +individual pull request, implementation plan, or chat. `Accepted` means only that +the Naruon decision governs its stated local scope; it does not transfer authority +to an external service or mean a future integration is implemented on protected +`develop`. `Proposed` records a discoverable target for later review and does not +govern implementation. + +| ADR | Decision | Status | Capability effect | +|---|---|---|---| +| [ADR-0001](0001-topic-measurement-authority.md) | Naruon-local policy for consuming structural topic measurement, never a keyword/label heuristic | Accepted | `ACCEPTED-NARUON-POLICY`; no runtime promotion | +| [ADR-0002](0002-fitted-topic-artifact-consumption.md) | Conditionally consume only a versioned fitted topic artifact through a fail-closed adapter | Proposed | Target `PLANNED`; runtime `BLOCKED-UPSTREAM` | +| [ADR-0003](0003-separate-topic-measurement-from-agenda-generation.md) | Keep statistical measurement separate from agenda generation | Proposed | Target and future capability `PLANNED`; no implementation authorization | +| [ADR-0004](0004-status-weighted-calendar-conflicts.md) | Evaluate CalDAV VEVENT overlaps by occupying status; cancelled does not occupy | Accepted | `ACCEPTED-NARUON-POLICY`; advisory evaluate API only | + +The complete topic-intelligence requirements, architecture, contract, UML, +conceptual ERD, security, test, and operability graph is indexed at +[`docs/topic-intelligence/README.md`](../topic-intelligence/README.md). +Its [canonical digest inventory](../topic-intelligence/README.md#canonical-digest-inventory) +is the single cross-document list for the planned adapter profile. + +## Change rule + +Create or update an ADR when a Naruon change adopts or declines an external service contract, introduces a new scientific/statistical inference contract, changes persistence or tenant authority, changes model/credential trust boundaries, or replaces a fail-closed product capability with a different production dependency. A Naruon ADR records Naruon's decision only; it cannot assign authority to, or accept a decision for, another service. + +Every implementing PR must keep the corresponding source, tests, doctoring, +architecture/operability contract, and CHANGELOG maturity truthful. An active PR, +accepted local policy, or proposed target must not be described as protected- +branch implementation before it is integrated and independently verified. diff --git a/docs/doctoring/kanban-task-keyboard-focus.md b/docs/doctoring/kanban-task-keyboard-focus.md new file mode 100644 index 000000000..00d067074 --- /dev/null +++ b/docs/doctoring/kanban-task-keyboard-focus.md @@ -0,0 +1,37 @@ +# Kanban task-card keyboard focus + +This note grounds the visible keyboard-focus treatment on task-card buttons in `frontend/src/components/TasksLayout.tsx` and the focused regression in `frontend/src/components/TasksLayout.focus-visible.test.ts`. + +## Accessibility boundary + +The Kanban cards are native `button` elements and therefore participate in sequential keyboard navigation. WCAG 2.2 Success Criterion 2.4.7 (Focus Visible, Level AA) requires a mode of operation in which keyboard focus is visible. W3C Technique C45 identifies CSS `:focus-visible` as a sufficient technique for providing keyboard-focus indication while allowing user agents to distinguish keyboard focus from ordinary pointer interaction. + +The card therefore preserves its existing hover treatment and adds the same explicit keyboard-focus token family already used by other Naruon interactive controls: + +- `focus-visible:outline-none` +- `focus-visible:ring-2` +- `focus-visible:ring-ring/40` + +The focused source contract locates the Kanban card button from the actual `tasksByStatus[col.id].map((task)` rendering path and requires all three tokens. It does not treat an unrelated focused control elsewhere in `TasksLayout` as evidence for the task card. + +## Research evidence + +Schrepp (2006) compared keyboard and mouse navigation in real websites and two small navigation studies, finding that common web designs imposed substantial efficiency disadvantages on keyboard navigation. The paper supports treating keyboard operability and orientation as concrete interaction-quality concerns rather than merely static markup properties. This bounded change addresses one necessary orientation cue—visible focus on the interactive Kanban card—without claiming that a focus ring alone removes the broader efficiency gap identified in the study. + +## Claim boundary + +This bounded change supports the WCAG 2.2 Focus Visible objective for the Kanban task-card control. It does not by itself claim whole-product WCAG 2.2 conformance, Focus Not Obscured conformance, or the Level AAA Focus Appearance area/contrast requirement. Those require rendered-browser assessment across supported themes, zoom levels, forced-colors/high-contrast modes, and viewport states. + +## References (APA 7th) + +Schrepp, M. (2006). On the efficiency of keyboard navigation in Web sites. *Universal Access in the Information Society, 5*(2), 180–188. https://doi.org/10.1007/s10209-006-0036-x + +World Wide Web Consortium. (2023). *Web Content Accessibility Guidelines (WCAG) 2.2* (W3C Recommendation). https://www.w3.org/TR/WCAG22/ + +World Wide Web Consortium, Web Accessibility Initiative. (n.d.). *C45: Using CSS `:focus-visible` to provide keyboard focus indication*. Retrieved August 15, 2026, from https://www.w3.org/WAI/WCAG22/Techniques/css/C45 + +World Wide Web Consortium, Web Accessibility Initiative. (n.d.). *Understanding Success Criterion 2.4.7: Focus Visible*. Retrieved August 15, 2026, from https://www.w3.org/WAI/WCAG22/Understanding/focus-visible + +## Verification boundary + +The branch is not merge-ready merely because this accessibility treatment, regression, and evidence note exist. Current-head repository CI, required organization workflows, security gates, resolved review threads, and qualifying independent approval remain authoritative. diff --git a/docs/doctoring/local-http-origin-port-validation.md b/docs/doctoring/local-http-origin-port-validation.md new file mode 100644 index 000000000..2e0b36a09 --- /dev/null +++ b/docs/doctoring/local-http-origin-port-validation.md @@ -0,0 +1,36 @@ +# Local HTTP origin port-validation boundary + +## Decision + +`validate_loopback_http_origin()` distinguishes an absent URI port from an explicitly supplied port value. Scheme defaults are applied only when the port subcomponent is absent. An explicit port `0`, a value outside the application's `1..65535` transport-port contract, or a malformed/non-numeric port is rejected rather than rewritten to the scheme default. + +This is an origin-integrity rule, not only input cleanup. A caller that supplied `:0` expressed a materially different authority from one that omitted the port. Replacing the explicit value with `80` or `443` changes caller intent and can convert malformed or attacker-controlled configuration into a valid local destination. + +The same validator continues to require the existing loopback-host allowlist and to reject credentials, path/query/fragment material outside the local-origin contract, control characters, and unsafe request-target traversal. + +## Standards basis + +RFC 3986 defines the URI authority as host plus an optional decimal port subcomponent and allows a scheme to define a default port. The default therefore belongs to the *absent-port* case; an explicitly parsed port must not be collapsed with absence merely because the application's language treats numeric zero as false. + +RFC 6335 defines the Service Name and Transport Protocol Port Number Registry and the port-number space used by transport protocols. Naruon's local-origin helper intentionally narrows its application contract to `1..65535`; port zero is not a usable destination for this product path. The validator preserves this product-level restriction without making a broader claim that RFC 3986 itself forbids the textual URI `:0`. + +## Verification contract + +Regression tests must keep these cases distinct: + +- `http://localhost` and `https://localhost` use their scheme defaults; +- explicit supported ports are preserved; +- explicit `:0` is rejected instead of defaulted; +- negative, out-of-range, and non-numeric ports fail closed; +- IPv4/IPv6 loopback canonicalization remains stable; +- userinfo, path/query/fragment material outside the origin contract, controls, and non-allowlisted hosts remain rejected. + +## Rollback + +If a future runtime genuinely needs port zero as a sentinel, introduce a separate typed configuration field or explicit sentinel contract. Do not reintroduce truthiness-based defaulting in URI parsing, because that again conflates an explicit authority with absence. + +## References (APA 7th) + +Berners-Lee, T., Fielding, R., & Masinter, L. (2005). *Uniform Resource Identifier (URI): Generic syntax* (RFC 3986). RFC Editor. https://doi.org/10.17487/RFC3986 + +Cotton, M., Eggert, L., Touch, J., Westerlund, M., & Cheshire, S. (2011). *Internet Assigned Numbers Authority (IANA) procedures for the management of the service name and transport protocol port number registry* (BCP 165, RFC 6335). RFC Editor. https://doi.org/10.17487/RFC6335 diff --git a/docs/doctoring/status-weighted-calendar-conflicts.md b/docs/doctoring/status-weighted-calendar-conflicts.md new file mode 100644 index 000000000..7a2b3ad3b --- /dev/null +++ b/docs/doctoring/status-weighted-calendar-conflicts.md @@ -0,0 +1,39 @@ +# Status-weighted calendar conflict policy + +## Shipped boundary in this slice + +Naruon evaluates a proposed calendar commitment against a bounded set of existing commitments and returns one of three deterministic outcomes: `available`, `blocked`, or `review_required`. The decision is advisory evidence only. It does not mutate, cancel, reschedule, accept, or decline any provider event. + +The public endpoint is `POST /api/calendar/conflicts/evaluate`. It is mounted behind Naruon's existing private API authentication dependency. Inputs are either structured commitments (`commitment_id`, timezone-aware `start_at`/`end_at`, status) or iCalendar/ICS `proposed_ics` / `existing_ics` VEVENT documents. Occupying statuses are `confirmed`, `tentative`, and `desired`. RFC 5545 `STATUS:CANCELLED` is accepted and does not occupy the interval. Existing evidence is capped at 500 commitments per request. The Calendar coordination view selects a signed, source-backed writeback source for the authenticated user/workspace and does not present canned ICS pairs as production coordination evidence. Known `.ics` pairs remain test fixtures only. + +## Standards traceability + +RFC 5545 defines `VEVENT` `DTSTART` as inclusive and `DTEND` as non-inclusive, and requires `DTEND` to be later than `DTSTART`. Naruon therefore evaluates conflicts as half-open intervals `[start_at, end_at)`: an event ending exactly when another begins is not a collision. The implementation compares timezone-aware instants, so equivalent instants represented with different UTC offsets still collide. + +RFC 5546 defines iTIP scheduling methods such as `REQUEST` and `REPLY`, including attendee participation status (`PARTSTAT`). It provides the interoperability basis for later RSVP/writeback integration, but it does **not** define Naruon's three-level scheduling priority. `confirmed > tentative > desired` is an explicit Naruon product policy required by roadmap issue #988, not a standards claim. + +## Decision policy + +- No occupying overlap, including overlap with only `STATUS:CANCELLED` events: `available`; the customer can proceed. +- Any equal- or higher-priority occupying overlap: `blocked`; the customer must choose another time or explicitly resolve that conflict first. +- Only lower-priority occupying overlaps: `review_required`; Naruon surfaces the lower-priority conflicts and requires explicit review instead of silently displacing them. +- An existing commitment with the same opaque identifier as the proposal is treated as the current representation of that event, not as a self-conflict. +- Conflict evidence is sorted by UTC start instant and then opaque identifier so provider response ordering cannot change the decision payload. + +This policy deliberately prevents a convenience feature from silently breaking an existing confirmed commitment. A later RSVP slice may consume the same deterministic policy, but this slice does not claim RSVP mutation support. + +## Security, privacy, and operability + +The decision path is deterministic and uses no LLM judgment. It accepts only scheduling evidence needed for the decision; it does not require email bodies, participant names, provider credentials, or calendar descriptions. The endpoint rejects naive timestamps, invalid/non-positive intervals, unsupported statuses, oversized evidence batches, extra request fields, and a missing proposed source through the transport/service validation layers. A missing proposal returns `calendar_proposed_source_missing` as HTTP 422; the handler does not use `assert`, so optimized bytecode cannot strip the guard. Customer-facing results include a concrete next action rather than a generic warning. + +No database objects or migrations are introduced. No provider is contacted. Rollback must disable or remove the frontend integration first (`frontend/src/components/calendar/types.ts`, `constants.ts`, `helpers.ts`, and `CalendarCoordinationView` wiring in `CalendarLayout`), then remove the backend route registration, ICS parser, and policy module. Existing calendar data is unaffected because the slice is read-only for provider and database state. + +## Verification evidence required before merge + +The exact unchanged PR head must prove known `.ics` pairs (cancelled allows, tentative review, confirmed blocks, adjacent allow), realistic overlap, adjacency, timezone-offset equivalence, deterministic ordering, self-update, invalid interval, unsupported status, API validation, authentication, and bounded-batch behavior. Repository-required CI, security, coverage, supply-chain, package, and independent current-head review gates remain authoritative; predecessor or queued evidence is non-passing. The policy decision is recorded in [ADR-0004](../adr/0004-status-weighted-calendar-conflicts.md). + +## References (APA 7th) + +Daboo, C. (Ed.). (2009). *iCalendar transport-independent interoperability protocol (iTIP)* (RFC 5546). RFC Editor. https://doi.org/10.17487/RFC5546 + +Desruisseaux, B. (Ed.). (2009). *Internet calendaring and scheduling core object specification (iCalendar)* (RFC 5545). RFC Editor. https://doi.org/10.17487/RFC5545 diff --git a/docs/doctoring/structural-topic-model-boundary.md b/docs/doctoring/structural-topic-model-boundary.md new file mode 100644 index 000000000..307856433 --- /dev/null +++ b/docs/doctoring/structural-topic-model-boundary.md @@ -0,0 +1,94 @@ +# Structural topic-model boundary + +**Architecture decision:** [`ADR-0001`](../adr/0001-topic-measurement-authority.md) defines Naruon's local policy for truthful topic-measurement consumption. This doctoring record supplies the scientific rationale and evidence; neither record assigns authority to TEPP, records TEPP acceptance, or promotes a future integration to protected-branch implementation. + +## Defect record + +Naruon previously exposed `email_categorizer` and +`meeting_agenda_generator`, whose outputs came from small hard-coded +Korean/English term lists rather than a fitted topic model. The traceable record +is deliberately narrow: commit +`c070c8d19f01ccfe46a5ee7e8a577b08e587bb14` described basic length/keyword +parsing and a 100%-coverage goal; commit +`699d7ef9d1285c8c2c5a1a38c6732117d0ff703e` made the tables deterministic; +commit `11a329fa3950a529d3df607e33ae09f55117a09d` established the later bound; and +the first merge to `develop` was +`eae74e215d99af49764a765b74e9679037b8fbbe` (PR #1075). These facts describe +the observable history, not unrecorded author intent. + +The two pseudo-model tools are now removed on PR #1297. `keyword_extractor` +remains because it honestly exposes deterministic term-frequency extraction. Its +output is lexical metadata, not topic-posterior evidence. Until PR #1297 merges, +this removal remains active-PR behavior rather than a protected-`develop` claim. + +## Measurement boundary + +Structural topic modeling estimates a mixed-membership vector +\(\theta_d\) for each document: multiple latent topics can contribute to one +document, and metadata may affect topic prevalence or content. A fixed-label +classifier instead selects or scores predefined business labels. Even when a +classifier uses keywords, embeddings, or an LLM, its label or score is not an +STM posterior and must not be presented as one. + +New-document STM inference also depends on a fitted corpus-level model and its +frozen vocabulary and preprocessing. Naruon must not fit a topic model inside a +single API request, substitute a larger dictionary, or degrade to embeddings or +LLM labels while calling the result STM. + +## Potential future Naruon consumption + +Naruon has no independently published TEPP production topic-measurement +artifact/API/contract or TEPP acceptance evidence to consume. The present change +therefore fails closed: when a fitted model is unavailable, no default `General` +label, agenda template, or synthetic posterior is returned. + +A future Naruon adapter remains blocked until TEPP independently publishes a +versioned production fitted-model artifact/API/contract and its own acceptance +evidence. If Naruon later evaluates such a published contract, its local +acceptance criteria include: + +- immutable document and model-artifact identifiers, model version, and content + and vocabulary digests; +- document, event, assertion, availability, and knowledge-cutoff times; +- frozen preprocessing, retained-token rules, frozen vocabulary, explicit OOV + handling, and language identification/support status; +- prevalence/content design specifications and relevant multilevel or + cross-classified multiple-membership covariates; +- mixed-membership topic proportions summing to one, inference method, + posterior uncertainty, diagnostics, and explicit abstention criteria/status; +- evidence-backed human-readable labels kept separate from numeric topic + identity; and +- explicit model-unavailable, incompatible-language, insufficient-retained- + token, and out-of-vocabulary errors. + +The integration is an optional typed service/model-artifact boundary. Naruon must +not read TEPP's private database, infer model compatibility from a display label, +or persist a generated human-readable topic label as the numeric topic identity. + +Agenda generation, if reintroduced, belongs behind a separate decision and +generation boundary that consumes source evidence and the versioned posterior. +No copyrighted paper is attached here; redistribution permission has not been +established. + +## References + +Roberts, M. E., Stewart, B. M., & Tingley, D. (2019). stm: An R package for +structural topic models. *Journal of Statistical Software, 91*(2), 1–40. +https://doi.org/10.18637/jss.v091.i02 + +This paper specifies the fitted STM workflow, prevalence/content covariates, +posterior quantities, and diagnostics implemented by the `stm` package. It +supports the boundary because a term lookup lacks those fitted-model and +uncertainty semantics. + +Roberts, M. E., Stewart, B. M., Tingley, D., Lucas, C., Leder-Luis, J., +Gadarian, S. K., Albertson, B., & Rand, D. G. (2014). Structural topic models +for open-ended survey responses. *American Journal of Political Science, +58*(4), 1064–1082. https://doi.org/10.1111/ajps.12103 + +This paper introduces STM for open-ended responses and demonstrates how +document metadata enters topic prevalence/content while documents remain mixed +memberships. It supports separating corpus-level measurement from fixed-label +classification. Redistribution permission for either article has not been +established, so this PR cites, links, and summarizes them without committing +copies. diff --git a/docs/operations/container-provenance-contract.md b/docs/operations/container-provenance-contract.md new file mode 100644 index 000000000..0d98c0863 --- /dev/null +++ b/docs/operations/container-provenance-contract.md @@ -0,0 +1,41 @@ +# Container provenance contract + +Naruon container images must be reproducible from reviewable, immutable base-image inputs. + +## Required invariants + +- Every production `FROM` instruction uses both a human-readable image tag and a full `sha256` digest. +- The root, backend, connector, and frontend Dockerfiles keep shared Python and Node base references synchronized where the runtime contract is shared. +- OCI `org.opencontainers.image.base.name` and `org.opencontainers.image.base.digest` annotations are derived from the actual first Dockerfile stage rather than duplicated constants. +- `OCI_IMAGE_BASE_DIGEST` and `OCI_IMAGE_BASE_NAME` are mandatory build arguments. Dockerfiles fail closed when a publishing or validation path omits either value. +- Published multi-platform images preserve annotations at both the manifest and index levels. +- Pull-request validation resolves the pinned Ollama manifest and fails closed when either `linux/amd64` or `linux/arm64` is absent. +- Dependency and image security pins remain governed by executable repository tests; a dependency upgrade must update its hash-locked artifact and the corresponding regression contract together. +- Backend `cryptography==50.0.0` and `protobuf==7.35.1`, Strix `cryptography==50.0.0` and `protobuf==6.33.6`, frontend source pins `postcss==8.5.24` and `jsdom==^30.0.1`, generated-lock resolutions `postcss==8.5.24` and `jsdom==30.0.1`, and the `brace-expansion==5.0.9` and `undici==8.9.0` overrides are parsed and checked structurally. + +## Change procedure + +1. Update the tag-and-digest reference in the canonical Dockerfile. +2. Synchronize every Dockerfile that shares that runtime. +3. Regenerate affected hash locks without weakening `--require-hashes` installation. +4. Update `CHANGELOG.md` when the runtime or published artifact changes. +5. Run release-governance, repository-hygiene, dependency-pin, application, image-build, and security checks on the exact pull-request head. +6. Merge only after independent review confirms that the OCI annotations describe the image that is actually built. + +A mutable tag by itself, a digest without its reviewable tag, an omitted mandatory base-metadata argument, or an annotation that does not match the first stage violates this contract. + +## Standards interpretation + +The OCI Image Format is the authoritative interoperability contract for image manifests, indexes, configurations, and descriptors. Naruon derives its base-image annotations from the Dockerfile actually used for the build so the published metadata cannot silently diverge from the reviewed build input. + +SLSA Build Provenance 1.2 describes provenance as verifiable information about where, when, and how an artifact was produced. It treats externally supplied build parameters as untrusted inputs that must be recorded and verified downstream. Naruon's tag-and-digest base references, exact workflow revision, and generated dependency locks are therefore reviewable build inputs rather than decorative metadata. This repository does not claim a SLSA level solely because it emits OCI annotations. + +NIST SP 800-218, SSDF 1.1, recommends protecting software and verifying third-party components throughout the development and delivery lifecycle. Naruon implements that guidance through immutable action and image pins, generated hash locks, exact-head tests, vulnerability scans, and independent review. The newer SSDF 1.2 document remains an initial public draft as of August 2026 and is informative rather than the formal conformance baseline. + +## References + +National Institute of Standards and Technology. (2022). *Secure software development framework (SSDF) version 1.1: Recommendations for mitigating the risk of software vulnerabilities* (NIST Special Publication 800-218). https://doi.org/10.6028/NIST.SP.800-218 + +Open Container Initiative. (2025). *OCI image format specification* (Version 1.1.1). https://github.com/opencontainers/image-spec/tree/v1.1.1 + +Supply-chain Levels for Software Artifacts. (2025). *Build provenance* (SLSA specification Version 1.2). https://slsa.dev/spec/v1.2/build-provenance diff --git a/docs/planning/naruon-platform-plan.md b/docs/planning/naruon-platform-plan.md index 9e7cdfaa0..7e93ecbd2 100644 --- a/docs/planning/naruon-platform-plan.md +++ b/docs/planning/naruon-platform-plan.md @@ -56,7 +56,12 @@ Every commitment carries a status on the axis **{confirmed | tentative | desired Contexts — **personal / work→{former employer, current employer} / per-project / per-band** — are **segregated by default**, classified by **content, not by account**. A private fact may affect another context only by propagating the **necessary consequence** (e.g., *"unavailable Tue–Thu"*), **never the private reason** (e.g., *"hospitalized"*). The user controls the disclosure level per bridge (minimum by default); data minimization and purpose limitation are enforced structurally at the boundary; every bridge is consent-gated, revocable, and audited. **Two further constants** apply everywhere and are folded into the above: -- **Language-agnostic (G6):** entity/relation extraction, resolution, and search work consistently across EN/KO/JA/ZH/VI via LLM extraction + multilingual embeddings + cross-lingual structured topic modeling — **no dependency on morphological analyzers** (Kiwi/Nori-style), which cause performance cliffs. +- **Language-agnostic (G6):** entity/relation extraction, resolution, and search + work consistently across EN/KO/JA/ZH/VI through language-agnostic lexical and + multilingual dense retrieval plus source-backed extraction — **no dependency + on morphological analyzers** (Kiwi/Nori-style), which cause performance + cliffs. Cross-lingual structural topic measurement is a **PLANNED** optional + TEPP integration, not a current Naruon search or inference signal. - **À-la-carte plugins:** nothing is mandatory; verticals/capabilities slot into fixed extension points; a user's enabled set reshapes their navigation. **Cross-cutting definition of done.** A unit of work is "done" only when it demonstrably honors all disciplines together: the happy path asked **zero questions** (CP-2); **no confirmed commitment was silently broken** (CP-4); every inference was made and labeled at the **correct level of analysis** (CP-3); and **no private reason crossed a context boundary** — only the necessary consequence, with consent and audit (CP-5). @@ -694,12 +699,12 @@ Hooks are **typed and ordered** (each point has a Pydantic input/output contract **Node & edge taxonomy.** Node types: `person`, `org`, `norm_group`, `project` (incl. Band), `thread`, `message`, `attachment`, `content_node`, `event`, `commitment`, `deliverable`, `requirement`, `wbs_item`, `erd_candidate`. Edge axes (density comes from many *simultaneous* relation axes): Social (`person—person`, `person—org`, `person—norm_group` **multi-membership**), Communication (`message—thread`, in_reply_to/references, sender/recipient), Temporal/event (`event—event` **enables/conflicts/unrelated**, resolved by density not asking; `event—commitment`), Commitment (status axis {confirmed|tentative|desired} + RSVP direction), Provenance (`object—content_segment` cited evidence, `object—extractor`, `correction—object`). Every semantic node/edge stores `confidence`, `extractor_name`, `extractor_version`, and cited `source_segment_uids` — auditable back to the exact DOM segment. -**Language-agnostic extraction.** LLM-based entity/relation extraction (via contextual-orchestrator) replaces today's deterministic rule extractor; extractors register through `kg.extractor`, emit candidates with confidence, cite segments; deterministic rules remain as a cheap first pass / offline-deterministic test fallback. Multilingual embeddings + subword/byte tokenization; **no morphological-analyzer dependency** (Kiwi/Nori cause performance cliffs); cross-lingual **structured topic modeling (STM)** feeds search and norm-group inference. Attachment DOM parsing is first-class (PDF→DOM via newsdom-api / MinerU Apache-2.0; audio/video via codec-carver), parsed into the same content_node/segment space so extraction and search treat body and attachment uniformly. +**Language-agnostic extraction.** LLM-based entity/relation extraction (via contextual-orchestrator) replaces today's deterministic rule extractor; extractors register through `kg.extractor`, emit candidates with confidence, cite segments; deterministic rules remain as a cheap first pass / offline-deterministic test fallback. Multilingual embeddings + subword/byte tokenization; **no morphological-analyzer dependency** (Kiwi/Nori cause performance cliffs). Cross-lingual **structural topic measurement is PLANNED, not LIVE**: it may feed search or norm-group research only after a separately accepted TEPP fitted artifact/API publishes frozen preprocessing and vocabulary, applicable multilevel/multiple-membership and temporal covariates, mixed-membership uncertainty and diagnostics, and fail-closed compatibility rules. The lexical `keyword_extractor` is never topic evidence. Attachment DOM parsing is first-class (PDF→DOM via newsdom-api / MinerU Apache-2.0; audio/video via codec-carver), parsed into the same content_node/segment space so extraction and search treat body and attachment uniformly. **Hybrid dense + sparse search.** LIVE: `api/search.hybrid_search` combines Postgres FTS (`to_tsvector`/`ts_rank_cd`) with pgvector `cosine_distance` (`_search_score = fts_score − vector_distance`), degrading gracefully to FTS-only; scoped over `email_records`/`email_attachments` bodies. TARGET: extend to `content_segments` and typed `project_graph_objects` (search the *meaning*); expose rank fusion (e.g., reciprocal-rank fusion) as a `search.ranker` extension point; move embedding from **inline per-import** to a **batch embedding pipeline** driven by contextual-orchestrator / pg-llm-batch (`batch_embedding_service` does not exist today) so re-embedding and high-volume ingest don't block the ingest transaction. **The Inference Layer** (turns a dense graph into judgment; where the architect-level rigor lives): -- **Norm-group resolution (before any inference)** — resolve which norm-group(s) an interaction belongs to by graph evidence (sender's `member_of` edges, thread project scope, account, STM topic, past patterns); a person is in **N overlapping groups** → a weighted set, not a label; all downstream norms evaluated relative to the resolved group(s). +- **Norm-group resolution (before any inference)** — resolve which norm-group(s) an interaction belongs to by implemented graph evidence (sender's `member_of` edges, thread project scope, account, past patterns); a person is in **N overlapping groups** → a weighted set, not a label; all downstream norms evaluated relative to the resolved group(s). A future fitted topic posterior may become an additional, non-causal signal only after the separately governed contract in `docs/topic-intelligence/` is implemented and validated; no lexical substitute is permitted. - **Ecological-fallacy-safe estimation** — `posterior ∝ prior(norm_group) × likelihood(individual content)`; never report a group base rate as an individual's property, never generalize an individual to their group; confidence is honest and propagated. - **Status-weighted conflict detection** — event↔event conflicts resolved by density (travel time, venue vs hotel, host = partner vs work), not by asking; `confirmed` wins and is never silently broken; `desired` over `confirmed` = surfaced conflict; RSVP direction matters; e-approval outcomes are first-class KG events linked to the events they enable (anticipatory). - **Output contract** — never a question; emits a **DecisionPoint** (resolved connection + recommendation + cited evidence + honest confidence, rendered by `DecisionPointCard.tsx`); the human corrects by exception; corrections land in `project_graph_corrections` and become training signal + higher-priority evidence. @@ -840,4 +845,4 @@ The single highest-leverage move: the semantic graph exists but is empty. ### Phase 5 — Verticals 15. **BandScope** (the flagship UC-09 demo — reuses Phase 3's conflict engine + Phase 4's per-band isolation), then **pg-erd-cloud**, **scopeweave**, **Inkspan**, **codec-carver** (audio minutes), plus **legal/contract** and **code-integration** capabilities — all as à-la-carte plugins on the Phase 1 SDK. -**Cross-cutting throughout every phase:** deepen OpenTelemetry distributed tracing and KG-quality/inference-confidence metrics; enforce the licensing gate (permissive-only), 2+word `snake_case` for new DB objects, and KV-registry (not `os.getenv`) secrets; route all LLM traffic through contextual-orchestrator; and hold the four disciplines (no-ask, status-weighted, ecological-fallacy-safe, minimal-disclosure) as the definition of done for every unit of work. \ No newline at end of file +**Cross-cutting throughout every phase:** deepen OpenTelemetry distributed tracing and KG-quality/inference-confidence metrics; enforce the licensing gate (permissive-only), 2+word `snake_case` for new DB objects, and KV-registry (not `os.getenv`) secrets; route all LLM traffic through contextual-orchestrator; and hold the four disciplines (no-ask, status-weighted, ecological-fallacy-safe, minimal-disclosure) as the definition of done for every unit of work. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md new file mode 100644 index 000000000..98bc17d2a --- /dev/null +++ b/docs/product-technical-gap-baseline.md @@ -0,0 +1,1016 @@ +# Naruon Product and Technical Gap Baseline + +**Baseline version:** 1.2 +**Observed on:** 2026-08-26 (Asia/Seoul) +**Observed protected branch (current scan; row Base-SHA values remain historical):** `develop@e5e99b4e3bb081b92c602358878856536030e2ca` +**Observed product version:** `0.14.4` +**Canonical completion issue:** [#1428](https://github.com/ContextualWisdomLab/naruon/issues/1428) + +**Inventory observation:** the 106-PR open surface below is a fresh live +scan captured at `2026-08-25T15:52:01Z`, which returned 106 open PRs after +PR #1337 merged into protected `develop` at `2026-08-25T00:10:39Z` and +the later governance stack merge (#1448) and additional PR wave opened since +the previous observation. The +v0.2 baseline's 93-row snapshot from `2026-08-21T19:25:43Z` remains historical +context, as do the earlier 92-PR state after PR #1442 merged and the initial +83-PR observation in issue #1428; all counts are point-in-time evidence, not +current merge state. + +**Follow-up delta observation:** a seventh live scan at +`2026-08-25T15:39:22Z` found the active exact-head work relevant to this +baseline: #1465 (`70596e26…`, tenant-archive import sanitization and duplicate +identity rejection), #1466 (`a3e6762f…`, origin-integrity port validation), +#1467 (`6816bc7f…`, utility-tool JSON boundary and Strix-trigger restoration), +#1468 (`1da167de…`, PostgreSQL smoke fixture schema alignment), #1469 +(`de6d7128…`, bounded 20–64 MiB deferred attachment parse-source admission), +and #1455 (`d8757e65…`, attachment filename traversal hardening). The +governance root #1443 now has exact head `62a0d645…` after child #1448 merged +normally into its stack branch. #1448 merged at +`2026-08-25T15:31:39Z` with merge commit `62a0d645…`; its tree retains the +parent gate fix and the multiline, stale-head, and current-finding regression +cases. The merge-result Checks for that commit remain queued and are tracked +as post-merge canary evidence, not success. +The newly opened #1470 is now `aba77cf5…` (NetworkGraph edge-description +lookup cleanup); the predecessor `f8a70d37…` was discarded after a remote +commit reintroduced the dead argument and its callers. The current head also +removes the tracked 452-line `NetworkGraph.tsx.out` pre-refactor copy found by +Devin, with the 11-test focused suite, TypeScript, zero-warning ESLint, and +diff checks passing locally. Hosted Checks and independent approval remain +required. +These PRs remain open until their current-head required Checks and qualifying +independent approval satisfy the protected ruleset; this follow-up does not +reuse predecessor evidence or claim a merge. #1448 is historical after its +normal merge; #1443's current head is `62a0d645…` and must be reviewed and +checked again from that exact head. + +**Live exact-head queue refresh (2026-08-25T17:51Z):** the following +post-snapshot states supersede only the matching historical SHA references +above; the full inventory remains a point-in-time record and is not silently +rewritten. #1468 is `1da167de26b442be6961622f15bb36ae9374e6c4`; its source, +backend, and frontend evidence is successful, while the retried Strix run is +queued after the earlier NVIDIA NIM provider failure. #1469 is now +`575b0c24fd9cb98106989eb101de74c5ce383db3`, after a remote follow-up removed +the unreachable document-size branch; 109 attachment/import/NewsDOM tests +pass locally, and the PR remains stacked behind #1468 because the selected +full suite exposes the base fixture defect repaired by #1468. #1470 remains +`aba77cf5b6a47985b352e8d2c2d76413579ea88a`; backend, frontend, Strix, and +metadata checks are successful, and its exact-head CodeRabbit approval is +present, but the ruleset still requires the second qualifying approval and +the exact-head OpenCode dispatch is queued. #1455 remains +`b2b4701366bc6bb2de1347b19eac9b4ed64cc614` with backend/frontend evidence +successful and Strix still running. #1429 remains +`b93caf0e00d09de0f836d8e4b869054d792e674b`; its refreshed hosted checks are +queued or in progress. #1347 remains +`f9e1751e2d5069841b279ecd2414fbbcad2e5692`; its newest metadata result is +not merge evidence and the remaining hosted checks are queued. None of these +states satisfies the two-approval protected ruleset, and none is a force-merge +candidate. + +**Live exact-head queue refresh (2026-08-25T18:06Z):** a subsequent +re-fetch supersedes only the matching queue entries above. PR #1436 moved to +exact head `034d2ff4e0d120d1e7b7669b35ea8eea7d8c1221`; its frontend, backend, +and Strix checks were recreated and queued, with zero unresolved review +threads and no qualifying approval. PR #1470 remains +`aba77cf5b6a47985b352e8d2c2d76413579ea88a`; its metadata, OpenCode, +frontend, backend, Strix, and security checks are successful, but only the +exact-head CodeRabbit approval exists and the protected ruleset requires a +second qualifying approval. PR #1467 remains +`6816bc7f938fe361f2eb7a0ecee427f87170fbcb`; source/security checks are +successful, metadata is still in progress, and only CodeRabbit has approved. +PR #1466 remains `a3e6762f01666f5b4e9d202932012de23b942c59` with all +observed required checks successful but no current approval. PR #1468 remains +`1da167de26b442be6961622f15bb36ae9374e6c4`; its source checks are successful, +the retried Strix check is queued, and its metadata gate is not a merge +result. PR #1469 remains `575b0c24fd9cb98106989eb101de74c5ce383db3`; its +source checks pass while image validation and Strix are running and metadata +is failed pending current review evidence. PR #1455 remains +`b2b4701366bc6bb2de1347b19eac9b4ed64cc614` with image checks successful, +coverage and metadata pending, and Strix running. PR #1347 remains +`f9e1751e2d5069841b279ecd2414fbbcad2e5692`; its three informational review +threads are resolved, but metadata is failed and Strix/coverage are still +running. PR #1429 remains `39d796d14b93484e004c13d7e2db4cc2eee5cdb1` with +the refreshed hosted suite queued. No entry satisfies the two-approval +ruleset, and no entry is a force-merge candidate. + +**Current Checks inventory:** the same live scan found 106 open PRs. Completed +failures were limited to `metadata-only gate evaluation` and `strix`; the +metadata gate reports the underlying Strix failure and, on some heads, a +current-head CodeRabbit quota/provider warning. The Naruon hosted Strix logs +for the observed #1468 run show NVIDIA NIM HTTP 429 rate limiting followed by +an unavailable direct fallback. This describes that run's provider evidence, +not the central Strix default documented in `AGENTS.md`; it is failed +infrastructure evidence rather than a clean security result or a source defect. +These PRs remain blocked and are not +force-merged; the exact head must receive a successful hosted security result. +PR #1466 has successful required Checks but remains in GitHub's protected +auto-merge queue with `REVIEW_REQUIRED` and no qualifying independent +approval, so it is also not treated as manually merged. + +The latest exact-head follow-up also records: #1347 at +`e97fa1e4…` with its governance regression tests passing but hosted required +Checks queued and no qualifying approval; #1433 at `b84255e5…` with source +checks passing but Strix failed closed after NVIDIA NIM HTTP 429 retries and a +configured direct OpenAI HTTP 404 fallback; #1450 at `073408ce…` with all +other Checks successful while the metadata gate remains in progress; #1455 at +`d8757e65…` with all hosted Checks successful but no current-head approval; +and #1466 at `a3e6762f…` with all hosted Checks successful but no qualifying +independent approval. These are live queue observations, not merge evidence. +The provider failures are retained as infrastructure evidence and are not +converted into source changes or clean security claims. + +**Exact-head maintenance ledger (2026-08-26):** PR #1347 now has exact head +`f9e1751e2d5069841b279ecd2414fbbcad2e5692` on this protected base. Its +governance normalization wrapper now fails closed when the review-unavailable +`jq` parser fails or emits a non-numeric count, and its early-exit cleanup trap +removes the wrapper-created comments snapshot when no PR number is available. +The exact-head local contract suite is 14 passed and the shell self-test remains +green. Hosted required Checks +were recreated and remain queued, so this is not a hosted pass or merge claim. +PR #1364 now has exact head +`3a3baa6b8dc9ea224f46395cb78c91a45be2090c` on this protected base. Its scoped +S3 document lifecycle, encrypted provider registry, compensation/orphan cleanup, +migrations, API, and LocalStack/PostgreSQL integration contract passed 195 +focused local tests, Ruff, compileall, and diff checks. Hosted required Checks +and independent approval remain required; protected auto-merge is enabled but +no merge is claimed. The attachment and UI work continued on independent +branches while hosted runners were queued. PR #1419 +now has exact head `2924b5598d4f527d493e1fc88cebd8fe87e1a3c4` on this protected +base, with 263 focused attachment/inline-image/email tests plus Ruff passing; +its hosted required checks were recreated for that head and remain queued, so +the normal protected squash auto-merge is enabled but not claimed as complete. +PR #1436 now has exact head +`1573be3332725bfbd05053943988d652df22a846` on the same base, with 446 frontend +tests, lint, typecheck, Next production build, Storybook build, and desktop / +mobile Storybook screenshots for source-open, low-confidence, +blocked-execution, and shared-scale states passing locally. It intentionally +remains pending because it carries a UI overlay at this document path while +PR #1429 owns the canonical commercial baseline; merge #1429 first, then +reconcile #1436 against the canonical file before enabling auto-merge. These +observations are exact-head evidence, not a release or hosted security claim. +PR #1415 now has exact head `994c6d40bb8a5a1de82e2f137300ea620bcdf933`; the +OIDC `kid` selection and strict administrator-role boundary passed 98 focused +authentication tests. PR #1417 now has exact head +`46f4b92a717361e3e4e42fcebc1d8c090a64c59b`; its PostgreSQL smoke seed now +explicitly supplies `is_read` after a real existing-schema NOT NULL failure, +and 180 focused tests pass. PR #1455 now has exact head +`b2b4701366bc6bb2de1347b19eac9b4ed64cc614`; bounded filename decoding and +Windows-separator traversal protection passed 130 focused parser/import tests. +All three have recreated hosted Checks and remain normal protected-merge +candidates; no hosted pass or merge is claimed. + +The protected-branch SHA in this header identifies the baseline's observation +point. The inventory's `Base-SHA` column is captured independently for each PR +at its scan time, so an older `develop` SHA in a row is expected snapshot +metadata rather than a second claim about the current protected branch. + +**Live exact-head queue refresh (2026-08-25T18:38Z):** PR #1468 remains at +`1da167de26b442be6961622f15bb36ae9374e6c4` and its current hosted check rollup +has no failed or pending run; it still has zero qualifying approvals and zero +unresolved threads, so protected auto-merge has not occurred. PR #1469 remains +at `575b0c24fd9cb98106989eb101de74c5ce383db3`; its source checks are passing, +while the metadata gate remains failed on current review evidence and +OpenCode is queued. PR #1429 remains at +`f2143f9d997736040eb3152f59ce058dc22ea72b` with the refreshed hosted suite +queued or in progress. PR #1436 remains at +`034d2ff4e0d120d1e7b7669b35ea8eea7d8c1221` with frontend image, coverage, and +Strix work still running. PR #1470 has a successful current check rollup and +one exact-head bot approval, but still lacks the second qualifying approval. +None of these observations authorizes a bypass merge. + +The owning upstream sidecar `Seongho-Bae/newsdom-api` PR #682 remains at +`585bb4e0fb719ab6a576cf46d1ef12b77872557b`. Its bounded 64 MiB source and +boundary tests are present, while its only failed hosted check is the +provider-infrastructure Strix run (NVIDIA NIM rate limiting followed by an +unavailable fallback); no source vulnerability report was produced and the +normal rerun workflow is unavailable. The provider PR therefore remains +`WAIT_AND_REMEDIATE`, not a clean security pass or a force-merge candidate. +NewsDOM issue [#707](https://github.com/Seongho-Bae/newsdom-api/issues/707) +owns the follow-up resumable-upload contract for documents above 64 MiB, so +the current synchronous fallback must not be mistaken for the target +commercial large-document UX. + +The central `ContextualWisdomLab/.github` control plane has two related +current-head repairs: PR #1331 (`a1408f52…`) separates the direct-OpenAI +fallback API base from the primary provider and has all observed checks passing +except queued coverage, while PR #1333 (`5454a196…`) adds bounded provider +retry/attempt-log handling with OpenCode still queued. Both have zero +qualifying approvals and remain normal protected-merge candidates; their +changes overlap in `.github/workflows/strix.yml`, so the first protected merge +must be re-fetched before the second is restacked. + +This document defines the evidence-backed boundary between what Naruon currently +ships on protected `develop`, what is present only in open pull requests, what is +still a product-plan aspiration, and what a buyer must be able to complete before +Naruon is described as a generally available commercial product. + +Counts, branch SHAs, checks, reviews, and pull-request state are point-in-time +evidence. They must be re-fetched before a merge or release decision. + +--- + +## 1. Executive decision + +Naruon is no longer a small prototype. The protected branch already contains a +substantial, security-conscious **customer-owned communication and context +control plane**: + +```text +customer-owned mail / calendar / contact / file systems +→ Naruon ingest, thread, search, context, evidence, task, and action control +→ explicit human approval or correction +→ conflict-aware provider writeback through an outbound connector +``` + +Naruon must **not** become an SMTP server, IMAP mailbox host, public MX provider, +calendar source of truth, or file source of truth. Customer providers remain +authoritative. Naruon owns scoped context, policy, recommendation, intent, +connector command state, retry/reconciliation evidence, and the user-visible +decision/action experience. + +The accurate current product classification is: + +> **Production-oriented pre-GA communication control plane with substantial +> protected-branch capability, an unconverged ~100-open-PR integration surface (102 at the 2026-08-25 snapshot), and an +> incomplete buyer-visible release/operations contract.** + +The first sellable boundary is **GA-1: Customer-owned Mail, Calendar, Contact, +and File Control Plane**. The complete dense knowledge graph, no-ask +correct-by-exception reasoning, minimal-disclosure bridge, and third-party plugin +platform remain the north-star after GA-1 rather than prerequisites for the +first commercial release. + +--- + +## 2. Evidence hierarchy + +When sources disagree, use this order: + +1. exact protected-branch code, migrations, tests, runtime contracts, and + security boundaries; +2. exact protected-branch architecture and operations documents; +3. exact current pull-request code and current-head evidence; +4. open Issues and accepted ADRs; +5. older product plans, README limitations, and historical PR descriptions. + +A plan marked `[LIVE]` is not proof if the protected implementation contradicts +it. Conversely, an old README statement that calls a protected implementation +“future work” must be corrected rather than used to hide shipped behavior. + +--- + +## 3. Point-in-time repository snapshot + +| Item | Observation | +|---|---| +| Protected branch | `develop@e5e99b4e3bb081b92c602358878856536030e2ca` | +| Product/package version | `0.14.4` | +| Open pull requests | **102** (live scan at `2026-08-25T00:26:45Z`, post-#1337 merge) | +| Open issues | **61** (live count at the same 2026-08-25 snapshot; 59 were open before this baseline program) | +| New completion issue | #1428 | +| Required backend runtime | Python 3.14 exact-head lane | +| Core runtime | Next.js frontend, FastAPI backend, PostgreSQL + pgvector | +| Default data authority | customer-owned mail, CalDAV/CardDAV, and WebDAV providers | +| Default merge posture | strict exact-head checks plus qualifying independent review | + +The **102-open-PR** count is the live inventory snapshot captured on +2026-08-25 against the protected branch shown above, after PR #1337 merged. +The v0.2 baseline recorded **93 open PRs** on 2026-08-21 after PR #1448 +opened, an earlier same-day snapshot recorded **92 open PRs** after +PR #1442 merged, and the initial completion issue #1428 recorded **83 open +PRs** on 2026-08-20 against `develop@c9bfba2...`; these are historical +baselines, not contradictions. Later live counts can change as PRs open, +close, or merge, so every release decision must re-fetch the REST state. + +The protected branch requires exact-head backend, frontend, security, CodeQL, +dependency review, Scorecard, OSV, Trivy, Strix, source/evidence coverage, +backend/frontend/combined image validation, and OpenCode review contexts. Pending, +queued, stale, predecessor-head, skipped-required, neutral, author-only, +model-only, or local-only evidence is not passing evidence. + +--- + +## 4. Protected-branch product truth + +### 4.1 Shipped communication and workspace surface + +Protected `develop` exposes buyer-recognizable product surfaces for: + +- Today execution dashboard; +- Mail, thread history, search, reply, and pending-reply work; +- Calendar views, source-backed coordination, and writeback intent; +- Tasks and source-linked ticket work; +- Projects, project graph, and evidence-linked records; +- Context Search and hybrid retrieval; +- AI Hub and provider-neutral AI workflows; +- Data/document ingestion and controlled materialization; +- Security/policy/audit views; +- Settings, identity, provider, and deployment controls. + +The product already distinguishes simulated local send from real delivery, +preserves `In-Reply-To` and `References`, scopes email/provider records by owner +and organization, keeps opaque public identifiers separate from sequential +surrogates, and applies deny-first RBAC/ABAC policies. + +### 4.2 Source-of-truth and writeback sovereignty + +Protected `develop` already enforces important commercial boundaries: + +- customer mail, calendars, contacts, and files remain durable provider truth; +- browser input selects an opaque source reference but cannot assert ownership, + region, credential, or capability; +- writeback is intent-only unless the user explicitly requests provider + execution; +- provider execution re-reads server-authoritative source records; +- CalDAV and WebDAV updates preserve ETag/If-Match conflict semantics; +- private-network provider access uses an outbound-only connector rather than + inbound firewall holes or public mail hosting; +- provider credentials and command payloads are excluded from browser and + aggregate observability surfaces. + +### 4.3 Durable writeback retry is implemented + +The current protected source-of-truth document records behavior that the root +README still describes as future work: + +- scoped `provider_writeback_retry_items` rows; +- encrypted retry command payloads; +- retry dispatch with retry enqueue disabled for the nested attempt; +- exponential backoff; +- `succeeded`, rescheduled retry, and `failed_exhausted` outcomes; +- persisted connector signal events for dispatch, timeout, transport, and + adapter outcomes; +- organization-admin aggregate queue-depth reads without exposing payload, + credential, runner, source, or retry identities. + +This is a material product-truth correction. The remaining gap is not “create a +retry queue.” It is **finish connector packaging, identity/enrollment, complete +protocol coverage, dead-letter/reconciliation operations, and buyer-visible +support evidence**. + +### 4.4 AI and scientific boundary + +Naruon has grounded content segments, hybrid search, named/versioned KG +extractor seams, deterministic fallback, contextual-orchestrator routing, and +batch-embedding integration boundaries. It does **not** have a protected live +Structural Topic Model endpoint or fitted topic artifact. Deterministic keyword +metadata must not be marketed as STM or temporal event psychometrics. + +TEPP may be consumed only through a separately accepted, immutable, versioned +scientific artifact/API with preprocessing, vocabulary, covariates, posterior +uncertainty, diagnostics, provenance, and abstention. Naruon owns identity, +authorization, adapter envelopes, and disclosure policy; TEPP owns the +scientific payload. + +--- + +## 5. Product-truth and release-truth inconsistencies + +| Inconsistency | Current evidence | Buyer risk | Required correction | +|---|---|---|---| +| README says durable retry/audit remains future work | protected operations document describes encrypted retry rows, retry worker, backoff, exhaustion, and aggregate visibility | buyers and contributors cannot tell what is shipped | merge a customer/operator README based on protected truth; keep unsupported behavior explicitly limited | +| Release architecture says first candidate should be `v0.1.0` | `VERSION` and backend package are `0.14.4` | release procedure may publish or validate the wrong identity | replace historical hypothesis with current release-train policy and immutable release manifest | +| Product plan marks typed Person/Event/Commitment/Plugin concepts as new/planned | current code search does not prove authoritative `graph_persons`, `graph_events`, `graph_commitments`, or `plugin_registrations` stores | UI/marketing can imply dense-KG/product-platform completion that does not exist | keep north-star language, implement typed domains through bounded PRs, and gate claims on protected code | +| The live open-PR population (93 on 2026-08-21; 102 on 2026-08-25) contains many overlapping, stacked, micro, dependency, governance, and broad integration changes | current GitHub inventory | predecessor evidence, writer collision, stale branches, and integration starvation | establish a release train, classify every PR, close duplicates, merge parent-first, and use one writer per authority cluster | +| Required independent review exists but the current human reviewer path is unresolved | #1371 | green automation cannot produce a lawful protected merge | resolve reviewer governance without weakening rulesets or self-approval | +| Connector is described through a self-hosted-runner analogy | protected code has protocol adapters and retry behavior but no complete released connector lifecycle | operators may deploy test infrastructure as production relay | deliver signed installable connector artifacts, enrollment/rotation, source health, fleet SLO, and runbooks | + +--- + +## 6. Current pull-request surface + +The current open PR count is too large to treat as one releasable integration +unit. This baseline does not claim that every one of the 102 PRs has been +line-by-line approved. It records the product-significant active lanes observed +and defines the inventory that must be completed before GA. + +### 6.1 Product-significant active lanes + +| PR | Lane | Baseline judgment | +|---:|---|---| +| #1364 | scoped S3 document-object backend | high-leverage GA durability lane; Draft until real PostgreSQL + S3 lifecycle, backfill, cleanup, failure, and exact-head evidence are complete | +| #1417 | shared PostgreSQL-backed email-send throttle | necessary multi-replica safety; keep isolated and merge only with current-head concurrency/security evidence | +| #1416 | provider-backed CalDAV create writeback | relevant to GA scheduling execution; preserve create vs update precondition distinction and integrate into the broader #978 contract | +| #1353 | HWP/HWPX deterministic recognition boundary | useful Korean enterprise document admission; does not complete parsing/conversion/search semantics | +| #1397 | inline-media admission/tracking-pixel classification | valid evidence-protection slice; remain Draft until the #1350 stack and independent review are coherent | +| #1419 | common image metadata | bounded local evidence extraction; no OCR/VLM claim | +| #1418 | auditable URL/contact hygiene | deterministic tool/evidence lane; ensure contact redaction is not represented as complete anonymization | +| #1317 | broad macOS/local-AI/runtime/governance integration | valuable evidence but unusually broad; must be decomposed or reconciled carefully because many active PRs overlap its surfaces | +| #1392 | customer/operator README rewrite | directly addresses product-truth debt and has reported exact-head checks; still requires independent current-head approval | +| #1300 | fail closed on unsafe global tool mutations | correct safety posture until durable tenant-scoped plugin/tool registry exists; links directly to #976 | +| #1264 | EgressWeave integration | correctly dependency-blocked on an immutable released EgressWeave package and hash lock; mutable VCS dependency is forbidden | +| #1390 / #1391 | 56- and 78-package dependency groups | excessive blast radius, including major runtime and OpenAI client changes; split by compatibility/authority and rehearse migrations before merge | +| #1426 / #1414 | review-governance gate refresh | metadata-only governance repair; must not dismiss review, weaken rulesets, or turn stale aggregate state into false success | +| #1241, #1320, #1408, #1410, #1411, #1421, #1422 | accessibility micro-lanes | useful but numerous; consolidate non-overlapping UI fixes into bounded component-level trains to reduce 17-check amplification | +| #1424, #1412, #1401 | micro performance lanes | require real benchmark or stable complexity contract; do not let automated micro-PRs displace GA integration work | +| #1455 | path-traversal attachment parser hardening | high-value Sentinel security lane; prioritize within the new wave and merge only with exact-head Strix evidence once the provider outage clears | +| #1347 | rate-limited review status governance | current head constrains repository API-scope identifiers and merges the protected base; local tests pass, while hosted Checks and independent review are still required | +| #1433 | Message-ID whitespace hardening | source/security checks pass; the observed Strix failure is provider infrastructure (NVIDIA NIM 429 and direct OpenAI 404), so retry exact-head evidence without weakening the gate | +| #1450 | stray scratch/debug cleanup | all source and security Checks passed; wait for the in-progress metadata gate and current-head independent review | +| #1465 | scoped tenant archive import hardening | portability slice now rejects duplicate identities before writes and sanitizes archive-controlled display fields; retain the bounded slice-1 query cost and require current-head hosted evidence | +| #1466 | origin-integrity URL validation | current head rejects explicit zero/out-of-range ports; keep the signed-session and SSRF contract tied to exact-head regression evidence | +| #1467 | utility-tool JSON and governance repair | deterministic URL/HTML/JSON utility surface; current head rejects non-standard JSON numbers and preserves the central Strix workflow trigger, while full smoke evidence still depends on #1468's schema fixture repair | +| #1468 | PostgreSQL smoke fixture schema alignment | small root-cause test/data-contract repair for the current `email_records.is_read` requirement; merge before dependent smoke-test PRs after exact-head hosted evidence; the observed Naruon Strix run failed at the provider boundary (NVIDIA NIM 429/OpenAI 404), not in this source change | +| #1443 | CodeRabbit approval-notice governance root | current source/test lane narrows approval-notice parsing to the exact current head and ignores pending-review prose while retaining explicit findings; the predecessor Strix provider failure is historical, while the current head requires fresh queued Checks and a qualifying independent approval | +| #1448 | stacked governance regression coverage | merged normally into #1443's stack branch at `62a0d645…`; parent gate logic plus multiline JSON, stale-head unrelated prose, mixed blocker, and explicit current-head finding fixtures passed locally; merge-result hosted Checks remain queued and are post-merge canary evidence | +| #1469 | deferred attachment parse-source admission | aligns the hidden 20 MiB parser bound with the authenticated 64 MiB import contract while keeping unsupported binaries metadata-only; current changelog states the supported 20–64 MiB range and the ADR-0006 contract remains required | +| #1470 | NetworkGraph lookup optimization | bounded frontend performance slice; current head `aba77cf5…` preserves first-instance duplicate-ID selection, removes the dead `describeEdge` input, and deletes the tracked pre-refactor `NetworkGraph.tsx.out` copy; local 11-test, TypeScript, zero-warning ESLint, and diff checks passed, while hosted Checks and independent approval remain required | +| #1456 | email-detail UX density | buyer-visible mail surface polish; hold to the responsive/accessibility evidence contract in the UI quality section before protected integration | +| #1462 | utility tool trio (URL codec, hash generator) | bounded deterministic tooling consistent with #1418/#1361; must not be represented as AI judgment or topic evidence | +| #1457–#1461 | refreshed dependency-group bumps | successor waves to the v0.2-flagged groups; the 64- and 86-package backend/CI bumps remain excessive blast radius and still require splitting and migration rehearsal | + +### 6.2 Required complete inventory + +Before a release candidate is cut, create a machine-readable and human-reviewed +inventory for **all** current open PRs with: + +```text +pr_number +head_sha +base_ref_and_sha +draft_state +mergeability +changed_authority_cluster +stack_parent +stack_children +current_review_state +unresolved_threads +required_check_summary +product_lane +disposition +next_action +``` + +Allowed dispositions: + +- direct GA-1 slice; +- ordered stacked child; +- dependency-blocked; +- governance-blocked; +- duplicate/superseded; +- experimental/north-star; +- unsafe or unrelated and to be closed. + +The inventory must be regenerated after every parent merge or branch movement. +It must not embed provider credentials, customer data, review-body secrets, or +large copied PR bodies. + +### 6.3 Merge-loop progress since v0.2 + +Point-in-time progress observed between the v0.2 snapshot (2026-08-21) and +this v0.3 observation (2026-08-25). None of this is merge evidence; every +claim requires a live exact-head re-fetch before any decision: + +- **Merge-gate progression:** #1337 completed its gate progression (CodeRabbit exact-head approval obtained, branch updated onto the base) and merged into protected `develop` at `2026-08-25T00:10:39Z`; #1438 obtained CodeRabbit exact-head approval and branch updates and remains open pending terminal required-check states. +- **Stale-snapshot repair:** #1241, #1368, #1412, and #1320 received systemic develop-baseline restores that preserved each PR's intended delta while removing predecessor-base drift from the diff surface. +- **Thread remediation waves:** unresolved review threads progressed through repeated remediation cycles on #1264, #1332, #1347, #1349, #1361, #1380, #1339, #1376, #1412, #1452, #1454, #1449, #1457, #1436, and #1441. +- **External blocker:** Naruon hosted Strix runs have observed NVIDIA NIM provider rate-limit failures (HTTP 429) since approximately 2026-08-24, emitting zero model-reported vulnerabilities before infrastructure failure. This is an observation of the affected hosted runs, not a change to the central GitHub Models default in `AGENTS.md`; per repository policy it is failed evidence, not clean-scan evidence, and Strix-dependent gates cannot pass until the provider path recovers. + +### 6.4 UI/UX quality contract and Storybook event inventory + +The UI is a buyer-facing control surface, not a decorative shell. The current +design-system implementation is carried by PR #1436 and ADR-0013; it uses the +production stylesheet as the Storybook token source and records Figma file ID +`68b5XB58w8nwT2LYOOnikK`. Until that PR is protected-branch code, its stories +are current-PR evidence rather than shipped capability. + +The UI/UX Pro Max checklist and Anti-Slop UI heuristics are adopted as review +inputs, not as normative standards or automatic approval. They help select one +coherent design direction, expose generic UI defaults, and force explicit +review of accessibility, touch targets, hierarchy, and state behavior. WCAG +2.2 and the repository's security/accessibility gates remain authoritative. + +The latest local fixed-origin capture for #1470 used `/` at desktop (1440×900), +tablet (834×1112), and mobile (390×844). Tablet and mobile presented the +responsive shell and an explicit mail-loading state. Desktop presented the +navigation shell but no data surface while the backend was intentionally not +running; the Next development server also reported a hydration mismatch for +the search-input caret style. This is local diagnostic evidence, not hosted +release evidence, and is tracked as a separate UX/runtime gap rather than +mixed into the bounded NetworkGraph change. + +| Quality axis | Required definition and applied evidence | Audit gate before GA-1 | +|---|---|---| +| Accessibility | WCAG 2.2 AA; keyboard order/focus-visible; accessible names; labels; live/status announcements; color is never the only signal; Storybook a11y test is `error` for applicable stories | axe/Vitest Storybook results, keyboard journey, screen-reader name assertions, and zero unresolved accessibility findings | +| Touch & interaction | primary pointer and keyboard paths; at least 44×44 CSS-pixel target or documented exception; 8px separation; loading/disabled/pressed feedback; no hover-only action | Storybook `play` events queried by role/label plus touch viewport browser test | +| Performance | reserved media dimensions, no avoidable layout shift, route/component splitting, virtualized long lists, and responsive feedback for async work | production build budget, responsive capture at 375/768/1024/1440, and measured CLS/input-latency evidence | +| Style selection | one documented design direction, consistent icon language, semantic tokens, deliberate radius/elevation, and no generic gradient/card/emoji defaults | ADR/Figma decision, token source review, and Anti-Slop heuristic checklist with human disposition | +| Layout & responsive | mobile-first hierarchy, no horizontal scroll, readable line length, safe-area/fixed-bar offsets, and synchronized desktop/tablet/mobile navigation | Storybook viewport stories and Playwright route/drawer assertions at each supported viewport | +| Typography & color | semantic foreground/surface/status tokens; body text and line-height contract; contrast ≥4.5:1 for normal text; wrapping/overflow for IDs and user content | token lint, contrast scan, long-content story, dark-mode story, and i18n expansion test | +| Animation | shared duration/easing tokens, causal motion, transform/opacity preference, interruptibility, and `prefers-reduced-motion` behavior | reduced-motion Storybook story and browser assertion that action remains usable during transitions | +| Forms & feedback | visible labels, field-local errors, helper text, async progress, retry/undo or next action, and server error preservation | valid/invalid/loading/success/timeout/permission stories with submit and recovery events | +| Navigation patterns | predictable back/deep links, stable route identity, focus restoration, drawer parity, and one primary action per surface | route matrix, keyboard navigation journey, refresh/deep-link test, and mobile drawer test | +| Charts & data | legends/tooltips or accessible table, empty/loading/error/partial states, textual values, and color-independent meaning | chart stories for every state, keyboard/tooltip test, screen-reader text, and deterministic snapshot/visual evidence | + +Each reusable component must have a Storybook story for its meaningful states. +Each interactive story must use a `play` function and user-like queries such as +role or accessible label; `data-testid` is a last resort. Storybook render, +interaction, accessibility, and visual tests are complementary: a passing +render story does not prove keyboard, async, responsive, or data correctness. + +The minimum scene/event matrix is: + +| Scene | Required event or assertion | Failure prevented | +|---|---|---| +| initial/ready | render, accessible name, primary action | dead or unnamed control | +| loading/pending | click or submit, disabled state, progress/status update | duplicate request and silent wait | +| empty/no-result | filter/search/reset, next-action copy | inert workspace | +| success/recognized | inspect, open, confirm, source/provenance text | unsupported product claim | +| validation/error/timeout | invalid input, server error, retry/recovery | error only in console or lost user work | +| unauthorized/forbidden | attempted action, denial explanation, no sensitive data | privilege disclosure | +| offline/connector unavailable | degraded read path, retry/backoff affordance | false provider success | +| long content/large dataset | wrap, scroll/virtualize, pagination or summary disclosure | layout collapse and browser lock-up | +| keyboard/touch/reduced motion | tab/enter/escape, pointer/touch, reduced-motion media query | inaccessible or motion-sensitive flow | + +The inventory must record component, story name, state, event, expected +customer-visible outcome, accessibility rule, token source, viewport, and test +command. A story that only renders a static screenshot is incomplete for a +button, form, navigation, chart, or asynchronous data surface. + +### 6.5 Queue convergence rules + +1. One active writer owns each overlapping file/authority cluster. +2. A stacked child is not promoted before its parent reaches protected + `develop` and the child is revalidated on the new exact base. +3. Predecessor-head checks and reviews never transfer. +4. Dependency changes that cross runtime majors are separated from unrelated + feature work. +5. Historical one-shot, repair, finalizer, and self-modifying workflow identities + are handled through #1324; do not add another write-capable cleanup workflow. +6. Independent approval remains mandatory; #1371 is resolved by establishing a + legitimate reviewer path, not by weakening protection. +7. A micro-optimization requires measured evidence or a stable tested complexity + invariant, not only an assertion that `Map` is faster than array lookup. +8. A PR that claims to close an umbrella issue must prove the full umbrella + acceptance journey, not one narrow slice. + +--- + +## 7. Buyer-visible Gap matrix + +### P0 — Release and integration control + +| Gap | Buyer problem | Protected/current evidence | Existing work | Completion evidence | +|---|---|---|---|---| +| Release-train convergence | no buyer can assess a product with ~100 unconverged open PRs (102 at the 2026-08-25 snapshot) | strict gates exist but queue topology is fragmented | #1428, #1371, #1324 | all PRs classified; duplicates closed; parent-first integration; one immutable RC SHA | +| Product/release truth | documentation conflicts with protected behavior/version | retry is shipped; release doc says `v0.1.0`; version is `0.14.4` | #1392, this PR | README, architecture, version, changelog, release manifest, and operator guide agree | +| Independent review path | automation cannot lawfully self-approve | effective rulesets require independent post-last-push approval | #1371 | verified reviewer route and normal protected merge without bypass | + +### P0 — Connector and provider action + +| Gap | Buyer problem | Protected/current evidence | Existing work | Completion evidence | +|---|---|---|---|---| +| Installable connector | adapters in source are not an operable product | outbound-only architecture and several adapters exist | #998 | signed packages/OCI, enrollment, rotation, upgrade/rollback, supported matrix | +| Source lifecycle | configuration does not equal observed provider capability | source IDs, eligibility, consent, and revisions exist in slices | #998, #978 | create/rotate/disable/delete, capability discovery, health, stale-capability invalidation | +| Durable reconciliation | retry exhaustion alone does not tell the buyer what happened remotely | retry/backoff/exhaustion and signal events exist | #998 | idempotent command, late-success reconciliation, dead-letter action, buyer receipt | +| Shared send safety | process-local throttles fail with multiple replicas | PR proposes PostgreSQL-backed atomic bucket | #1379, #1417 | concurrency/expiry/isolation/DB-unavailable tests and protected integration | + +### P0 — Data durability, portability, and customer exit + +| Gap | Buyer problem | Protected/current evidence | Existing work | Completion evidence | +|---|---|---|---|---| +| Binary object lifecycle | large/deferred document bytes cannot remain an inline database strategy | S3-compatible implementation is Draft | #1076, #1364 | upload/read/recognize/retain/delete/backfill/orphan round trip with real integration | +| Disaster recovery | a release is not enterprise-ready without restore evidence | HA evaluation exists; production WAL/PITR policy remains incomplete | #1428 | WAL archive/PITR, failover fencing, backup and clean restore rehearsal | +| Tenant export/reimport | customers need exit and migration without losing provenance | no single demonstrated full tenant round trip | #1428 | export → clean instance import preserving source, opaque IDs, history, evidence, policy | +| Retention/legal hold/disposition | deletion and evidence preservation conflict unless modeled | partial security/key/retention work exists across repository history | #1428, #1364 | purpose-scoped retention, legal hold, verified disposition, object/DB reconciliation | +| Attachment parser admission and unsupported formats | a file above 20 MiB can pass import transport but fail later at a hidden parser limit, while unsupported binaries are not searchable | Naruon import transport and generic deferred parser admission are bounded at 64 MiB; the NewsDOM `/parse` provider contract remains 20 MiB, so PDF bytes from 20–64 MiB are admitted and retained fail-closed but are not sent to NewsDOM; unsupported types remain metadata-only | #1427, #1469, #1353, #1419, NewsDOM #682/#707 | one documented bounded admission contract, provider-side PDF limit alignment or an explicit large-PDF fallback, parser/status evidence, deferred recognition, and object-backed retention before increasing the bound again | + +### P0 — Evidence-based AI and document intelligence + +| Gap | Buyer problem | Protected/current evidence | Existing work | Completion evidence | +|---|---|---|---|---| +| Canonical evidence identity | OCR/media/attachment/model slices can disagree about the same source | source segments and several deterministic admission slices exist | #1350, #1353, #1397, #1419 | one source identity/provenance chain across email, thread, document, attachment, media, model result | +| Judgment explanation | model output without evidence/calibration is not defensible | grounded extractor seam exists; wider evidence pipeline incomplete | #1350 | evidence IDs, claim support, abstention, correction, verifier result, prompt/model/version receipt | +| Provider-neutral route | raw provider coupling spreads credentials and failure behavior | contextual-orchestrator boundary exists; EgressWeave integration is blocked on release | #1262, #1264 | released hash-locked adapter, route/fallback evidence, no raw secret in products | +| Scientific claim discipline | keyword labels can be mistaken for topic/event measurement | architecture explicitly says no live STM | TEPP dependency path | accepted immutable TEPP artifact/API or explicit feature absence; no lexical-as-STM claim | + +### P1 — Typed context and scheduling differentiation + +| Gap | Buyer problem | Protected/current evidence | Existing work | Completion evidence | +|---|---|---|---|---| +| Responsive shell hydration and unavailable state | a buyer can see a polished navigation shell but no actionable content when a data request is unavailable, and hydration drift can produce inconsistent controls | local fixed-origin capture showed tablet/mobile loading feedback, desktop blank content without the backend, and a development-server caret-style hydration mismatch; this is not a hosted release result | follow-up required; keep separate from #1470's bounded lookup optimization | deterministic server/client markup, explicit desktop unavailable/error state, backend-backed responsive Playwright evidence, and no hydration warnings | +| Stacked PR current-head review dispatch | a dependent PR can show only metadata while the central OpenCode/required checks are still being materialized on a non-default base branch | #1448 exact head `068aefdf…` received a targeted scheduler/ OpenCode dispatch and then merged normally; its merge-result checks on `62a0d645…` remain queued | #1443, #1448, ContextualWisdomLab/.github scheduler | every supported stack base receives a bounded exact-head OpenCode/Noema/required-check run, with queued/provider states observable and no false merge readiness | +| Typed Person/Event/Commitment graph | generic string graph cannot safely drive high-stakes action | planning spec marks types as new/planned | #977, #978, #1000 | normalized temporal/multi-membership identities, evidence/confidence/correction on every inferred edge | +| Status-weighted scheduling | calendar CRUD does not prevent harmful double booking | CalDAV source/writeback/retry foundation exists | #978, #988, #989, #990, #1416 | confirmed/tentative/desired + organizer/attendee + recurrence/free-busy/resource end-to-end | +| Minimal-disclosure bridge | personal context can influence work availability without exposing private reason | policy substrate exists; product bridge is planned | #979, #991 | consented consequence-only propagation, revocation, audit, regression tests | +| Correct-by-exception inference | asking users to reconstruct context defeats the product mission | extractor/search foundations exist; dense-KG resolution remains incomplete | #977, #992, #1001 | one recommendation, evidence/calibration, hold/override, correction learning, no silent irreversible action | + +### P1 — Platform and ecosystem + +| Gap | Buyer problem | Protected/current evidence | Existing work | Completion evidence | +|---|---|---|---|---| +| Plugin lifecycle | internal extension seams do not create an enterprise platform | extractor/parser seams exist; durable custom tool mutation fails closed | #976, #1300 | signed manifest/release, tenant grant, sandbox, compatibility, upgrade/rollback/uninstall | +| Stable cross-repository contracts | copying sibling code creates a distributed monolith | several adapters are planned or dependency-blocked | #976, #1262, #1350 | released SDK/API/event/OCI contracts pinned by version/digest; no direct sibling SQL | +| Buyer administration | operators need source, connector, policy, health, retention, and support controls | settings/security surfaces exist but not one completed admin lifecycle | #998, #1428 | role-specific admin console with next-action explanations and audited high-risk changes | + +--- + +## 8. GA-1 product definition + +GA-1 is complete only when a buyer can perform the following journey on one +released, immutable product version: + +```text +install or access Naruon +→ authenticate through the supported enterprise identity boundary +→ install and enroll a signed outbound connector +→ register customer-owned mail/calendar/contact/file sources +→ verify observed source capabilities and health +→ synchronize source records with provenance +→ receive a source-cited judgment or action recommendation +→ inspect evidence, confidence, authorization, privacy, and provider impact +→ approve, hold, or correct the recommendation +→ execute an idempotent provider action with current revision protection +→ observe success, retry, conflict, exhaustion, or reconciliation evidence +→ restore, export, or migrate the tenant without losing provenance +``` + +Naruon is not GA if the demonstration substitutes mocked provider success, +process-local state, local-only credentials, predecessor-head checks, synthetic +review approval, hidden manual database edits, or an unreleased sibling branch. + +--- + +## 9. Delivery sequence + +### Wave 0 — Product truth and queue convergence + +1. Merge this baseline after exact-head documentation checks and independent + review. +2. Keep #1428 as the single completion gate. +3. Maintain the complete open-PR inventory (section 13) and classify every PR as it arrives. +4. Resolve #1371 without weakening protection. +5. Merge/close governance and stale-workflow lanes through normal protected + integration. +6. Correct README/release/version contradictions. + +### Wave 1 — GA-1 runtime and operations + +1. Finish #998 connector artifact, enrollment, capability, fleet, retry, and + reconciliation lifecycle. +2. Integrate shared send throttling (#1379/#1417). +3. Finish binary object lifecycle (#1076/#1364). +4. Complete PostgreSQL WAL/PITR/failover/restore evidence. +5. Complete OIDC/SCIM/tenant administration and privacy-safe OpenTelemetry/SLO. +6. Publish independent backend, frontend, connector, and compatibility artifacts. + +### Wave 2 — Buyer differentiation + +1. Implement typed temporal/multi-membership Person/Event/Commitment graph. +2. Complete status-weighted scheduling (#978/#988/#989/#990). +3. Complete evidence-based mail/document/media resolution (#1350). +4. Add tenant export/reimport and customer-exit evidence. +5. Run the full buyer journey and failure/restore variants. + +### Wave 3 — North-star platform + +1. Minimal-disclosure privacy bridge (#979/#991). +2. Dense-KG correct-by-exception resolution (#977/#992/#1001). +3. Signed plugin platform (#976) with one real independently released CWL + plugin. +4. Optional TEPP and other ecosystem adapters through accepted immutable + contracts. + +--- + +## 10. Release and quality gate + +### Product correctness + +- real IMAP/SMTP and DAV interoperability fixtures; +- duplicate/replay, late response, partial failure, provider conflict, connector + restart, network partition, and source-capability-change tests; +- recurrence, timezone, DST, organizer/attendee, free/busy, and resource + scheduling tests; +- full tenant export/reimport and backup/restore rehearsals; +- buyer-visible provenance completeness and unsupported-claim rate gates; +- no silent confirmed-commitment break; +- no private reason crossing a context boundary without explicit authorized + disclosure. + +### Code and documentation + +- production statement coverage 100% for owned production code; +- production branch coverage 100%; +- public API/module/class/function docstrings 100%; +- frontend component, interaction, action-edge, design-token, accessibility, and + i18n tests; +- no documentation that represents planned behavior as shipped or shipped + behavior as future work; +- ADR, PRD, TRD, architecture, data model, runbook, and standard traceability + updated with each release-relevant decision. + +### Database + +- normalized ownership and provider mappings; +- descriptive two-or-more-word `snake_case` object names; +- no direct cross-service SQL; +- bitemporal/effective-dated facts where provider state, membership, consent, + policy, or source mapping changes over time; +- tenant enforcement and hot-partition/load evidence; +- clean install, N-1 upgrade, expand/backfill/contract, rollback, and restore. + +### Security and supply chain + +- OAuth/OIDC threat controls aligned with the current OAuth 2.0 Security BCP; +- signed connector and release artifacts; +- digest-pinned OCI images; +- SPDX 3.0.1 SBOM; +- SLSA 1.2 provenance; +- dependency/license/vulnerability evidence; +- secret, prompt, message, event, contact, and file-content exclusion from + ordinary telemetry; +- no mutable VCS dependency in production; +- no self-modifying or broad-token workflow used to compensate for product code. + +### Operations + +- liveness, startup, readiness, and drain semantics per independently deployable + component; +- OpenTelemetry trace, metric, and log acceptance; +- low-cardinality label contract; +- SLO, error budget, burn-rate alert, dashboard, on-call, incident, and support + bundle; +- connector offline/backpressure and safe rolling upgrade; +- backup/PITR/failover/restore and customer-exit runbooks; +- versioned support, deprecation, compatibility, and security-fix policy. + +### Protected integration + +- one exact current head and one exact protected base; +- all live required contexts terminal-success; +- zero actionable unresolved review threads; +- qualifying independent non-author post-last-push approval; +- no bypass, self-approval, force push, dummy commit, empty requeue, or stale + predecessor evidence; +- immutable release source SHA and artifact digests recorded in release evidence. + +--- + +## 11. Issues established or strengthened by this baseline + +| Issue | Purpose | +|---:|---| +| [#1428](https://github.com/ContextualWisdomLab/naruon/issues/1428) | new umbrella: GA scope, release train, and buyer-visible acceptance | +| [#976](https://github.com/ContextualWisdomLab/naruon/issues/976) | strengthened: signed plugin SDK, registry, permissions, compatibility, sandbox, lifecycle | +| [#978](https://github.com/ContextualWisdomLab/naruon/issues/978) | strengthened: typed Event/Commitment model, iTIP/CalDAV scheduling, free/busy/resource, conflict and privacy contract | +| [#998](https://github.com/ContextualWisdomLab/naruon/issues/998) | strengthened: installable connector, enrollment/identity, protocol capability, reconciliation, OpenTelemetry/SLO | + +Existing linked implementation and blocker issues remain authoritative for their +bounded scopes, including #1022, #1076, #1229, #1262, #1324, #1350, #1371, and +#1379. + +--- + +## 12. Standards and research traceability + +The following standards are not decorative references. They define protocol, +security, accessibility, observability, or supply-chain acceptance tests in the +issues above. + +For the DiskSage boundary specifically, PROV-O supplies the provenance +relations, DCAT supplies catalog/dataset/service discovery terms, OpenLineage +supplies run and facet vocabulary, and NIST SP 800-209 supplies storage +protection and recovery controls. The implementation must keep these as +metadata relationships and receipts; a provider upload or local path alone is +not an eviction or lineage claim. + +### APA 7th references + +Crispin, M. (2003). *Internet Message Access Protocol—Version 4rev1* (RFC +3501). RFC Editor. https://doi.org/10.17487/RFC3501 + +Daboo, C. (2010). *iCalendar transport-independent interoperability protocol +(iTIP)* (RFC 5546). RFC Editor. https://doi.org/10.17487/RFC5546 + +Daboo, C. (2011). *vCard extensions to WebDAV (CardDAV)* (RFC 6352). RFC +Editor. https://doi.org/10.17487/RFC6352 + +Daboo, C., Desruisseaux, B., & Dusseault, L. M. (2007). *Calendaring extensions +to WebDAV (CalDAV)* (RFC 4791). RFC Editor. https://doi.org/10.17487/RFC4791 + +Daboo, C., & Desruisseaux, B. (2012). *Scheduling extensions to CalDAV* (RFC +6638). RFC Editor. https://doi.org/10.17487/RFC6638 + +Daboo, C., & Quillaud, A. (2012). *Collection synchronization for Web +Distributed Authoring and Versioning (WebDAV)* (RFC 6578). RFC Editor. +https://doi.org/10.17487/RFC6578 + +Gellens, R., & Klensin, J. (2011). *Message submission for mail* (RFC 6409). +RFC Editor. https://doi.org/10.17487/RFC6409 + +Jenkins, N., & Newman, C. (2019). *The JSON Meta Application Protocol (JMAP) +for Mail* (RFC 8621). RFC Editor. https://doi.org/10.17487/RFC8621 + +Lodderstedt, T., Bradley, J., Labunets, A., & Fett, D. (2025). *Best current +practice for OAuth 2.0 security* (BCP 240; RFC 9700). RFC Editor. +https://doi.org/10.17487/RFC9700 + +Melnikov, A., & Leiba, B. (2021). *Internet Message Access Protocol (IMAP) +version 4rev2* (RFC 9051). RFC Editor. https://doi.org/10.17487/RFC9051 + +OpenSSF. (2025). *Supply-chain Levels for Software Artifacts specification, +version 1.2*. https://slsa.dev/spec/v1.2/ + +OpenTelemetry Authors. (2026). *OpenTelemetry specification, version 1.60.0*. +https://opentelemetry.io/docs/specs/otel/ + +Elkady, H. (2026). *Anti-Slop UI: A Deterministic State-Machine Architecture +for Eliminating Design Hallucinations in LLM-Generated Interfaces*. Local Over. +https://local-over.github.io/Anti-Slop-UI/research_paper.pdf + +NextLevelBuilder. (2026). *UI/UX Pro Max skill* (Version 2.5.0) [Computer +software]. GitHub. https://github.com/nextlevelbuilder/ui-ux-pro-max-skill + +Storybook. (n.d.). *Accessibility testing*. Retrieved August 21, 2026, from +https://storybook.js.org/docs/writing-tests/accessibility-testing + +Storybook. (n.d.). *Interaction tests*. Retrieved August 21, 2026, from +https://storybook.js.org/docs/writing-tests/interaction-testing + +SPDX Workgroup. (2024). *SPDX specification, version 3.0.1*. +https://spdx.github.io/spdx-spec/ + +World Wide Web Consortium. (2024). *Web Content Accessibility Guidelines +(WCAG) 2.2*. https://www.w3.org/TR/WCAG22/ + +National Institute of Standards and Technology. (2020). *Security guidelines +for storage infrastructure* (NIST Special Publication 800-209). +https://doi.org/10.6028/NIST.SP.800-209 + +OpenLineage. (n.d.). *OpenLineage documentation: Object model, run cycle, and +facets*. Retrieved August 25, 2026, from https://openlineage.io/docs/ + +World Wide Web Consortium. (2013). *PROV-O: The PROV ontology*. +https://www.w3.org/TR/prov-o/ + +World Wide Web Consortium. (2024). *Data Catalog Vocabulary (DCAT) — Version +3*. https://www.w3.org/TR/vocab-dcat-3/ + +--- + +## 13. Live open-PR identity inventory + +This table was generated from a live GitHub pull-request collection whose +response was captured at `2026-08-25T00:26:45Z`. It records every currently open Naruon +PR's immutable head, +base, draft state, authority cluster, stack parent reference, and provisional +disposition. Review decisions, unresolved threads, mergeability, and Checks are +volatile and must be fetched again for the exact head immediately before any +merge; GraphQL rate-limit failures are not treated as approval or success. + +The 102-row inventory is a fresh refresh later than baseline v0.2's 93-row +snapshot captured at `2026-08-21T19:25:43Z`: PR #1337 merged into protected +`develop` at `2026-08-25T00:10:39Z` and PRs #1449–#1463 opened since that +observation. The v0.2 93-row snapshot, the earlier 92-row snapshot taken +after PR #1442 merged, and their exact-head observations remain historical +and are retained in baseline v0.2 and in git history for audit +traceability. The self-row reflects the head observed at scan time and may +trail this document's own commit; all review decisions, Checks, and +mergeability still require a live exact-head fetch before merge. +Base-SHA columns record the protected-branch tip each row was scanned +against (`develop@81c10564...` for rows captured before PR #1337 +merged; `develop@e5e99b4e...` for rows captured after). A few older rows +retain `develop@dd8d1519...` from an earlier scan window; that value is also +historical scan provenance, not a second claim about the current protected +branch. These values are therefore not one shared base declaration. + +| PR | Title | Exact head SHA | Base ref and SHA | Draft | Authority cluster | Stack parent ref | Disposition | Next action | +|---:|---|---|---|:---:|---|---|---|---| +| 1463 | 🎨 Palette: 비동기 작업 버튼의 로딩 상태 시각적 피드백 개선 | d0f51274dc86271e365aa1384e1362daf93f1d7d | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | frontend/a11y | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1462 | feat: 신규 유틸리티 도구 3종 추가 (URL 인코더, URL 디코더, 해시 생성기) | dba144b01be8a7e332c9fc7b390e1d9b105536c4 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | other | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1461 | chore(deps): bump the ci-python group across 1 directory with 86 updates | f012853e6123017c290a7782cc7cf8e4801e4bb7 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | dependency | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1460 | chore(deps): bump the backend-python group across 1 directory with 64 updates | dec4329986b261607204c0711d62f3bcd782d181 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | dependency | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1459 | chore(deps): bump the frontend-npm group across 1 directory with 14 updates | 3edb8320b0288f2bba54ce7e28083164e0d51966 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | dependency | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1458 | chore(deps): bump the github-actions group across 1 directory with 3 updates | f7099f951574a01f1333cb72a5e48ac91e7f4ea5 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | dependency | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1457 | chore(deps): bump the docker-base-images group across 1 directory with 2 updates | 245c8f1d2177e451e9812525ef6bbd7db9be3e6e | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | ingest/storage | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1456 | 🎨 Palette: 이메일 상세 화면 UX 밀도 개선 | 20b1c7f4798343d6949ab57ee9ab015d928b04bf | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | frontend/a11y | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1455 | 🛡️ Sentinel: [HIGH] 첨부 파일 파서의 경로 탐색 취약점 수정 | 40c5b7087a92a23eba787682fe32604861a9474b | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | security/governance | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1454 | ⚡ Bolt: Optimize NetworkGraph edge description lookups to strictly O(1) | 9cb7883cd0a41b9634f4f74e77f3595d754ab7e7 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | performance | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1453 | 🎨 Palette: SettingsLayout 비활성화 버튼 접근성 툴팁 개선 | eef7e8d2eb25199afeb13c6b8a24f0f73bbd0fe5 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | frontend/a11y | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1452 | 🎨 Palette: 비활성화 버튼 접근성(Tooltip) 개선 추가 | ad32c8d35c11a76f52e189d7a5e83fb74fb603df | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | frontend/a11y | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1451 | feat: 텍스트 유틸리티 도구 추가 및 테스트 보완 | a2a20e3eb14bb65fab8ee2f2f9199dd76dd71bce | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | other | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1450 | chore: remove stray scratch/debug files committed to develop | 7d1f57a83a34312bea265ae3095bc67ff4e0493e | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | other | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1449 | 🎨 Palette: 데이터 저장소 버튼 액션 로딩 UX 및 접근성 개선 | 90248b26fff37f395c6918525c4eef041d683e09 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | frontend/a11y | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1448 | test(governance): exercise multiline CodeRabbit pending notice | 874c098548e6794217393e0338074ba2f292d080 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | security/governance | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1443 | fix: ignore CodeRabbit approval pending notices | 41e48413cffefa8a5393d6af1d5ad16be3c5de7c | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | other | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1441 | 🎨 Palette: [UX 개선] 메일 상세 고밀도 컴포넌트 추가 | ce6ff8f26e4cfdc76be7d667c7b159c57f8e0ac5 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | frontend/a11y | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1439 | ⚡ Bolt: 프론트엔드 O(N) Array.find() 룩업을 O(1) Map 룩업으로 성능 개선 | 68106d13175d7ff67978de66e31d385237ca53b1 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | performance | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1438 | fix(governance): supersede stale review decisions safely | 5cc9ca3348575931b5d2ec35d1277436d1eece63 | develop@e5e99b4e3bb081b92c602358878856536030e2ca | no | security/governance | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1436 | feat(frontend): add Storybook UI inventory | becc4e9e56bb30e511e812e8c66b19d094b28de0 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | frontend/a11y | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1434 | fix: accept DiskSage cloud-readiness schema 7 | 85e3eb43f9a7c3e87848df1307549d4efd3d29de | develop@e5e99b4e3bb081b92c602358878856536030e2ca | no | other | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1433 | fix(security): reject ambiguous Message-ID whitespace | 0d71272d6ec5420afefa98cd5ae57b91efc31007 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | security/governance | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1432 | fix(compose): harden optional pg-llm-batch database | e3dbed9a0d4e08348f94d26de09a2fabbdcfa96b | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | llm/orchestration | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1431 | test(core): cover operator env path resolution | e058f8c6e50256194d19be617f8df54f60bd1c27 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | other | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1430 | 🎨 Palette: 키보드 내비게이션을 위한 focus-visible 스타일 추가 | 49fc3eabd8e94a95dee2af3c1254c4d46a294399 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | frontend/a11y | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1429 | docs: establish Naruon product completion gap baseline | 5a72475506e8ec76692abfa233f3205c645568eb | develop@e5e99b4e3bb081b92c602358878856536030e2ca | no | docs/product | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1427 | fix(data): align PDF DOM upload budget with sidecar | 29be15e4ec5e29dc1f62ac636928c9307a6f520f | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | ingest/storage | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1426 | fix(governance): wait on stale aggregate review state | 5cc148e1f2f84d1afcd2d3cf3dabaade616c01d0 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | security/governance | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1424 | ⚡ Bolt: [성능 개선] 네트워크 그래프에서 O(N) 노드 라벨 조회를 O(1) Map 조회로 대체 | 32c7edc11fd6faf8ae6918dae8b00de7c5c0b773 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | performance | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1421 | 🎨 [UX] 설정 화면 장식용 아이콘에 aria-hidden 추가 | 719c1b347aae52e77ae7e40b0eb60769fd7178cb | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | frontend/a11y | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1420 | feat: URL 코덱 및 엄격한 JSON 포매터 추가 | bf7b741ea0b73a146ce9bcd323ca621c1562cb4e | develop@dd8d15191338b841f9e6f3a06507c6a5643b95d0 | yes | other | — | experimental/draft | validate parent and promote only after scope proof | +| 1419 | feat(attachments): index common image metadata | a0f5e03107e6ec3e85eea029bb11c8c8e784b907 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | ingest/storage | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1418 | feat(tools): add auditable URL and contact hygiene | 19adb3e74c66837c5fb2d0a11a7ac030bbbfe3c4 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | other | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1417 | fix(email): enforce shared send throttling | 69fb72d30c71ab7a9c2c6e09413292a05278148d | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | mail/calendar | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1416 | fix(calendar): allow provider-backed create writeback | d8b3df7d19def826a5b92abbcaec043377ceb3a4 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | mail/calendar | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1415 | fix(auth): select OIDC signing key by kid | e0a1f166221790e7ba4f0df37b328ac3cb896092 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | security/governance | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1414 | fix(governance): refresh gate after OpenCode review | df1642c473011e935ab7501f4012fb58b8d06e21 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | security/governance | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1412 | ⚡ Bolt: [성능 개선] tools 배열 탐색을 Map 기반 O(1)로 변경 | b5032d7fb428189da86c10221d58d093d3abcc6e | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | performance | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1411 | 🎨 Palette: 검색어 지우기 ARIA 라벨 통일 | 6369e91f2f10ed9b6b436a41a08d89b7aedf9008 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | frontend/a11y | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1410 | 🎨 [OIDC 로그인/로그아웃 버튼 로딩 상태 및 접근성 개선] | fac7a5377bd7c5bc7bde89a6f0f05b3fd2c47632 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | security/governance | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1408 | fix(a11y): expose keyboard focus on AI Hub tabs | cda57c26e75788eaa350d0faeb898349818da074 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | frontend/a11y | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1407 | feat(mail): fail-closed Inkspan edit handoff for recognized HWPX | 22e4909dd13623190f61eae47baf74d70fa2b83a | cursor/mail-hwpx-attachment-preview-7b5e@b83a0da03b46a447f9710b5f91d245f5b1783dfa | yes | ingest/storage | cursor/mail-hwpx-attachment-preview-7b5e | experimental/draft | validate parent and promote only after scope proof | +| 1406 | feat(mail): open recognized HWPX text from email attachments | b83a0da03b46a447f9710b5f91d245f5b1783dfa | cursor/hwpx-recognized-text-preview-b246@f21811379c1cc2435eadb41bb2746b4887947d53 | yes | ingest/storage | cursor/hwpx-recognized-text-preview-b246 | experimental/draft | validate parent and promote only after scope proof | +| 1404 | feat(data): show recognized HWPX paragraph text in attachment preview | f21811379c1cc2435eadb41bb2746b4887947d53 | feat/hwpx-section-text-recognition@0fcf4d85dd70d4f2ee9dd0296fc454f764ae5326 | yes | security/governance | feat/hwpx-section-text-recognition | experimental/draft | validate parent and promote only after scope proof | +| 1403 | fix(oidc): fail closed on malformed token-endpoint escapes | 7a0b0e443ae31a941c5a2139a2093b1af876458c | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | security/governance | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1402 | feat(email-writing): add independent criterion Judge | 3d6b3341c5dd15512d5d60cd5f8d95a1bbc6d846 | feat/llm-email-writing-candidate-task6@fa844bd035ab1f188a28c58e0ed2dc45fa31d0f3 | yes | mail/calendar | feat/llm-email-writing-candidate-task6 | experimental/draft | validate parent and promote only after scope proof | +| 1401 | ⚡ Bolt: ProjectsLayout 인라인 배열 맵핑 렌더링 최적화 | 1221fc848a086db721ef1dde0a26eb6af77fe035 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | performance | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1400 | feat(email): Slice 3 buyer-visible withheld-media next actions | db7ca961de800a514cf9bee34d324f1c5cf233bb | cursor/email-media-quarantine-persist-0ad6@ff1dc18cd9de5e06649ac516b163af2db4bbde83 | yes | ingest/storage | cursor/email-media-quarantine-persist-0ad6 | experimental/draft | validate parent and promote only after scope proof | +| 1399 | feat(email): Slice 3 persist quarantined inline media | ff1dc18cd9de5e06649ac516b163af2db4bbde83 | cursor/email-media-admission-wiring-cd1a@1af546dbb01964e9a620ed341ae0dd3dab9439fd | yes | ingest/storage | cursor/email-media-admission-wiring-cd1a | experimental/draft | validate parent and promote only after scope proof | +| 1398 | feat(email): Slice 3 wire admission so only document_image continues | 1af546dbb01964e9a620ed341ae0dd3dab9439fd | cursor/email-media-admission-slice3-c9de@5a80583bcabc22609e8677864ae86f867d85fd45 | yes | ingest/storage | cursor/email-media-admission-slice3-c9de | experimental/draft | validate parent and promote only after scope proof | +| 1397 | feat(email): Slice 3 inline-media admission and tracking-pixel classification | 5a80583bcabc22609e8677864ae86f867d85fd45 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | ingest/storage | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1392 | docs: make README customer and operator focused | c0ac8c01d58473680b89a225107366fec4fae986 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | docs/product | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1387 | fix(a11y): keep unavailable calendar actions discoverable | 37917799e8d27c07e29eeed87a52d5be41528330 | develop@dd8d15191338b841f9e6f3a06507c6a5643b95d0 | yes | frontend/a11y | — | experimental/draft | validate parent and promote only after scope proof | +| 1384 | feat(noema): route LLM through contextual-orchestrator | 0fd330137cdd19068fa8903dc70e1dc88f42cdc9 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | llm/orchestration | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1380 | fix(dav): land capability honesty with tomllib CI import | 658f69accc627b99e379835593c2b9e49b514d00 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | security/governance | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1376 | fix(email): expose header-derived media pixel dimensions | aae34d0a9e7d607070bc98e7b0d03e17f607dd6c | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | ingest/storage | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1375 | feat(email-writing): parse contextual review candidates | fa844bd035ab1f188a28c58e0ed2dc45fa31d0f3 | feat/llm-email-writing-orchestrator-task5@9cd9b953a2dd236aebe1fcdc25e59ba3e9388505 | yes | security/governance | feat/llm-email-writing-orchestrator-task5 | experimental/draft | validate parent and promote only after scope proof | +| 1373 | feat(hwpx): recognize ordered section text with provenance | 32099709bafcee19fb32c385bbe89e0df15fe102 | feat/hwp-hwpx-attachment-recognition@70683266b93233dae62faec6cbd4df118be41383 | yes | ingest/storage | feat/hwp-hwpx-attachment-recognition | experimental/draft | validate parent and promote only after scope proof | +| 1370 | feat(supply-chain): verify locked hashes against PyPI releases | 1a6ac604e159d98631b3996eb3f74d036e4a760b | feat/dependency-lock-provenance-receipt@f6eeb69f561e94cd50ae38fb1f43faa6cd2c52d7 | no | security/governance | feat/dependency-lock-provenance-receipt | stacked-child | re-fetch exact review/check state, then fix or protected-merge | +| 1369 | feat(supply-chain): attest Python lock provenance before install | f6eeb69f561e94cd50ae38fb1f43faa6cd2c52d7 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | security/governance | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1368 | ⚡ Bolt: [성능 개선] EmailDetail 개별 메시지 컴포넌트 메모이제이션 | 5c2f048c0e9fc97545e1d6f09d1379b3ae8f8b68 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | mail/calendar | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1366 | fix(threading): honor RFC 5256 References ancestry | be0237714e373052b57d73e1168087da3adfda34 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | mail/calendar | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1365 | fix(containers): publish explicit split runtime targets | 43666bed6214ce724d4dc50810d9f65f3d77d3f3 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | security/governance | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1364 | feat(storage): add scoped S3 document object backend | 3a3baa6b8dc9ea224f46395cb78c91a45be2090c | develop@e5e99b4e3bb081b92c602358878856536030e2ca | no | ingest/storage | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1363 | fix(governance): audit orphaned Actions workflow identities | 48593a1cab22cca86e2dbfb7e6d5cb89cf298f3c | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | security/governance | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1361 | feat(tools): add bounded content checksum generator | 5859a8f3f5e9dddf20a43313b53d7aa6453f8cd7 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | other | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1356 | feat(email-writing): add hardened contextual-orchestrator boundary | 9cd9b953a2dd236aebe1fcdc25e59ba3e9388505 | feat/llm-email-writing-context-task4@4570747ccebd57ccaab30ffc68239f0c9d2f1ca0 | yes | mail/calendar | feat/llm-email-writing-context-task4 | experimental/draft | validate parent and promote only after scope proof | +| 1355 | fix(email): preserve deterministic descending thread order | fddc883ca2911c138bd9fcc3a9bd8257e1036124 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | mail/calendar | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1354 | feat(ui): add Storybook design-token contract | 84edbbf152d257cd05777bf0b007fcfec2ac1d18 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | frontend/a11y | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1353 | feat(attachments): recognize HWP and HWPX parser boundaries | dd501dae0fc03d813f4a65aa21318cc89d1a193c | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | ingest/storage | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1352 | fix(a11y): expose async button busy states | 8b7731da7063c39651aa9e3debfaa2052c476c35 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | frontend/a11y | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1349 | docs(product): define evidence-based workspace task contract | 559d091a9a75d8e79ab9608c04931c5a1e82e173 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | docs/product | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1347 | fix(governance): reject rate-limited review status as semantic evidence | cd973fc364efb8d150786f4c2bceec54187eb806 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | security/governance | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1345 | fix(dav): reject ambiguous nested authorization encodings | 07954b2e4d402fa2fd1e9775c52ec6b9483de52d | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | security/governance | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1339 | fix(host-policy): normalize dotted bracketed IPv6 safely | 44b65cac131ab5c4be25cfdebc026c4a1cf3bc35 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | security/governance | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1333 | feat: persist DiskSage file lineage ontology | 017cdef392385571acfc5abc177882724d6026b9 | develop@e5e99b4e3bb081b92c602358878856536030e2ca | no | other | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1332 | feat(email): surface calendar writeback If-Match conflicts | d53598dc7e45b470907fd97dffb9d64e954f2731 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | mail/calendar | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1329 | feat(email-writing): build authorized thread context | 4570747ccebd57ccaab30ffc68239f0c9d2f1ca0 | feat/llm-email-writing-review-evidence-task3@51fb5e8543247b1e5c790f3fdf98424c8fbed669 | yes | security/governance | feat/llm-email-writing-review-evidence-task3 | experimental/draft | validate parent and promote only after scope proof | +| 1328 | feat(email-writing): persist privacy-minimized review evidence | 51fb5e8543247b1e5c790f3fdf98424c8fbed669 | feat/llm-email-writing-contracts-task2@fb7c406ee1328a6ac42dbaf54bb6852c199d8b0a | yes | security/governance | feat/llm-email-writing-contracts-task2 | experimental/draft | validate parent and promote only after scope proof | +| 1327 | feat(email-writing): define strict review contracts | fb7c406ee1328a6ac42dbaf54bb6852c199d8b0a | feat/inkspan-email-writing-guide@bfc2df112136bb9fe358778d701e78bf9e78b685 | yes | security/governance | feat/inkspan-email-writing-guide | experimental/draft | validate parent and promote only after scope proof | +| 1322 | docs(adr): design Inkspan-based LLM email writing guidance | d943203afc0afae0c9a6190681675f4d30dcf257 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | frontend/a11y | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1321 | fix(auth): require issued-at in Keyverse OIDC sessions | d06eff3875543b1afa28570f9269571a63a81983 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | security/governance | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1320 | fix(calendar): expose proposal context to screen readers | 1a9afa0cf845a49db4c2eb2372f9f36eaf4c8c4e | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | mail/calendar | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1317 | feat: harden live macOS runtime and governance | 25d80197ee0eb8cb2aafc7d205ac7b98ccccba0c | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | security/governance | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1302 | fix(tools): remove canned source-derived tools | 2ee0c65097c78a99849fc749a3a848440c50271c | fix/remove-unsafe-phishing-detector@646a2401de35529425163fdefa7ad5e6355c349f | yes | other | fix/remove-unsafe-phishing-detector | experimental/draft | validate parent and promote only after scope proof | +| 1301 | fix(tools): remove unsafe phishing detector | 646a2401de35529425163fdefa7ad5e6355c349f | fix/fail-closed-tool-mutations@67dbfddb01bacb604a0533ce486a550115ff0d64 | yes | other | fix/fail-closed-tool-mutations | experimental/draft | validate parent and promote only after scope proof | +| 1300 | fix(tools): fail closed on unsafe global tool mutations | 7ea4bad69cf36acb9c8fbca48d32333907cce55f | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | other | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1288 | test(email): consolidate thread identity and folder visibility coverage | b7db9a316162a70bf8e594faf4f2e73766d9dc6c | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | mail/calendar | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1287 | 🧪 테스트: runtime_secrets.py의 build_encryption_keyring 누락된 테스트 추가 | 4de4f5d5850baf1abc05203f204c489650ba9624 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | frontend/a11y | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1284 | test(pdf): cover pending document decode success | 68da2a53084751361f6332ca4f3fe82c34443964 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | ingest/storage | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1280 | test(search): cover configured fusion settings | 89742939fe8b9a9a33a9018d6863050f5c7a7fc7 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | other | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1279 | test(tools): consolidate webhook validation and execution coverage | c6a50e97b7742a7172d80de01159516796d4d1a0 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | other | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1277 | test(core): cover connector scope statement | aa04c9392c86c29fd39cae80149fbbd02681cff8 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | other | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1267 | perf(mail): memoize email list element mapping | 8fccadb727ac54a81422642022ccc2b31723bab9 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | mail/calendar | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1264 | integration: route LLM egress through EgressWeave | 5a46a85f75dfae94dd8c1f8df0b88150b40ab3c8 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | llm/orchestration | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1257 | chore(deps): update connector websockets to 17.0.1 | 4613ff5a1a85e4882af1a3a4abd0e56b3b574187 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | dependency | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1245 | fix(email-detail): make responsive evidence actions functional | 796b34c5a1322f09c6f00b8cf24591ae04b89b6b | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | yes | frontend/a11y | — | experimental/draft | validate parent and promote only after scope proof | +| 1244 | chore(deps): update hash-locked aiohttp to 3.14.3 | c1d4c7fd2b98e464d3ff7e92f26d92a6c0f1e6e8 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | security/governance | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1241 | fix(a11y): show keyboard focus on OIDC actions | fb7e63dee1d72365db595edb1bc49e097202e707 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | security/governance | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1206 | fix(security,api): opaque prompt IDs and CardDAV single-decode | d7ae4768b7c30be7bac19fb9425d40a66e8fda05 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | security/governance | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | +| 1195 | feat(email): deterministic dedupe provenance — gate strong fingerprints on genuine Date (naruon#1086) | c7aedc6a6a09bc91156e9c62e44f18cf8b4d3846 | develop@81c105645ca6e680f5f8c15ba9c33b67eb63c48b | no | frontend/a11y | — | normal-or-stacked-root | re-fetch exact review/check state, then fix or protected-merge | + +The inventory is intentionally identity-first: + +It represents the 2026-08-25 102-row refresh described above, which supersedes +the v0.2 post-#1442 93-row snapshot as current evidence while that snapshot +remains historical context: no review body, credential, +customer data, or copied provider payload is stored in this document. A parent +merge or head movement invalidates the affected row and requires regeneration. + +### Exact-head check observations + +The following failures were also read from the live Checks API during this +snapshot and are retained as RCA pointers, not as reusable merge evidence: + +| PR | Exact head | Check/run evidence | Observed cause or disposition | +|---:|---|---|---| +| #1347 | `3f6932026fbef281a373d792518058e4aaf5178f` | Strix run `32440010004`, job `96648648553` | provider/model infrastructure returned NVIDIA NIM 404 and no complete Strix report; rerun after the central fail-closed workflow repair, without weakening the gate | +| #1442 | `94e10a6188a1b96ac162fa659ae4025bc00895bd` | metadata gate `96719432050`; merged `2026-08-21T09:18:28Z` | historical pre-merge observation only; the post-merge 93-row inventory excludes this PR | +| #1443 | `62a0d645b619bcd2eac8f0db87460c5c1990d128` | new parent-head Checks queued; predecessor Strix job `97705801302` | child #1448 merged normally into this stack branch, invalidating all predecessor-head evidence. The prior provider failure was NVIDIA NIM HTTP 429 with unavailable fallback/direct OpenAI 404; current head requires fresh Checks and qualifying approval | +| #1448 | `068aefdfa48122bc73cb85a1dc23614bb09ebc04` → merge `62a0d645b619bcd2eac8f0db87460c5c1990d128` | merge-result Checks queued; Devin passed on PR head | normal stack merge at `2026-08-25T15:31:39Z`; local governance and parent synchronization tests passed. Delayed merge-result Checks are being tracked and are not represented as successful hosted evidence | + +Queued or pending Checks are not treated as source failures, and completed +predecessor-head evidence is never reused. + +--- + +## 14. Claim boundary + +This baseline is a product and technical decision record, not a certification, +security attestation, market valuation, or claim that Naruon is already GA. + +The existence of 100% coverage gates, many PRs, or detailed documentation does +not itself demonstrate commercial completeness. GA is demonstrated only by the +end-to-end buyer journey, current exact-head protected integration, released +artifacts, provider interoperability, recovery/customer-exit evidence, and +operational support contract defined here. diff --git a/docs/research/email-authentication-xoauth2/README.md b/docs/research/email-authentication-xoauth2/README.md new file mode 100644 index 000000000..d764ddf66 --- /dev/null +++ b/docs/research/email-authentication-xoauth2/README.md @@ -0,0 +1,54 @@ +# Email authentication — XOAUTH2 delimiter integrity + +This note grounds Naruon's SASL XOAUTH2 payload construction at +`backend/services/email_client.py` and the hostile-input regression in +`backend/tests/test_email_client.py`. + +## Protocol boundary + +RFC 7628 defines OAuth SASL key/value fields as being separated by the octet +`%x01` (Control-A). Google's Gmail XOAUTH2 documentation uses the same wire +shape for the initial client response: one `user` field, one +`auth=Bearer ...` field, and a final empty field, each separated by Control-A. +The delimiter is therefore protocol structure, not ordinary caller-controlled +field data. + +Naruon's helper previously interpolated the supplied user identity and access +token into that attribute stream before base64 encoding. A Control-A embedded +inside either value created an additional protocol field boundary. Base64 does +not remove that ambiguity; it only encodes the already-constructed octet +sequence. + +## Decision + +`generate_oauth2_string()` rejects `\x01` in either the user identity or access +token before the SASL response is constructed. The ordinary response format is +unchanged. The function does not log credentials, repair malformed values, +percent-encode the delimiter, introduce a fallback authentication mechanism, or +broaden the allowed IMAP/SMTP destinations. + +The regression corpus covers delimiter injection through both caller-controlled +fields and preserves the existing valid-payload test. This is a structural +protocol validation rule rather than a keyword/security-score heuristic. + +## Claim boundary + +This change prevents caller data from introducing extra XOAUTH2 field +separators at this construction boundary. It does not by itself claim complete +OAuth, SASL, Gmail, IMAP, or SMTP security; token issuance, audience/scope, +transport security, server policy, credential storage, TLS identity, egress +allowlisting, and provider behavior remain separate controls. + +## References (APA 7) + +- Mills, W., Showalter, T., & Tschofenig, H. (2015). *A set of Simple + Authentication and Security Layer (SASL) mechanisms for OAuth* (RFC 7628). + RFC Editor. https://www.rfc-editor.org/rfc/rfc7628.html +- Google. (n.d.). *OAuth 2.0 mechanism*. Google Workspace. Retrieved August 14, + 2026, from https://developers.google.com/workspace/gmail/imap/xoauth2-protocol + +## Verification boundary + +The branch is not merge-ready merely because this note and the narrow fix +exist. Current-head repository CI, security, coverage, independent review, and +protected-branch gates remain authoritative. diff --git a/docs/superpowers/plans/2026-08-09-structural-topic-boundary.md b/docs/superpowers/plans/2026-08-09-structural-topic-boundary.md new file mode 100644 index 000000000..1dd902260 --- /dev/null +++ b/docs/superpowers/plans/2026-08-09-structural-topic-boundary.md @@ -0,0 +1,228 @@ +# Structural Topic Boundary Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Remove keyword-triggered pseudo-topic classification from Naruon's +product surface and document the fail-closed TEPP STM boundary. + +**Architecture:** Naruon's generic tool registry will retain honest lexical +utilities but will no longer expose fixed dictionaries as topic inference or +agenda generation. Corpus-level STM remains an external Rust-first TEPP +measurement boundary whose future posterior contract is documented here rather +than simulated in the request handler. + +**Tech Stack:** Python 3.12+, FastAPI tool registry, pytest, Ruff, Markdown. + +## Global Constraints + +- Do not introduce keyword, embedding, or LLM fallback topic classification. +- Do not claim that a fixed business label is an STM posterior probability. +- Preserve `ANALYSIS_TEXT_MAX_CHARS` enforcement for the retained lexical tool. +- Treat every warning as a verification failure. +- Keep the change atomic and avoid unrelated tool-registry refactoring. + +- [x] **Preflight: read the repository root `AGENTS.md` completely before any + change.** + +--- + +### Task 1: Lock out lexical pseudo-topic tools + +**Files:** +- Modify: `backend/tests/test_tools_api.py` +- Modify: `backend/api/tools.py` + +**Interfaces:** +- Consumes: the existing module-level `registry: ToolRegistry`. +- Produces: a registry without `email_categorizer` or + `meeting_agenda_generator`; `keyword_extractor` remains registered and is + explicitly described as term-frequency extraction. + +- [x] **Step 1: Write the failing registry-contract test** + +```python +@pytest.mark.parametrize( + "tool_code", ["email_categorizer", "meeting_agenda_generator"] +) +def test_registry_omits_lexical_pseudo_topic_tools(tool_code): + assert registry.get(tool_code) is None + + +def test_keyword_extractor_is_disclosed_as_lexical_term_frequency(): + tool = registry.get("keyword_extractor") + assert tool is not None + assert tool.description == ( + "텍스트 본문에서 빈도와 최초 출현 순으로 반복 용어를 추출합니다." + ) +``` + +- [x] **Step 2: Run the focused tests and verify RED** + +Run: +`PYTHONWARNINGS=error DISABLE_BACKGROUND_WORKERS=1 python -m pytest backend/tests/test_tools_api.py::test_registry_omits_lexical_pseudo_topic_tools backend/tests/test_tools_api.py::test_keyword_extractor_is_disclosed_as_lexical_term_frequency -q` + +Expected: the first test fails because both pseudo-topic tools are registered; +the second fails because the current description overclaims importance. + +- [x] **Step 3: Remove the pseudo-model implementation** + +Delete `_CATEGORY_TERMS`, `_AGENDA_TOPICS`, `_contains_analysis_term`, both +handlers, both `registry.register(...)` blocks, and their behavior-locking tests. +Change the retained handler docstring to +`"""Extract deterministic lexical terms by frequency and first occurrence."""` +and its tool description to +`"텍스트 본문에서 빈도와 최초 출현 순으로 반복 용어를 추출합니다."`. + +- [x] **Step 4: Run the focused test file and verify GREEN** + +Run: +`PYTHONWARNINGS=error DISABLE_BACKGROUND_WORKERS=1 python -m pytest backend/tests/test_tools_api.py -q` + +Expected: all tests pass with no warning-class output. + +### Task 2: Record the scientific and governance boundary + +**Files:** +- Modify: `AGENTS.md` +- Modify: `CHANGELOG.md` +- Modify: `docs/adr/README.md` +- Create: `docs/adr/0001-topic-measurement-authority.md` +- Create: `docs/doctoring/structural-topic-model-boundary.md` + +**Interfaces:** +- Consumes: the decision in + `docs/superpowers/specs/2026-08-09-structural-topic-boundary-design.md`. +- Produces: a durable anti-pattern rule, user-visible change record, and APA 7 + research note. + +- [x] **Step 1: Add the anti-pattern rule** + +State that topic inference must not be implemented with hard-coded term lists, +term frequency, embeddings, or LLM labels presented as STM; unavailability of a +fitted TEPP model must fail closed. + +- [x] **Step 2: Add the changelog entry** + +Under the current unreleased section, record removal of the two misleading tools +and preservation of the honest lexical utility. + +- [x] **Step 3: Add the doctoring note** + +Document the defect history, distinction between STM and classification, future +TEPP contract, and APA 7 references. Check redistribution permission for each +relevant paper: commit the PDF only when redistribution is permitted; otherwise +include its citation, DOI link, and a concise summary of how it supports the +boundary. Permission was not established for the two cited articles, so this PR +uses citations, links, and summaries rather than copies. + +### Task 3: Complete the decision-to-operation documentation graph + +**Files:** +- Modify: `README.md`, `ARCHITECTURE.md`, `CLAUDE.md`, `CHANGELOG.md` +- Modify: `docs/planning/naruon-platform-plan.md` +- Modify: the boundary design and this implementation plan +- Create: companion ADRs and `docs/topic-intelligence/` requirements, + architecture, contract/schema, UML, conceptual ERD, security, threat, test, + operability, traceability, references, and fitness records +- Create: `backend/tests/test_topic_intelligence_documentation.py` + +- [x] **Step 1: Audit the pre-change documentation set** + +Record whether each requested artifact exists, is discoverable, is internally +consistent, and distinguishes implemented behavior from a planned contract. + +- [x] **Step 2: Add the missing or stale records** + +Make the deletion decision reviewable and the future integration discoverable, +without claiming a runtime endpoint, physical topic persistence, accepted TEPP +contract, or reproducible replay where only digests are available. + +- [x] **Step 3: Add machine-readable fitness checks** + +Validate required files and links, schema revision/ownership/status markers, +error-versus-abstention semantics, conceptual-only data modeling, sensitive +digest treatment, and removal of stale platform-plan claims. + +### Task 4: Verify and publish + +**Files:** +- Verify all files changed by Tasks 1 and 2. + +**Interfaces:** +- Consumes: the completed atomic diff. +- Produces: exact local evidence and a GitHub pull request based on the current + `develop` head. + +- [x] **Step 1: Run Ruff** + +Run: +`python -m ruff check backend/api/tools.py backend/tests/test_tools_api.py backend/tests/test_topic_intelligence_documentation.py` + +Expected: exit 0 and no diagnostics. + +Observed on the complete candidate tree: Ruff passed for the affected tool and +documentation-fitness test files. + +- [x] **Step 2: Run the complete backend suite** + +Run: +`PYTHONWARNINGS=error DISABLE_BACKGROUND_WORKERS=1 python -m pytest backend -q` + +Expected: exit 0 with no `Timeout`, `Fatal`, `Warn`, or `Denied` output. + +Observed after merging the protected-base security remediation, with proxy +variables removed: `1711 passed, 33 skipped`; the focused tool/documentation +suite reported `79 passed`. + +- [x] **Step 3: Inspect the exact diff** + +Run: `git diff --check`, `git diff --stat`, and compare the exact changed paths +against this allowlist (including every file under `docs/topic-intelligence/`): + +```text +AGENTS.md +ARCHITECTURE.md +CHANGELOG.md +CLAUDE.md +README.md +backend/api/tools.py +backend/tests/test_tools_api.py +backend/tests/test_topic_intelligence_documentation.py +docs/adr/0001-topic-measurement-authority.md +docs/adr/0002-fitted-topic-artifact-consumption.md +docs/adr/0003-separate-topic-measurement-from-agenda-generation.md +docs/adr/README.md +docs/doctoring/structural-topic-model-boundary.md +docs/planning/naruon-platform-plan.md +docs/superpowers/plans/2026-08-09-structural-topic-boundary.md +docs/superpowers/specs/2026-08-09-structural-topic-boundary-design.md +docs/topic-intelligence/API_CONTRACT.md +docs/topic-intelligence/ARCHITECTURE.md +docs/topic-intelligence/DATA_MODEL.md +docs/topic-intelligence/DOCUMENTATION_FITNESS.md +docs/topic-intelligence/OPERABILITY.md +docs/topic-intelligence/PRD.md +docs/topic-intelligence/README.md +docs/topic-intelligence/REFERENCES.md +docs/topic-intelligence/SECURITY.md +docs/topic-intelligence/TEST_STRATEGY.md +docs/topic-intelligence/THREAT_MODEL.md +docs/topic-intelligence/TRACEABILITY.md +docs/topic-intelligence/TRD.md +docs/topic-intelligence/UML.md +docs/topic-intelligence/schema/topic-inference-result-v1.schema.json +``` + +Expected: no whitespace errors; only the scoped source, tests, governance, and +research/design documents changed. + +Observed: `git diff --check` passed and the complete base-to-candidate plus +working-tree path set exactly matched all 31 allowlisted paths. + +- [x] **Step 4: Commit and open a pull request** + +The predecessor source/test head passed local validation and PR #1297 was +opened. Its body must distinguish predecessor evidence from eventual exact-head +evidence and link the current-head CI, security, and review results before +merge. Push documentation and review fixes to the same +`fix/remove-lexical-topic-heuristics` branch; do not open a duplicate PR. diff --git a/docs/superpowers/specs/2026-08-09-structural-topic-boundary-design.md b/docs/superpowers/specs/2026-08-09-structural-topic-boundary-design.md new file mode 100644 index 000000000..0e68e4778 --- /dev/null +++ b/docs/superpowers/specs/2026-08-09-structural-topic-boundary-design.md @@ -0,0 +1,127 @@ +# Structural Topic Boundary Design + +**Status:** Active PR deletion design for PR #1297; removal is not +protected-`develop` behavior until merge. This file's former future-integration +summary is `SUPERSEDED` by the canonical package below. The Naruon-local policy +is accepted, the target decisions remain proposed, and runtime topic inference +is `BLOCKED-UPSTREAM` and unimplemented. + +**Canonical documentation graph:** +[`docs/topic-intelligence/README.md`](../../topic-intelligence/README.md) + +That package and its linked ADR index govern maturity, authority, requirements, +the 14-field digest inventory, and error-versus-abstention semantics. This legacy +design remains useful for the deletion history only. It does not assign product +or scientific authority to TEPP or another producer, impose an external +obligation, record upstream acceptance, or establish a production contract. + +## Context + +Protected `develop` exposes `email_categorizer` and +`meeting_agenda_generator` as analysis tools, but both derive their outputs from +small hard-coded Korean/English term lists. The behavior entered in commit +`c070c8d19f01ccfe46a5ee7e8a577b08e587bb14` as demonstration logic and was +later made deterministic and better tested without correcting the underlying +measurement error. The tests consequently canonized lexical hits as topic +evidence. + +Structural topic modeling (STM) is not fixed-label keyword classification. It +estimates mixed-membership topic proportions over a corpus and can model how +document metadata affects topic prevalence or content. Inference for a new +document requires a previously fitted model and its frozen vocabulary; the +result is a topic mixture with uncertainty, not a calibrated probability for a +business label. + +## Decision + +1. Remove `email_categorizer` and `meeting_agenda_generator` from Naruon's tool + registry. They have no callers outside their own tests, so removal eliminates + misleading product behavior without breaking an integrated workflow. +2. Remove `_CATEGORY_TERMS`, `_AGENDA_TOPICS`, and the substring matcher that + exists only to support those pseudo-models. +3. Retain `keyword_extractor` as an explicitly lexical utility, but describe it + honestly as deterministic term-frequency extraction. Its output must never + be treated as topic posterior evidence. +4. Do not add an embedding, LLM, or larger dictionary fallback and do not call + any such fallback STM. +5. Keep corpus-level topic estimation outside this Naruon deletion change. + TEPP's Rust-first `topic_measurement` architecture is directional evidence, + not an assignment of authority. Naruon may evaluate any independently + published, compatible fitted-model boundary only after its publisher releases + a versioned, source-backed artifact/inference contract and acceptance evidence, + and Naruon separately accepts the integration. Until then, absence of a fitted + model fails closed rather than returning `General` or a template agenda. + +## Conditional future Naruon acceptance profile + +The accepted local policy is [ADR-0001](../../adr/0001-topic-measurement-authority.md). +The fitted-artifact and agenda target decisions remain proposed in +[ADR-0002](../../adr/0002-fitted-topic-artifact-consumption.md) and +[ADR-0003](../../adr/0003-separate-topic-measurement-from-agenda-generation.md). +The following are conditions Naruon would apply to its own consumption decision; +they do not govern an upstream publisher. + +Any later Naruon integration must bind all 14 exact fields in the canonical +[digest inventory](../../topic-intelligence/README.md#canonical-digest-inventory), +including the schema, source snapshot, complete scientific payload, artifact, +manifest, vocabulary, preprocessing, design, lineage, model card, validation +report, evidence-time manifest, covariate snapshot, and design row. It must also +carry, at minimum: + +- immutable document, snapshot, model, artifact, and contract identities; +- document, event, assertion, availability, and knowledge-cutoff times; +- language and multilevel/cross-classified membership covariates; +- the frozen preprocessing and prevalence/content design specifications; +- topic proportions that sum to one, posterior uncertainty, inference method, + model version, and diagnostic status; +- evidence-backed topic labels kept separate from the numeric topic identity; +- explicit input, incompatibility, integrity, availability, and protocol errors; + and +- `abstained` only for a compatible active model's declared posterior or + diagnostic-policy rejection, never for an error or fallback. + +If Naruon later accepts and implements agenda generation, that capability must +consume authorized source evidence and, optionally, a versioned posterior through +a separate decision/generation boundary. It must not map raw words directly to +agenda templates. Proposed ADR-0003 is not implementation authorization. + +## Alternatives rejected + +- **Expand the dictionaries:** deterministic but still lexical, brittle across + language and domain, and unable to represent mixed membership or uncertainty. +- **Use embeddings or an LLM as a drop-in replacement:** potentially useful for + semantic labeling, but neither is STM and neither supplies the required + corpus-level estimand or covariate effects. +- **Fit a model inside each API request:** statistically invalid for a single + document, operationally expensive, and incompatible with reproducible model + artifacts. + +## Verification + +- The pre-change regression test failed because the two misleading tool codes + were registered. +- On PR #1297, the registry omits both codes while retaining the + explicitly lexical term-frequency utility. +- Focused tools tests, the complete backend test suite with warnings promoted to + errors, and Ruff must pass. + +## References + +Roberts, M. E., Stewart, B. M., & Tingley, D. (2019). stm: An R package for +structural topic models. *Journal of Statistical Software, 91*(2), 1–40. +https://doi.org/10.18637/jss.v091.i02 + +The package paper defines a fitted STM workflow with covariate-aware prevalence +and content, posterior quantities, and diagnostics. Those requirements are why +a deterministic term table cannot satisfy the measurement contract. + +Roberts, M. E., Stewart, B. M., Tingley, D., Lucas, C., Leder-Luis, J., +Gadarian, S. K., Albertson, B., & Rand, D. G. (2014). Structural topic models +for open-ended survey responses. *American Journal of Political Science, +58*(4), 1064–1082. https://doi.org/10.1111/ajps.12103 + +The application paper establishes mixed-membership topics whose prevalence or +content can vary with document metadata. It grounds the design's separation of +corpus-level inference from fixed business labels. Redistribution permission +for either article has not been established; citations, links, and summaries +are supplied instead of paper copies. diff --git a/docs/topic-intelligence/API_CONTRACT.md b/docs/topic-intelligence/API_CONTRACT.md new file mode 100644 index 000000000..6ee28e047 --- /dev/null +++ b/docs/topic-intelligence/API_CONTRACT.md @@ -0,0 +1,361 @@ +# Planned topic-inference API contract + +- **Capability maturity:** `BLOCKED-UPSTREAM` +- **Document status:** `PRESENT-CURRENT` +- **Contract version:** `topic-inference-result-v1` +- **Contract revision:** `2026-08-09.1` + +There is no shipped topic-inference endpoint in Naruon today. This document +defines the Naruon-side contract that may be implemented only after TEPP +independently publishes a compatible production fitted-model artifact and +inference boundary with its own acceptance evidence. + +## Contract layers + +Three representations must not be conflated: + +1. **Authenticated Naruon API.** A browser or API client submits opaque source + references. Naruon reauthorizes and resolves server-authoritative evidence. +2. **Internal Naruon adapter envelope.** Naruon binds an immutable snapshot, + schema/deployment pins, sensitive canonical digests, error mapping, and + scientific acceptance policy. The result schema in this package applies to + this internal envelope. +3. **Expected upstream scientific payload.** Naruon expects the independently + accepted producer to own fitted-model inference evidence nested under + `tepp_payload`. TEPP is only the expected producer if it separately publishes + a compatible production contract and accepts that responsibility; this local + schema does not assign it or claim adoption. + +The authenticated API returns a redacted projection of the internal envelope. +Sensitive digests, tenant/workspace bindings, covariate values, design rows, and +raw evidence locations must not cross the public boundary. + +## Planned endpoint + +`POST /api/topic-intelligence/inferences` + +The route is private and signed-session authenticated. Its final implementation +must scope every lookup by the authenticated owner, organization, and workspace. +Elevated platform roles do not automatically become the mailbox/document owner. + +### Request + +```json +{ + "document_ref": "doc_Q7mWQVp1jJHq5J3e", + "evidence_ref": "ev_q9BH7d4eMG3A2x6n", + "request_revision": "reqrev_01", + "idempotency_key": "a-client-generated-bounded-opaque-value", + "expected_snapshot_revision": "snaprev_184", + "language": "ko-KR", + "purpose": "topic_assistance" +} +``` + +| Field | Rule | +|---|---| +| `document_ref` | Opaque source identifier. It is not a database primary key, provider message ID, URL, or path. | +| `evidence_ref` | Opaque, audience-bound, snapshot-bound, tenant/workspace-bound, expiring capability reference. Naruon reauthorizes it at use time and never dereferences client-controlled URLs or paths. | +| `request_revision` | Bounded opaque revision used for optimistic request compatibility. Reusing it for different canonical content is a conflict. | +| `idempotency_key` | Bounded opaque client token. Naruon stores/compares only a protected tenant-keyed representation; reuse with a different canonical request returns `409`. | +| `expected_snapshot_revision` | Optional optimistic pin. The route compares it with the newly authorized server snapshot; mismatch returns `409`. | +| `language` | Required BCP 47 tag selected or confirmed by the caller. A homemade keyword/script detector must not silently override it. Model support is checked during preflight. | +| `purpose` | Must equal an allowlisted, consented purpose. Revision `2026-08-09.1` defines only `topic_assistance`. | + +The public request never contains raw email/document content, topic labels, +tenant identifiers, model display names, covariate values, membership labels, +or upstream endpoints. Naruon derives the canonical internal request from the +reauthorized snapshot and active deployment. + +### Successful public projection + +HTTP `200` has exactly two semantic states: + +- `inferred`: an accepted mixed-membership posterior is available; +- `abstained`: compatible input and model reached inference, but the attempted + posterior or a declared diagnostic/acceptance rule declined publication. + +```json +{ + "request_id": "tir_3FQn1v8H2zK6cP4m", + "status": "inferred", + "model_id": "opaque-versioned-model-id", + "model_version": "opaque-versioned-model-ref", + "analysis_context": { + "analysis_unit": "document", + "estimand_id": "document_topic_mixture", + "causal_design": "non_causal", + "covariate_level": "analysis_unit" + }, + "credible_level": 0.95, + "interval_method": "upstream-declared-method", + "uncertainty_scope": "conditional_on_fitted_artifact", + "topics": [ + { + "topic_id": 17, + "rank": 1, + "proportion": 0.62, + "credible_interval": {"lower": 0.51, "upper": 0.72} + }, + { + "topic_id": 4, + "rank": 2, + "proportion": 0.38, + "credible_interval": {"lower": 0.28, "upper": 0.49} + } + ], + "diagnostic_status": "accepted", + "completed_at": "2026-08-09T12:00:00Z" +} +``` + +Human-readable labels, if later exposed, use a separate `presentation` object +with their own version and evidence references. They do not replace `topic_id` +or alter posterior values. + +An abstention has no usable topic components: + +```json +{ + "request_id": "tir_B7p4K2m9Q5x1V8nD", + "status": "abstained", + "model_id": "opaque-versioned-model-id", + "model_version": "opaque-versioned-model-ref", + "analysis_context": { + "analysis_unit": "document", + "estimand_id": "document_topic_mixture", + "causal_design": "non_causal", + "covariate_level": "analysis_unit" + }, + "credible_level": 0.95, + "interval_method": "upstream-declared-method", + "uncertainty_scope": "conditional_on_fitted_artifact", + "topics": [], + "diagnostic_status": "rejected", + "abstention_reasons": ["posterior_uncertainty_exceeds_policy"], + "completed_at": "2026-08-09T12:00:00Z" +} +``` + +The response must carry `Cache-Control: no-store`. The UI must render an +abstention as unavailable evidence, not as a zero-probability topic, successful +classification, or reason to invoke agenda generation. + +`model_id`, `model_version`, and `analysis_context` are required safe semantic +context, not optional UI decoration. Consumers join numeric topic IDs only within +that model identity and must display/use the analysis unit, estimand, coarse +covariate level, and `non_causal` designation so group-level effects cannot be +presented as an individual's trait or causal outcome. No group value, membership +identifier, raw covariate, or sensitive digest is exposed. + +## Internal adapter envelope + +The internal response validates against +[`topic-inference-result-v1.schema.json`](schema/topic-inference-result-v1.schema.json), +whose immutable identifier is: + +`https://naruon.net/schemas/topic-intelligence/topic-inference-result-v1/2026-08-09.1` + +The adapter configuration pins both that `$id` and the `schema_digest` +construction defined in [Digest contract](#digest-contract). A response repeats +the ID, revision, and complete canonical-digest record. The schema intentionally +does not embed its own expected digest value because that value is distributed +out of band with the adapter configuration. + +The envelope is Naruon-owned and requires: + +- opaque request, document, snapshot, expiring evidence-reference identity, and + matching server-created snapshot/scope bindings; +- schema revision/digest and adapter version; +- source-snapshot and nested scientific-payload digests; +- the independently accepted expected-upstream scientific payload nested under + `tepp_payload`, including all scientific provenance, design, posterior, + uncertainty, and diagnostics fields; and +- optional versioned presentation labels outside the scientific payload. + +The envelope and its digests are internal validation data. The public projection +above deliberately omits them. + +## HTTP and abstention semantics + +| HTTP | `error_code` or status | When it applies | Retry rule | +|---:|---|---|---| +| `200` | `inferred` | Compatible request, active verified deployment, valid payload, and accepted posterior/diagnostics | Normal result | +| `200` | `abstained` | Compatible request/model reached inference, but posterior uncertainty, diagnostics, or pinned publication policy rejected release | Do not retry unchanged input/model/policy | +| `401` | `topic_authentication_required` | No valid signed session or service identity is present | Authenticate; do not reveal resource existence | +| `403` | `topic_evidence_forbidden` | Evidence authorization fails for the current owner/tenant/workspace | Do not retry without a new authorization decision | +| `403` | `topic_purpose_forbidden`, `topic_consent_required`, `topic_region_forbidden` | Purpose, consent, or region policy denies processing | Policy/consent remediation; no upstream call | +| `409` | `topic_source_snapshot_conflict` | Expected and current authorized snapshot revisions differ | Refresh source state | +| `409` | `topic_request_revision_conflict` | A trusted request revision is incompatible with the canonical request | Create a new revision after refresh | +| `409` | `topic_idempotency_conflict` | An idempotency key was previously bound to different canonical request material | Use a new key only for a genuinely new operation | +| `409` | `topic_schema_revision_conflict` | Client/adapter revision pin conflicts with the active immutable contract | Negotiate a supported revision; never coerce | +| `408` | `topic_deadline_exceeded` | Naruon's bounded request budget expires before an upstream timeout can be classified | Retry only within bounded client policy | +| `422` | `topic_input_invalid` | Bounded shape or source snapshot cannot satisfy the inference request | Correct the request/source | +| `422` | `topic_language_unsupported` | Active fitted artifact does not support the declared language/profile | Select a compatible model only through deployment policy | +| `422` | `topic_input_insufficient_tokens` | Frozen preprocessing retains fewer tokens than the active threshold | More evidence is required | +| `422` | `topic_input_out_of_vocabulary` | OOV count/ratio violates the active artifact policy | Use compatible source evidence/model; no fallback | +| `422` | `topic_temporal_context_invalid` | Event, availability, assertion, or knowledge-cutoff evidence violates the temporal policy | Correct authoritative time evidence | +| `422` | `topic_covariate_contract_invalid` | Required covariate, level, membership weight, or design row is missing/incompatible | Correct authoritative covariate evidence | +| `429` | `topic_rate_limited` | Tenant/workspace quota, concurrency, or repeated-query policy denies work | Honor `Retry-After`; do not change measurement method | +| `503` | `topic_deployment_unavailable` | No verified active deployment exists | Retry only after operator activation | +| `503` | `topic_model_artifact_unavailable` | Pinned artifact or required retained manifest cannot be resolved | Operator/upstream remediation | +| `503` | `topic_model_artifact_integrity_failed` | Artifact/provenance digest validation fails | Quarantine deployment; do not retry blindly | +| `502` | `topic_upstream_inference_failed` | Compatible request reached the upstream boundary but transport/runtime failed | Retry according to bounded service policy | +| `502` | `topic_upstream_protocol_error` | Upstream response fails schema, digest, asserted date-time format, known code-registry, or cross-field validation | Quarantine/review; never turn into abstention | +| `504` | `topic_upstream_timeout` | The bounded expected-upstream deadline expires and work is cancelled | Retry only within bounded service policy | +| `500` | `topic_adapter_internal_error` | Unexpected Naruon defect after safe classification | Incident handling; no internal detail in response | + +A client disconnect or explicit cancellation may make an HTTP response +impossible. The adapter must cancel bounded work, emit no result, and record only +the internal stable outcome `topic_request_cancelled` in approved redacted +telemetry. It must not serialize a partial posterior or retry after cancellation. + +Authentication/authorization/purpose/consent/region denial, rate limiting, +unsupported language, token/OOV insufficiency, temporal/covariate incompatibility, +missing deployment/artifact, integrity failure, trusted conflicts, timeout, +cancellation, and upstream protocol failures must never return HTTP `200` or +`status=abstained`. +These error conditions must never return HTTP `200` or `status=abstained`. + +## RFC 9457 problem details + +Every non-`200` response uses `application/problem+json` and a stable RFC 9457 +problem type. `error_code` is a required Naruon extension; clients branch on the +code, not on localized `title` or `detail` text. + +```json +{ + "type": "https://naruon.net/problems/topic-language-unsupported", + "title": "Topic inference language is unsupported", + "status": 422, + "detail": "The active fitted model cannot infer this language profile.", + "instance": "/api/topic-intelligence/inferences/tir_3FQn1v8H2zK6cP4m", + "error_code": "topic_language_unsupported", + "request_id": "tir_3FQn1v8H2zK6cP4m", + "retryable": false +} +``` + +Problem responses must not include raw source content, topic candidates, +canonical digests, tenant/workspace IDs, covariates, membership identities, +upstream URLs, stack traces, provider errors, or arbitrary evidence references. + +## Scientific payload requirements + +The nested `tepp_payload` contains no presentation label. It must provide: + +- fitted model ID/version/topic count; +- artifact, manifest, vocabulary, preprocessing, design, lineage, model-card, + validation-report, evidence-time manifest, covariate snapshot, and design-row + canonical digests; +- estimator, analysis unit, estimand, prevalence/content formulas, contrasts, + versioned covariate schema, typed covariate level/missingness policy, + membership structure/normalization, unseen-level policy, validation profile, + temporal policy version, explicit document/event/assertion/availability/ + knowledge-cutoff time values, the pinned temporal missingness rule, an asserted + availability-at-cutoff result, and `causal_design=non_causal`; +- inference method, implementation/version, numerical backend, credible level, + interval method, and + `uncertainty_scope=conditional_on_fitted_artifact`; +- non-negative integer topic IDs, ranks, proportions, and per-topic intervals for + accepted results; +- input diagnostics for language, original/retained/OOV tokens and their pinned + thresholds, temporal context, and covariates; +- posterior diagnostics with an immutable diagnostic-code registry version, + convergence and its stable known code, numerical status, bounded stable known + quality codes, iteration count, finite values, interval + validity, observed component count, posterior sum, and normalization tolerance; + and +- an acceptance-policy version, immutable reason-code registry version, boolean + decision, and stable known reason codes. + +The adapter recomputes and cross-checks unique topic IDs/ranks; for `inferred`, +equality of fitted, declared, observed, and actual component counts; sum, +interval containment, finite values, method copies, status, and diagnostic +consistency. It also checks request/evidence snapshot and scope-binding equality, +current tenant/workspace/purpose/consent/region authorization, input thresholds, +membership-structure/normalization coupling, typed covariate level/missingness, +RFC 3339 format assertion, and `availability_time <= knowledge_cutoff_time`. +Unknown code registry versions or codes and any failed cross-check are `502` +protocol errors. Schema validation alone is insufficient. + +## Digest contract + +The complete internal inventory is the schema, source snapshot, scientific +payload, artifact descriptor, artifact manifest, vocabulary, preprocessing, +design, lineage, model card, validation report, evidence-time manifest, +covariate snapshot, and design row. Each of those exactly 14 canonical digest +fields, including `schema_digest`, is a `canonicalDigest` record whose `value` +is: + +`SHA-256(UTF8(domain) || 0x00 || UTF8(RFC8785(value)))` + +with lowercase hexadecimal output. `schema_digest` has one construction: +`domain` is exactly `naruon.topic-inference.schema.v1`, and the formula's +`value` input is the complete parsed JSON value of the immutable schema resource +identified by the pinned `$id`, including its annotations and definitions. +Whitespace, JSON member order, source-file encoding, and other raw-file +serialization details are therefore not separate schema-digest inputs. The +adapter's out-of-band pin stores the resulting canonical-digest record, and the +response repeats that record. + +All 14 contract fields bind canonical JSON values. In particular, +`artifact_digest` binds the fitted-artifact **descriptor**, not the raw fitted +artifact bytes. An independently published artifact manifest may additionally +contain a distinct optional raw-byte hash record, but that record must declare +its algorithm and the exact byte serialization or package it covers. It is +manifest content protected through `manifest_digest`; it is neither +`artifact_digest`, a substitute for any canonical field, nor a fifteenth field +in the inventory. Without that distinct record, the contract makes no raw +artifact-byte integrity claim. RFC 8785 does not normalize Unicode, so +normalization belongs only to the pinned preprocessing contract. + +The no-covariate canonical representations are fixed: + +- `{"covariates":[],"memberships":[]}` under + `naruon.topic-inference.covariate-snapshot.v1`; +- `{"columns":[],"values":[]}` under + `tepp.topic-measurement.design-row.v1`. + +A canonical digest verifies equality with the exact retained canonical JSON +value; it does not prove that a descriptor is truthful or complete and cannot +reconstruct or retrieve the described material. Reproduction requires the +authorized source snapshot and every pinned model, vocabulary, preprocessing, +design, lineage, temporal, covariate, and design-row object to remain resolvable +under retention policy. Raw artifact-byte equality additionally requires the +separate manifest-owned byte hash described above. + +All content-, evidence-, covariate-, membership-, temporal-, design-, and +label-derived digests are sensitive pseudonymous linkage data. They are never +public and never appear in ordinary logs, metrics, traces, or audit events. A +restricted audit record may be referenced only by a tenant-keyed opaque handle. + +## Evidence reference rules + +An `evidence_ref`: + +- is opaque and contains no source/provider identifier, URL, or path; +- is bound to one audience, tenant, workspace, document snapshot, and purpose; +- has an expiry and is rejected when expired; +- is reauthorized on every use instead of treated as a bearer shortcut; +- cannot be exchanged across organizations or workspaces; and +- resolves only through a server-side registry that returns the retained + immutable snapshot or fails closed. + +The internal `evidence_ref.snapshot_revision` must equal +`request.source_snapshot_revision`, and its `scope_binding_ref` must equal the +request binding. Resolution must reproduce the current authenticated tenant, +workspace, purpose, consent, region, and authorization binding; equality of the +opaque strings alone is insufficient. + +## Compatibility and change control + +Revision `2026-08-09.1` is immutable. Backward-compatible clarifications require +a new revision and schema digest; semantic changes to topic identity, +uncertainty, abstention, provenance, error mapping, or ownership require a new +contract version and a superseding Naruon ADR. Naruon must not silently coerce a +expected-upstream payload from an unrecognized revision. + +No route can move from `BLOCKED-UPSTREAM` to implemented until the requirements +and evidence in [Traceability](TRACEABILITY.md) and the operability/security +gates are satisfied on the exact candidate revision. diff --git a/docs/topic-intelligence/ARCHITECTURE.md b/docs/topic-intelligence/ARCHITECTURE.md new file mode 100644 index 000000000..e2a268267 --- /dev/null +++ b/docs/topic-intelligence/ARCHITECTURE.md @@ -0,0 +1,249 @@ +# Topic intelligence architecture + +- **Capability maturity:** `BLOCKED-UPSTREAM` +- **Document status:** `PRESENT-CURRENT` +- **Contract revision:** `2026-08-09.1` + +This is a proposed target profile governed by Naruon's accepted local policy for +a future topic-intelligence adapter. It is not a description of a shipped route, +active TEPP deployment, or published TEPP production contract. Naruon currently +has no fitted topic model to call and therefore exposes no topic-inference +fallback. + +The governing local decision is +[`ADR-0001`](../adr/0001-topic-measurement-authority.md). The adapter remains +blocked until TEPP independently publishes a compatible, versioned fitted-model +artifact and inference contract with its own acceptance evidence. + +## Authority and ownership + +The integration deliberately separates the product envelope from scientific +authority. `TEPP` below is an expected upstream producer, not a present owner or +commitment: that responsibility exists only if TEPP independently publishes a +compatible production contract, artifact, and acceptance evidence. + +| Boundary | Owner | Responsibilities | Must not do | +|---|---|---|---| +| Authenticated product request | Naruon | Reauthorize the tenant/workspace-scoped source, resolve an immutable snapshot, enforce purpose/consent policy, and run preflight checks | Accept a browser-supplied body, tenant identifier, path, URL, or model label as authoritative | +| Adapter envelope | Naruon | Pin schema revision/digest, assign opaque request identity, map failures, validate the upstream payload, redact public projections, and enforce acceptance policy | Refit a model, synthesize a posterior, or reinterpret a label as numeric topic identity | +| Scientific payload | Expected upstream producer; TEPP only after independent publication | Identify the fitted artifact and frozen preprocessing/vocabulary/design, perform new-document inference, and return mixed-membership estimates, uncertainty, provenance, and diagnostics | Depend on Naruon's UI labels or agenda templates as model inputs, or treat this Naruon acceptance profile as an assigned TEPP obligation | +| Presentation labels | Naruon, from versioned evidence | Attach evidence-backed human-readable labels after inference | Mutate topic identifiers, proportions, intervals, or diagnostic outcomes | +| Agenda/action generation | A separate future contract | Consume an authorized source snapshot and an accepted posterior | Infer topics from raw keywords or run when topic inference abstained | + +The JSON Schema in +[`schema/topic-inference-result-v1.schema.json`](schema/topic-inference-result-v1.schema.json) +defines Naruon's **internal adapter envelope** and the scientific payload shape +that Naruon would require before consumption. It does not claim that TEPP has +adopted that schema. A public Naruon API may expose only a redacted projection; +canonical digests and scope-binding evidence are internal validation material. +That projection still carries the opaque model ID/version plus analysis unit, +estimand, coarse covariate level, and causal/non-causal designation needed to +prevent ecological or causal over-interpretation. + +## Planned components + +```mermaid +flowchart TD + Client["Authenticated Naruon client"] --> API["Naruon topic API"] + API --> Snapshot["Authorized immutable snapshot"] + API --> Adapter["Naruon topic adapter"] + Adapter --> TEPP["Expected upstream inference boundary"] + TEPP --> Artifact["Versioned fitted artifact"] + Snapshot --> Adapter + Adapter --> API +``` + +- The client supplies only opaque source and evidence references plus a bounded + request revision. It never selects a model by display label. +- The Naruon API reauthorizes the evidence reference on every request and + resolves the current server-authoritative snapshot. +- The adapter pins the accepted schema, deployment, model artifact, + preprocessing, vocabulary, design, temporal policy, and validation policy. +- The independently accepted upstream producer would perform inference against + an already fitted artifact. Training or per-request refitting is outside this + request path. +- Naruon validates contract and scientific invariants before producing either + `inferred` or the narrowly defined `abstained` result. + +No component reads another service's private database. The future integration +must use a versioned typed boundary and an explicitly deployed artifact. + +## Request and result flow + +```mermaid +sequenceDiagram + participant C as Client + participant N as Naruon API + participant A as Naruon adapter + participant T as Expected upstream producer + + C->>N: Opaque evidence ref + request revision + N->>N: Reauthorize and freeze snapshot + N->>A: Canonical internal request + A->>A: Preflight and deployment pin + alt Input is ineligible + A-->>N: RFC 9457 problem (422) + else Deployment or artifact is unavailable + A-->>N: RFC 9457 problem (503) + else Trusted revision or idempotency conflicts + A-->>N: RFC 9457 problem (409) + else Compatible request + A->>T: Versioned inference request + alt Upstream deadline expires + A-->>N: RFC 9457 problem (504) + else Scientific payload returned + T-->>A: Scientific payload + A->>A: Verify digests, schema, codes, cross-fields + alt Transport or payload validation fails + A-->>N: RFC 9457 problem (502) + else Posterior and policy accept + A-->>N: 200 inferred + else Posterior or diagnostic policy declines + A-->>N: 200 abstained + end + end + end + N-->>C: Redacted response or safe problem detail +``` + +`200 abstained` is not a generic failure bucket. It is permitted only after the +request, language, snapshot, model deployment, artifact, temporal inputs, and +covariates are compatible and an attempted posterior or its diagnostic policy +does not meet the declared acceptance criteria. Preflight failures never appear +as abstentions. + +## Trust boundaries and fail-closed behavior + +| Condition | Boundary that detects it | Contract result | +|---|---|---| +| Missing/expired/wrong-audience evidence reference | Naruon authorization | Authentication/authorization failure; no upstream call | +| Purpose, consent, region, tenant, or workspace denial | Naruon authorization | `403` RFC 9457 problem; existence remains undisclosed | +| Tenant/workspace quota or rate policy exceeded | Naruon edge/adapter | `429` RFC 9457 problem; no upstream call | +| Unsupported language, insufficient retained tokens, excessive OOV, invalid temporal context, or invalid covariates | Naruon/expected-upstream preflight | `422` RFC 9457 problem with stable `error_code` | +| No active deployment, missing artifact, or failed artifact-integrity validation | Naruon adapter | `503` RFC 9457 problem | +| Snapshot revision, request revision, schema pin, or idempotency conflict | Naruon adapter | `409` RFC 9457 problem | +| Upstream transport fails or a schema, digest, format, code-registry, or cross-field result cannot be validated | Naruon adapter | `502` problem; never an abstention or fabricated posterior | +| Upstream deadline expires | Naruon adapter | `504` RFC 9457 problem; bounded cancellation and no result | +| Client cancels or disconnects | Naruon edge/adapter | Cancel work and record only a stable internal cancellation outcome; no HTTP result may be deliverable | +| Compatible inference produces a posterior rejected by declared diagnostic/acceptance policy | Naruon adapter | `200`, `status=abstained`, no usable topic components | +| Valid posterior satisfies the pinned policy | Naruon adapter | `200`, `status=inferred` | + +There is no keyword, embedding, zero-shot, LLM-label, default-topic, or template- +agenda fallback under this contract. + +## Scientific invariants + +The adapter must enforce invariants that JSON Schema alone cannot express: + +1. Topic identifiers are non-negative JSON integers. For `inferred`, + `fitted_topic_count`, `topic_count`, `observed_topic_count`, and the number of + components are equal; topic IDs and ranks are independently unique. For + `abstained`, the latter three counts are zero while `fitted_topic_count` + remains the active artifact's topic count. +2. Every proportion and credible-interval bound is finite and in `[0, 1]`. +3. Each estimate lies within its own interval. +4. For an `inferred` result, component proportions sum to one within the pinned + `normalization_tolerance`; the diagnostic `posterior_sum` agrees with the + recomputed value. +5. `credible_level`, `interval_method`, and + `uncertainty_scope=conditional_on_fitted_artifact` apply to every component. + They do not claim to include model-selection, corpus-selection, or label + uncertainty. +6. `inferred` requires accepted diagnostics, a non-empty component vector, and + no abstention reasons. Its posterior diagnostics include a stable + `convergence_code`, `numerical_status=valid`, and bounded stable + `quality_codes`. `abstained` requires rejected diagnostics, at least one + posterior/policy reason, and an empty component vector. +7. The declared fitted topic count, design row, membership structure, temporal + policy, and preprocessing/vocabulary identities match the active deployment. +8. Any multilevel, multiple-membership, cross-classified, temporal, prevalence, + or content extension names its estimator, analysis unit, estimand, formulas, + contrasts, membership-weight normalization, unseen-level policy, and + validation profile. `causal_design` remains `non_causal` unless a separate + causal design is approved and documented. +9. Multiple-membership structures require weights that sum to one per analysis + unit. A structure without multiple membership requires + `membership_weight_normalization=not_applicable`. Covariate schema version, + level, and typed missingness policy must match the retained covariate snapshot + and model card; missing values are never silently assigned to a default level. +10. The evidence reference snapshot and scope bindings equal the enclosing + request bindings, and use-time reauthorization resolves the same current + tenant, workspace, purpose, consent, region, and authorization context. +11. Validators assert RFC 3339 `date-time` formats, recompute + `availability_time <= knowledge_cutoff_time`, and enforce the pinned temporal + missingness policy. A producer's `availability_at_knowledge_cutoff=true` + assertion is evidence to verify, not a substitute for that check. +12. The inference-method copy and every diagnostic or reason code agree with the + exact pinned immutable code registries. An unknown registry version or code + is a `502` protocol error, never an abstention. + +## Canonical provenance and reproduction + +All contract digest fields are SHA-256 over an RFC 8785 canonical JSON +descriptor with domain separation: + +`SHA-256(UTF8(domain) || 0x00 || UTF8(RFC8785(value)))` + +RFC 8785 does not normalize Unicode. Producers must apply the frozen +preprocessing contract before constructing a value to digest; consumers must +not add an undocumented normalization pass. + +For a model with no covariates, the canonical empty values are: + +- covariate snapshot: + `{"covariates":[],"memberships":[]}` with domain + `naruon.topic-inference.covariate-snapshot.v1`; +- design row: `{"columns":[],"values":[]}` with domain + `tepp.topic-measurement.design-row.v1`. + +Digests prove equality with retained material; they do not make deleted or +unavailable material reproducible. Reproduction additionally requires an +authorized, resolvable retained source snapshot, model artifact, manifests, +vocabulary, preprocessing/design specifications, temporal evidence, and +covariate/design-row material. + +Every content-, evidence-, covariate-, membership-, temporal-, design-, and +label-derived digest is sensitive pseudonymous linkage data. It is for internal +validation only and must never be +placed in a public response, ordinary audit event, application log, metric, or +trace. Restricted audit records may hold a tenant-keyed opaque reference to a +protected validation record, subject to retention and deletion policy. + +## Deployment and operability gates + +A deployment may become active only when all of the following are pinned and +verified as one compatible set. The complete canonical-digest inventory is the +schema, source snapshot, nested scientific payload, artifact descriptor, +artifact manifest, vocabulary, preprocessing, design, lineage, model card, +validation report, evidence-time manifest, covariate snapshot, and design row: + +- immutable schema ID, revision, and externally configured schema digest; +- independently accepted upstream service and scientific contract version; +- model artifact, artifact manifest, vocabulary, preprocessing, design, + lineage, model-card, and validation-report digests; +- temporal policy, temporal missingness rule, evidence-time manifest digest, + and asserted/recomputed availability-at-cutoff ordering; +- covariate-schema version, typed level/missingness policy, covariate-snapshot + digest, design-row digest, and scope-binding policy; +- estimator, analysis unit, estimand, formulas/contrasts, membership and unseen- + level policy, plus the validation profile; +- language, retained-token, OOV, posterior-normalization, uncertainty, + diagnostic acceptance thresholds, immutable diagnostic/reason-code registry + versions, and a validator with `date-time` format assertion enabled; and +- tenant/workspace purpose, consent, retention, deletion, evidence-reference, + log-redaction, and restricted-audit controls. + +Activation, rollback, artifact revocation, validation drift, latency/error SLOs, +and incident playbooks belong to the operability contract. Until those gates +and the independently published upstream capability exist, the runtime maturity +remains `BLOCKED-UPSTREAM`. + +## Related records + +- [Product requirements](PRD.md) +- [Technical requirements](TRD.md) +- [API contract](API_CONTRACT.md) +- [UML views](UML.md) +- [Conceptual data model](DATA_MODEL.md) +- [Requirements traceability](TRACEABILITY.md) +- [ADR-0001](../adr/0001-topic-measurement-authority.md) diff --git a/docs/topic-intelligence/DATA_MODEL.md b/docs/topic-intelligence/DATA_MODEL.md new file mode 100644 index 000000000..0cc805347 --- /dev/null +++ b/docs/topic-intelligence/DATA_MODEL.md @@ -0,0 +1,302 @@ +# Topic intelligence conceptual data model + +- **Capability maturity:** `BLOCKED-UPSTREAM` +- **Document status:** `PRESENT-CURRENT` +- **Persistence status:** `NOT-APPLICABLE` + +This document is a conceptual integration model, not a physical database model. +There is no current Naruon table or persistence authorized for any entity below. +The names describe messages, immutable artifacts, and bounded references needed +to reason about the planned adapter. + +Names prefixed `TEPP_` denote the expected shape of independently published +upstream evidence. They do not assign present ownership to TEPP; TEPP becomes the +producer only if it separately publishes and accepts a compatible production +contract and artifact. + +Any future persistence requires a separate accepted ADR, threat model, retention +and deletion design, tenant/workspace row-level authorization, migration, rollback +plan, and database tests. A diagram here must never be used as permission to add +tables or columns. + +## Integration entities + +```mermaid +erDiagram + NARUON_DOCUMENT_SNAPSHOT ||--o{ TOPIC_INFERENCE_REQUEST : supplies + TEPP_MODEL_ARTIFACT ||--o{ TEPP_MODEL_DEPLOYMENT : realizes + TEPP_MODEL_DEPLOYMENT ||--o{ TOPIC_INFERENCE_REQUEST : selected_for + TOPIC_INFERENCE_REQUEST ||--o| TOPIC_INFERENCE_RESULT : produces + + NARUON_DOCUMENT_SNAPSHOT { + string snapshot_ref PK + string document_ref + string snapshot_revision + string source_snapshot_digest + datetime knowledge_cutoff_time + } + TOPIC_INFERENCE_REQUEST { + string request_id PK + string request_revision + string snapshot_ref FK + string deployment_ref FK + string evidence_ref + string scope_binding_ref + string purpose_code + } + TEPP_MODEL_ARTIFACT { + string model_artifact_ref PK + string model_id + string model_version + string artifact_descriptor_digest + string schema_revision + } + TEPP_MODEL_DEPLOYMENT { + string deployment_ref PK + string model_artifact_ref FK + string validation_profile_version + string deployment_state + } + TOPIC_INFERENCE_RESULT { + string request_id PK + string result_status + string payload_digest + datetime completed_at + } +``` + +These relationships express authority, not storage foreign keys: + +- `NARUON_DOCUMENT_SNAPSHOT` is a server-authoritative immutable view. Its + opaque `snapshot_ref` binds the exact document reference, snapshot revision, + and source-snapshot digest resolved after owner, organization, workspace, + purpose, and consent checks. +- `TOPIC_INFERENCE_REQUEST` binds exactly one snapshot revision to one active + deployment and one idempotent request revision. +- `TEPP_MODEL_ARTIFACT` is a conditional expected-upstream evidence role. Naruon + may consume it only after independent publication and compatibility review; it + does not currently assign TEPP an obligation or own/mutate such an artifact. + Its opaque `model_artifact_ref` binds the exact model ID, model version, + artifact-descriptor digest, and schema revision. +- `TEPP_MODEL_DEPLOYMENT` is Naruon's compatibility/activation record for a + particular immutable upstream evidence set. A mutable display tag is not an + identity. +- `TOPIC_INFERENCE_RESULT` exists only for HTTP `200` outcomes (`inferred` or + narrowly defined `abstained`). RFC 9457 problems are errors, not result rows. + +## Scientific result entities + +```mermaid +erDiagram + TOPIC_INFERENCE_RESULT ||--o{ TOPIC_POSTERIOR_COMPONENT : contains + TOPIC_INFERENCE_RESULT ||--|| SCIENTIFIC_PROVENANCE : validates_with + TOPIC_INFERENCE_RESULT ||--|| DIAGNOSTIC_BUNDLE : qualifies + TOPIC_POSTERIOR_COMPONENT ||--o{ TOPIC_LABEL_EVIDENCE : may_present_as + + TOPIC_INFERENCE_RESULT { + string request_id PK + string model_id + string model_version + string result_status + string scientific_payload_digest + } + TOPIC_POSTERIOR_COMPONENT { + string component_ref PK + string request_id FK + string model_id + string model_version + int topic_id + int rank + number proportion + number interval_lower + number interval_upper + } + SCIENTIFIC_PROVENANCE { + string artifact_descriptor_digest + string artifact_manifest_digest + string vocabulary_digest + string preprocessing_digest + string design_digest + string lineage_digest + string model_card_digest + string validation_report_digest + string evidence_time_manifest_digest + string covariate_snapshot_digest + string design_row_digest + string analysis_unit + string estimand_id + string causal_design + } + DIAGNOSTIC_BUNDLE { + string diagnostic_status + string diagnostic_code_registry_version + string reason_code_registry_version + number posterior_sum + boolean policy_accepted + string policy_version + } + TOPIC_LABEL_EVIDENCE { + string label_evidence_ref PK + string component_ref FK + string model_id + string model_version + int topic_id + string label_id + string label_version + string opaque_evidence_refs + string review_method + } +``` + +The `PK` and `FK` labels above are conceptual message identities, not proposed +SQL columns or additions to the public wire contract. Each opaque reference is +immutable and resolves only when every bound scope value agrees: + +| Entity | Required immutable identity binding | Forbidden unscoped shortcut | +|---|---|---| +| Document snapshot | `snapshot_ref` -> (`document_ref`, `snapshot_revision`, `source_snapshot_digest`) | `document_ref` alone | +| Model artifact | `model_artifact_ref` -> (`model_id`, `model_version`, `artifact_descriptor_digest`, `schema_revision`) | `model_id` or a display tag alone | +| Posterior component | `component_ref` -> (`request_id`, `model_id`, `model_version`, `topic_id`) | `topic_id`, rank, or label alone | +| Label evidence | `label_evidence_ref` -> (`model_id`, `model_version`, `topic_id`, `label_id`, `label_version`, `opaque_evidence_refs`) | `topic_id`, `label_id`, or label text alone | + +A resolver must fail closed when an opaque reference and its supplied scope tuple +disagree. Numeric topic identity is reusable only within its exact model ID and +model version; a result component adds request/result scope, and presentation +evidence additionally adds label ID and label version. + +An `abstained` result has zero `TOPIC_POSTERIOR_COMPONENT` instances, rejected +diagnostics, and one or more posterior/diagnostic-policy reason codes. Input, +language, temporal, covariate, deployment, artifact, revision, and protocol +failures are not represented as abstained results. + +For `inferred`, the fitted artifact count, declared inference count, observed +diagnostic count, and number of components are equal. For `abstained`, the latter +three are zero while the fitted artifact count remains unchanged. Numeric topic +identity is a non-negative JSON integer scoped by model ID and model version; +joins to a result also require its request/result scope. + +`TOPIC_LABEL_EVIDENCE` is presentation metadata owned by Naruon. Its relationship +to a component is referential only: the model ID, model version, numeric topic +ID, label ID, and label version must all agree, and labels cannot become the +topic identifier or alter any estimate. Agenda generation is not an entity in +this model because it belongs to a separate downstream authorized contract. + +## Covariate and temporal evidence + +```mermaid +erDiagram + NARUON_DOCUMENT_SNAPSHOT ||--o| COVARIATE_SNAPSHOT : contextualizes + COVARIATE_SNAPSHOT ||--o{ MEMBERSHIP_WEIGHT : contains + COVARIATE_SNAPSHOT ||--|| DESIGN_ROW : compiles_to + DESIGN_ROW ||--|| EVIDENCE_TIME_MANIFEST : constrained_by + + COVARIATE_SNAPSHOT { + string snapshot_digest PK + string covariate_schema_version + string covariate_level + string missingness_policy + string membership_structure + string unseen_level_policy + } + MEMBERSHIP_WEIGHT { + string membership_ref PK + number weight + string level_ref + } + DESIGN_ROW { + string design_row_digest PK + string estimator_id + string analysis_unit + string estimand_id + } + EVIDENCE_TIME_MANIFEST { + string manifest_digest PK + string temporal_policy_version + string temporal_missingness_policy + datetime document_time + datetime event_time + datetime assertion_time + datetime availability_time + datetime knowledge_cutoff_time + boolean availability_at_knowledge_cutoff + } +``` + +Membership weights must obey the model's pinned normalization rule. New or +unknown levels follow only the declared unseen-level policy; they are never +silently mapped to a familiar group. The design row must be reproducible from +the retained authorized covariate snapshot and pinned design specification. +`multiple_membership` and `cross_classified_multiple_membership` require weights +that sum to one per analysis unit; all other structures require the explicit +`not_applicable` normalization value. Covariates carry a versioned typed level +and missingness policy, and missing state is never inferred from an absent field. + +The adapter enables RFC 3339 `date-time` format assertion and independently +checks `availability_time <= knowledge_cutoff_time`. Only `document_time` and +`event_time` may be null under revision `2026-08-09.1`; the pinned temporal +missingness policy governs their interpretation. + +For a model with no covariates, the entities still have deterministic canonical +empty values rather than missing or implementation-specific sentinels: + +| Concept | RFC 8785 value | Digest domain | +|---|---|---| +| Covariate snapshot | `{"covariates":[],"memberships":[]}` | `naruon.topic-inference.covariate-snapshot.v1` | +| Design row | `{"columns":[],"values":[]}` | `tepp.topic-measurement.design-row.v1` | + +The digest input is +`UTF8(domain) || 0x00 || UTF8(RFC8785(value))`, hashed with SHA-256 and encoded +as lowercase hexadecimal. + +## Concept glossary and ownership + +| Concept | Authority | Identity and lifecycle | +|---|---|---| +| Document snapshot | Naruon source boundary | Opaque document reference plus immutable snapshot revision and canonical digest; resolvable only under current authorization | +| Evidence reference | Naruon authorization boundary | Opaque, tenant/snapshot/audience-bound, expiring, and reauthorized on every use; never a URL or filesystem path | +| Model artifact | Expected upstream producer; TEPP only after independent publication | Immutable fitted artifact with independently published version, manifest, scientific validation, and digest evidence | +| Deployment | Naruon adapter | Compatibility and activation decision binding one exact upstream artifact/contract set to one Naruon validation profile | +| Scientific payload | Expected upstream producer; TEPP only after independent publication | Mixed-membership estimate, uncertainty, scientific provenance, and diagnostics returned from the fitted artifact | +| Adapter envelope | Naruon | Request identity, schema pin, payload digest, result status, acceptance decision, and safe error mapping | +| Presentation label | Naruon from versioned evidence | Human-readable aid with separate version and evidence references; never numeric topic identity | + +The request and evidence reference each carry the same opaque scope-binding and +snapshot revision. Equality is a runtime invariant; use-time reauthorization +must resolve that binding to the current tenant, workspace, purpose, consent, +region, and authorization context. Neither the binding nor its protected record +is a public identifier. + +The complete internal digest inventory is: schema, source snapshot, scientific +payload, artifact descriptor, artifact manifest, vocabulary, preprocessing, +design, lineage, model card, validation report, evidence-time manifest, +covariate snapshot, and design row. Every item uses its schema-defined domain. +The public projection omits those digests but retains opaque model ID/version, +analysis unit, estimand, coarse covariate level, and causal/non-causal status so +consumers cannot silently reinterpret a group-level or non-causal estimand. + +## Privacy classification + +Source text is not part of this data model and must not be copied into request +logs, metrics, traces, errors, or unrestricted audit events. Every content-, +evidence-, covariate-, membership-, temporal-, design-, and label-derived digest +is sensitive pseudonymous linkage data even though it is one-way. Such digests +are internal validation material only. + +Where auditability is required, an audit event may contain a tenant-keyed opaque +reference to a restricted validation record. Resolution must re-check owner, +organization, workspace, purpose, consent, retention, and deletion policy. The +public API and UI receive a redacted projection without canonical digests, +tenant bindings, raw covariates, membership identifiers, or arbitrary evidence +locations. + +## Non-persistence decision + +At revision `2026-08-09.1`: + +- no Alembic migration is authorized; +- none of these conceptual names is a SQL table or ORM model; +- no posterior, label, covariate row, or digest is retained by default; +- a request may be processed transiently only after the upstream and runtime + gates in [Architecture](ARCHITECTURE.md) are satisfied; and +- a future persistence proposal must prove why transient processing and a + restricted audit reference are insufficient before adding durable storage. diff --git a/docs/topic-intelligence/DOCUMENTATION_FITNESS.md b/docs/topic-intelligence/DOCUMENTATION_FITNESS.md new file mode 100644 index 000000000..48e0b43b7 --- /dev/null +++ b/docs/topic-intelligence/DOCUMENTATION_FITNESS.md @@ -0,0 +1,105 @@ +# Documentation fitness assessment + +- **Assessment date:** 2026-08-09 +- **Protected-base snapshot:** `develop@5425ce4f55b2cf16b2c82a4fd661c9d0bd0660c7` +- **Candidate:** PR #1297 +- **Verdict before this package:** insufficient +- **Verdict after this package:** design-sufficient for deletion review and + future contract discovery; partial for runtime implementation; insufficient + to claim a live STM capability + +Fitness terms are `PRESENT-CURRENT`, `PRESENT-STALE`, `PARTIAL`, `MISSING`, +`NOT-APPLICABLE`, and `SUPERSEDED`. These terms assess documentation fitness, +not implementation maturity. + +The maturity split is explicit: ADR-0001 is an +`ACCEPTED-NARUON-POLICY`; ADR-0002 and ADR-0003 are `Proposed` Naruon target +decisions; the target acceptance profile is `PLANNED`; and the runtime capability +remains `BLOCKED-UPSTREAM`. A proposed target or complete document package is +not an accepted runtime architecture and cannot promote the capability. + +## Assessment matrix + +| Artifact | Before | After | Evidence and remaining limit | +| --- | --- | --- | --- | +| Topic-specific PRD | `PRESENT-STALE` | `PRESENT-CURRENT` | Requirement IDs, users, non-goals, failure/abstention journeys, and explicit maturity are consolidated. | +| Topic-specific TRD | `PARTIAL` | `PRESENT-CURRENT` | Naruon ownership, upstream non-authority, artifact, input/result, error/abstention, provenance, security, compatibility, and gates are explicit. | +| Naruon ADR | `PARTIAL` | `PRESENT-CURRENT` | ADR-0001 records only Naruon's accepted local policy; the ADR index and package separately expose ADR-0002 and ADR-0003 as proposed targets, not upstream acceptance or runtime implementation. | +| Future adapter decisions | `MISSING` | `PARTIAL` | Proposed ADR-0002 covers conditional fitted-artifact consumption and proposed ADR-0003 covers agenda separation. Transport/authentication, artifact signing/registry, cache, retention/deletion, rate limit, sensitive-covariate, and downstream-authorization ADRs still await a real upstream boundary. | +| Architecture | `PARTIAL` | `PRESENT-CURRENT` | Current/candidate/target ownership, trust, and failure boundaries are separated; the target views are a proposed acceptance profile governed only by the accepted local policy. | +| UML | `PARTIAL` | `PRESENT-CURRENT` | Conceptual component, class, success/error/abstention, artifact-state, and deployment views are available without claiming runtime code. | +| ERD/data model | `PRESENT-STALE` | `PRESENT-CURRENT` | Contract concepts are modeled; physical Naruon persistence remains correctly `NOT-APPLICABLE`. | +| API/schema/versioning | `PARTIAL` | `PRESENT-CURRENT` | Planned Naruon adapter validation shape, closed revision rules, errors, abstention, and cross-field invariants are documented; no live transport is claimed. | +| Canonical digest inventory | `MISSING` | `PRESENT-CURRENT` | One 14-field inventory names the three envelope and eleven scientific-provenance digests, including model card, validation report, covariate snapshot, and design row; schema/API remain the machine-readable and formula authorities. | +| Security and threat model | `PARTIAL` | `PRESENT-CURRENT` | Assets, misuse cases, privacy/statistical risks, controls, residual decisions, and refresh triggers are explicit. | +| Test strategy | `PARTIAL` | `PRESENT-CURRENT` | Naruon product/integration evidence is separated from upstream scientific validation. | +| Operability | `PARTIAL` | `PRESENT-CURRENT` | Readiness, safe signals, promotion, incidents, rollback, recovery, and replay gates are documented without invented SLOs. | +| Traceability | `PARTIAL` | `PRESENT-CURRENT` | Requirements map to the Naruon decision, design/contract, code/tests, and maturity. | +| References | `PARTIAL` | `PRESENT-CURRENT` | Scientific, standards, and dated repository evidence are separated from implementation claims. | +| Machine documentation fitness | `MISSING` | `PARTIAL` | File/link/schema-maturity/source-absence checks are useful, but balanced fences are not Mermaid parsing and JSON parsing is not Draft 2020-12 metaschema or fixture validation. | + +## Why the verdict is not “implementation-ready” + +The package defines what Naruon would require; it does not supply the upstream +dependency or implementation evidence. In particular: + +- Naruon has no independently published upstream production topic artifact, + inference API/contract, or publisher acceptance evidence to consume. +- The planned Naruon envelope is not an upstream publisher's canonical payload + and cannot assign obligations or ownership to TEPP or another producer. +- Transport, service authentication, artifact signing/registry, cache, + retention/deletion, rate limit, sensitive-covariate, and downstream- + authorization decisions are unresolved. +- No physical Naruon topic persistence, migration, retention contract, or + resolvable replay snapshot has been approved. Digests support verification, + not reconstruction. +- No fitted production artifact, model card, validation thresholds, interval + calibration/coverage evidence, drift baseline, signed promotion record, + representative capacity study, or numeric SLO exists. +- The 14 digest fields specify verification bindings only. No retained object, + upstream adoption, scientific validity, or replay capability follows from the + inventory itself. +- No live OpenAPI route, adapter, real-service E2E evidence, or topic UI exists. + +These are intentional gates while runtime integration is `BLOCKED-UPSTREAM`, +not permission to describe the capability as implemented. + +## Completeness decision + +The deletion change is adequately specified when reviewers can verify all of the +following: + +1. The two lexical pseudo-topic tools disappear from registry and source on the + candidate branch. +2. The retained keyword utility remains bounded and explicitly lexical. +3. No substitute topic handler, default label, template agenda, or network/model + dependency is introduced. +4. Protected-base, active-PR, accepted-local-policy, planned, and upstream- + blocked claims remain distinct. +5. Error and scientific-abstention semantics do not overlap. +6. Any future integration is blocked on an independently published compatible + fitted artifact/API/contract and scientific acceptance evidence. + +The documentation is therefore sufficient for PR #1297's deletion decision and +for initiating later contract discovery. It is insufficient to authorize a +runtime adapter, persistence, downstream topic use, or product UI. + +## Reassessment triggers + +Re-run this assessment when any of the following occurs: + +- an upstream publisher independently publishes or changes a production topic- + measurement artifact/API/contract or its acceptance evidence; +- Naruon selects a transport, schema revision, fitted artifact, cache, audit, or + persistence design; +- topic output is consumed by search, norm-group inference, labels, agenda + generation, or another downstream decision; +- a UI, sensitive covariate, temporal/multilevel estimator, or causal claim is + proposed; or +- an incident, drift result, validation result, or retention requirement changes + the accepted Naruon boundary. + +When maturity changes, update the PRD requirement row, TRD, ADR status/scope, +architecture and contract, tests/evidence, traceability, changelog, and this +fitness matrix in the same PR. A document-only status promotion without +protected-branch runtime evidence is invalid. diff --git a/docs/topic-intelligence/OPERABILITY.md b/docs/topic-intelligence/OPERABILITY.md new file mode 100644 index 000000000..773dc8b20 --- /dev/null +++ b/docs/topic-intelligence/OPERABILITY.md @@ -0,0 +1,122 @@ +# Topic intelligence operability + +**Status:** target operating design `PLANNED`; runtime integration +`BLOCKED-UPSTREAM`; no runtime runbook, dashboard, threshold, or SLO is claimed + +The safest current operating state is “integration absent.” The pseudo-topic +removal introduces no external dependency. Everything below is a release gate +for a future adapter, not evidence that a TEPP topic service or fitted model is +available. + +## Readiness gates + +- Naruon receives and accepts an independently published production contract, + immutable fitted-artifact manifest, validation packet, model card, promotion + evidence, and named upstream owners. This is a Naruon consumption gate, not an + assignment of work or ownership to TEPP. +- Naruon approves transport/service-authentication, evidence-reference, + artifact-signing/registry, cache/idempotency, retention/deletion, + sensitive-covariate, privacy/rate-limit, and downstream-authorization ADRs. +- Representative capacity tests establish request bytes, retained tokens, + concurrency, queue, deadline, cancellation, retry, circuit-breaker, and quota + limits. +- Authentication/authorization denial, rate limiting, deadline expiry, and + cancellation have tested stable non-`200` mappings, bounded retry rules, and no + scientific-abstention or fallback transition. +- Dashboards and alerts are verified with synthetic traffic and contain no raw + content, labels, sensitive covariates, direct identifiers, or unkeyed derived + digests. +- Operators drill artifact promotion, signer revocation, quarantine, rollback, + tenant disable, deletion propagation, cache eviction, and full service disable. +- Every disabled, unavailable, timeout, schema, artifact, or policy state is + verified to have no keyword, embedding, LLM, cached-other-model, category, or + agenda fallback. + +## Planned signals + +| Signal | Safe dimensions | Excluded dimensions | +| --- | --- | --- | +| Request and result counts | contract/schema revision, model version, coarse status/error/abstention code, tenant-safe aggregate | content, label, direct user/source ID, raw request/result ID | +| Latency and deadline | operation, model version, coarse outcome | raw content size or rare tenant dimensions unless privacy-reviewed | +| Scientific diagnostics | pass/abstain code, privacy-reviewed aggregate retained-token/OOV bands, artifact version | terms, excerpts, per-user/group values, posterior vector | +| Artifact state | candidate/validated/approved/active/quarantined/retired and opaque registry reference | mutable filesystem path, model bytes, signing secret, raw manifest/content digest | +| Policy and audit | opaque restricted reference, purpose and decision code | credentials, provider URL, body, label, sensitive membership, raw derived digest | + +Every content-, evidence-, covariate-, membership-, temporal-, design-, and +label-derived digest is a sensitive pseudonymous linkage value. It is excluded +from ordinary logs, metrics, traces, dashboards, and product payloads. +Restricted audit uses an opaque reference or tenant-scoped keyed digest with a +documented canonical representation, domain separator, TTL, deletion, and key +rotation. + +Numeric objectives and alert thresholds remain `TBD` until a production TEPP +service and representative workload produce measurements. Placeholder 99.x% +targets would be false precision. + +## Model and contract promotion + +1. Register immutable candidate model bytes, manifest, schema, validation packet, + model card, and build/signing provenance. +2. Verify exact schema ID/revision/digest and code-registry versions, raw artifact + bytes, manifest, vocabulary, preprocessing, design, lineage, model-card, + validation-report, build, signer, and promotion identities and digests. +3. Verify scientific, security, privacy, temporal, and extended-STM evidence for + the exact candidate; reject any unreviewed method or covariate change. +4. Complete independent approval of the exact artifact and signer state. +5. Exercise shadow or restricted-tenant validation without using output for + product decisions. +6. Promote by immutable reference; never mutate an artifact or reuse `latest`. +7. Monitor version-specific errors, abstention, diagnostic-code registry, drift, + privacy, and capacity signals using safe dimensions. +8. Quarantine immediately on integrity, isolation, signer, material validity, + harmful-label, temporal, or deletion concern. + +Schema deployment is coordinated: producers must not send a new closed revision +until consumers pin and negotiate it. A revision uses an immutable schema +identifier and digest; cache identity must not collapse different revisions. + +## Incident response + +| Incident | Immediate action | Recovery evidence | +| --- | --- | --- | +| Artifact/validation-report/digest/signature/signer failure | Quarantine the exact deployment and signer, disable affected inference, preserve minimal restricted evidence | Root cause, key disposition, clean rebuilt artifact and validation report, every binding reverified, full revalidation and authorized promotion | +| Cross-tenant or purpose disclosure | Disable integration, invoke security/privacy response, stop downstream use, propagate deletion | Isolation fix, notification/deletion disposition, adversarial regression tests and controlled re-enable | +| Raw content or derived digest in telemetry | Stop emission and access, preserve only necessary incident evidence, rotate keyed material if applicable | Purge/retention disposition, redaction fix, historical search, rotation and regression evidence | +| Invalid posterior, interval, diagnostics, or unknown code | Reject as a protocol error and disable the exact model/schema/code-registry combination | Producer evidence, closed code-registry review, schema/numerical/scientific revalidation | +| Temporal or ecological misuse | Stop affected consumer and result presentation | Estimand/temporal correction, model-card review, consumer and copy tests | +| Elevated timeout/error/resource use | Open circuit, cancel bounded work, return unavailable | Capacity/root-cause evidence and controlled re-enable | +| Drift, poisoning, or harmful labels | Stop downstream use; retire label or quarantine model independently as applicable | Corpus/model/label review and new immutable version | +| TEPP unavailable | Return stable unavailable error | Health, authorization, schema, artifact, and compatibility verified before re-enable | + +## Rollback principle + +Rollback means disabling the adapter or selecting a previously approved, +compatible immutable artifact under an explicit audited policy. It never means +restoring removed keyword tables, calling an LLM, returning `General`, reusing a +posterior from another tenant/artifact/purpose, or generating an agenda template. +Topic identity remains model-version scoped; consumers must not compare or join +topics across model versions without a separately validated alignment. + +## Recovery, replay, and deletion + +A request is replayable only when the exact authorized snapshot, purpose, +consent, artifact, vocabulary, preprocessing, design, inference version, +analysis unit, estimand, temporal policy, and knowledge cutoff remain valid. An +idempotency key binds retries to that tuple and tenant scope. Replaying after +retention, consent, source access, tenant, model, signer, or policy invalidation +is forbidden even if bytes remain technically available. + +Deletion must cover transient snapshots, queues, caches, persisted results, +restricted audit references, and derived linkage material under their approved +policies. Key rotation is not a substitute for deleting retained content, and +deleting product output does not by itself prove that TEPP-side state is gone. + +## Ownership and handoff + +Naruon operators own Naruon tenant policy, adapter enablement, product projection, +and incident coordination. Before Naruon consumes any external capability, its +published evidence must identify upstream ownership for service health, artifact +promotion/quarantine, scientific validation, deletion, incident escalation, and +a tested disable path. This document assigns no responsibility to TEPP. Naruon +keeps its own full-disable path, and a protocol failure at the ownership seam +fails closed rather than being assigned to the user or hidden by a fallback. diff --git a/docs/topic-intelligence/PRD.md b/docs/topic-intelligence/PRD.md new file mode 100644 index 000000000..9ed5980c4 --- /dev/null +++ b/docs/topic-intelligence/PRD.md @@ -0,0 +1,129 @@ +# Product requirements: topic intelligence + +- **Status:** removal `ACTIVE-PR`; local policy `ACCEPTED-NARUON-POLICY`; + runtime integration `BLOCKED-UPSTREAM` +- **Date:** 2026-08-09 +- **Related change:** PR #1297 +- **Accepted local decision:** [ADR-0001](../adr/0001-topic-measurement-authority.md) +- **Proposed target decisions:** + [ADR-0002](../adr/0002-fitted-topic-artifact-consumption.md) and + [ADR-0003](../adr/0003-separate-topic-measurement-from-agenda-generation.md) + +## Problem + +Naruon exposed `email_categorizer` and `meeting_agenda_generator` through product +names that suggested topic understanding, although both used small fixed Korean +and English term tables. Deterministic output made those rules reproducible; it +did not make them a fitted topic model. The behavior hid uncertainty, confused +business labels with latent topic identity, and failed across languages and +domains. + +Users need an honest boundary between lexical utilities and corpus-derived topic +measurement. They also need Naruon to withhold a topic result when the required +fitted model, contract, input support, or evidence is absent. + +## Users and needs + +| User | Need | +| --- | --- | +| Knowledge worker | Know whether a result is lexical metadata, an evidence-valid posterior, an abstention, or an error. | +| Workspace administrator | Ensure tenant content is purpose-bound and never sent to an unapproved model or corpus. | +| Analyst or research owner | Verify a result against a versioned model, frozen preprocessing/vocabulary, design, times, and diagnostics. | +| Operator | Detect incompatibility, integrity failure, abstention, drift, and service failure without logging message bodies. | +| Developer or reviewer | Prevent lexical, embedding, clustering, or LLM shortcuts from being mislabeled as STM. | + +## Goals + +1. Remove executable product behavior that implies topic inference without a + fitted corpus-level model. +2. Preserve useful keyword extraction only under an explicit lexical contract. +3. Establish a Naruon-local, fail-closed acceptance boundary for any future + independently published fitted-model integration. +4. Keep numeric topic identity, human labels, downstream decisions, and agenda + generation separate. +5. Make future results verifiable against authorized source evidence and an + immutable compatible model artifact. + +## Non-goals + +- Fitting a topic model inside an API request or training models inside Naruon. +- Assigning responsibilities to TEPP or claiming that TEPP accepted this PRD, + ADR, or a Naruon-authored contract. +- Calling keyword counts, embeddings, clustering, classifiers, zero-shot output, + or LLM labels “STM.” +- Adding a Naruon topic route, table, migration, public response, or UI before a + real upstream contract and release evidence exist. +- Reintroducing agenda generation as a topic-measurement side effect. +- Claiming causal effects or individual attributes from group-level topic + prevalence. + +## Product requirements + +| ID | Requirement | Acceptance evidence | Maturity | +| --- | --- | --- | --- | +| `TI-REQ-001` | Remove `email_categorizer` and `meeting_agenda_generator` from the tool registry and implementation. | Registry regression tests and source absence. | `ACTIVE-PR` | +| `TI-REQ-002` | Describe retained `keyword_extractor` output as deterministic lexical frequency/first-occurrence metadata, never topic evidence. | Registry description and handler tests. | `ACTIVE-PR` | +| `TI-REQ-003` | Fail closed until an independently published, compatible fitted-model artifact/API/contract exists; never substitute a default category, synthetic posterior, keyword/embedding/LLM result, or agenda template. | Accepted ADR-0001, proposed ADR-0002, and future negative-path adapter tests. | `ACCEPTED-NARUON-POLICY`; target `PLANNED`; runtime `BLOCKED-UPSTREAM` | +| `TI-REQ-004` | A future valid result returns a mixed-membership topic vector with diagnostics and intervals whose level, method, and uncertainty scope are explicit; compatible-model abstention is a distinct vector-free state. | Independently published calibration/coverage evidence plus Naruon schema and invariant tests. | `BLOCKED-UPSTREAM` | +| `TI-REQ-005` | Any temporal, multilevel, multiple-membership, or cross-classified STM extension names its estimator, analysis unit, estimand, prevalence/content formula and contrasts, membership weights/normalization/unseen-level policy, non-causal status, and validation evidence. | Model card, design manifest, known-truth simulation, and downstream suppression tests. | `BLOCKED-UPSTREAM` | +| `TI-REQ-006` | Bind every result to all 14 fields in the [canonical digest inventory](README.md#canonical-digest-inventory), including `model_card_digest`, `validation_report_digest`, `covariate_snapshot_digest`, and `design_row_digest`. Treat digests as verification, not reconstruction; later reproducibility also requires a resolvable retained snapshot. | Fourteen-field schema/inventory parity, digest/provenance, temporal-leakage, retention, and replay tests. | `BLOCKED-UPSTREAM` | +| `TI-REQ-007` | Keep numeric topic identity separate from evidence-backed, language-aware, versioned human-readable labels. | Schema, presentation, and UI contract tests. | `BLOCKED-UPSTREAM` | +| `TI-REQ-008` | Enforce tenant, workspace, source, purpose, consent, region, retention, deletion, digest-handling, and log/metric/trace-redaction controls before inference. | Authorization, isolation, deletion, restricted-audit, and redaction tests. | `BLOCKED-UPSTREAM` | +| `TI-REQ-009` | Treat agenda generation as a separately authorized downstream decision/generation capability with its own evidence and audit contract. | Accepted separation policy in ADR-0001; proposed ADR-0003 plus separate product/technical contract, endpoint, permissions, and tests before release. | `ACCEPTED-NARUON-POLICY`; target decision and future capability `PLANNED` | +| `TI-REQ-010` | Expose a product UI only after the runtime contract, compatible artifact, uncertainty language, abstention/error states, security controls, and operational gates are real. | Release-readiness review and source-backed E2E tests. | `PLANNED` | + +## User journeys + +### Current candidate + +1. A user or agent lists available analysis tools. +2. The two pseudo-topic tools are absent. +3. Keyword extraction, when selected, is described as lexical frequency rather + than inferred topics. + +### Future valid inference + +1. An authorized user requests topic intelligence for a bounded document + snapshot and declared purpose. +2. Naruon validates scope, minimization, language metadata, time semantics, and + an operator-approved upstream model policy. +3. An independently published upstream interface supplies publisher-accepted + evidence for a compatible active fitted artifact and returns either a mixed- + membership result or a narrowly defined scientific abstention. +4. Naruon validates the pinned contract and numerical invariants before + presenting permitted posterior, provenance, uncertainty, and label evidence. + +### Future input or operational error + +Unsupported language, insufficient retained tokens, excessive OOV input, +invalid temporal/covariate data, missing/incompatible model, integrity failure, +authorization denial, or timeout returns a stable error. No posterior, label, or +fallback is produced. + +### Future scientific abstention + +Only after a compatible active model accepts the input contract may its declared +posterior or diagnostic acceptance rule return `abstained`. The result contains +a stable reason and no topic vector or label. + +## Success and release gates + +- Zero registered pseudo-topic tools and zero production references to their + handlers or fixed dictionaries on the candidate head. +- Lexical extraction remains bounded, deterministic, and honestly named. +- Every future result can be verified against exact source/artifact identities + and digests; any replay or reproducibility claim also proves the authorized + immutable snapshot remains resolvable under an approved retention contract. +- Topic proportions, interval coverage/calibration, diagnostics, and any + extended-STM structures have independently published scientific-validation + evidence before a Naruon adapter is enabled. This is a Naruon acceptance gate, + not an assignment of duties to the publisher. +- Tenant isolation, purpose, redaction, incompatibility, integrity, timeout, + rollback, and no-fallback tests pass with warnings treated as failures. +- Product copy distinguishes lexical terms, numeric topics, human labels, + uncertainty, scientific abstention, and operational/input errors. + +No numeric latency, availability, or scientific-quality target is invented in +this PR. Such targets require an independently published production contract, +representative corpus/workload, capacity study, model card, and approved release +evidence. diff --git a/docs/topic-intelligence/README.md b/docs/topic-intelligence/README.md new file mode 100644 index 000000000..17d9b89de --- /dev/null +++ b/docs/topic-intelligence/README.md @@ -0,0 +1,159 @@ +# Topic intelligence decision package + +- **Snapshot:** 2026-08-09 +- **Change candidate:** [PR #1297](https://github.com/ContextualWisdomLab/naruon/pull/1297) +- **Protected-base evidence:** `develop@5425ce4f55b2cf16b2c82a4fd661c9d0bd0660c7` +- **Scope owner:** Naruon maintainers + +This directory is Naruon's authority graph for removing lexical pseudo-topic +tools and for evaluating any later structural-topic-model (STM) integration. It +does not govern TEPP, assign scientific authority to TEPP, record TEPP acceptance, +or claim that an upstream production contract exists. + +The package is design-sufficient for the deletion review and for future contract +discovery. It is intentionally partial for runtime implementation and is not +evidence that STM is available in Naruon. + +## Maturity vocabulary + +| Term | Meaning | +| --- | --- | +| `IMPLEMENTED-ON-PROTECTED-DEVELOP` | Observable behavior on the pinned protected-base snapshot | +| `ACTIVE-PR` | Implemented only on PR #1297's candidate branch | +| `ACCEPTED-NARUON-POLICY` | An accepted Naruon-local architecture/product rule; not runtime evidence or upstream acceptance | +| `PLANNED` | Designed or required, but not implemented | +| `BLOCKED-UPSTREAM` | Naruon work cannot start until an independently published, versioned upstream production contract and acceptance evidence exist | +| `OUT-OF-SCOPE` | Deliberately excluded from this change | + +Documentation fitness uses a separate vocabulary: +`PRESENT-CURRENT`, `PRESENT-STALE`, `PARTIAL`, `MISSING`, `NOT-APPLICABLE`, +and `SUPERSEDED`. + +ADR status is separate again. [ADR-0001](../adr/0001-topic-measurement-authority.md) +is the accepted Naruon-local policy. [ADR-0002](../adr/0002-fitted-topic-artifact-consumption.md) +and [ADR-0003](../adr/0003-separate-topic-measurement-from-agenda-generation.md) +are proposed target decisions, not accepted architecture or runtime evidence. +`PLANNED` may describe their design work, but it never overrides the runtime +capability gate `BLOCKED-UPSTREAM`. + +## Current truth + +| Concern | Maturity | Evidence-backed statement | +| --- | --- | --- | +| Protected `develop` behavior at the pinned base | `IMPLEMENTED-ON-PROTECTED-DEVELOP` | `email_categorizer` and `meeting_agenda_generator` are registered lexical heuristics. | +| Candidate behavior | `ACTIVE-PR` | PR #1297 removes both tools and retains `keyword_extractor` only as an explicitly lexical utility. | +| Naruon consumption rule | `ACCEPTED-NARUON-POLICY` | Naruon does not present keywords, embeddings, clustering, zero-shot output, or LLM labels as an STM posterior. The ADR becomes protected-branch authority only when the candidate is accepted and merged. | +| Upstream fitted-model dependency | `BLOCKED-UPSTREAM` | TEPP architecture provides direction, but Naruon has no independently published TEPP production topic artifact/API/contract or TEPP acceptance evidence to consume. | +| Proposed Naruon STM target profile | `PLANNED`; capability `BLOCKED-UPSTREAM` | The acceptance profile is design material. No production handler, endpoint, table, model artifact, migration, or UI exists, and runtime work cannot start without the independently published upstream dependency. | +| Agenda generation from topic evidence | `PLANNED` | It is a separate downstream decision/generation capability, never part of topic measurement. | + +## Authority graph + +```mermaid +flowchart TD + ADR1["ADR-0001: accepted local policy"] --> PRD["PRD: product intent"] + ADR2["ADR-0002: proposed adapter"] -.-> TRD + ADR3["ADR-0003: proposed agenda boundary"] -.-> PRD + PRD --> TRD["TRD: technical obligations"] + TRD --> DESIGN["Architecture, UML, and data model"] + TRD --> CONTRACT["Planned adapter contract"] + DESIGN --> ASSURANCE["Security, tests, and operations"] + CONTRACT --> ASSURANCE + ASSURANCE --> TRACE["Traceability and fitness"] +``` + +Solid arrows descend from the accepted local policy. Dotted arrows identify +proposed Naruon decisions whose acceptance triggers have not been satisfied. + +When documents conflict, the accepted [Naruon-local +ADR](../adr/0001-topic-measurement-authority.md) governs Naruon's decision, the +PRD governs product intent, and the TRD governs proposed implementation +obligations. Runtime code and deployed OpenAPI remain the authority for shipped +behavior. A checked planned schema is not a deployed API and cannot stand in for +the future upstream contract. + +## Document map + +| Document | Purpose | +| --- | --- | +| [PRD](PRD.md) | Product problem, users, requirements, non-goals, and release gates | +| [TRD](TRD.md) | Technical ownership, artifact, result, failure, security, and implementation obligations | +| [Documentation fitness](DOCUMENTATION_FITNESS.md) | Before/after completeness assessment and intentional gaps | +| [ADR index](../adr/README.md) | Status and change rules for all Naruon architecture decisions | +| [Naruon ADR-0001](../adr/0001-topic-measurement-authority.md) | Accepted local consumption policy and upstream non-authority boundary | +| [Proposed ADR-0002](../adr/0002-fitted-topic-artifact-consumption.md) | Conditional fitted-artifact consumption and fail-closed adapter decision | +| [Proposed ADR-0003](../adr/0003-separate-topic-measurement-from-agenda-generation.md) | Conditional downstream agenda-generation separation decision | +| [Architecture](ARCHITECTURE.md) | Current and target components, trust boundaries, and failure architecture | +| [UML](UML.md) | Conceptual component, class, sequence, state, and deployment views | +| [Conceptual ERD](DATA_MODEL.md) | Contract relationships without inventing physical persistence | +| [Planned adapter contract](API_CONTRACT.md) | Closed-version envelope, errors, abstention, and compatibility semantics | +| [Security](SECURITY.md) | Data protection and control requirements | +| [Threat model](THREAT_MODEL.md) | Design-time misuse cases, mitigations, and residual decisions | +| [Test strategy](TEST_STRATEGY.md) | Naruon integration evidence separated from upstream scientific validation | +| [Operability](OPERABILITY.md) | Promotion, monitoring, incident, rollback, and recovery gates | +| [Traceability](TRACEABILITY.md) | Requirement-to-decision-to-contract-to-evidence mapping | +| [References](REFERENCES.md) | Scientific, standards, and repository evidence | + +## Canonical digest inventory + +Revision `2026-08-09.1` has exactly 14 canonical digest fields. This table is the +single cross-document inventory; the planned +[JSON Schema](schema/topic-inference-result-v1.schema.json) is the machine-readable +definition, and the [API contract](API_CONTRACT.md#digest-contract) defines the +canonicalization formula. The Naruon-authored field name `tepp_payload_digest` +is part of the local acceptance profile and does not assert that an upstream +publisher adopted the name or assigned ownership to TEPP. + +| Scope | Exact field | Bound evidence | +| --- | --- | --- | +| Envelope | `schema_digest` | Complete parsed immutable schema JSON value named by the pinned `$id`; its sole construction is defined in the API contract | +| Envelope | `source_snapshot_digest` | Authorized immutable source-snapshot descriptor | +| Envelope | `tepp_payload_digest` | Complete nested scientific-payload descriptor | +| Scientific provenance | `artifact_digest` | Canonical fitted-artifact descriptor, not raw artifact bytes | +| Scientific provenance | `manifest_digest` | Canonical artifact manifest, including any separately declared optional raw-byte hash record | +| Scientific provenance | `vocabulary_digest` | Frozen vocabulary | +| Scientific provenance | `preprocessing_digest` | Frozen preprocessing contract | +| Scientific provenance | `design_digest` | Statistical design specification | +| Scientific provenance | `lineage_digest` | Training and build lineage descriptor | +| Scientific provenance | `model_card_digest` | Model card | +| Scientific provenance | `validation_report_digest` | Scientific validation report | +| Scientific provenance | `evidence_time_manifest_digest` | Evidence-time manifest | +| Scientific provenance | `covariate_snapshot_digest` | Authorized covariate and membership snapshot | +| Scientific provenance | `design_row_digest` | Compiled design row | + +Aliases and shortened subsets are not contract-equivalent. Any field addition, +removal, rename, canonicalization change, or domain-separator change requires a +new immutable schema revision and synchronized updates to requirements, +decisions, tests, traceability, and this inventory. + +These 14 fields verify equality with exact canonical JSON values under the API +formula. They do not by themselves verify descriptor truth or completeness, +evidence availability, authorization, or raw fitted-artifact bytes. Raw-byte +integrity exists only when an independently published manifest carries a +distinct optional hash record that declares both its algorithm and the exact +byte serialization or package covered. That record is not `artifact_digest` +and does not add a canonical digest field to this inventory. + +## Non-negotiable behavior + +No keyword table, term-frequency score, embedding cluster, zero-shot label, or +LLM-generated label may be represented as an STM posterior. New-document STM +inference requires an independently published, compatible fitted artifact with +frozen preprocessing and vocabulary, declared input/covariate semantics, +uncertainty, diagnostics, provenance, and validation evidence. + +Model or service unavailability, incompatibility, integrity failure, unsupported +language, insufficient retained tokens, excessive out-of-vocabulary input, +invalid temporal/covariate input, authorization denial, and timeout are explicit +errors. `abstained` is reserved for a compatible active model that accepted the +input contract but withheld a posterior under a declared diagnostic or posterior +acceptance rule. Neither path may invoke a lexical, embedding, LLM, default-label, +or agenda fallback. + +Canonical contract digests verify equality with retained canonical JSON values; +they do not establish the truth of those values, prove raw-byte equality, or +reconstruct source content. Every content-, evidence-, covariate-, membership-, +temporal-, design-, and label-derived digest is sensitive pseudonymous linkage +data. Later reproducibility requires a separately approved, resolvable immutable +snapshot/evidence reference and retention contract. Raw fitted-artifact bytes +also require the separate manifest-owned byte hash described above. diff --git a/docs/topic-intelligence/REFERENCES.md b/docs/topic-intelligence/REFERENCES.md new file mode 100644 index 000000000..31062e7e3 --- /dev/null +++ b/docs/topic-intelligence/REFERENCES.md @@ -0,0 +1,100 @@ +# Topic intelligence references + +**Snapshot date:** 2026-08-09 (Asia/Seoul) + +**Maturity:** reference design `PLANNED`; runtime integration +`BLOCKED-UPSTREAM` + +These sources ground the scientific, provenance, risk, security-development, +and wire-contract boundaries. A citation does not establish Naruon or TEPP +conformity, certification, production readiness, or implementation. + +## Structural topic modeling + +Roberts, M. E., Stewart, B. M., Tingley, D., Lucas, C., Leder-Luis, J., +Gadarian, S. K., Albertson, B., & Rand, D. G. (2014). Structural topic models +for open-ended survey responses. *American Journal of Political Science, 58*(4), +1064–1082. https://doi.org/10.1111/ajps.12103 + +Roberts, M. E., Stewart, B. M., & Tingley, D. (2019). stm: An R package for +structural topic models. *Journal of Statistical Software, 91*(2), 1–40. +https://doi.org/10.18637/jss.v091.i02 + +These works support mixed-membership topic estimation, document-level metadata, +and uncertainty-aware analysis. They do not by themselves establish a +multilevel, multiple-membership, cross-classified, longitudinal, multilingual, +or production-serving estimator. Naruon remains blocked from accepting any such +upstream extended-STM design unless independently published evidence names the +method and estimand, freezes formulas/contrasts, and supplies separate known-truth +validation. This acceptance condition assigns no obligation to TEPP. + +## Risk, security, and provenance standards + +National Institute of Standards and Technology. (2023). *Artificial +Intelligence Risk Management Framework (AI RMF 1.0)* (NIST AI 100-1). +https://doi.org/10.6028/NIST.AI.100-1 + +National Institute of Standards and Technology. (2022). *Secure Software +Development Framework (SSDF) version 1.1: Recommendations for mitigating the +risk of software vulnerabilities* (NIST SP 800-218). +https://doi.org/10.6028/NIST.SP.800-218 + +National Institute of Standards and Technology. (2024). *Secure software +development practices for generative AI and dual-use foundation models: An SSDF +community profile* (NIST SP 800-218A). +https://doi.org/10.6028/NIST.SP.800-218A + +International Organization for Standardization. (2023). *ISO/IEC 42001:2023— +Information technology—Artificial intelligence—Management system*. +https://www.iso.org/standard/42001.html + +International Organization for Standardization. (2023). *ISO/IEC 23894:2023— +Information technology—Artificial intelligence—Guidance on risk management*. +https://www.iso.org/standard/77304.html + +World Wide Web Consortium. (2013). *PROV-O: The PROV ontology* (W3C +Recommendation). https://www.w3.org/TR/prov-o/ + +These sources inform risk ownership, lifecycle evidence, secure development, +and provenance. The documents in this package use them as design guidance and +make no audit or certification claim. + +## Wire-contract standards + +Nottingham, M., Wilde, E., & Dalal, S. (2023). *Problem details for HTTP APIs* +(RFC 9457). RFC Editor. https://www.rfc-editor.org/rfc/rfc9457.html + +Rundgren, A., Jordan, B., & Erdtman, S. (2020). *JSON Canonicalization Scheme +(JCS)* (RFC 8785). RFC Editor. https://www.rfc-editor.org/rfc/rfc8785.html + +JSON Schema. (2020). *JSON Schema specification: Draft 2020-12*. +https://json-schema.org/draft/2020-12 + +JSON Schema. (2020). *JSON Schema validation: A vocabulary for structural +validation of JSON* (Draft 2020-12). +https://json-schema.org/draft/2020-12/json-schema-validation + +RFC 9457 grounds the planned HTTP problem-details shape only after a transport +ADR selects HTTP. RFC 8785 grounds deterministic canonical JSON bytes for the +planned domain-separated digest contract; it does not normalize Unicode. Draft +2020-12 grounds the planned Naruon adapter schema. Its `format` keyword does not +by itself prove that a chosen validator asserts date-time validity; Naruon's +future validator and fixtures must exercise the required format behavior. Exact +schema identity, revision, and digest must be immutable and pinned; the checked +schema is not a deployed OpenAPI component or TEPP's canonical payload. + +## Inspected TEPP repository evidence + +- Repository: [ContextualWisdomLab/tepp](https://github.com/ContextualWisdomLab/tepp) +- Exact inspected protected-`main` revision: + [`b8e26aae334397daa1974d4a24c9015cfd682600`](https://github.com/ContextualWisdomLab/tepp/commit/b8e26aae334397daa1974d4a24c9015cfd682600) +- Commit timestamp: `2026-08-06T11:33:18+09:00` +- Inspection date: `2026-08-09` (Asia/Seoul) + +At that exact revision, `crates/evidence_core/` contains immutable evidence +domain primitives and the JSON wire boundary. `ARCHITECTURE.md` names +`topic_measurement` only in the target architecture. There is no corresponding +production topic-measurement crate or endpoint, and +`crates/tepp_api/src/lib.rs` explicitly states that the foundation slice exposes +no production behavior. The observation is revision-bound and must be refreshed +before any implementation or maturity claim. diff --git a/docs/topic-intelligence/SECURITY.md b/docs/topic-intelligence/SECURITY.md new file mode 100644 index 000000000..01b004112 --- /dev/null +++ b/docs/topic-intelligence/SECURITY.md @@ -0,0 +1,115 @@ +# Topic intelligence security requirements + +**Status:** pseudo-topic removal `ACTIVE-PR`; target security design `PLANNED`; +runtime integration `BLOCKED-UPSTREAM` + +This document supplements the repository-wide [security policy](../../SECURITY.md). +It does not claim NIST or ISO conformity. The safest current state is that no +Naruon-to-TEPP topic-inference boundary exists. The current change removes +misleading local behavior and introduces no new network, persistence, or model +execution surface. + +## Protected assets + +- message and document content, exact source evidence, and bounded snapshots; +- tenant, workspace, user, purpose, consent, region, time, group, and membership + metadata; +- fitted model bytes, manifests, frozen vocabulary and preprocessing/design + specifications, validation reports, label evidence, and promotion decisions; +- posterior topic mixtures, uncertainty, diagnostics, and downstream decisions; +- service credentials, artifact-signing and verification material, audit events, + retention/deletion records; and +- every content-, evidence-, covariate-, membership-, temporal-, design-, or + label-derived digest. Such digests are sensitive pseudonymous linkage values, + not anonymous or generally safe telemetry. + +## Input authority + +| Input class | Authority and treatment | +| --- | --- | +| User intent | Attacker-controlled until a verified Naruon session and policy authorize the exact source and purpose. | +| Document content | Attacker-controlled data, never instructions; bounded before crossing the service boundary. | +| Tenant/workspace/source scope | Resolved from verified server-side identity and records, never trusted from public headers or caller ownership fields. | +| Covariates and membership | Server-resolved, typed, purpose-approved, level-aware, and explicit about observed/missing state. Caller-supplied group identity or weight is not authority. | +| Model and artifact selection | Operator-controlled allowlist of immutable versions and digests. A tenant cannot provide a URL, path, mutable alias, or signing key. | +| `evidence_ref` | If a future contract permits it, an opaque, audience-bound, tenant-bound, expiring capability that is reauthorized at resolution; never an arbitrary URI or filesystem path. | + +## Required controls + +| Control area | Requirement | +| --- | --- | +| Identity | Accept only verified Naruon session and mutually authenticated service identity. Public identity headers and payload ownership claims are not authority. | +| Authorization | Apply deny-first RBAC/ABAC for tenant, workspace, user, source, purpose, consent, region, group, and customer policy before snapshot creation. On every use, require the evidence reference, document reference, source-snapshot revision, audience, expiry, authorization-policy version, and opaque authorization binding to resolve to the same current server-verified tenant/workspace/source/purpose scope; a schema-valid reference is never authority by itself. | +| Minimization | Send only bounded evidence and covariates required by the approved estimand. Exclude unrelated history, credentials, provider URLs, and sequential database identifiers. | +| Tenant isolation | Partition authorization, caches, idempotency, artifact policy, telemetry, rate limits, audit, retention, and deletion. Never reuse a content-bearing or derived-result cache entry across tenants. | +| Transport | Encrypt in transit, mutually authenticate services, bind audience and operation, enforce deadlines, and reject replay outside the idempotency contract. | +| Evidence references | Resolve only opaque server-issued references after rechecking tenant/source/purpose scope. Do not fetch caller URLs or follow redirects. | +| Artifact integrity | Resolve an allowlisted immutable artifact; verify raw artifact bytes and the manifest, vocabulary, preprocessing, design, lineage, model-card, validation-report, build-provenance, and promotion-state identities and digests before inference. Tampering with any one binding, signer state, or retained validation report quarantines the exact deployment. Support signer revocation, downgrade prevention, quarantine, and key rotation. | +| Contract integrity | Pin the exact contract major, immutable schema revision/identifier and digest, diagnostic/quality/reason-code registry versions, and acceptance-policy version. Reject unknown fields, unnegotiated revisions, unknown codes, incompatible runtime state, and malformed numerical output; an unknown upstream code is a protocol error, never an inferred result or abstention. | +| Output control | Keep model-scoped non-semantic topic identity separate from labels, authorize label evidence through its own audience- and model/topic-bound reference, validate uncertainty and diagnostics, and project only product-approved fields. A public projection retains the non-sensitive safety semantics needed to interpret it: model identity/version, analysis unit, versioned estimand, covariate level when applicable, and non-causal designation. It must not expose raw covariates, tenant bindings, or sensitive digests. | +| Derived digests | Treat all content/evidence/covariate/design/label digests as sensitive. Keep raw digests out of product responses, logs, metrics, and traces. Restricted audit should prefer opaque references or tenant-scoped keyed digests with domain separation. | +| Logging | Record only opaque request/result/model references, versions, outcome codes, latency, and redacted aggregate diagnostics. Never log raw content, excerpts, direct identifiers, credentials, sensitive covariates, group values, or unkeyed derived digests. | +| Retention/deletion | Establish purpose-specific TTLs, deletion propagation, cache eviction, audit retention, and keyed-digest rotation before persisting a snapshot or posterior. No topic-specific Naruon persistence is approved today. | +| Availability | Use size/token/time/concurrency bounds, quotas, cancellation, circuit breaking, and bounded retry. Authentication/authorization denial, rate limiting, deadline expiry, and cancellation have stable non-`200` error semantics and never become scientific abstention. Degradation fails closed and never activates keyword, embedding, LLM, cached-other-model, category, or agenda fallback. | +| Secrets and supply chain | Use the repository's operator-managed credential path; pin TEPP build and schema provenance; never place credentials or signing keys in model manifests. | + +## Privacy and statistical safety + +Topic mixtures can disclose health, political, labor, legal, financial, or other +sensitive themes without exposing source text. Membership covariates and rare +groups increase re-identification, stigmatization, and ecological-fallacy risk. +Therefore: + +- every result and model card binds an analysis unit, a versioned estimand, the + covariate level, membership semantics, and an explicit causal/non-causal + designation; +- every public result preserves those non-sensitive interpretation fields, or the + public endpoint is constrained by a versioned contract to one fixed analysis + unit and estimand and communicates that constraint explicitly; +- group-level prevalence or covariate effects MUST NOT be presented as an + individual's trait, intent, diagnosis, or causal outcome; +- individual content MUST NOT be generalized back to a group without a separate + approved estimand, privacy review, and downstream authorization; +- sensitive covariates require a documented purpose, minimum necessary fields, + explicit missingness, access policy, and model-card disclosure; +- multiple-membership weights require an opaque membership set and group/level, + a frozen normalization rule, and an unseen-level policy; missing membership is + never converted to a default group; +- evidence availability, knowledge cutoff, assertion time, and any nullable event + or document time follow an immutable temporal policy with explicit missingness, + canonical time-zone handling, and validated ordering; a declared `valid` status + is not a substitute for recomputing those relations; +- aggregate displays require approved minimum-cell and sparse-group suppression + rules, with tests against differencing and repeated-query attacks; +- training membership, representative documents, source excerpts, and label + evidence are not exposed to ordinary users; and +- a display label or excerpt requires separate authorization and safe rendering. + Its evidence reference has a label-specific audience and is bound to the exact + model, topic, label version, and language. It never changes model-scoped numeric + topic identity or becomes executable HTML. + +## Audit evidence + +A future restricted audit record may contain an opaque actor/workspace scope, +purpose code, opaque request/result reference, selected model and contract +versions, verified artifact-manifest reference, outcome/abstention/error code, +policy-decision reference, assertion time, and a redacted diagnostic summary. +It must not contain source text, label excerpts, sensitive group values, or raw +derived digests. When exact binding is required, use an opaque audit reference or +a tenant-scoped keyed digest with a documented algorithm, domain separator, +canonical empty representation, retention, deletion propagation, and key +rotation. Audit access and retention are separate from product-result access. + +## Security release gate + +No adapter can be enabled until the real transport, service authentication, +artifact registry/signing, result retention, cache, rate-limit, covariate, and +downstream-consumer decisions have approved ADRs and this threat model is +refreshed against them. Tenant-isolation, confused-deputy, authorization and +reference cross-binding, artifact/validation-report/digest tamper and downgrade, +schema and diagnostic-code confusion, public scientific-semantics projection, +temporal ordering, digest-linkage, log-redaction, deletion, authentication, +rate-limit, deadline, cancellation, retry, cache, label-evidence/rendering, and +rollback tests must pass. +An operator must be able to quarantine one artifact or disable the entire +integration immediately without reactivating pseudo-topic behavior. diff --git a/docs/topic-intelligence/TEST_STRATEGY.md b/docs/topic-intelligence/TEST_STRATEGY.md new file mode 100644 index 000000000..72edb970b --- /dev/null +++ b/docs/topic-intelligence/TEST_STRATEGY.md @@ -0,0 +1,156 @@ +# Topic intelligence test strategy + +**Status:** pseudo-topic removal tests `ACTIVE-PR`; target integration-test design +`PLANNED`; runtime integration evidence `BLOCKED-UPSTREAM` + +## Test ownership + +Naruon verifies product/API correctness, authorization, tenant isolation, +integration safety, contract enforcement, and honest presentation. Before +consumption, Naruon requires independently published upstream evidence covering +model estimation, new-document inference, conditional uncertainty, temporal and +extended-STM behavior, artifact reproducibility, and implementation parity. +Naruon may consume signed validation evidence; it must not duplicate a toy +estimator and claim that it proves upstream scientific validity. This requirement +governs Naruon's acceptance decision and assigns no obligation to TEPP. + +## Current removal evidence + +| Contract | Test or evidence | +| --- | --- | +| Pseudo-topic tools absent | `test_registry_omits_lexical_pseudo_topic_tools` | +| Lexical utility described honestly | `test_keyword_extractor_is_disclosed_as_lexical_term_frequency` | +| Lexical determinism, language, and empty input | Existing `keyword_extractor_handler` tests | +| Analysis input bound retained | Existing oversized-analysis-text tests | +| Documentation authority graph and planned schema | `test_topic_intelligence_documentation.py` | + +All Python checks run with warnings promoted to failures. Ruff, balanced +documentation checks, and `git diff --check` are required. These checks prove +the deletion and documentation contract only; they do not prove STM behavior. + +## Future Naruon contract tests + +- exact accepted/rejected contract majors, immutable schema IDs/revisions and + schema digests; unknown fields and unnegotiated revisions fail closed; +- inferred versus abstained result shapes, stable error classes, malformed + diagnostics, exact accepted diagnostic-code registry versions, unknown-code + rejection as a protocol error, and top-level/diagnostic status agreement; +- topic proportion range and sum tolerance; model-scoped non-semantic topic IDs; + rank uniqueness and ordering; equality among fitted, declared, observed, and + serialized component counts; credible-interval ordering and containment; and + declared interval level, method, and uncertainty scope; +- exact model, manifest, artifact, vocabulary, preprocessing, design, lineage, + model-card, evidence-time, covariate-snapshot, and design-row binding; +- canonical empty covariate/design representation and domain-separated digest + behavior when the model uses no covariates; +- model unavailable, trusted-request conflict, deployment incompatibility, + artifact integrity failure, unsupported language, insufficient tokens, + excessive OOV, temporal/covariate invalidity, diagnostic abstention, + authentication/authorization denial, rate limiting, deadline expiry, + cancellation, bounded retry, and idempotency mismatch, each with its stable + non-`200` mapping and retry rule where applicable; +- no keyword, embedding, LLM, cached-other-artifact, category, or agenda fallback + on every failure, cancellation, disabled, quarantine, and rollback path; +- verified identity and tenant/workspace/user/source scope, purpose, consent, + region, role, group, and customer-policy deny precedence; +- cross-tenant request/result/cache/idempotency/rate-limit/audit isolation; +- opaque evidence-reference audience, tenant, workspace, source, purpose, expiry, + replay, reauthorization, redirect, SSRF, and file-path rejection behavior; + mismatched document/evidence snapshot revisions or authorization bindings fail + before any upstream call; +- analysis-unit, estimand, non-causal designation, covariate level, membership + structure/normalization conditional combinations, missingness and unseen-level + policy, minimum-cell/sparse-group suppression, public projection of required + non-sensitive semantics, and prohibition on individual-attribute claims from + group effects; +- immutable temporal-policy identity; asserted date-time format; nullable-time + missingness; and ordering tests including evidence unavailable at the knowledge + cutoff; +- safe label rendering, rejection of semantic labels as topic IDs, component/topic + referential integrity, and independent label-evidence audience/authorization; +- one-at-a-time tampering of payload, source-snapshot, schema, raw artifact, + manifest, vocabulary, preprocessing, design, lineage, model-card, + validation-report, evidence-time, covariate-snapshot, design-row, signature, + signer-state, build-provenance, and promotion-state bindings; and +- fixtures captured from the exact production TEPP implementation. Mocks alone + are not release evidence. + +## Future independently published scientific evidence + +Naruon's acceptance decision requires an independently published validation +packet that includes: + +- known-truth corpus simulation with label switching resolved explicitly before + topic-wise comparison; +- topic-proportion bias and RMSE plus interval coverage/calibration for the + declared interval level, method, and uncertainty scope; +- disclosure that new-document intervals are conditional on the frozen fitted + artifact unless broader model/training uncertainty is separately implemented + and validated; +- prevalence and content covariate recovery with explicit missingness; +- multilevel, multiple-membership, cross-classified, or longitudinal recovery + only for a documented upstream extended-STM estimator, analysis unit, estimand, + formula/contrasts, weight normalization, and unseen-level policy; +- temporal train/validation splits and knowledge-cutoff leakage checks using + evidence availability rather than only event time; +- preprocessing, vocabulary, artifact, design, and new-document inference + reproducibility from immutable manifests; +- unsupported-language, low-token, OOV, degenerate-document, adversarial input, + covariate, and temporal rejection, separately from posterior/diagnostic + abstention; +- convergence and diagnostic rejection plus immutable promotion thresholds; +- determinism within declared tolerance and CPU/GPU/alternate-runtime parity; + and +- corpus drift and label-evidence review without silently changing numeric topic + identity across model versions. + +Scientific thresholds must come from representative data and be recorded in the +model card. This document intentionally invents no quality target. + +## Security and privacy adversarial tests + +- attempt cross-tenant source, result, cache, idempotency, and deletion access; +- tamper independently with the schema, payload and snapshot digests, raw artifact, + manifest, vocabulary, preprocessing, design, lineage, model-card, + validation-report, evidence-time, covariate/design-row, signature, signer state, + build provenance, and promotion state; +- request a mutable alias, older artifact, revoked signer, arbitrary endpoint, + provider URL, redirecting evidence reference, local/private address, or file + path; +- submit oversized, multilingual, prompt-like, low-token, high-OOV, crafted + membership, non-finite weight, missingness, and future-availability inputs; +- inspect logs, metrics, traces, problem details, audit, fixtures, and snapshots + for raw content, excerpts, direct identifiers, credentials, sensitive group + values, or any unkeyed content/evidence/covariate/design/label digest; +- prove tenant-keyed digests use canonical bytes and domain separation, cannot be + compared across tenants, rotate safely, and disappear under deletion policy; +- measure repeated-query membership/model inference risk and confirm rate/query + controls and aggregate suppression resist differencing; and +- quarantine/disable the integration during in-flight work and prove no fallback + or stale result reaches a consumer. + +## Test data + +Use synthetic or appropriately licensed and de-identified corpora for CI. +Production message bodies, tenant identifiers, secrets, sensitive membership +attributes, and production-derived digests must not enter fixtures, snapshots, +logs, or external evaluation services. Multilingual and code-switching fixtures +are allowed only after the artifact declares support. Redistributable research +PDFs may be committed; otherwise cite and summarize the official source. + +## Release matrix + +| Gate | Removal PR | Future adapter | Future UI/downstream consumer | +| --- | --- | --- | --- | +| Focused unit and contract tests | Required | Required | Required | +| Full warnings-as-errors suite | Required | Required | Required | +| Independently published scientific validation packet | Not applicable | Required | Required | +| Tenant/security/privacy/threat tests | No new runtime boundary | Required | Required | +| Exact real-service contract E2E | Not applicable | Required | Required | +| Real PostgreSQL smoke path if persistence is added | Not applicable | Required when applicable | Required when applicable | +| Load, capacity, and numeric SLO evidence | Not applicable | Required before enablement | Required | +| Artifact quarantine, service disable, and recovery drill | No integration | Required | Required | + +An unavailable external reviewer or pending GitHub check is a wait state, not +permission to weaken evidence. Merge remains subject to the repository's +current-head branch-protection and review contract. diff --git a/docs/topic-intelligence/THREAT_MODEL.md b/docs/topic-intelligence/THREAT_MODEL.md new file mode 100644 index 000000000..b408df320 --- /dev/null +++ b/docs/topic-intelligence/THREAT_MODEL.md @@ -0,0 +1,113 @@ +# Threat model: topic intelligence + +- **Status:** design-time model `PLANNED`; runtime integration `BLOCKED-UPSTREAM` +- **Scope:** the future Naruon-to-TEPP topic-intelligence boundary +- **Review trigger:** a real TEPP transport, artifact store, persistence design, + covariate, downstream consumer, or UI + +## Overview + +Naruon is a tenant-scoped email/PIM hub. A future topic-intelligence path may +authorize a bounded document snapshot, send it to a separately deployed TEPP +measurement service, validate a fitted-model result, and expose a policy-filtered +posterior or explicit abstention. That runtime path does not exist today. The +current pseudo-topic removal reduces attack surface and introduces no new +network or persistence boundary. + +This model is intentionally narrower than the repository-wide security policy. +It covers confidentiality, tenant isolation, scientific integrity, statistical +misuse, provenance, model supply chain, and availability at the planned boundary. +TEPP training internals remain out of scope until TEPP implements and publishes +their production contracts, but Naruon release gates still require evidence +about those controls. + +## Threat model, trust boundaries, and assumptions + +### Actors + +- an ordinary or malicious tenant user, including a tenant administrator; +- an attacker controlling imported email/document content; +- a compromised Naruon or TEPP service or service credential; +- a compromised artifact publisher, registry, signer, or verification key; +- an insider with model, corpus, label-evidence, or audit access; and +- a network attacker capable of observing, replaying, or tampering with traffic. + +### Trust boundaries + +| Boundary | Data crossing | Security invariant | +| --- | --- | --- | +| Browser/client to Naruon | Opaque source selection and processing intent | Verified signed session; server-resolved tenant/workspace/source/purpose; document/evidence/snapshot/audience/expiry/policy bindings cross-checked on every use; deny before snapshot creation. | +| Naruon records to snapshot | Minimized content, times, and approved covariates | Re-read ownership and policy; enforce bounds; content remains attacker-controlled data. | +| Naruon to TEPP | Bounded content or opaque evidence capability, model policy, provenance, idempotency | Mutual authentication, audience binding, encryption, schema and deadline bounds, no arbitrary URL/path. | +| Artifact registry to TEPP | Immutable manifest, model, vocabulary, preprocessing/design, validation evidence | Allowlist, signature/digest verification, signer revocation, downgrade protection, quarantine. | +| TEPP to Naruon | Posterior or abstention, diagnostics, model and provenance binding | Exact schema and code-registry revisions/digests, request/result binding, numerical and scientific invariant checks, unknown-code rejection, no fallback. | +| Result to product/audit | Policy-filtered output and restricted metadata | Preserve model-scoped topic identity, analysis unit, estimand, covariate level when applicable, and non-causal status; separate permissions and retention; no source text or raw derived digest in ordinary telemetry. | + +Attacker-controlled inputs include document bytes, language-like content, +prompt-like strings, repeated query patterns, oversized/OOV documents, and user +intent. Operator-controlled inputs include allowed service endpoints, model and +schema versions, verification keys, promotion state, quotas, and feature-disable +controls. Developer-controlled inputs include code, contract fixtures, migrations, +and release configuration; they are not trusted merely because they are local. + +## Attack surface, mitigations, and attacker stories + +| ID | Threat | Example impact | Required mitigation | Residual disposition | +| --- | --- | --- | --- | --- | +| `TI-T01` | Identity or scope spoofing | A caller references another tenant's source or model policy. | Verified session/service identity; server-side scope re-read; deny-first RBAC/ABAC. | Reassess with real auth protocol. | +| `TI-T02` | Artifact, validation-evidence, digest, downgrade, or signer compromise | A poisoned or stale model or substituted validation report is served as approved. | Verify raw artifact bytes and every manifest, vocabulary, preprocessing, design, lineage, model-card, validation-report, build, signer, and promotion binding; signer revocation, monotonic policy, quarantine, and rollback. | Signing/registry ADR required. | +| `TI-T03` | Repudiation | An operator cannot prove which model, purpose, and policy produced a result. | Append-only restricted audit with opaque refs, versions, artifact-manifest ref, policy decision, and times. | Durable audit design is planned. | +| `TI-T04` | Content or posterior disclosure | Logs, labels, caches, responses, or evidence reveal sensitive themes or cross-tenant data. | Minimization, output projection, tenant-partitioned caches, separate label/evidence permission, deletion tests. | Corpus-specific sensitivity review required. | +| `TI-T05` | Digest linkage or dictionary attack | A raw content, covariate, membership, evidence, design, or label digest links records or reveals a low-entropy value. | Exclude raw derived digests from product/telemetry; use restricted opaque refs or tenant-keyed, domain-separated digests with TTL and rotation. | Canonicalization/key design required. | +| `TI-T06` | Denial of service | Oversized/OOV documents, expensive inference, or retry storms exhaust capacity. | Input/token/concurrency limits, quotas, deadlines, cancellation, bounded retry, circuit breaker, and stable rate/deadline/cancellation errors that cannot become abstention. | Numeric limits require load evidence. | +| `TI-T07` | Privilege escalation | A member invokes an admin-only model/purpose or selects an arbitrary artifact/endpoint. | Server-owned policy allowlist, role and purpose checks, no caller URL/path or mutable alias. | Policy mapping is planned. | +| `TI-T08` | Training or label poisoning | Malicious corpus data shifts topics, labels, or downstream decisions. | Corpus provenance, quality checks, held-out and known-truth validation, independent promotion, label evidence review, rollback. | Naruon requires concrete independently published upstream controls and evidence before consumption. | +| `TI-T09` | Membership or model inference | Repeated queries reveal corpus membership or reconstruct model properties. | Per-principal/tenant query controls, coarse diagnostics, no exemplars, abuse monitoring, empirical privacy tests before exposure. | Privacy test method is unresolved. | +| `TI-T10` | Semantic or diagnostic-code confusion | A keyword, embedding, LLM label, old model, truncated vector, unknown quality code, or another tenant's cache is accepted as an STM posterior. | Strict model-scoped non-semantic topic identity, exact fitted/result/observed component counts, versioned closed diagnostic-code registries, artifact and request binding, label separation, partitioned cache, compatibility tests, no fallback. | Guarded by ADR and contract tests. | +| `TI-T11` | Ecological fallacy or stigmatization | A group prevalence estimate becomes an asserted individual trait or individual content stigmatizes a group. | Public and internal results preserve analysis unit/estimand/covariate level/non-causal status; model-card review, minimum-cell/sparse-group suppression, product-copy and downstream tests. | Human governance remains required. | +| `TI-T12` | Temporal leakage | A model or covariate uses evidence unavailable at the asserted knowledge cutoff. | Immutable temporal-policy identity/digest, evidence-time/covariate/design-row binding, explicit missingness and canonical time parsing, availability-at-cutoff ordering recomputed by Naruon, time-sliced validation. | Naruon requires independently published upstream temporal-validation evidence before consumption. | +| `TI-T13` | Confused deputy or unsafe evidence reference | An external service uses Naruon's authority to fetch unrelated content, follows an attacker URL, or Naruon accepts an unbound result. | Push bounded content or use opaque audience/tenant/workspace/source/purpose/snapshot-bound expiring capabilities; cross-check document and snapshot revisions, reauthorize resolution, no redirects, exact request/result binding. | Reassess with transport. | +| `TI-T14` | Covariate or membership manipulation | A caller supplies a privileged group, fabricated missingness, or weights that change the estimate. | Server-resolved typed covariates; frozen formula/contrast, normalization and unseen-level policy; finite/range checks. | Extended-STM contract is planned. | +| `TI-T15` | Label/evidence injection | Prompt-like corpus text manipulates a generated label, a semantic label is smuggled in as topic identity, or active markup reaches a UI. | Model-scoped non-semantic topic IDs; label-specific evidence reference and audience bound to model/topic/version/language; component referential checks; constrained output, escaping/sanitization, provenance and human review. | UI/label pipeline does not exist. | +| `TI-T16` | Incomplete deletion or cross-purpose replay | Revoked content remains in snapshots, caches, audit, or an idempotent replay. | Purpose TTL, deletion propagation, cache eviction, consent/policy recheck before replay, keyed-digest rotation. | Retention ADR required. | + +Representative abuse cases include crafted multilingual/OOV documents intended +to force a convenient default label, repeated near-duplicate queries intended to +extract corpus membership, a deprecated artifact requested through a mutable +alias, and timeouts intended to activate a cheaper keyword path. Every case must +end in denial, a stable error, or an explicit model-governed abstention. None may +change the measurement method. + +Out of scope for the current removal are attacks requiring a deployed TEPP +endpoint, model registry, topic store, or topic UI because none exists. They are +still release blockers for the future integration, not evidence that the threat +is impossible. + +## Severity calibration + +- **Critical:** cross-tenant source/posterior disclosure at scale; compromise of + an artifact-signing root that silently promotes attacker-controlled models; + service identity compromise that grants unrestricted tenant corpus access. +- **High:** unauthorized inference of sensitive themes; persistent raw content + or low-entropy derived digests in broadly accessible logs; poisoning that + materially alters product decisions; bypass of purpose, consent, or region + policy; arbitrary evidence-reference network/file access. +- **Medium:** tenant-local resource exhaustion with bounded recovery; harmful or + misleading labels that do not alter numeric identity; incomplete redaction in + a restricted operator surface; reproducibility or temporal defects that block + scientific use but do not expose another tenant. +- **Low:** documentation-only inconsistency while the runtime remains disabled, + or a non-sensitive diagnostic formatting defect with no policy, integrity, + availability, or disclosure impact. + +Repository policy requires remediation of Medium-and-higher validated findings. +Severity must be reassessed against the real transport, data volume, privileges, +and downstream decisions. + +## Security decisions still required + +Before implementation, approve ADRs for service authentication, transport and +evidence-reference semantics, artifact signing/registry and signer revocation, +cache/idempotency partitioning, result/audit retention and deletion, sensitive +covariates, privacy testing, rate limits, and downstream-consumer authorization. +The conceptual schema silently decides none of these. diff --git a/docs/topic-intelligence/TRACEABILITY.md b/docs/topic-intelligence/TRACEABILITY.md new file mode 100644 index 000000000..9b2875964 --- /dev/null +++ b/docs/topic-intelligence/TRACEABILITY.md @@ -0,0 +1,116 @@ +# Topic intelligence requirements traceability + +- **Document status:** `PRESENT-CURRENT` +- **Assessment date:** 2026-08-09 +- **Contract revision:** `2026-08-09.1` + +This matrix connects the product requirements to decisions, planned contracts, +verification, and release evidence. A documentation link proves only that a +requirement is specified. It is not evidence that a runtime capability, TEPP +production contract, fitted artifact, scientific validation, or UI exists. + +Capability maturity uses only: +`IMPLEMENTED-ON-PROTECTED-DEVELOP`, `ACTIVE-PR`, +`ACCEPTED-NARUON-POLICY`, `PLANNED`, `BLOCKED-UPSTREAM`, and `OUT-OF-SCOPE`. + +## Requirement matrix + +| ID | Requirement summary | Decision and contract coverage | Verification or release evidence | Current maturity | +|---|---|---|---|---| +| `TI-REQ-001` | Remove `email_categorizer` and `meeting_agenda_generator`. | [ADR-0001](../adr/0001-topic-measurement-authority.md); [PRD](PRD.md); `backend/api/tools.py` candidate diff | `backend/tests/test_tools_api.py::test_registry_omits_lexical_pseudo_topic_tools`; source-symbol absence; [PR #1297](https://github.com/ContextualWisdomLab/naruon/pull/1297) exact-head checks | `ACTIVE-PR` | +| `TI-REQ-002` | Retain `keyword_extractor` only as lexical frequency/first-occurrence metadata. | [ADR-0001](../adr/0001-topic-measurement-authority.md); [PRD](PRD.md); [Architecture](ARCHITECTURE.md) no-fallback boundary | `backend/tests/test_tools_api.py::test_keyword_extractor_is_disclosed_as_lexical_term_frequency`; bounded-input handler tests | `ACTIVE-PR` | +| `TI-REQ-003` | Fail closed until a compatible independently published fitted-model contract exists; no default or substitute method. | [ADR-0001](../adr/0001-topic-measurement-authority.md); [TRD](TRD.md); [API errors](API_CONTRACT.md#http-and-abstention-semantics); [UML state model](UML.md#result-state-model) | Current registry omissions; future no-deployment, incompatible-input, upstream-fault, and no-fallback adapter tests | `ACCEPTED-NARUON-POLICY`; runtime `BLOCKED-UPSTREAM` | +| `TI-REQ-004` | Return a complete mixed-membership vector with numeric identity, explicit interval level/method/scope and diagnostics; narrowly define vector-free abstention. | [Architecture scientific invariants](ARCHITECTURE.md#scientific-invariants); [API result semantics](API_CONTRACT.md#successful-public-projection); schema `$defs.inferenceResult`, `$defs.posteriorComponent`, `$defs.diagnosticBundle` | Future schema fixtures, fitted/declared/observed/actual count equality, unique numeric-ID/rank, sum/interval-containment, code-registry, calibration/coverage, and status/diagnostic cross-checks | `BLOCKED-UPSTREAM` | +| `TI-REQ-005` | Fully specify temporal, multilevel, multiple-membership, and cross-classified extensions and keep claims non-causal. | [TRD fitted-artifact requirements](TRD.md#fitted-artifact-requirements); [Architecture scientific invariants](ARCHITECTURE.md#scientific-invariants); schema `$defs.scientificProvenance` and `$defs.designContract`; [Data model](DATA_MODEL.md#covariate-and-temporal-evidence) | Future model card/design manifest, known-truth simulation, estimator/formula/contrast tests, membership-weight normalization, unseen-level, temporal-leakage, and downstream-suppression tests | `BLOCKED-UPSTREAM` | +| `TI-REQ-006` | Bind the result to schema/source/payload/artifact/manifest/vocabulary/preprocessing/design/lineage/model-card/validation-report/evidence-time/covariate/design-row provenance; digest verifies but does not reconstruct. | [Architecture canonical provenance](ARCHITECTURE.md#canonical-provenance-and-reproduction); [API digest contract](API_CONTRACT.md#digest-contract); schema `$defs.requestIdentity` and `$defs.scientificProvenance`; [Data model](DATA_MODEL.md) | Future RFC 8785 known-answer/domain-separation, complete inventory, digest mismatch, immutable-artifact, snapshot/scope-binding, retained-snapshot replay, and deletion/retention tests | `BLOCKED-UPSTREAM` | +| `TI-REQ-007` | Keep numeric topic identity separate from versioned evidence-backed labels. | [ADR-0001](../adr/0001-topic-measurement-authority.md); [UML contract structure](UML.md#contract-structure); schema `$defs.posteriorComponent`, `$defs.presentation`, and `$defs.presentationLabel` | Future label/topic join tests, label-version/evidence tests, absent-label tests, and tests proving labels cannot alter numeric posterior fields | `BLOCKED-UPSTREAM` | +| `TI-REQ-008` | Enforce tenant/workspace/source/purpose/consent/region/retention/deletion/digest/redaction controls. | [Security](SECURITY.md); [Threat model](THREAT_MODEL.md); [API evidence rules](API_CONTRACT.md#evidence-reference-rules); schema `$defs.opaqueEvidenceRef`; [Data privacy classification](DATA_MODEL.md#privacy-classification) | Future cross-tenant/workspace denial, expiry/audience/snapshot binding, reauthorization, region/purpose/consent, deletion, cache isolation, restricted-audit, and no-log/metric/trace leakage tests | `BLOCKED-UPSTREAM` | +| `TI-REQ-009` | Keep agenda generation in a separately authorized downstream contract. | [ADR-0001](../adr/0001-topic-measurement-authority.md); [Architecture ownership](ARCHITECTURE.md#authority-and-ownership); [PRD non-goals](PRD.md#non-goals) | A separate future ADR/PRD/TRD/API/threat model plus source authorization, abstention suppression, evidence, audit, and E2E tests | `ACCEPTED-NARUON-POLICY`; future capability `PLANNED` | +| `TI-REQ-010` | Add UI only after the real runtime, uncertainty, abstention/error, security, and operational evidence exists. | [PRD success gates](PRD.md#success-and-release-gates); [Operability](OPERABILITY.md); [API public projection](API_CONTRACT.md#successful-public-projection) | Future source-backed E2E tests for loading, inferred, abstained, each error family, permission denial, rollback, accessibility, redaction, and no-fallback copy | `PLANNED` | + +## Contract-to-test map + +The names below are proposed acceptance tests, not current test functions unless +an existing path is explicitly named. + +| Contract obligation | Proposed verification | Expected evidence owner | +|---|---|---| +| Immutable schema ID/revision and out-of-band digest pin | `test_topic_schema_id_revision_and_digest_pin`; RFC 8785 canonical known-answer fixture | Naruon adapter | +| Closed envelope and required scientific payload | `test_topic_result_schema_rejects_unknown_or_missing_fields` | Naruon adapter | +| Expected producer is conditional, not assigned ownership | Assert `x-owner=NARUON`, absence of `x-upstream-owner`, and conditional `x-expected-upstream-producer=TEPP` copy | Naruon architecture review | +| `inferred` status consistency | `test_inferred_requires_components_and_accepted_diagnostics` | Naruon adapter | +| `abstained` status consistency | `test_abstained_requires_empty_vector_and_posterior_policy_reason` | Naruon adapter | +| Numeric topic IDs/ranks and complete component count | `test_topic_components_use_numeric_identity`; `test_fitted_declared_observed_and_actual_counts_match` | Naruon adapter | +| Proportions sum to one within pinned tolerance | `test_topic_proportions_and_reported_sum_match` | Naruon adapter plus upstream numerical evidence | +| Interval containment and explicit uncertainty semantics | `test_topic_estimates_lie_inside_declared_intervals`; calibration/coverage report | Naruon adapter and independently published expected-upstream scientific evidence | +| Unsupported language/token/OOV/temporal/covariate input is an error | Parameterized route tests asserting `422` and stable RFC 9457 `error_code` | Naruon adapter | +| No active deployment/artifact/integrity is unavailable | Parameterized route tests asserting `503` and no substitute fallback | Naruon adapter/operator | +| Snapshot/revision/schema/idempotency conflict | Parameterized route tests asserting `409` | Naruon adapter | +| Invalid upstream schema/digest/cross-field result | `test_invalid_upstream_payload_is_502_not_abstention` | Naruon adapter | +| Unknown diagnostic/reason registry or code | `test_unknown_diagnostic_code_fails_closed_as_502`; exact registry-version fixtures | Naruon adapter and expected upstream producer | +| Snapshot/scope binding | Mismatched snapshot and scope refs plus current tenant/workspace/purpose/consent/region reauthorization tests | Naruon authorization boundary | +| Temporal assertion and ordering | RFC 3339 format-assertion, nullable-field policy, and availability-at-knowledge-cutoff tests | Naruon adapter and expected upstream evidence | +| Covariate/membership coupling | Typed level/missingness fixtures and invalid structure/normalization combinations | Naruon adapter and expected upstream evidence | +| No keyword/embedding/LLM/default-label/agenda fallback | `test_every_topic_failure_path_has_no_substitute_result` | Naruon adapter | +| Canonical no-covariate representation | Known-answer hashes for `{"covariates":[],"memberships":[]}` and `{"columns":[],"values":[]}` under their fixed domains | Naruon adapter and contract fixture producer | +| Retained evidence is required for replay | `test_digest_without_resolvable_snapshot_cannot_replay` | Naruon retention boundary | +| Evidence reference binding | Expired, wrong audience/snapshot/tenant/workspace/purpose tests plus reauthorization-on-use assertion | Naruon authorization boundary | +| Sensitive digest handling | Log/metric/trace/public response capture tests and restricted-audit opaque-reference test | Naruon security/observability | +| Multilevel/membership/design contract | Known-truth recovery, weight normalization, unseen-level rejection, formula/contrast and temporal-leakage fixtures | Independently published expected-upstream evidence plus Naruon compatibility validator | +| Labels remain presentation-only | Mutation and serialization tests proving labels cannot change component identity/posterior | Naruon adapter/UI | + +## Error-code traceability + +| Family | HTTP | Stable codes | Requirement | +|---|---:|---|---| +| Trusted request conflict | `409` | `topic_source_snapshot_conflict`, `topic_request_revision_conflict`, `topic_idempotency_conflict`, `topic_schema_revision_conflict` | `TI-REQ-003`, `TI-REQ-006`, `TI-REQ-008` | +| Authentication/authorization policy | `401`, `403` | `topic_authentication_required`, `topic_evidence_forbidden`, `topic_purpose_forbidden`, `topic_consent_required`, `topic_region_forbidden` | `TI-REQ-008` | +| Naruon request deadline | `408` | `topic_deadline_exceeded` | `TI-REQ-003`, `TI-REQ-008` | +| Input/model preflight | `422` | `topic_input_invalid`, `topic_language_unsupported`, `topic_input_insufficient_tokens`, `topic_input_out_of_vocabulary`, `topic_temporal_context_invalid`, `topic_covariate_contract_invalid` | `TI-REQ-003`, `TI-REQ-005`, `TI-REQ-008` | +| Deployment/artifact availability | `503` | `topic_deployment_unavailable`, `topic_model_artifact_unavailable`, `topic_model_artifact_integrity_failed` | `TI-REQ-003`, `TI-REQ-006` | +| Upstream execution/protocol | `502` | `topic_upstream_inference_failed`, `topic_upstream_protocol_error` | `TI-REQ-003`, `TI-REQ-004`, `TI-REQ-006` | +| Quota/rate policy | `429` | `topic_rate_limited` | `TI-REQ-008` | +| Upstream deadline | `504` | `topic_upstream_timeout` | `TI-REQ-003`, `TI-REQ-008` | +| Client cancellation | no deliverable response | internal redacted outcome `topic_request_cancelled` | `TI-REQ-003`, `TI-REQ-008` | +| Adapter defect | `500` | `topic_adapter_internal_error` | `TI-REQ-003`, `TI-REQ-008` | +| Scientific publication decline | `200` | `status=abstained` plus `posterior_*` policy reason | `TI-REQ-004` | + +Authorization failures use Naruon's existing authenticated API security contract +and intentionally do not reveal whether a document, evidence reference, tenant, +workspace, or deployment exists. + +## Current evidence versus blockers + +| Claim | Evidence available on 2026-08-09 | Missing before runtime/UI release | +|---|---|---| +| Pseudo-topic behavior is removed | Candidate source/tests and PR #1297 | Merge and exact protected-`develop` verification | +| Lexical keyword extraction is honestly scoped | Candidate description and handler tests | Merge and protected-branch verification | +| Naruon has a local no-fallback policy | Accepted ADR-0001, AGENTS rule, PRD/TRD/architecture package | Runtime adapter negative-path tests after upstream capability exists | +| A planned Naruon envelope is specified | Revisioned JSON Schema, API/architecture/UML/data-model documents | Independently published compatible expected-upstream production contract and joint fixture review; TEPP only if it accepts that role | +| A fitted model can serve Naruon | No | Published artifact/deployment/API, model card, scientific validation, signatures/integrity, capacity and operability evidence | +| Topic results are scientifically valid | No | Representative and known-truth validation, interval calibration/coverage, diagnostics, temporal/membership validation, model promotion evidence | +| Multi-tenant handling is production safe | No topic runtime exists | Implemented authorization, evidence-reference, isolation, consent/region/retention/deletion/redaction tests | +| Topic UI is releasable | No | Real runtime, safe public projection, E2E states, accessibility, security and operational release gates | + +## Release evidence bundle + +Promotion from `BLOCKED-UPSTREAM` requires one exact-revision evidence bundle: + +1. independently published expected-upstream production contract, fitted + artifact, manifest, model card, scientific-validation report, and acceptance + evidence; TEPP occupies that role only if it separately publishes and accepts + the compatible responsibility; +2. Naruon ADR review of that exact upstream revision, including any differences + from this planned acceptance profile; +3. schema fixtures and all cross-field/error/abstention tests above; +4. tenant/workspace/source/purpose/consent/region/retention/deletion/evidence- + reference and sensitive-digest security evidence; +5. activation, revocation, rollback, drift, latency, availability, rate-limit, + capacity, incident, and deletion operability evidence; +6. exact-head CI, security scans, warning-free full tests, and independent code + review; and +7. only after items 1–6, source-backed UI and E2E evidence. + +If any item is unavailable, the product remains useful without topic inference +and the topic capability stays disabled. Documentation completeness must never +be used as a substitute for upstream or runtime evidence. diff --git a/docs/topic-intelligence/TRD.md b/docs/topic-intelligence/TRD.md new file mode 100644 index 000000000..94186a0b2 --- /dev/null +++ b/docs/topic-intelligence/TRD.md @@ -0,0 +1,206 @@ +# Technical requirements: topic intelligence + +- **Status:** deletion `ACTIVE-PR`; Naruon-local policy + `ACCEPTED-NARUON-POLICY`; runtime adapter `BLOCKED-UPSTREAM` +- **Normative language:** MUST, MUST NOT, SHOULD, and MAY express obligations for + a future Naruon implementation. +- **Accepted local decision:** [ADR-0001](../adr/0001-topic-measurement-authority.md) +- **Proposed target decisions:** + [ADR-0002](../adr/0002-fitted-topic-artifact-consumption.md) and + [ADR-0003](../adr/0003-separate-topic-measurement-from-agenda-generation.md) + +## Current deliverable + +PR #1297 MUST remove `email_categorizer`, `meeting_agenda_generator`, their fixed +dictionaries, matching helpers used only by them, registry entries, and +behavior-locking tests. It MUST retain the existing input bound for the honest +lexical utility and describe that utility as deterministic lexical frequency and +first-occurrence metadata. + +This change MUST NOT add a replacement topic handler, route, table, migration, +model fit, network dependency, embedding/LLM fallback, default label, template +agenda, or simulated success response. + +## Authority and ownership + +This TRD records Naruon requirements only. It does not govern an upstream +publisher, transfer scientific authority, or claim that TEPP or another producer +accepted a Naruon envelope. A Naruon adapter remains blocked until a publisher +independently publishes a versioned production fitted artifact/API/contract and +its own acceptance evidence. Every reference below to published upstream +evidence is a condition on Naruon's consumption decision, not an obligation this +TRD assigns to the publisher. + +| Boundary | Naruon responsibility | Published upstream evidence Naruon requires before consumption | +| --- | --- | --- | +| Authorization | Tenant/workspace/user/source/purpose checks | Documented service authentication and authorization at upstream ingress | +| Input | Bounded, minimized, immutable authorized snapshot or evidence reference | Published input schema and frozen-preprocessing compatibility rules | +| Model | Select only an operator-approved published model policy | Published fitting, validation, versioning, promotion, and serving evidence | +| Result | Pin schema revision; validate, policy-filter, present, and audit metadata | Published posterior/abstention, uncertainty, diagnostic, and scientific-validation contract | +| Downstream action | Govern search, norm-group use, labels, or agenda generation separately | No implied Naruon product action | + +Naruon MUST NOT read an upstream private database or mount a mutable model path as +an implicit contract. A versioned authenticated service, event, or artifact +boundary MUST be the only integration seam. + +## Input requirements + +A future request MUST include or resolve server-side: + +- an opaque document snapshot/evidence ID, content digest, one bounded content or + evidence representation, language support signal, declared purpose, and + event/assertion/availability/knowledge-cutoff times where applicable; +- tenant/workspace/source authority derived from verified server-side identity, + never public identity headers or caller-supplied ownership; +- an operator-approved model policy and exact contract/schema compatibility + requirement; and +- only purpose-approved covariates, with typed observed/missing state and, when + applicable, level, membership-set, weight, normalization, and unseen-level + semantics. + +Credentials, provider URLs, sequential internal database IDs, unrelated +messages, and unbounded conversation history MUST NOT cross the boundary. + +## Fitted-artifact requirements + +Naruon MUST accept a published fitted artifact only when it is immutable and +content-addressed. Naruon MUST require the integrity-protected evidence bundle to +contain and match every field in the [canonical 14-field digest +inventory](README.md#canonical-digest-inventory), including +`model_card_digest`, `validation_report_digest`, +`covariate_snapshot_digest`, and `design_row_digest`. Beyond those canonical +bindings, the published evidence must identify at least: + +- model ID/version, training-corpus lineage, training cutoff, and knowledge + policy; +- preprocessing implementation/version, token-retention rules, supported + languages, and frozen vocabulary; +- topic count and numeric identities, prevalence/content designs, covariate and + missing-value schemas; +- inference implementation/version, numerical backend, diagnostics, validation + report, model-card identity, and promotion state; and +- separately versioned label evidence, never used as numeric topic identity. + +Naruon MUST reject any digest, schema, language, vocabulary, design, runtime, or +signature mismatch. Naruon MUST NOT silently choose an older or “closest” model +unless a separately accepted, audited compatibility policy names it. + +Standard STM references do not establish temporal, multilevel, multiple- +membership, or cross-classified estimation automatically. To satisfy Naruon's +acceptance criteria, Naruon MUST accept a published model claiming any extension +only when its artifact, model card, and validation evidence name the estimator, +analysis unit, estimand, prevalence/content formula and contrasts, opaque level +and membership semantics, weight normalization, unseen-level policy, non-causal +status unless a causal design is independently established, and known-truth +validation for the extension. + +## Result requirements + +An `inferred` result MUST contain non-negative topic proportions that sum to one +within a versioned tolerance; unique numeric topic IDs; inference implementation +and numerical backend; diagnostic status, stable convergence code, explicit +numerical status, and bounded stable quality codes; and exact request, model, +analysis-unit, estimand, purpose, and knowledge-cutoff provenance. It MUST carry +all 14 canonical digest fields, rather than a shortened or aliased subset, so the +schema, source snapshot, complete scientific payload, artifact, manifest, +vocabulary, preprocessing, design, lineage, model card, validation report, +evidence-time manifest, covariate snapshot, and design row are each bound. + +Each credible interval MUST state its level, method, and uncertainty scope. The +default scope is conditional on the frozen fitted artifact. Product copy MUST NOT +imply that it covers model selection, training-corpus, label, or all parameter- +estimation uncertainty unless the published model card and calibration evidence +support that broader claim. + +Labels MAY be absent. If present, each label MUST carry a label identity, +version, language, and evidence reference/digest separate from +`(model_id, model_version, topic_id)`. Consumers MUST join on numeric topic +identity and model version, not display text. + +## Error and abstention requirements + +The future adapter MUST distinguish: + +- model/service unavailable; +- unsupported contract or schema revision; +- request/idempotency/model-policy conflict; +- deployment, preprocessing, vocabulary, design, or runtime incompatibility; +- artifact/manifest integrity failure; +- unsupported language, insufficient retained tokens, excessive OOV input, or + invalid temporal/covariate input; +- authorization/purpose/consent/region denial; and +- timeout or cancellation. + +Those conditions are errors and MUST produce no posterior, label, or agenda. +HTTP bindings SHOULD use RFC 9457 problem details with a stable Naruon-defined +`error_code` extension and redacted public detail. + +`abstained` is a successful scientific state only after a compatible active +model accepts the input contract but a declared posterior or diagnostic +acceptance rule declines. It MUST contain a stable reason and MUST NOT contain a +topic vector or label. + +Every error and abstention path MUST preserve the no-fallback boundary. Naruon +MUST NOT change the measurement method to keywords, embeddings, clustering, +zero-shot/LLM labels, a cached result from another artifact, a default category, +or a template agenda. + +## Verification, replay, and retention + +All 14 fields in the canonical digest inventory verify that available bytes and +definitions match approved evidence; they do not reconstruct missing content. +Every content-, evidence-, +covariate-, membership-, temporal-, design-, and label-derived digest is +sensitive pseudonymous linkage data. A later claim of replay or +reproducibility MUST additionally prove that the exact authorized snapshot or +evidence reference, model artifact, manifest, vocabulary, preprocessing, design, +lineage, model card, validation report, evidence-time manifest, covariate +snapshot, design row, inference version, purpose, consent, retention, and +knowledge-cutoff context remain resolvable and valid. + +An idempotency key MUST bind retries to that tuple. Replay after consent, +retention, tenant, source, model, or policy invalidation is forbidden even if +the original bytes remain technically accessible. + +## Security and privacy requirements + +- Enforce deny-first RBAC/ABAC, tenant isolation, purpose limitation, consent, + region, retention, and deletion before snapshot materialization. +- Mutually authenticate and authorize the service boundary and protect content + and metadata in transit. +- Resolve only allowlisted immutable artifact references and verify integrity + before inference. +- Exclude raw content, plain content digests, excerpts, credentials, direct user + identifiers, and sensitive covariate values from ordinary logs, metrics, + traces, and public errors. +- Treat plain content digests as sensitive pseudonymous linkage values. A + restricted audit store SHOULD prefer opaque references or tenant-scoped keyed + digests with bounded retention, rotation, and deletion propagation. +- Partition caches, artifact policy, idempotency, telemetry, and deletion work by + tenant and purpose. + +## Compatibility and implementation gates + +The future adapter MUST use an explicit contract major and exact closed schema +revision. Unknown fields are rejected. An additive field may stay in one major +only after a new closed revision is published, pinned, deployed to consumers, +and explicitly negotiated before the producer sends it. Changed meaning, topic +identity, required fields, or preprocessing semantics requires a new major or +artifact version. + +Runtime work remains blocked until all of the following are true: + +1. An upstream publisher independently publishes a production fitted-model + artifact/API/contract and its own acceptance evidence. +2. Naruon reviews that exact contract against ADR-0001 and updates this package + without treating the current planned envelope as upstream authority. +3. Upstream scientific evidence covers estimation, calibration/coverage, + diagnostics, model card, promotion, and any extended-STM claims. +4. Naruon accepts separate transport/authentication, artifact-signing/registry, + retention/deletion, cache, rate-limit, sensitive-covariate, and downstream- + authorization decisions. +5. Contract fixtures from the exact upstream implementation pass Naruon schema, + invariant, failure, abstention, isolation, redaction, timeout, rollback, and + real-service E2E tests with warnings treated as failures. +6. Representative load establishes numeric limits and SLOs. +7. A UI is considered only after all preceding gates are real. diff --git a/docs/topic-intelligence/UML.md b/docs/topic-intelligence/UML.md new file mode 100644 index 000000000..b6f2ebed2 --- /dev/null +++ b/docs/topic-intelligence/UML.md @@ -0,0 +1,253 @@ +# Topic intelligence UML views + +- **Capability maturity:** `BLOCKED-UPSTREAM` +- **Document status:** `PRESENT-CURRENT` +- **Contract revision:** `2026-08-09.1` + +These diagrams describe a planned integration boundary. They do not represent +deployed classes, routes, tables, or an accepted TEPP production API. TEPP is +only the expected upstream producer if it independently publishes a compatible +production contract, fitted artifact, and acceptance evidence. + +## Contract structure + +The Naruon-owned envelope controls authorization, revisioning, validation, and +safe projection. The nested scientific payload carries fitted-model evidence +and estimates; presentation labels stay outside that payload. + +```mermaid +classDiagram + class TopicInferenceEnvelope { + +ContractIdentity contract + +RequestIdentity request + +ResultStatus status + +datetime completed_at + +CanonicalDigest tepp_payload_digest + } + class TEPPScientificPayload { + +ScientificProvenance provenance + +InferenceResult inference + +DiagnosticBundle diagnostics + } + class InferenceResult { + +number credible_level + +string interval_method + +string uncertainty_scope + +integer topic_count + } + class PosteriorComponent { + +integer topic_id + +integer rank + +number proportion + +CredibleInterval credible_interval + } + class PresentationLabel { + +integer topic_id + +string label_id + +string label_version + +string language + +string label + +OpaqueEvidenceRef[] evidence_refs + } + + TopicInferenceEnvelope *-- TEPPScientificPayload + TEPPScientificPayload *-- InferenceResult + InferenceResult *-- PosteriorComponent + TopicInferenceEnvelope o-- PresentationLabel +``` + +`PresentationLabel.topic_id` may reference a posterior component but cannot +change its identifier, rank, proportion, interval, or diagnostic outcome. +The public projection preserves opaque model ID/version, analysis unit, +estimand, coarse covariate level, and causal/non-causal designation while +redacting canonical digests, scope bindings, raw covariates, and group values. + +## Provenance and diagnostics + +```mermaid +classDiagram + class ScientificProvenance { + +string model_id + +string model_version + +integer fitted_topic_count + +string temporal_policy_version + +string estimator_id + +string analysis_unit + +string estimand_id + +string causal_design + } + class CanonicalDigest { + +string algorithm + +string canonicalization + +string domain + +string value + } + class DesignContract { + +string covariate_schema_version + +string covariate_level + +string covariate_missingness_policy + +string prevalence_formula + +string content_formula + +string contrast_specification + +string membership_structure + +string membership_weight_normalization + +string unseen_level_policy + } + class DiagnosticBundle { + +string diagnostic_status + +InputDiagnostics input + +PosteriorDiagnostics posterior + +PolicyDiagnostics policy + } + class PosteriorDiagnostics { + +string diagnostic_code_registry_version + +boolean converged + +string convergence_code + +string numerical_status + +string[] quality_codes + } + class PolicyDiagnostics { + +string policy_version + +string reason_code_registry_version + +boolean accepted + +string[] reason_codes + } + + ScientificProvenance *-- CanonicalDigest + ScientificProvenance *-- DesignContract + DiagnosticBundle *-- PosteriorDiagnostics + DiagnosticBundle *-- PolicyDiagnostics +``` + +The single `CanonicalDigest` association represents the required schema, +snapshot, scientific payload, artifact descriptor, artifact manifest, vocabulary, +preprocessing, design, lineage, model-card, validation-report, evidence-time, +covariate-snapshot, and design-row digests. Each use has its own fixed domain +separator. + +## Planned request sequence + +```mermaid +sequenceDiagram + participant U as Naruon client + participant R as Naruon route + participant A as Topic adapter + participant T as Expected upstream boundary + + U->>R: document_ref, evidence_ref, revision + R->>R: Authenticate and reauthorize + alt Authentication or scope denied + R-->>U: 401 or 403 Problem + error_code + else Rate policy exceeded + R-->>U: 429 Problem + error_code + else Authorized + R->>A: Immutable canonical snapshot request + A->>A: Preflight and pin deployment + alt Preflight ineligible + A-->>R: 422 Problem + error_code + else No active model or artifact + A-->>R: 503 Problem + error_code + else Revision or idempotency conflict + A-->>R: 409 Problem + error_code + else Compatible + A->>T: Versioned scientific request + alt Upstream deadline expires + A-->>R: 504 Problem + bounded cancellation + else Scientific payload returned + T-->>A: Expected scientific payload + A->>A: Verify schema, digests, codes, cross-fields + alt Payload validation fails + A-->>R: 502 Protocol Problem + error_code + else Accepted posterior + A-->>R: 200 inferred envelope + else Posterior or policy rejected + A-->>R: 200 abstained envelope + end + end + end + R-->>U: Redacted safe projection or Problem + end +``` + +An expired, wrong-audience, wrong-snapshot, or wrong-tenant evidence reference +is rejected before the adapter call. The route must resolve the reference +server-side; it must never dereference an arbitrary client URL or path. + +## Result state model + +```mermaid +stateDiagram-v2 + [*] --> Received + Received --> Rejected422: Ineligible preflight + Received --> RejectedAuth: Authentication or scope denial + Received --> RateLimited429: Quota or rate denial + Received --> Conflict409: Trusted request conflict + Received --> Unavailable503: No active deployment + Received --> Eligible: Compatible input and model + Eligible --> Inferring + Inferring --> ProtocolFault502: Unusable upstream response + Inferring --> Deadline504: Upstream deadline + Inferring --> Cancelled: Client cancellation + Inferring --> Validating: Scientific payload returned + Validating --> ProtocolFault502: Schema, digest, code, or invariant fails + Validating --> Inferred: Posterior accepted + Validating --> Abstained: Posterior or policy rejected + Rejected422 --> [*] + RejectedAuth --> [*] + RateLimited429 --> [*] + Conflict409 --> [*] + Unavailable503 --> [*] + ProtocolFault502 --> [*] + Deadline504 --> [*] + Cancelled --> [*] + Inferred --> [*] + Abstained --> [*] +``` + +The state model intentionally has no fallback transition from an error or +abstention to a default topic, lexical classifier, embedding cluster, LLM label, +or agenda template. + +## Deployment compatibility state + +```mermaid +stateDiagram-v2 + [*] --> Discovered + Discovered --> Quarantined: Missing upstream evidence + Discovered --> Verifying: Published contract found + Verifying --> Quarantined: Digest or validation failure + Verifying --> Inactive: Compatible evidence verified + Inactive --> Active: Operator activation + Active --> Revoked: Artifact or policy revoked + Active --> Inactive: Controlled rollback + Revoked --> Verifying: New immutable revision +``` + +Only `Active` can serve inference. A display name, mutable tag, or previously +seen model ID is not sufficient deployment evidence. + +## Cross-field validation obligations + +The schema validates local types, bounds, and status-dependent shape. The +adapter must additionally validate: + +- non-negative integer topic IDs, rank uniqueness, and, for `inferred`, equality + of fitted, declared, observed, and actual component counts; +- component sum within the pinned tolerance; +- estimate containment within each credible interval; +- equality between recomputed and reported diagnostic counts/sums; +- status, diagnostic acceptance, and reason-code consistency; +- request/evidence snapshot equality, scope-binding equality, expiry, and current + tenant/workspace/purpose/consent/region reauthorization; +- RFC 3339 format assertion, availability-at-knowledge-cutoff ordering, and the + pinned temporal missingness rule; +- exact diagnostic/reason-code registry versions and known-code membership, with + unknown versions or codes mapped to `502`; +- deployment identity and every pinned provenance digest; +- design formula/contrast, estimator, analysis-unit, estimand, covariate schema, + level/missingness, membership structure/normalization coupling, unseen-level, + temporal, and validation-profile compatibility; and +- reauthorization of each opaque evidence reference at the time of use. + +See [API contract](API_CONTRACT.md) for HTTP semantics and +[conceptual data model](DATA_MODEL.md) for ownership relationships. diff --git a/docs/topic-intelligence/schema/topic-inference-result-v1.schema.json b/docs/topic-intelligence/schema/topic-inference-result-v1.schema.json new file mode 100644 index 000000000..f86f0b6e3 --- /dev/null +++ b/docs/topic-intelligence/schema/topic-inference-result-v1.schema.json @@ -0,0 +1,1228 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://naruon.net/schemas/topic-intelligence/topic-inference-result-v1/2026-08-09.1", + "title": "Naruon internal topic-inference result envelope", + "description": "PLANNED Naruon adapter envelope for consuming an independently published compatible scientific payload. TEPP is only the expected upstream producer if it separately publishes and accepts that responsibility; this is not a shipped Naruon endpoint or an assertion that such a TEPP contract exists. A candidate payload that fails this schema or any declared runtime invariant is an upstream protocol error mapped to HTTP 502, not an inferred or abstained result.", + "x-maturity": "PLANNED", + "x-capability-status": "BLOCKED-UPSTREAM", + "x-owner": "NARUON", + "x-expected-upstream-producer": "TEPP", + "x-runtime-status": "NOT_IMPLEMENTED", + "x-schema-digest-required": true, + "x-validator-requirements": [ + "JSON Schema Draft 2020-12", + "date-time format assertion enabled", + "all x-runtime-invariants enforced after schema validation" + ], + "x-runtime-invariants": [ + "For inferred results: provenance.fitted_topic_count equals inference.topic_count, diagnostics.posterior.observed_topic_count, and the number of topic_components; topic_id and rank are each unique.", + "For abstained results: inference.topic_count, diagnostics.posterior.observed_topic_count, and the number of topic_components are zero while fitted_topic_count remains the deployed artifact topic count.", + "request.evidence_ref.snapshot_revision equals request.source_snapshot_revision and request.evidence_ref.scope_binding_ref equals request.scope_binding_ref; reauthorization resolves the same current tenant, workspace, purpose, and authorization binding.", + "request.language equals diagnostics.input.language_tag; retained_token_count meets its minimum and out_of_vocabulary_ratio does not exceed its maximum.", + "availability_time is at or before knowledge_cutoff_time and nullable temporal fields follow the pinned temporal missingness policy.", + "diagnostic and reason codes are members of the exact pinned registries; any unknown registry version or code is an upstream protocol error, never abstention.", + "inference.inference_method equals diagnostics.posterior.inference_method and all reported counts, sums, intervals, and status-dependent diagnostics agree with recomputed values." + ], + "type": "object", + "additionalProperties": false, + "required": [ + "contract", + "request", + "status", + "completed_at", + "tepp_payload_digest", + "tepp_payload" + ], + "properties": { + "contract": { + "$ref": "#/$defs/contractIdentity" + }, + "request": { + "$ref": "#/$defs/requestIdentity" + }, + "status": { + "type": "string", + "enum": [ + "inferred", + "abstained" + ] + }, + "completed_at": { + "type": "string", + "format": "date-time" + }, + "tepp_payload_digest": { + "allOf": [ + { + "$ref": "#/$defs/canonicalDigest" + }, + { + "properties": { + "domain": { + "const": "naruon.topic-inference.tepp-payload.v1" + } + } + } + ] + }, + "tepp_payload": { + "$ref": "#/$defs/teppScientificPayload" + }, + "presentation": { + "$ref": "#/$defs/presentation" + } + }, + "allOf": [ + { + "if": { + "properties": { + "status": { + "const": "inferred" + } + }, + "required": [ + "status" + ] + }, + "then": { + "properties": { + "tepp_payload": { + "allOf": [ + { + "properties": { + "inference": { + "properties": { + "topic_count": { + "minimum": 1 + }, + "topic_components": { + "minItems": 1 + } + } + } + } + }, + { + "properties": { + "diagnostics": { + "properties": { + "diagnostic_status": { + "const": "accepted" + }, + "posterior": { + "properties": { + "converged": { + "const": true + }, + "numerical_status": { + "const": "valid" + }, + "finite_values": { + "const": true + }, + "intervals_valid": { + "const": true + } + } + }, + "policy": { + "properties": { + "accepted": { + "const": true + }, + "reason_codes": { + "maxItems": 0 + } + } + } + } + } + } + } + ] + } + } + } + }, + { + "if": { + "properties": { + "status": { + "const": "abstained" + } + }, + "required": [ + "status" + ] + }, + "then": { + "not": { + "required": [ + "presentation" + ] + }, + "properties": { + "tepp_payload": { + "allOf": [ + { + "properties": { + "inference": { + "properties": { + "topic_count": { + "const": 0 + }, + "topic_components": { + "maxItems": 0 + } + } + } + } + }, + { + "properties": { + "diagnostics": { + "properties": { + "diagnostic_status": { + "const": "rejected" + }, + "policy": { + "properties": { + "accepted": { + "const": false + }, + "reason_codes": { + "minItems": 1 + } + } + } + } + } + } + } + ] + } + } + } + } + ], + "$defs": { + "contractIdentity": { + "type": "object", + "additionalProperties": false, + "required": [ + "schema_id", + "schema_revision", + "schema_digest", + "adapter_name", + "adapter_version", + "tepp_contract_version" + ], + "properties": { + "schema_id": { + "const": "https://naruon.net/schemas/topic-intelligence/topic-inference-result-v1/2026-08-09.1" + }, + "schema_revision": { + "const": "2026-08-09.1" + }, + "schema_digest": { + "allOf": [ + { + "$ref": "#/$defs/canonicalDigest" + }, + { + "properties": { + "domain": { + "const": "naruon.topic-inference.schema.v1" + } + } + } + ] + }, + "adapter_name": { + "const": "naruon-topic-intelligence-adapter" + }, + "adapter_version": { + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "tepp_contract_version": { + "type": "string", + "minLength": 1, + "maxLength": 128 + } + } + }, + "requestIdentity": { + "type": "object", + "additionalProperties": false, + "required": [ + "request_id", + "request_revision", + "idempotency_binding_ref", + "document_ref", + "source_snapshot_revision", + "source_snapshot_digest", + "evidence_ref", + "scope_binding_ref", + "language", + "purpose" + ], + "properties": { + "request_id": { + "type": "string", + "pattern": "^tir_[A-Za-z0-9_-]{16,64}$" + }, + "request_revision": { + "type": "string", + "pattern": "^reqrev_[A-Za-z0-9_-]{1,64}$" + }, + "idempotency_binding_ref": { + "type": "string", + "pattern": "^idem_[A-Za-z0-9_-]{16,128}$" + }, + "document_ref": { + "type": "string", + "pattern": "^doc_[A-Za-z0-9_-]{16,128}$" + }, + "source_snapshot_revision": { + "type": "string", + "pattern": "^snaprev_[A-Za-z0-9_-]{1,128}$" + }, + "source_snapshot_digest": { + "allOf": [ + { + "$ref": "#/$defs/canonicalDigest" + }, + { + "properties": { + "domain": { + "const": "naruon.topic-inference.source-snapshot.v1" + } + } + } + ] + }, + "evidence_ref": { + "$ref": "#/$defs/opaqueEvidenceRef" + }, + "scope_binding_ref": { + "type": "string", + "pattern": "^scopebind_[A-Za-z0-9_-]{16,128}$", + "description": "Opaque server-created binding for the currently authorized tenant, workspace, and purpose. It must equal evidence_ref.scope_binding_ref at runtime." + }, + "language": { + "type": "string", + "pattern": "^[A-Za-z]{2,8}(?:-[A-Za-z0-9]{1,8})*$", + "maxLength": 63 + }, + "purpose": { + "const": "topic_assistance" + } + } + }, + "opaqueEvidenceRef": { + "type": "object", + "additionalProperties": false, + "required": [ + "ref", + "audience", + "snapshot_revision", + "scope_binding_ref", + "authorization_binding_ref", + "purpose", + "expires_at" + ], + "properties": { + "ref": { + "type": "string", + "pattern": "^ev_[A-Za-z0-9_-]{16,128}$" + }, + "audience": { + "const": "naruon-topic-intelligence-adapter" + }, + "snapshot_revision": { + "type": "string", + "pattern": "^snaprev_[A-Za-z0-9_-]{1,128}$", + "description": "Must equal the enclosing request source_snapshot_revision at runtime." + }, + "scope_binding_ref": { + "type": "string", + "pattern": "^scopebind_[A-Za-z0-9_-]{16,128}$", + "description": "Opaque tenant/workspace/purpose binding. It must equal the enclosing request scope_binding_ref and be reauthorized at use time." + }, + "authorization_binding_ref": { + "type": "string", + "pattern": "^authz_[A-Za-z0-9_-]{16,128}$" + }, + "purpose": { + "const": "topic_assistance" + }, + "expires_at": { + "type": "string", + "format": "date-time" + } + } + }, + "teppScientificPayload": { + "type": "object", + "additionalProperties": false, + "required": [ + "provenance", + "inference", + "diagnostics" + ], + "properties": { + "provenance": { + "$ref": "#/$defs/scientificProvenance" + }, + "inference": { + "$ref": "#/$defs/inferenceResult" + }, + "diagnostics": { + "$ref": "#/$defs/diagnosticBundle" + } + } + }, + "scientificProvenance": { + "type": "object", + "additionalProperties": false, + "required": [ + "deployment_ref", + "model_id", + "model_version", + "fitted_topic_count", + "artifact_digest", + "manifest_digest", + "vocabulary_digest", + "preprocessing_digest", + "design_digest", + "lineage_digest", + "model_card_digest", + "validation_report_digest", + "temporal_policy_version", + "evidence_times", + "evidence_time_manifest_digest", + "covariate_snapshot_digest", + "design_row_digest", + "estimator_id", + "analysis_unit", + "estimand_id", + "causal_design", + "design_contract" + ], + "properties": { + "deployment_ref": { + "type": "string", + "pattern": "^deploy_[A-Za-z0-9_-]{16,128}$" + }, + "model_id": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "model_version": { + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "fitted_topic_count": { + "type": "integer", + "minimum": 1, + "maximum": 10000 + }, + "artifact_digest": { + "allOf": [ + { + "$ref": "#/$defs/canonicalDigest" + }, + { + "properties": { + "domain": { + "const": "tepp.topic-measurement.artifact-descriptor.v1" + } + } + } + ] + }, + "manifest_digest": { + "allOf": [ + { + "$ref": "#/$defs/canonicalDigest" + }, + { + "properties": { + "domain": { + "const": "tepp.topic-measurement.artifact-manifest.v1" + } + } + } + ] + }, + "vocabulary_digest": { + "allOf": [ + { + "$ref": "#/$defs/canonicalDigest" + }, + { + "properties": { + "domain": { + "const": "tepp.topic-measurement.vocabulary.v1" + } + } + } + ] + }, + "preprocessing_digest": { + "allOf": [ + { + "$ref": "#/$defs/canonicalDigest" + }, + { + "properties": { + "domain": { + "const": "tepp.topic-measurement.preprocessing.v1" + } + } + } + ] + }, + "design_digest": { + "allOf": [ + { + "$ref": "#/$defs/canonicalDigest" + }, + { + "properties": { + "domain": { + "const": "tepp.topic-measurement.design.v1" + } + } + } + ] + }, + "lineage_digest": { + "allOf": [ + { + "$ref": "#/$defs/canonicalDigest" + }, + { + "properties": { + "domain": { + "const": "tepp.topic-measurement.lineage.v1" + } + } + } + ] + }, + "model_card_digest": { + "allOf": [ + { + "$ref": "#/$defs/canonicalDigest" + }, + { + "properties": { + "domain": { + "const": "tepp.topic-measurement.model-card.v1" + } + } + } + ] + }, + "validation_report_digest": { + "allOf": [ + { + "$ref": "#/$defs/canonicalDigest" + }, + { + "properties": { + "domain": { + "const": "tepp.topic-measurement.validation-report.v1" + } + } + } + ] + }, + "temporal_policy_version": { + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "evidence_times": { + "$ref": "#/$defs/temporalEvidence" + }, + "evidence_time_manifest_digest": { + "allOf": [ + { + "$ref": "#/$defs/canonicalDigest" + }, + { + "properties": { + "domain": { + "const": "naruon.topic-inference.evidence-time-manifest.v1" + } + } + } + ] + }, + "covariate_snapshot_digest": { + "allOf": [ + { + "$ref": "#/$defs/canonicalDigest" + }, + { + "properties": { + "domain": { + "const": "naruon.topic-inference.covariate-snapshot.v1" + } + } + } + ] + }, + "design_row_digest": { + "allOf": [ + { + "$ref": "#/$defs/canonicalDigest" + }, + { + "properties": { + "domain": { + "const": "tepp.topic-measurement.design-row.v1" + } + } + } + ] + }, + "estimator_id": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "analysis_unit": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "estimand_id": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "causal_design": { + "const": "non_causal" + }, + "design_contract": { + "$ref": "#/$defs/designContract" + } + } + }, + "designContract": { + "type": "object", + "additionalProperties": false, + "required": [ + "covariate_schema_version", + "covariate_level", + "covariate_missingness_policy", + "prevalence_formula", + "content_formula", + "contrast_specification", + "membership_structure", + "membership_weight_normalization", + "unseen_level_policy", + "validation_profile_version" + ], + "properties": { + "covariate_schema_version": { + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "covariate_level": { + "type": "string", + "enum": [ + "not_applicable", + "analysis_unit", + "group", + "multiple_membership", + "cross_classified" + ] + }, + "covariate_missingness_policy": { + "type": "string", + "enum": [ + "not_applicable", + "reject_missing", + "explicit_missing_indicator", + "explicit_missing_level" + ] + }, + "prevalence_formula": { + "type": "string", + "minLength": 1, + "maxLength": 4096 + }, + "content_formula": { + "type": "string", + "minLength": 1, + "maxLength": 4096 + }, + "contrast_specification": { + "type": "string", + "minLength": 1, + "maxLength": 4096 + }, + "membership_structure": { + "type": "string", + "enum": [ + "none", + "multilevel", + "multiple_membership", + "cross_classified", + "cross_classified_multiple_membership" + ] + }, + "membership_weight_normalization": { + "type": "string", + "enum": [ + "not_applicable", + "sum_to_one_per_analysis_unit" + ] + }, + "unseen_level_policy": { + "type": "string", + "enum": [ + "reject", + "predeclared_other_level" + ] + }, + "validation_profile_version": { + "type": "string", + "minLength": 1, + "maxLength": 128 + } + }, + "allOf": [ + { + "if": { + "properties": { + "membership_structure": { + "enum": [ + "multiple_membership", + "cross_classified_multiple_membership" + ] + } + }, + "required": [ + "membership_structure" + ] + }, + "then": { + "properties": { + "membership_weight_normalization": { + "const": "sum_to_one_per_analysis_unit" + } + } + }, + "else": { + "properties": { + "membership_weight_normalization": { + "const": "not_applicable" + } + } + } + }, + { + "if": { + "properties": { + "covariate_level": { + "const": "not_applicable" + } + }, + "required": [ + "covariate_level" + ] + }, + "then": { + "properties": { + "covariate_missingness_policy": { + "const": "not_applicable" + } + } + }, + "else": { + "properties": { + "covariate_missingness_policy": { + "enum": [ + "reject_missing", + "explicit_missing_indicator", + "explicit_missing_level" + ] + } + } + } + } + ] + }, + "temporalEvidence": { + "type": "object", + "additionalProperties": false, + "required": [ + "document_time", + "event_time", + "assertion_time", + "availability_time", + "knowledge_cutoff_time", + "temporal_missingness_policy", + "availability_at_knowledge_cutoff" + ], + "properties": { + "document_time": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "event_time": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "assertion_time": { + "type": "string", + "format": "date-time" + }, + "availability_time": { + "type": "string", + "format": "date-time" + }, + "knowledge_cutoff_time": { + "type": "string", + "format": "date-time" + }, + "temporal_missingness_policy": { + "const": "document_event_nullable_assertion_availability_cutoff_required" + }, + "availability_at_knowledge_cutoff": { + "const": true, + "description": "Expected-upstream producer assertion that availability_time is at or before knowledge_cutoff_time. Naruon must parse and recompute this ordering; the assertion alone is insufficient." + } + } + }, + "inferenceResult": { + "type": "object", + "additionalProperties": false, + "required": [ + "inference_method", + "inference_implementation", + "inference_version", + "numerical_backend", + "credible_level", + "interval_method", + "uncertainty_scope", + "topic_count", + "topic_components" + ], + "properties": { + "inference_method": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "inference_implementation": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "inference_version": { + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "numerical_backend": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "credible_level": { + "type": "number", + "exclusiveMinimum": 0, + "exclusiveMaximum": 1 + }, + "interval_method": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "uncertainty_scope": { + "const": "conditional_on_fitted_artifact" + }, + "topic_count": { + "type": "integer", + "minimum": 0, + "maximum": 10000 + }, + "topic_components": { + "type": "array", + "maxItems": 10000, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/posteriorComponent" + } + } + } + }, + "posteriorComponent": { + "type": "object", + "additionalProperties": false, + "required": [ + "topic_id", + "rank", + "proportion", + "credible_interval" + ], + "properties": { + "topic_id": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "rank": { + "type": "integer", + "minimum": 1, + "maximum": 10000 + }, + "proportion": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "credible_interval": { + "$ref": "#/$defs/credibleInterval" + } + } + }, + "credibleInterval": { + "type": "object", + "additionalProperties": false, + "required": [ + "lower", + "upper" + ], + "properties": { + "lower": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "upper": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + } + }, + "diagnosticBundle": { + "type": "object", + "additionalProperties": false, + "required": [ + "diagnostic_status", + "input", + "posterior", + "policy" + ], + "properties": { + "diagnostic_status": { + "type": "string", + "enum": [ + "accepted", + "rejected" + ] + }, + "input": { + "$ref": "#/$defs/inputDiagnostics" + }, + "posterior": { + "$ref": "#/$defs/posteriorDiagnostics" + }, + "policy": { + "$ref": "#/$defs/policyDiagnostics" + } + } + }, + "inputDiagnostics": { + "type": "object", + "additionalProperties": false, + "required": [ + "language_tag", + "language_support_status", + "original_token_count", + "retained_token_count", + "minimum_retained_token_count", + "out_of_vocabulary_token_count", + "out_of_vocabulary_ratio", + "maximum_out_of_vocabulary_ratio", + "temporal_context_status", + "covariate_contract_status" + ], + "properties": { + "language_tag": { + "type": "string", + "pattern": "^[A-Za-z]{2,8}(?:-[A-Za-z0-9]{1,8})*$", + "maxLength": 63 + }, + "language_support_status": { + "const": "supported" + }, + "original_token_count": { + "type": "integer", + "minimum": 1 + }, + "retained_token_count": { + "type": "integer", + "minimum": 1 + }, + "minimum_retained_token_count": { + "type": "integer", + "minimum": 1 + }, + "out_of_vocabulary_token_count": { + "type": "integer", + "minimum": 0 + }, + "out_of_vocabulary_ratio": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "maximum_out_of_vocabulary_ratio": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "temporal_context_status": { + "const": "valid" + }, + "covariate_contract_status": { + "const": "valid" + } + } + }, + "posteriorDiagnostics": { + "type": "object", + "additionalProperties": false, + "required": [ + "inference_method", + "diagnostic_code_registry_version", + "converged", + "convergence_code", + "numerical_status", + "quality_codes", + "iteration_count", + "finite_values", + "intervals_valid", + "observed_topic_count", + "posterior_sum", + "normalization_tolerance" + ], + "properties": { + "inference_method": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "diagnostic_code_registry_version": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "description": "Exact immutable registry for convergence_code and quality_codes. Unknown versions or codes fail closed as an upstream protocol error." + }, + "converged": { + "type": "boolean" + }, + "convergence_code": { + "type": "string", + "pattern": "^[a-z][a-z0-9_]{0,95}$" + }, + "numerical_status": { + "type": "string", + "enum": [ + "valid", + "invalid" + ] + }, + "quality_codes": { + "type": "array", + "maxItems": 32, + "uniqueItems": true, + "items": { + "type": "string", + "pattern": "^[a-z][a-z0-9_]{0,95}$" + } + }, + "iteration_count": { + "type": "integer", + "minimum": 0 + }, + "finite_values": { + "type": "boolean" + }, + "intervals_valid": { + "type": "boolean" + }, + "observed_topic_count": { + "type": "integer", + "minimum": 0, + "maximum": 10000 + }, + "posterior_sum": { + "type": "number", + "minimum": 0, + "maximum": 10000 + }, + "normalization_tolerance": { + "type": "number", + "exclusiveMinimum": 0, + "maximum": 0.1 + } + } + }, + "policyDiagnostics": { + "type": "object", + "additionalProperties": false, + "required": [ + "policy_version", + "reason_code_registry_version", + "accepted", + "reason_codes" + ], + "properties": { + "policy_version": { + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "reason_code_registry_version": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "description": "Exact immutable registry for reason_codes. Unknown versions or codes fail closed as an upstream protocol error." + }, + "accepted": { + "type": "boolean" + }, + "reason_codes": { + "type": "array", + "maxItems": 32, + "uniqueItems": true, + "items": { + "type": "string", + "pattern": "^posterior_[a-z0-9_]{1,96}$" + } + } + } + }, + "presentation": { + "type": "object", + "additionalProperties": false, + "required": [ + "labels" + ], + "properties": { + "labels": { + "type": "array", + "minItems": 1, + "maxItems": 10000, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/presentationLabel" + } + } + } + }, + "presentationLabel": { + "type": "object", + "additionalProperties": false, + "required": [ + "topic_id", + "label_id", + "label_version", + "language", + "label", + "review_method", + "evidence_refs" + ], + "properties": { + "topic_id": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "label_id": { + "type": "string", + "pattern": "^label_[A-Za-z0-9_-]{16,128}$" + }, + "label_version": { + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "language": { + "type": "string", + "pattern": "^[A-Za-z]{2,8}(?:-[A-Za-z0-9]{1,8})*$", + "maxLength": 63 + }, + "label": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "review_method": { + "type": "string", + "enum": [ + "human_curated", + "model_assisted_human_reviewed" + ] + }, + "evidence_refs": { + "type": "array", + "minItems": 1, + "maxItems": 64, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/opaqueEvidenceRef" + } + } + } + }, + "canonicalDigest": { + "type": "object", + "additionalProperties": false, + "required": [ + "algorithm", + "canonicalization", + "domain", + "value" + ], + "properties": { + "algorithm": { + "const": "sha-256" + }, + "canonicalization": { + "const": "RFC8785" + }, + "domain": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9._-]+\\.v[0-9]+$", + "maxLength": 256 + }, + "value": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + } + } +} diff --git a/frontend/.Jules/palette.md b/frontend/.Jules/palette.md index e4d2c050f..3cb9bb8eb 100644 --- a/frontend/.Jules/palette.md +++ b/frontend/.Jules/palette.md @@ -9,3 +9,7 @@ ## 2026-06-08 - WorkspaceHome unused import investigation **Learning:** Investigating unused import reports should first verify the current file because the codebase may already have evolved. The repo lint entrypoint is `eslint`, and the focused check for this investigation was `npx eslint src/components/WorkspaceHome.tsx`. **Action:** Use the focused `npx eslint src/components/WorkspaceHome.tsx` check when confirming WorkspaceHome import health, and reserve broader `eslint` runs for full frontend lint validation. + +## 2026-06-08 - Accessible Tooltips on Disabled Buttons +**Learning:** Adding a `title` tooltip directly to a natively `disabled` `

오래된 메시지부터 최신 메시지 순서로 보여줍니다. 답장은 선택된 메시지를 기준으로 작성됩니다.

{threadLoading &&

대화 흐름을 불러오는 중입니다...

} @@ -773,11 +770,6 @@ export const EmailDetail = memo(function EmailDetail({ emailId, actionCommand = {toMailDisplayText(msg.sender, '보낸 사람')}
{formatEmailDate(msg.date)} - {msg.id !== conversationMessages[0]?.id && ( - - )}
{msg.id === email.id && 선택된 메시지} diff --git a/frontend/src/components/NetworkGraph.map-lookup.test.ts b/frontend/src/components/NetworkGraph.map-lookup.test.ts new file mode 100644 index 000000000..3ba76c75c --- /dev/null +++ b/frontend/src/components/NetworkGraph.map-lookup.test.ts @@ -0,0 +1,66 @@ +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; + +import { describe, expect, it } from "vitest"; + +const networkGraphSource = readFileSync( + fileURLToPath(new URL("./NetworkGraph.tsx", import.meta.url)), + "utf8", +); + +function sourceBetween(startMarker: string, endMarker: string): string { + const startIndex = networkGraphSource.indexOf(startMarker); + const endIndex = networkGraphSource.indexOf(endMarker, startIndex); + + expect(startIndex).toBeGreaterThanOrEqual(0); + expect(endIndex).toBeGreaterThan(startIndex); + + return networkGraphSource.slice(startIndex, endIndex); +} + +describe("NetworkGraph constant-time selection lookup contract", () => { + it("keeps graph event selection on memoized maps without linear fallback scans", () => { + const edgeSelection = sourceBetween("const selectEdge =", "const selectNode ="); + const nodeSelection = sourceBetween("const selectNode =", "const handleEdgeSelection ="); + + expect(edgeSelection).toContain("edgeMap.get(String(edgeId))"); + expect(edgeSelection).not.toContain(".find("); + + expect(nodeSelection).toContain("nodeMap.get(String(nodeId))"); + expect(nodeSelection).toContain("?? String(nodeId)"); + expect(nodeSelection).not.toContain("findNodeLabel("); + expect(nodeSelection).not.toContain(".find("); + }); + + it("keeps select controls on memoized maps without rescanning nodes or edges", () => { + const graphNodeSelection = sourceBetween( + "const selectGraphNode =", + "const handleSelectFirstRelationship =", + ); + const relationshipControl = sourceBetween( + "const handleRelationshipOptionChange =", + "const handleNodeOptionChange =", + ); + const nodeControl = sourceBetween( + "const handleNodeOptionChange =", + "const handleZoomGraph =", + ); + + expect(graphNodeSelection).toContain("nodeMap.get(String(node.id))"); + expect(graphNodeSelection).toContain("?? String(node.id)"); + expect(graphNodeSelection).not.toContain("findNodeLabel("); + expect(graphNodeSelection).not.toContain(".find("); + + expect(relationshipControl).toContain("edgeMap.get(value)"); + expect(relationshipControl).not.toContain(".find("); + + expect(nodeControl).toContain("nodeInstanceMap.get(value)"); + expect(nodeControl).not.toContain(".find("); + }); + + it("builds edge and node instance maps as first-wins lookups", () => { + expect(networkGraphSource).toContain("firstGraphEntryById(edges"); + expect(networkGraphSource).toContain("firstGraphEntryById(nodes"); + expect(networkGraphSource).not.toMatch(/new Map\((edges|nodes)\.map\(/); + }); +}); diff --git a/frontend/src/components/NetworkGraph.test.tsx b/frontend/src/components/NetworkGraph.test.tsx index 061e162e2..328d7c543 100644 --- a/frontend/src/components/NetworkGraph.test.tsx +++ b/frontend/src/components/NetworkGraph.test.tsx @@ -108,6 +108,39 @@ describe("NetworkGraph", () => { expect(Network).not.toHaveBeenCalled(); }); + it("describes the unavailable first-relationship action programmatically", async () => { + const fetchMock = vi.fn(() => + Promise.resolve( + jsonResponse({ + nodes: [{ id: "node-1", label: "노드" }], + edges: [], + }), + ), + ); + vi.stubGlobal("fetch", fetchMock); + + await renderGraph(); + await flushAsyncWork(); + + const mountedContainer = getMountedContainer(); + const wrapper = mountedContainer.querySelector('span[tabindex="0"]'); + const button = mountedContainer.querySelector('button[disabled]'); + const descriptionId = wrapper?.getAttribute("aria-describedby"); + + expect(wrapper).toBeInstanceOf(HTMLSpanElement); + expect(wrapper?.className).toContain("cursor-not-allowed"); + expect(wrapper?.getAttribute("title")).toBe("표시할 관계 데이터가 없습니다."); + expect(wrapper?.className).toContain("focus-visible:ring-2"); + expect(descriptionId).toBeTruthy(); + expect(document.getElementById(descriptionId ?? "")?.textContent).toBe( + "표시할 관계 데이터가 없습니다.", + ); + expect(button).toBeInstanceOf(HTMLButtonElement); + expect((button as HTMLButtonElement).disabled).toBe(true); + expect(button?.className).toContain("disabled:cursor-not-allowed"); + expect(button?.className).toContain("pointer-events-none"); + }); + it("announces graph loading failures as a polite alert", async () => { const consoleError = vi.spyOn(console, "error").mockImplementation(() => undefined); const fetchMock = vi.fn(() => Promise.reject(new Error("network unavailable"))); @@ -290,6 +323,116 @@ describe("NetworkGraph", () => { expect(mountedContainer.textContent).toContain("그래프 맞춤 완료"); }); + function registeredGraphHandler(eventName: string) { + const handler = onMock.mock.calls.find((call) => call[0] === eventName)?.[1]; + if (typeof handler !== "function") { + throw new Error(`${eventName} handler was not registered.`); + } + return handler as (event: { + nodes?: Array; + edges?: Array; + }) => void; + } + + it("resolves vis-network selection events for mixed numeric and string ids", async () => { + const fetchMock = vi.fn(() => + Promise.resolve( + jsonResponse({ + nodes: [ + { id: 101, label: "발신자", title: "PM" }, + { id: "recipient-1", label: "수신자", title: "Owner" }, + ], + edges: [ + { id: 7, from: 101, to: "recipient-1", title: "메일 1건" }, + ], + }), + ), + ); + vi.stubGlobal("fetch", fetchMock); + + await renderGraph(); + await flushAsyncWork(); + + const mountedContainer = getMountedContainer(); + const selectNode = registeredGraphHandler("selectNode"); + const selectEdge = registeredGraphHandler("selectEdge"); + + await act(async () => { + selectNode({ nodes: [101] }); + }); + + const nodeSelect = mountedContainer.querySelector('select[aria-label="노드 선택"]'); + expect(nodeSelect).toBeInstanceOf(HTMLSelectElement); + expect((nodeSelect as HTMLSelectElement).value).toBe("101"); + expect(mountedContainer.textContent).toContain("선택된 노드: 발신자"); + expect(mountedContainer.textContent).toContain("그래프에서 노드를 선택했습니다."); + + await act(async () => { + selectEdge({ edges: [7] }); + }); + + const relationshipSelect = mountedContainer.querySelector('select[aria-label="관계 선택"]'); + expect(relationshipSelect).toBeInstanceOf(HTMLSelectElement); + expect((relationshipSelect as HTMLSelectElement).value).toBe("7"); + expect(mountedContainer.textContent).toContain("선택된 관계: 발신자 -> 수신자 (메일 1건)"); + expect(mountedContainer.textContent).toContain("그래프에서 관계를 선택했습니다."); + }); + + it("keeps the first edge instance when duplicate relationship ids collide", async () => { + const fetchMock = vi.fn(() => + Promise.resolve( + jsonResponse({ + nodes: [ + { id: "sender-1", label: "김지현", title: "PM" }, + { id: "recipient-1", label: "사용자", title: "Owner" }, + { id: "calendar-1", label: "일정", title: "Schedule" }, + ], + edges: [ + { id: "rel-shared", from: "sender-1", to: "recipient-1", title: "메일 2건" }, + { id: "rel-shared", from: "sender-1", to: "calendar-1", title: "일정 후보 1건" }, + ], + }), + ), + ); + vi.stubGlobal("fetch", fetchMock); + + await renderGraph(); + await flushAsyncWork(); + + const mountedContainer = getMountedContainer(); + const selectEdge = registeredGraphHandler("selectEdge"); + + await act(async () => { + selectEdge({ edges: ["rel-shared"] }); + }); + + expect(mountedContainer.textContent).toContain("선택된 관계: 김지현 -> 사용자 (메일 2건)"); + expect(mountedContainer.textContent).not.toContain("선택된 관계: 김지현 -> 일정 (일정 후보 1건)"); + expect(selectEdgesMock).not.toHaveBeenCalled(); + + const relationshipSelect = mountedContainer.querySelector('select[aria-label="관계 선택"]'); + expect(relationshipSelect).toBeInstanceOf(HTMLSelectElement); + + await act(async () => { + if (relationshipSelect instanceof HTMLSelectElement) { + relationshipSelect.value = "rel-shared"; + relationshipSelect.dispatchEvent(new Event("change", { bubbles: true })); + } + }); + + expect(selectEdgesMock).toHaveBeenCalledWith(["rel-shared"]); + expect(fitMock).toHaveBeenCalledWith({ + nodes: ["sender-1", "recipient-1"], + animation: false, + }); + expect(fitMock).not.toHaveBeenCalledWith({ + nodes: ["sender-1", "calendar-1"], + animation: false, + }); + expect(mountedContainer.textContent).toContain("선택된 관계: 김지현 -> 사용자 (메일 2건)"); + expect(mountedContainer.textContent).toContain("선택한 관계를 열었습니다."); + }); + it("normalizes backend source target edges before rendering the graph", async () => { const fetchMock = vi.fn(() => Promise.resolve( @@ -320,6 +463,62 @@ describe("NetworkGraph", () => { expect(edges[0]).not.toHaveProperty("target"); }); + it("does not expose graph records without an id as selectable nodes", async () => { + const fetchMock = vi.fn(() => + Promise.resolve( + jsonResponse({ + nodes: [ + { id: null, label: "식별자 없는 노드" }, + { id: "person-1", label: "김지현" }, + ], + edges: [{ from: "person-1", to: "person-1", title: "관련 메일" }], + }), + ), + ); + vi.stubGlobal("fetch", fetchMock); + + await renderGraph(); + await flushAsyncWork(); + + const nodeSelect = getMountedContainer().querySelector( + 'select[aria-label="노드 선택"]', + ); + expect(nodeSelect).toBeInstanceOf(HTMLSelectElement); + expect(nodeSelect?.textContent).toContain("김지현"); + expect(nodeSelect?.textContent).not.toContain("식별자 없는 노드"); + }); + + it("keeps raw endpoint ids visible when a relationship has no matching node", async () => { + const fetchMock = vi.fn(() => + Promise.resolve( + jsonResponse({ + nodes: [{ id: "known-node", label: "확인된 노드" }], + edges: [{ from: "missing-from", to: "missing-to", title: "고립 관계" }], + }), + ), + ); + vi.stubGlobal("fetch", fetchMock); + + await renderGraph(); + await flushAsyncWork(); + + const mountedContainer = getMountedContainer(); + expect(mountedContainer.textContent).toContain( + "관계 1: missing-from -> missing-to (고립 관계)", + ); + + const relationshipButton = Array.from(mountedContainer.querySelectorAll("button")).find( + (button) => button.textContent === "첫 관계 보기", + ); + await act(async () => { + relationshipButton?.dispatchEvent(new MouseEvent("click", { bubbles: true })); + }); + + expect(mountedContainer.textContent).toContain( + "선택된 관계: missing-from -> missing-to (고립 관계)", + ); + }); + it("refits the graph when the viewport changes", async () => { const fetchMock = vi.fn(() => Promise.resolve( @@ -342,6 +541,7 @@ describe("NetworkGraph", () => { vi.useFakeTimers(); try { await act(async () => { + resizeObserverCallback?.([] as ResizeObserverEntry[], {} as ResizeObserver); resizeObserverCallback?.([] as ResizeObserverEntry[], {} as ResizeObserver); vi.advanceTimersByTime(49); }); diff --git a/frontend/src/components/NetworkGraph.tsx b/frontend/src/components/NetworkGraph.tsx index d33dc04fd..f9eb61c71 100644 --- a/frontend/src/components/NetworkGraph.tsx +++ b/frontend/src/components/NetworkGraph.tsx @@ -1,6 +1,6 @@ 'use client'; -import { useEffect, useMemo, useRef, useState } from 'react'; +import { useEffect, useId, useMemo, useRef, useState } from 'react'; import { Network } from 'vis-network'; interface Node { @@ -85,10 +85,6 @@ function isGraphId(value: unknown): value is number | string { return typeof value === 'number' || typeof value === 'string'; } -function graphIdEquals(left: unknown, right: unknown) { - return isGraphId(left) && isGraphId(right) && String(left) === String(right); -} - function stableEdgeId(edge: Edge, index: number) { if (isGraphId(edge.id)) return edge.id; return `relationship-${index}-${String(edge.from)}-${String(edge.to)}`; @@ -127,20 +123,34 @@ function titleText(value: unknown) { return value == null ? '' : String(value).trim(); } -function findNodeLabel(nodes: Node[], id: number | string) { - const node = nodes.find((candidate) => graphIdEquals(candidate.id, id)); - return String(node?.label ?? id); +/** + * Index graph records by public id, keeping the first instance. + * + * `new Map(items.map((item) => [String(item.id), item]))` is last-wins and + * desynchronizes first-wins label maps from the selected node or edge when + * the API repeats an id. The previous `.find()` selection path was first-wins. + */ +function firstGraphEntryById( + items: readonly T[], + readId: (item: T) => unknown, +): Map { + const map = new Map(); + for (const item of items) { + const rawId = readId(item); + if (!isGraphId(rawId)) { + continue; + } + const key = String(rawId); + if (!map.has(key)) { + map.set(key, item); + } + } + return map; } -function describeEdge(edge: Edge, nodes: Node[], nodeMap?: Map) { - let fromLabel, toLabel; - if (nodeMap) { - fromLabel = nodeMap.get(String(edge.from)) ?? String(edge.from); - toLabel = nodeMap.get(String(edge.to)) ?? String(edge.to); - } else { - fromLabel = findNodeLabel(nodes, edge.from); - toLabel = findNodeLabel(nodes, edge.to); - } +function describeEdge(edge: Edge, nodeMap: Map) { + const fromLabel = nodeMap.get(String(edge.from)) ?? String(edge.from); + const toLabel = nodeMap.get(String(edge.to)) ?? String(edge.to); const title = titleText(edge.title); return title ? `${fromLabel} -> ${toLabel} (${title})` : `${fromLabel} -> ${toLabel}`; } @@ -150,6 +160,7 @@ import { apiClient } from '@/lib/api-client'; export default function NetworkGraph() { const containerRef = useRef(null); const networkRef = useRef(null); + const unavailableRelationshipDescriptionId = useId(); const [nodes, setNodes] = useState([]); const [edges, setEdges] = useState([]); @@ -159,6 +170,8 @@ export default function NetworkGraph() { const [graphActionStatus, setGraphActionStatus] = useState('그래프 준비 완료'); const [relationshipOptionId, setRelationshipOptionId] = useState(''); const [nodeOptionId, setNodeOptionId] = useState(''); + const edgeMap = useMemo(() => firstGraphEntryById(edges, (edge) => edge.id), [edges]); + const nodeInstanceMap = useMemo(() => firstGraphEntryById(nodes, (node) => node.id), [nodes]); const nodeMap = useMemo(() => { const map = new Map(); for (const node of nodes) { @@ -202,18 +215,18 @@ export default function NetworkGraph() { }; const selectEdge = (edgeId: number | string) => { - const edge = edges.find((candidate) => graphIdEquals(candidate.id, edgeId)); + const edge = edgeMap.get(String(edgeId)); if (!edge) return; setRelationshipOptionId(String(edge.id)); setNodeOptionId(''); - setSelectedGraphDetail(`선택된 관계: ${describeEdge(edge, nodes, nodeMap)}`); + setSelectedGraphDetail(`선택된 관계: ${describeEdge(edge, nodeMap)}`); setGraphActionStatus('그래프에서 관계를 선택했습니다.'); }; const selectNode = (nodeId: number | string) => { setRelationshipOptionId(''); setNodeOptionId(String(nodeId)); - setSelectedGraphDetail(`선택된 노드: ${findNodeLabel(nodes, nodeId)}`); + setSelectedGraphDetail(`선택된 노드: ${nodeMap.get(String(nodeId)) ?? String(nodeId)}`); setGraphActionStatus('그래프에서 노드를 선택했습니다.'); }; @@ -262,7 +275,7 @@ export default function NetworkGraph() { network.destroy(); }; } - }, [nodes, edges, nodeMap]); + }, [nodes, edges, nodeMap, edgeMap]); const nodeLabels = useMemo(() => { return nodes @@ -273,25 +286,25 @@ export default function NetworkGraph() { const firstEdge = edges[0] ?? null; const relationshipOptions = useMemo(() => { - return edges.slice(0, 5).map((edge, index) => ({ + return Array.from(edgeMap.values()).slice(0, 5).map((edge, index) => ({ edge, id: String(edge.id), - label: `관계 ${index + 1}: ${describeEdge(edge, nodes, nodeMap)}`, + label: `관계 ${index + 1}: ${describeEdge(edge, nodeMap)}`, })); - }, [edges, nodes, nodeMap]); + }, [edgeMap, nodeMap]); const nodeOptions = useMemo(() => { - return nodes.slice(0, 8).map((node) => ({ + return Array.from(nodeInstanceMap.values()).slice(0, 8).map((node) => ({ id: String(node.id), label: `노드: ${String(node.label ?? node.id)}`, node, })); - }, [nodes]); + }, [nodeInstanceMap]); const selectRelationship = (edge: Edge, status: string) => { setRelationshipOptionId(String(edge.id)); setNodeOptionId(''); - setSelectedGraphDetail(`선택된 관계: ${describeEdge(edge, nodes, nodeMap)}`); + setSelectedGraphDetail(`선택된 관계: ${describeEdge(edge, nodeMap)}`); setGraphActionStatus(status); if (isGraphId(edge.id)) { networkRef.current?.selectEdges?.([edge.id]); @@ -303,7 +316,7 @@ export default function NetworkGraph() { if (!isGraphId(node.id)) return; setRelationshipOptionId(''); setNodeOptionId(String(node.id)); - setSelectedGraphDetail(`선택된 노드: ${findNodeLabel(nodes, node.id)}`); + setSelectedGraphDetail(`선택된 노드: ${nodeMap.get(String(node.id)) ?? String(node.id)}`); setGraphActionStatus(status); networkRef.current?.selectNodes?.([node.id]); networkRef.current?.fit?.({ nodes: [node.id], animation: false }); @@ -315,13 +328,13 @@ export default function NetworkGraph() { }; const handleRelationshipOptionChange = (value: string) => { - const edge = edges.find((candidate) => String(candidate.id) === value); + const edge = edgeMap.get(value); if (!edge) return; selectRelationship(edge, '선택한 관계를 열었습니다.'); }; const handleNodeOptionChange = (value: string) => { - const node = nodes.find((candidate) => String(candidate.id) === value); + const node = nodeInstanceMap.get(value); if (!node) return; selectGraphNode(node, '선택한 노드를 열었습니다.'); }; @@ -377,14 +390,30 @@ export default function NetworkGraph() {

- + {!firstEdge && ( + + 표시할 관계 데이터가 없습니다. + + )} + + - +

+ 서명된 고객 일정 원본을 선택합니다. 고정 ICS 예시나 미리 정해 둔 충돌 결과는 + 조율 증거가 아닙니다. 원본 VEVENT 읽기는 커넥터 조회가 준비될 때까지 대기합니다. +

+
+ {writebackSources.map((source, index) => { + const sourceLabel = getCalendarSourceLabel(index); + const sourceSelected = selectedSource?.source_id === source.source_id; + return ( + + ); + })}
+

+ {sourceLoadStatus === 'loading' && '서명된 일정 원본을 확인하는 중입니다.'} + {sourceLoadStatus === 'error' && '서명 세션으로 일정 원본을 확인할 수 없습니다. 공개 헤더로는 조율할 수 없습니다.'} + {sourceLoadStatus === 'ready' && writebackSources.length === 0 && '서명된 고객 일정 원본이 없어 조율 결과를 보여 주지 않습니다.'} + {sourceLoadStatus === 'ready' && selectedSource !== null && '선택한 일정 원본의 서명된 증거만 조율에 사용합니다.'} + {sourceLoadStatus === 'ready' && writebackSources.length > 0 && selectedSource === null && '조율에 사용할 서명된 일정 원본을 선택하세요.'} +

- + ); } diff --git a/frontend/src/components/calendar/constants.ts b/frontend/src/components/calendar/constants.ts index f7440bfdd..aa418bc12 100644 --- a/frontend/src/components/calendar/constants.ts +++ b/frontend/src/components/calendar/constants.ts @@ -1,4 +1,9 @@ -import type { CalendarCandidateEvent, CalendarDefinition, CalendarMonthEvent, CalendarWeekEvent } from './types'; +import type { + CalendarCandidateEvent, + CalendarDefinition, + CalendarMonthEvent, + CalendarWeekEvent, +} from './types'; export const calendarDefinitions: CalendarDefinition[] = [ { id: 'personal', name: '김나루 (나)', colorClass: 'bg-primary' }, diff --git a/frontend/src/components/calendar/helpers.ts b/frontend/src/components/calendar/helpers.ts index 1f6578aa9..2b4e627f1 100644 --- a/frontend/src/components/calendar/helpers.ts +++ b/frontend/src/components/calendar/helpers.ts @@ -1,5 +1,9 @@ import { calendarDefinitions } from "./constants"; -import { CalendarWritebackSource, CalendarWritebackIntentResponse } from "./types"; +import { + CalendarConflictDecisionCode, + CalendarWritebackSource, + CalendarWritebackIntentResponse, +} from "./types"; export function buildInitialCalendarVisibility() { return Object.fromEntries(calendarDefinitions.map((calendar) => [calendar.id, true])); @@ -66,6 +70,36 @@ export function getProviderRetryLabel(result: CalendarWritebackIntentResponse) { return '실행 요청 없음'; } +export function getConflictDecisionLabel(decisionCode: CalendarConflictDecisionCode): string { + switch (decisionCode) { + case 'available': + return '진행 가능'; + case 'blocked': + return '이중 예약 차단'; + case 'review_required': + return '검토 필요'; + default: { + const exhaustiveCheck: never = decisionCode; + return exhaustiveCheck; + } + } +} + +export function getConflictNextActionLabel(decisionCode: CalendarConflictDecisionCode): string { + switch (decisionCode) { + case 'available': + return '이 시간은 비어 있습니다. 일정을 계속 진행하세요.'; + case 'blocked': + return '확정된 일정이 겹칩니다. 다른 시간을 고르거나 기존 확정 일정을 먼저 조정하세요.'; + case 'review_required': + return '잠정 일정이 겹칩니다. 잠정 일정을 조정하거나 유지할지 확인한 뒤 진행하세요.'; + default: { + const exhaustiveCheck: never = decisionCode; + return exhaustiveCheck; + } + } +} + export function getApiErrorStatus(error: unknown) { const shapedError = error as { status?: unknown; response?: { status?: unknown } } | null; if (typeof shapedError?.status === 'number') return shapedError.status; diff --git a/frontend/src/components/calendar/types.ts b/frontend/src/components/calendar/types.ts index 29006ba01..5481cd8ab 100644 --- a/frontend/src/components/calendar/types.ts +++ b/frontend/src/components/calendar/types.ts @@ -30,6 +30,23 @@ export type WritebackStatus = 'idle' | 'loading' | 'success' | 'no_source' | 'co export type CalendarWritebackActionKey = 'create' | 'update' | 'execute'; +export type CalendarConflictDecisionCode = 'available' | 'blocked' | 'review_required'; + +export type CalendarConflictEvidence = { + commitment_id: string; + start_at: string; + end_at: string; + status: 'confirmed' | 'tentative' | 'desired' | 'cancelled'; +}; + +export type CalendarConflictResponse = { + decision_code: CalendarConflictDecisionCode; + reason_code: string; + conflicts: CalendarConflictEvidence[]; + recommended_action: string; + policy_version: string; +}; + export type CalendarDefinition = { id: string; name: string; diff --git a/frontend/tests/e2e/helpers.ts b/frontend/tests/e2e/helpers.ts index 4042c15eb..d98fac63d 100644 --- a/frontend/tests/e2e/helpers.ts +++ b/frontend/tests/e2e/helpers.ts @@ -1023,6 +1023,40 @@ export async function mockDashboardApi(page: Page, onApiRequest?: (path: string, return; } + if (path === '/api/calendar/conflicts/evaluate' && request.method() === 'POST') { + const payload = JSON.parse(request.postData() || '{}') as { + existing_ics?: string; + }; + if (payload.existing_ics?.includes('STATUS:CANCELLED')) { + await fulfillJson(route, { + decision_code: 'available', + reason_code: 'no_overlapping_commitment', + conflicts: [], + recommended_action: 'Proceed with scheduling.', + policy_version: 'status-weighted-v1', + }); + return; + } + if (payload.existing_ics?.includes('STATUS:TENTATIVE')) { + await fulfillJson(route, { + decision_code: 'review_required', + reason_code: 'lower_priority_conflict_requires_explicit_resolution', + conflicts: [], + recommended_action: 'Review the lower-priority conflict.', + policy_version: 'status-weighted-v1', + }); + return; + } + await fulfillJson(route, { + decision_code: 'blocked', + reason_code: 'equal_or_higher_priority_conflict', + conflicts: [], + recommended_action: 'Choose another time.', + policy_version: 'status-weighted-v1', + }); + return; + } + if (path === '/api/calendar/writeback-intent' && request.method() === 'POST') { await fulfillJson(route, { workspace_id: 'default', diff --git a/plan.md b/plan.md new file mode 100644 index 000000000..bbc5ddc29 --- /dev/null +++ b/plan.md @@ -0,0 +1,21 @@ +# NetworkGraph constant-time lookup plan + +1. Pre-compute `edgeMap` and `nodeInstanceMap` with `useMemo`, and keep the existing `nodeMap` as the authoritative node-label lookup for rendered selections. + - `selectEdge` uses `edgeMap.get(String(edgeId))`. + - `selectNode` uses `nodeMap.get(String(nodeId))` with the node identifier as the no-entry fallback. + - `selectGraphNode` uses `nodeMap.get(String(node.id))` with the node identifier as the no-entry fallback. + - `handleRelationshipOptionChange` uses `edgeMap.get(value)`. + - `handleNodeOptionChange` uses `nodeInstanceMap.get(value)`. + - Selection handlers must not fall back to `Array.prototype.find()` or `findNodeLabel()` scans. + - `edgeMap` and `nodeInstanceMap` are first-wins, matching `nodeMap` and the previous `.find()` path. Last-wins `new Map(items.map(...))` construction is rejected. + +2. Verify the exact branch head from `frontend/` with these commands: + + ```bash + pnpm test -- src/components/NetworkGraph.test.tsx src/components/NetworkGraph.map-lookup.test.ts + pnpm exec eslint src/components/NetworkGraph.tsx src/components/NetworkGraph.test.tsx src/components/NetworkGraph.map-lookup.test.ts + pnpm typecheck + pnpm build + ``` + +3. Keep the pull request open until the unchanged exact head has terminal-success required checks, all addressed review threads are resolved, and protected-branch review requirements are satisfied without bypass. From b609301994a6071c483233cd9cc993c1c3e16805 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 09:48:08 +0900 Subject: [PATCH 03/16] experiment: add PR-scoped concurrency to Bandit scan --- .github/workflows/bandit.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/bandit.yml b/.github/workflows/bandit.yml index c5c613c08..d0ee88755 100644 --- a/.github/workflows/bandit.yml +++ b/.github/workflows/bandit.yml @@ -10,6 +10,10 @@ on: permissions: contents: read +concurrency: + group: Bandit Security Scan-${{ github.repository }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: security: runs-on: ubuntu-latest From f6ad4dde7666d5c21383ed3fa735c56edca59d31 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 09:52:15 +0900 Subject: [PATCH 04/16] experiment: scope Docker PR validation to job-level concurrency --- .github/workflows/docker-publish.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index fc7058413..9328c1ccb 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -25,6 +25,9 @@ jobs: name: validate ${{ matrix.component }} image if: github.event_name == 'pull_request' runs-on: ubuntu-latest + concurrency: + group: Build and Publish Docker Images-pr-validation-${{ github.repository }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true strategy: fail-fast: false matrix: From 2aa1235caaf7cd894e28ca6c7bcc6b05473d4673 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 09:52:58 +0900 Subject: [PATCH 05/16] experiment: standardize PR concurrency groups to workflow-repository-PR/ref --- .github/workflows/app-ci.yml | 2 +- .github/workflows/dependency-review.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/app-ci.yml b/.github/workflows/app-ci.yml index e8f445748..5219faa64 100644 --- a/.github/workflows/app-ci.yml +++ b/.github/workflows/app-ci.yml @@ -15,7 +15,7 @@ permissions: contents: read concurrency: - group: application-ci-${{ github.event.pull_request.number || github.ref }} + group: ${{ github.workflow }}-${{ github.repository }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index c303d1e61..12918e563 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -13,7 +13,7 @@ permissions: pull-requests: read concurrency: - group: dependency-review-${{ github.event.pull_request.number || github.ref }} + group: ${{ github.workflow }}-${{ github.repository }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: From 10329934768ef309e157e6c49e192e04f86d5170 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 10:08:33 +0900 Subject: [PATCH 06/16] fix(ci): serialize Docker tag publishes per ref, never cancel releases --- .github/workflows/docker-publish.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 9328c1ccb..7465c87d4 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -178,6 +178,9 @@ jobs: name: publish ${{ matrix.component }} image if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') runs-on: ubuntu-latest + concurrency: + group: Build and Publish Docker Images-publish-${{ github.repository }}-${{ github.ref }} + cancel-in-progress: false permissions: contents: read packages: write From 22f3532f36b2d7a78933122bc993feee9166cc08 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 10:11:05 +0900 Subject: [PATCH 07/16] fix(ci): scope Docker PR validation concurrency per matrix component --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 7465c87d4..b510dec70 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -26,7 +26,7 @@ jobs: if: github.event_name == 'pull_request' runs-on: ubuntu-latest concurrency: - group: Build and Publish Docker Images-pr-validation-${{ github.repository }}-${{ github.event.pull_request.number || github.ref }} + group: Build and Publish Docker Images-pr-validation-${{ github.repository }}-${{ github.event.pull_request.number || github.ref }}-${{ matrix.component }} cancel-in-progress: true strategy: fail-fast: false From 562338ec6c51c0b50044ad3f9ceccd303aaaa1cd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 10:34:40 +0900 Subject: [PATCH 08/16] fix(ci): isolate matrix image publication groups --- .github/workflows/docker-publish.yml | 2 +- AGENTS.md | 4 ++++ backend/tests/test_release_governance.py | 10 ++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index b510dec70..b8b8b8882 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -179,7 +179,7 @@ jobs: if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') runs-on: ubuntu-latest concurrency: - group: Build and Publish Docker Images-publish-${{ github.repository }}-${{ github.ref }} + group: Build and Publish Docker Images-publish-${{ github.repository }}-${{ github.ref }}-${{ matrix.component }} cancel-in-progress: false permissions: contents: read diff --git a/AGENTS.md b/AGENTS.md index 9104dd1f4..8e7eca308 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -110,6 +110,10 @@ in this repo. ## Release governance defaults +- Matrix release workflows must include the matrix component in both validation + and publication concurrency groups. Keep PR validation cancellable only + within the same workflow, repository, PR, and component; keep publication + non-cancellable while allowing independent components to publish in parallel. - GitHub Actions used by governed workflows must be pinned to full commit SHAs with a trailing version comment, for example `# v6`; major-only refs such as `@v6` are not allowed in release or security workflows. diff --git a/backend/tests/test_release_governance.py b/backend/tests/test_release_governance.py index a23c70746..172950818 100644 --- a/backend/tests/test_release_governance.py +++ b/backend/tests/test_release_governance.py @@ -716,6 +716,16 @@ def test_docker_publish_validates_pr_images_and_publishes_semver_images_only_on_ assert workflow.count("image: naruon") == 2 assert "push: false" in workflow assert "push: true" in workflow + assert ( + "Build and Publish Docker Images-pr-validation-${{ github.repository }}-" + "${{ github.event.pull_request.number || github.ref }}-${{ matrix.component }}" + in workflow + ) + assert ( + "Build and Publish Docker Images-publish-${{ github.repository }}-" + "${{ github.ref }}-${{ matrix.component }}" in workflow + ) + assert workflow.count("cancel-in-progress: false") == 1 assert workflow.count("base_dockerfile: Dockerfile") == 4 assert workflow.count("base_dockerfile: frontend/Dockerfile") == 2 assert workflow.count('base_digest="${base_reference##*@}"') == 2 From 12d8cac36562b27c9f23e10eb4e75fbf6535e9be Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 10:46:45 +0900 Subject: [PATCH 09/16] fix(ci): queue release image publishes without cross-component eviction --- .github/workflows/docker-publish.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index f72a542be..65845ff23 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -183,6 +183,10 @@ jobs: name: publish ${{ matrix.component }} image if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') runs-on: ubuntu-latest + concurrency: + group: Build and Publish Docker Images-publish-${{ github.repository }}-${{ github.ref }}-${{ matrix.component }} + queue: max + cancel-in-progress: false permissions: contents: read packages: write From 86ea64021487c6b2c9897832d5645db7fbe3b7d7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 14:47:05 +0900 Subject: [PATCH 10/16] test(ci): require whole-release image serialization --- .../tests/test_docker_workflow_concurrency.py | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/backend/tests/test_docker_workflow_concurrency.py b/backend/tests/test_docker_workflow_concurrency.py index 4ef4274ac..e6350f66a 100644 --- a/backend/tests/test_docker_workflow_concurrency.py +++ b/backend/tests/test_docker_workflow_concurrency.py @@ -30,6 +30,19 @@ def test_docker_pr_concurrency_isolates_reruns_from_first_attempts() -> None: def test_docker_release_publication_queues_each_component_per_ref() -> None: """Serialize same-ref image publication without evicting pending release jobs.""" + release_workflow = ( + REPO_ROOT / ".github/workflows/docker-release-images.yml" + ).read_text(encoding="utf-8") + publish_section = release_workflow.split("jobs:\n", 1)[1] + + assert "workflow_call:" in release_workflow + assert "matrix.component" in publish_section + assert "push: true" in publish_section + assert "sbom: true" in publish_section + + +def test_docker_release_publication_serializes_whole_image_set_per_ref() -> None: + """Hold one same-ref release lock until every component publication completes.""" workflow = (REPO_ROOT / ".github/workflows/docker-publish.yml").read_text( encoding="utf-8" ) @@ -37,15 +50,12 @@ def test_docker_release_publication_queues_each_component_per_ref() -> None: "\n deploy_preflight:", 1 )[0] expected_group = ( - "group: Build and Publish Docker Images-publish-${{ github.repository }}-" - "${{ github.ref }}-${{ matrix.component }}" - ) - bare_group = ( - "group: Build and Publish Docker Images-publish-${{ github.repository }}-" - "${{ github.ref }}" + "group: Build and Publish Docker Images-publish-set-" + "${{ github.repository }}-${{ github.ref }}" ) + assert "uses: ./.github/workflows/docker-release-images.yml" in publish_section assert expected_group in publish_section - assert bare_group not in publish_section.splitlines() assert "queue: max" in publish_section assert "cancel-in-progress: false" in publish_section + assert "matrix.component" not in publish_section From c8b565ec7b38cd84bf980be9a6672fafe3bd126c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 14:48:39 +0900 Subject: [PATCH 11/16] fix(ci): serialize whole release image sets --- .github/workflows/docker-publish.yml | 197 +------------------ .github/workflows/docker-release-images.yml | 205 ++++++++++++++++++++ 2 files changed, 210 insertions(+), 192 deletions(-) create mode 100644 .github/workflows/docker-release-images.yml diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 65845ff23..c80cb5481 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -17,7 +17,8 @@ concurrency: # First-attempt PR validations share a generation key so a newer event # supersedes only another first attempt. Manual reruns add their stable # run_id and therefore cannot cancel, or be cancelled by, a newer PR event. - # Tag publication is unique per run and cancel-in-progress remains false. + # Tag runs remain unique here because release-set serialization is held by + # the reusable-workflow caller job for the full publication duration. group: docker-publish-${{ github.repository }}-${{ github.event.pull_request.number || github.ref }}-${{ github.event_name == 'pull_request' && github.run_attempt == 1 && 'first-attempt' || github.run_id }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} @@ -180,204 +181,16 @@ jobs: sbom: false publish_images: - name: publish ${{ matrix.component }} image + name: publish release image set if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') - runs-on: ubuntu-latest concurrency: - group: Build and Publish Docker Images-publish-${{ github.repository }}-${{ github.ref }}-${{ matrix.component }} + group: Build and Publish Docker Images-publish-set-${{ github.repository }}-${{ github.ref }} queue: max cancel-in-progress: false permissions: contents: read packages: write - strategy: - fail-fast: false - matrix: - include: - - component: backend - image: ai_email_client-backend - dockerfile: Dockerfile - base_dockerfile: Dockerfile - context: . - build_args: | - BUILDKIT_INLINE_CACHE=1 - - component: naruon - image: naruon - dockerfile: Dockerfile - base_dockerfile: Dockerfile - context: . - build_args: | - BUILDKIT_INLINE_CACHE=1 - - component: frontend - image: ai_email_client-frontend - dockerfile: frontend/Dockerfile - base_dockerfile: frontend/Dockerfile - context: . - build_args: | - BUILDKIT_INLINE_CACHE=1 - steps: - - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 - with: - egress-policy: audit - - - name: Checkout repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Read release version - id: version - run: | - VERSION="$(cat VERSION)" - TAG_VERSION="${GITHUB_REF_NAME#v}" - if [ "$TAG_VERSION" != "$VERSION" ]; then - printf 'Tag %s does not match VERSION %s\n' "$GITHUB_REF_NAME" "$VERSION" >&2 - exit 1 - fi - printf 'version=%s\n' "$VERSION" >> "$GITHUB_OUTPUT" - - - name: Set up QEMU - uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 - - - name: Prepare OCI annotation values - id: oci - env: - BASE_DOCKERFILE: ${{ matrix.base_dockerfile }} - GIT_REF_NAME: ${{ github.ref_name }} - IMAGE_COMPONENT: ${{ matrix.component }} - IMAGE_NAME: ${{ matrix.image }} - REPOSITORY: ${{ github.repository }} - REVISION: ${{ github.sha }} - VERSION_VALUE: ${{ steps.version.outputs.version }} - run: | - version="${VERSION_VALUE:-$(cat VERSION)}" - created="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" - vendor="${REPOSITORY%%/*}" - base_reference="$(awk 'toupper($1) == "FROM" { print $2; exit }' "$BASE_DOCKERFILE")" - if ! printf '%s\n' "$base_reference" | grep -Eq '^[A-Za-z0-9._/-]+:[A-Za-z0-9._-]+@sha256:[0-9a-f]{64}$'; then - printf '::error file=%s,line=1::Expected an exact tagged sha256 base pin; found %s\n' "$BASE_DOCKERFILE" "$base_reference" - exit 1 - fi - base_digest="${base_reference##*@}" - base_repository="${base_reference%@*}" - case "$base_repository" in - */*) base_name="$base_reference" ;; - *) base_name="docker.io/library/$base_reference" ;; - esac - case "$IMAGE_COMPONENT" in - frontend) - title="naruon frontend" - description="Naruon Next.js frontend runtime image" - ;; - backend) - title="naruon backend" - description="Naruon FastAPI backend runtime image" - ;; - *) - title="naruon" - description="Naruon combined FastAPI and Next.js runtime image" - ;; - esac - { - printf 'created=%s\n' "$created" - printf 'authors=%s\n' "Seongho Bae" - printf 'url=https://github.com/%s/pkgs/container/%s\n' "$REPOSITORY" "$IMAGE_NAME" - printf 'documentation=https://github.com/%s#readme\n' "$REPOSITORY" - printf 'source=https://github.com/%s\n' "$REPOSITORY" - printf 'version=%s\n' "$version" - printf 'revision=%s\n' "$REVISION" - printf 'vendor=%s\n' "$vendor" - printf 'licenses=%s\n' "LicenseRef-Naruon-Proprietary" - printf 'ref_name=%s\n' "$GIT_REF_NAME" - printf 'title=%s\n' "$title" - printf 'description=%s\n' "$description" - printf 'base_digest=%s\n' "$base_digest" - printf 'base_name=%s\n' "$base_name" - } >> "$GITHUB_OUTPUT" - - - name: Log in to GHCR - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract Docker metadata - id: meta - uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 - env: - DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index - with: - images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ matrix.image }} - tags: | - type=semver,pattern={{version}} - type=raw,value=${{ steps.version.outputs.version }} - type=raw,value=latest - labels: | - org.opencontainers.image.created=${{ steps.oci.outputs.created }} - org.opencontainers.image.authors=${{ steps.oci.outputs.authors }} - org.opencontainers.image.url=${{ steps.oci.outputs.url }} - org.opencontainers.image.documentation=${{ steps.oci.outputs.documentation }} - org.opencontainers.image.source=${{ steps.oci.outputs.source }} - org.opencontainers.image.version=${{ steps.oci.outputs.version }} - org.opencontainers.image.revision=${{ steps.oci.outputs.revision }} - org.opencontainers.image.vendor=${{ steps.oci.outputs.vendor }} - org.opencontainers.image.licenses=${{ steps.oci.outputs.licenses }} - org.opencontainers.image.ref.name=${{ steps.oci.outputs.ref_name }} - org.opencontainers.image.title=${{ steps.oci.outputs.title }} - org.opencontainers.image.description=${{ steps.oci.outputs.description }} - org.opencontainers.image.base.digest=${{ steps.oci.outputs.base_digest }} - org.opencontainers.image.base.name=${{ steps.oci.outputs.base_name }} - - - name: Build and publish Docker image - id: build - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 - with: - context: ${{ matrix.context }} - file: ${{ matrix.dockerfile }} - platforms: linux/amd64,linux/arm64 - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - annotations: ${{ steps.meta.outputs.annotations }} - build-args: | - ${{ matrix.build_args }} - OCI_IMAGE_CREATED=${{ steps.oci.outputs.created }} - OCI_IMAGE_AUTHORS=${{ steps.oci.outputs.authors }} - OCI_IMAGE_URL=${{ steps.oci.outputs.url }} - OCI_IMAGE_DOCUMENTATION=${{ steps.oci.outputs.documentation }} - OCI_IMAGE_SOURCE=${{ steps.oci.outputs.source }} - OCI_IMAGE_VERSION=${{ steps.oci.outputs.version }} - OCI_IMAGE_REVISION=${{ steps.oci.outputs.revision }} - OCI_IMAGE_VENDOR=${{ steps.oci.outputs.vendor }} - OCI_IMAGE_LICENSES=${{ steps.oci.outputs.licenses }} - OCI_IMAGE_REF_NAME=${{ steps.oci.outputs.ref_name }} - OCI_IMAGE_TITLE=${{ steps.oci.outputs.title }} - OCI_IMAGE_DESCRIPTION=${{ steps.oci.outputs.description }} - OCI_IMAGE_BASE_DIGEST=${{ steps.oci.outputs.base_digest }} - OCI_IMAGE_BASE_NAME=${{ steps.oci.outputs.base_name }} - provenance: true - sbom: true - - - name: Record image digest - env: - IMAGE_COMPONENT: ${{ matrix.component }} - IMAGE_DIGEST: ${{ steps.build.outputs.digest }} - IMAGE_NAME: ${{ matrix.image }} - IMAGE_REGISTRY: ${{ env.REGISTRY }} - IMAGE_VERSION: ${{ steps.version.outputs.version }} - REPO_OWNER: ${{ github.repository_owner }} - run: | - { - printf '### %s image\n' "$IMAGE_COMPONENT" - printf -- '- Image: %s/%s/%s\n' "$IMAGE_REGISTRY" "$REPO_OWNER" "$IMAGE_NAME" - printf -- '- Version: %s\n' "$IMAGE_VERSION" - printf -- '- Digest: %s\n' "$IMAGE_DIGEST" - } >> "$GITHUB_STEP_SUMMARY" + uses: ./.github/workflows/docker-release-images.yml deploy_preflight: name: Detect AKS deploy configuration diff --git a/.github/workflows/docker-release-images.yml b/.github/workflows/docker-release-images.yml new file mode 100644 index 000000000..fe389b2f6 --- /dev/null +++ b/.github/workflows/docker-release-images.yml @@ -0,0 +1,205 @@ +name: Publish Docker Release Images + +on: + workflow_call: + +permissions: + contents: read + packages: write + +env: + REGISTRY: ghcr.io + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + +jobs: + publish_images: + name: publish ${{ matrix.component }} image + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - component: backend + image: ai_email_client-backend + dockerfile: Dockerfile + base_dockerfile: Dockerfile + context: . + build_args: | + BUILDKIT_INLINE_CACHE=1 + - component: naruon + image: naruon + dockerfile: Dockerfile + base_dockerfile: Dockerfile + context: . + build_args: | + BUILDKIT_INLINE_CACHE=1 + - component: frontend + image: ai_email_client-frontend + dockerfile: frontend/Dockerfile + base_dockerfile: frontend/Dockerfile + context: . + build_args: | + BUILDKIT_INLINE_CACHE=1 + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: audit + + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Read release version + id: version + run: | + VERSION="$(cat VERSION)" + TAG_VERSION="${GITHUB_REF_NAME#v}" + if [ "$TAG_VERSION" != "$VERSION" ]; then + printf 'Tag %s does not match VERSION %s\n' "$GITHUB_REF_NAME" "$VERSION" >&2 + exit 1 + fi + printf 'version=%s\n' "$VERSION" >> "$GITHUB_OUTPUT" + + - name: Set up QEMU + uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + + - name: Prepare OCI annotation values + id: oci + env: + BASE_DOCKERFILE: ${{ matrix.base_dockerfile }} + GIT_REF_NAME: ${{ github.ref_name }} + IMAGE_COMPONENT: ${{ matrix.component }} + IMAGE_NAME: ${{ matrix.image }} + REPOSITORY: ${{ github.repository }} + REVISION: ${{ github.sha }} + VERSION_VALUE: ${{ steps.version.outputs.version }} + run: | + version="${VERSION_VALUE:-$(cat VERSION)}" + created="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" + vendor="${REPOSITORY%%/*}" + base_reference="$(awk 'toupper($1) == "FROM" { print $2; exit }' "$BASE_DOCKERFILE")" + if ! printf '%s\n' "$base_reference" | grep -Eq '^[A-Za-z0-9._/-]+:[A-Za-z0-9._-]+@sha256:[0-9a-f]{64}$'; then + printf '::error file=%s,line=1::Expected an exact tagged sha256 base pin; found %s\n' "$BASE_DOCKERFILE" "$base_reference" + exit 1 + fi + base_digest="${base_reference##*@}" + base_repository="${base_reference%@*}" + case "$base_repository" in + */*) base_name="$base_reference" ;; + *) base_name="docker.io/library/$base_reference" ;; + esac + case "$IMAGE_COMPONENT" in + frontend) + title="naruon frontend" + description="Naruon Next.js frontend runtime image" + ;; + backend) + title="naruon backend" + description="Naruon FastAPI backend runtime image" + ;; + *) + title="naruon" + description="Naruon combined FastAPI and Next.js runtime image" + ;; + esac + { + printf 'created=%s\n' "$created" + printf 'authors=%s\n' "Seongho Bae" + printf 'url=https://github.com/%s/pkgs/container/%s\n' "$REPOSITORY" "$IMAGE_NAME" + printf 'documentation=https://github.com/%s#readme\n' "$REPOSITORY" + printf 'source=https://github.com/%s\n' "$REPOSITORY" + printf 'version=%s\n' "$version" + printf 'revision=%s\n' "$REVISION" + printf 'vendor=%s\n' "$vendor" + printf 'licenses=%s\n' "LicenseRef-Naruon-Proprietary" + printf 'ref_name=%s\n' "$GIT_REF_NAME" + printf 'title=%s\n' "$title" + printf 'description=%s\n' "$description" + printf 'base_digest=%s\n' "$base_digest" + printf 'base_name=%s\n' "$base_name" + } >> "$GITHUB_OUTPUT" + + - name: Log in to GHCR + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 + env: + DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index + with: + images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ matrix.image }} + tags: | + type=semver,pattern={{version}} + type=raw,value=${{ steps.version.outputs.version }} + type=raw,value=latest + labels: | + org.opencontainers.image.created=${{ steps.oci.outputs.created }} + org.opencontainers.image.authors=${{ steps.oci.outputs.authors }} + org.opencontainers.image.url=${{ steps.oci.outputs.url }} + org.opencontainers.image.documentation=${{ steps.oci.outputs.documentation }} + org.opencontainers.image.source=${{ steps.oci.outputs.source }} + org.opencontainers.image.version=${{ steps.oci.outputs.version }} + org.opencontainers.image.revision=${{ steps.oci.outputs.revision }} + org.opencontainers.image.vendor=${{ steps.oci.outputs.vendor }} + org.opencontainers.image.licenses=${{ steps.oci.outputs.licenses }} + org.opencontainers.image.ref.name=${{ steps.oci.outputs.ref_name }} + org.opencontainers.image.title=${{ steps.oci.outputs.title }} + org.opencontainers.image.description=${{ steps.oci.outputs.description }} + org.opencontainers.image.base.digest=${{ steps.oci.outputs.base_digest }} + org.opencontainers.image.base.name=${{ steps.oci.outputs.base_name }} + + - name: Build and publish Docker image + id: build + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: ${{ matrix.context }} + file: ${{ matrix.dockerfile }} + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + annotations: ${{ steps.meta.outputs.annotations }} + build-args: | + ${{ matrix.build_args }} + OCI_IMAGE_CREATED=${{ steps.oci.outputs.created }} + OCI_IMAGE_AUTHORS=${{ steps.oci.outputs.authors }} + OCI_IMAGE_URL=${{ steps.oci.outputs.url }} + OCI_IMAGE_DOCUMENTATION=${{ steps.oci.outputs.documentation }} + OCI_IMAGE_SOURCE=${{ steps.oci.outputs.source }} + OCI_IMAGE_VERSION=${{ steps.oci.outputs.version }} + OCI_IMAGE_REVISION=${{ steps.oci.outputs.revision }} + OCI_IMAGE_VENDOR=${{ steps.oci.outputs.vendor }} + OCI_IMAGE_LICENSES=${{ steps.oci.outputs.licenses }} + OCI_IMAGE_REF_NAME=${{ steps.oci.outputs.ref_name }} + OCI_IMAGE_TITLE=${{ steps.oci.outputs.title }} + OCI_IMAGE_DESCRIPTION=${{ steps.oci.outputs.description }} + OCI_IMAGE_BASE_DIGEST=${{ steps.oci.outputs.base_digest }} + OCI_IMAGE_BASE_NAME=${{ steps.oci.outputs.base_name }} + provenance: true + sbom: true + + - name: Record image digest + env: + IMAGE_COMPONENT: ${{ matrix.component }} + IMAGE_DIGEST: ${{ steps.build.outputs.digest }} + IMAGE_NAME: ${{ matrix.image }} + IMAGE_REGISTRY: ${{ env.REGISTRY }} + IMAGE_VERSION: ${{ steps.version.outputs.version }} + REPO_OWNER: ${{ github.repository_owner }} + run: | + { + printf '### %s image\n' "$IMAGE_COMPONENT" + printf -- '- Image: %s/%s/%s\n' "$IMAGE_REGISTRY" "$REPO_OWNER" "$IMAGE_NAME" + printf -- '- Version: %s\n' "$IMAGE_VERSION" + printf -- '- Digest: %s\n' "$IMAGE_DIGEST" + } >> "$GITHUB_STEP_SUMMARY" From e4ab2abd5e1108fbf2acfdd06e6ce42f61d30495 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 14:53:50 +0900 Subject: [PATCH 12/16] test(ci): follow release workflow decomposition --- backend/tests/test_release_governance.py | 53 ++++++++++++++---------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/backend/tests/test_release_governance.py b/backend/tests/test_release_governance.py index 3b2eb610c..15fd37ed8 100644 --- a/backend/tests/test_release_governance.py +++ b/backend/tests/test_release_governance.py @@ -105,18 +105,20 @@ def test_release_version_sources_are_synchronized() -> None: def test_container_images_cover_all_oci_predefined_image_annotations() -> None: root_dockerfile = read_repo_text("Dockerfile") frontend_dockerfile = read_repo_text("frontend/Dockerfile") - docker_publish_workflow = read_repo_text(".github/workflows/docker-publish.yml") + docker_release_workflow = read_repo_text( + ".github/workflows/docker-release-images.yml" + ) for annotation_key in OCI_PREDEFINED_IMAGE_ANNOTATION_KEYS: assert annotation_key in root_dockerfile assert annotation_key in frontend_dockerfile - assert annotation_key in docker_publish_workflow + assert annotation_key in docker_release_workflow assert ( - "DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index" in docker_publish_workflow + "DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index" in docker_release_workflow ) assert ( - "annotations: ${{ steps.meta.outputs.annotations }}" in docker_publish_workflow + "annotations: ${{ steps.meta.outputs.annotations }}" in docker_release_workflow ) assert_oci_metadata_matches_first_base(root_dockerfile) assert_oci_metadata_matches_first_base(frontend_dockerfile) @@ -385,6 +387,7 @@ def test_stepsecurity_remediation_adds_pinned_audit_hardening() -> None: ".github/workflows/app-ci.yml", ".github/workflows/bandit.yml", ".github/workflows/docker-publish.yml", + ".github/workflows/docker-release-images.yml", ".github/workflows/pr-governance.yml", ] @@ -674,32 +677,35 @@ def test_docker_publish_validates_pr_images_and_publishes_semver_images_only_on_ None ): workflow = read_repo_text(".github/workflows/docker-publish.yml") + release_workflow = read_repo_text(".github/workflows/docker-release-images.yml") + combined_workflows = f"{workflow}\n{release_workflow}" assert "pull_request:" in workflow assert "push:" in workflow - assert "FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true" in workflow + assert "workflow_call:" in release_workflow + assert "FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true" in combined_workflows assert ( - workflow.count( + combined_workflows.count( "docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0" ) == 2 ) assert ( - workflow.count( + combined_workflows.count( "docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0" ) == 2 ) assert ( "docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0" - in workflow + in release_workflow ) assert ( "docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0" - in workflow + in release_workflow ) assert ( - workflow.count( + combined_workflows.count( "docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0" ) == 2 @@ -711,26 +717,27 @@ def test_docker_publish_validates_pr_images_and_publishes_semver_images_only_on_ assert "tags:" in push_block assert "branches:" not in push_block assert "develop" in pull_request_block - assert "ai_email_client-backend" in workflow - assert "ai_email_client-frontend" in workflow - assert workflow.count("image: naruon") == 2 + assert "ai_email_client-backend" in combined_workflows + assert "ai_email_client-frontend" in combined_workflows + assert combined_workflows.count("image: naruon") == 2 assert "push: false" in workflow - assert "push: true" in workflow - assert workflow.count("base_dockerfile: Dockerfile") == 4 - assert workflow.count("base_dockerfile: frontend/Dockerfile") == 2 - assert workflow.count('base_digest="${base_reference##*@}"') == 2 - assert workflow.count('base_name="docker.io/library/$base_reference"') == 2 + assert "push: true" in release_workflow + assert combined_workflows.count("base_dockerfile: Dockerfile") == 4 + assert combined_workflows.count("base_dockerfile: frontend/Dockerfile") == 2 + assert combined_workflows.count('base_digest="${base_reference##*@}"') == 2 + assert combined_workflows.count('base_name="docker.io/library/$base_reference"') == 2 assert "Resolve pinned Ollama base manifest" in workflow assert "docker buildx imagetools inspect" in workflow assert "Platform:[[:space:]]+${platform}[[:space:]]*$" in workflow assert "Pinned Ollama manifest is missing %s" in workflow assert "linux/amd64 linux/arm64" in workflow - assert "sha256:44dd04494ee8f3b538294360e7c4b3acb87c8268e4d0a4828a6500b1eff50061" not in workflow - assert "sha256:191ef878ecb351d68b78219593de18bd8942afd59af59f29960dc4b24805a3f1" not in workflow + assert "sha256:44dd04494ee8f3b538294360e7c4b3acb87c8268e4d0a4828a6500b1eff50061" not in combined_workflows + assert "sha256:191ef878ecb351d68b78219593de18bd8942afd59af59f29960dc4b24805a3f1" not in combined_workflows assert "sbom: false" in workflow - assert workflow.count("sbom: true") == 1 - assert "type=semver" in workflow - assert "type=ref,event=branch" not in workflow + assert release_workflow.count("sbom: true") == 1 + assert "type=semver" in release_workflow + assert "type=ref,event=branch" not in release_workflow + assert "uses: ./.github/workflows/docker-release-images.yml" in workflow assert "deploy_preflight:" in workflow assert "AKS_KUBECONFIG_CONTENT: ${{ secrets.AKS_KUBECONFIG }}" in workflow assert "configured=false" in workflow From 2e09118db01af40264ab2b6bd76734ea0100e961 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 15:02:13 +0900 Subject: [PATCH 13/16] docs(ci): record release-set serialization decision --- .../tests/test_docker_workflow_concurrency.py | 28 ++++++- ...whole-release-publication-serialization.md | 79 +++++++++++++++++++ docs/adr/README.md | 3 +- .../release-deployment-architecture.md | 73 +++++++++++++---- 4 files changed, 167 insertions(+), 16 deletions(-) create mode 100644 docs/adr/0005-whole-release-publication-serialization.md diff --git a/backend/tests/test_docker_workflow_concurrency.py b/backend/tests/test_docker_workflow_concurrency.py index e6350f66a..83d825cad 100644 --- a/backend/tests/test_docker_workflow_concurrency.py +++ b/backend/tests/test_docker_workflow_concurrency.py @@ -29,7 +29,7 @@ def test_docker_pr_concurrency_isolates_reruns_from_first_attempts() -> None: def test_docker_release_publication_queues_each_component_per_ref() -> None: - """Serialize same-ref image publication without evicting pending release jobs.""" + """Keep all image components inside the reusable release boundary.""" release_workflow = ( REPO_ROOT / ".github/workflows/docker-release-images.yml" ).read_text(encoding="utf-8") @@ -39,6 +39,9 @@ def test_docker_release_publication_queues_each_component_per_ref() -> None: assert "matrix.component" in publish_section assert "push: true" in publish_section assert "sbom: true" in publish_section + assert "provenance: true" in publish_section + assert "packages: write" in release_workflow + assert "password: ${{ secrets.GITHUB_TOKEN }}" in publish_section def test_docker_release_publication_serializes_whole_image_set_per_ref() -> None: @@ -58,4 +61,27 @@ def test_docker_release_publication_serializes_whole_image_set_per_ref() -> None assert expected_group in publish_section assert "queue: max" in publish_section assert "cancel-in-progress: false" in publish_section + assert "packages: write" in publish_section assert "matrix.component" not in publish_section + + +def test_release_serialization_decision_and_operability_docs_match_active_pr() -> None: + """Keep release-set semantics documented without claiming protected acceptance.""" + adr = ( + REPO_ROOT / "docs/adr/0005-whole-release-publication-serialization.md" + ).read_text(encoding="utf-8") + adr_index = (REPO_ROOT / "docs/adr/README.md").read_text(encoding="utf-8") + operations = ( + REPO_ROOT / "docs/operations/release-deployment-architecture.md" + ).read_text(encoding="utf-8") + + assert "**Status:** Proposed" in adr + assert "PR #1621; not protected-branch authority" in adr + assert "queue: max" in adr + assert "cancel-in-progress: false" in adr + assert "docker-release-images.yml" in adr + assert "ADR-0005" in adr_index + assert "PR #1621 `ACTIVE-PR`; no protected-release acceptance yet" in adr_index + assert "active-PR evidence, not protected-branch authority" in operations + assert "docker-release-images.yml" in operations + assert "A failed image publication blocks deployment" in operations diff --git a/docs/adr/0005-whole-release-publication-serialization.md b/docs/adr/0005-whole-release-publication-serialization.md new file mode 100644 index 000000000..2537aa088 --- /dev/null +++ b/docs/adr/0005-whole-release-publication-serialization.md @@ -0,0 +1,79 @@ +# ADR-0005: Serialize Docker release image sets at the reusable-workflow caller + +- **Status:** Proposed +- **Date:** 2026-09-09 +- **Owner:** Naruon release publication +- **Implementation:** PR #1621; not protected-branch authority until normally merged + +## Problem + +Naruon publishes backend, combined `naruon`, and frontend container images from one version tag. Per-component concurrency prevents two backend jobs, two frontend jobs, or two combined-image jobs for the same ref from evicting one another, but it does not make the three-image release a serialized set. Two workflow runs for the same tag/ref can otherwise interleave their component publications. A downstream deployment that depends only on completion of its own matrix can then run while another same-ref publication is also mutating release tags. + +The release boundary therefore needs one lock whose lifetime covers all three image publications while preserving parallelism inside a single release. + +## Constraints + +- PR image validation keeps #1592's first-attempt cancellation identity; manual reruns must not be cancelled by a newer first-attempt event. +- Tag publication must never cancel an in-progress release for the same repository/ref. +- A queued same-ref release must not replace an earlier pending release merely because it arrived later. +- Backend, combined, and frontend images should still build in parallel inside one release. +- Existing tag/`VERSION` equality checks, multi-architecture builds, OCI metadata, SBOM, provenance, digest evidence, and AKS deployment ordering must remain intact. +- The design must use supported GitHub Actions primitives rather than a repository-local lock service or mutable external coordination record. + +## Decision + +Keep `.github/workflows/docker-publish.yml` as the event-facing caller. PR image validation remains there. Move tag image publication into `.github/workflows/docker-release-images.yml` as a local reusable workflow invoked with `workflow_call`. + +The caller `publish_images` job holds one concurrency group for the entire called workflow: + +```yaml +concurrency: + group: Build and Publish Docker Images-publish-set-${{ github.repository }}-${{ github.ref }} + queue: max + cancel-in-progress: false +``` + +The called workflow owns the backend/naruon/frontend matrix and therefore keeps those three builds parallel after the caller acquires the release-set lock. `deploy_preflight` continues to need the caller job, so deployment remains downstream of completion of every matrix child in the called workflow. + +GitHub's current Actions contract documents `queue: max` for workflow/job concurrency, with up to 100 pending jobs or runs in one concurrency group, and disallows combining it with `cancel-in-progress: true`. GitHub also documents `jobs..concurrency` and `jobs..permissions` as supported keywords for jobs that call reusable workflows. + +## Alternatives considered + +### Keep per-component release groups + +Rejected as the complete solution. It prevents a backend publication from evicting another backend publication, but release A and release B can still interleave different components. That is component safety, not release-set serialization. + +### Put one queued group on the entire mixed PR/tag workflow + +Rejected. PR validation intentionally cancels superseded first attempts while tag publication must queue without cancellation. `queue: max` and `cancel-in-progress: true` are incompatible in one concurrency mapping, and weakening #1592's PR-rerun identity would reintroduce a repaired CI invariant. + +### Serialize all three image builds inside one non-matrix job + +Rejected. It would establish a lock but unnecessarily removes safe component parallelism and lengthens release publication without improving the release-set invariant. + +### Use an environment or external lock service + +Rejected for this boundary. Environments introduce deployment/protection semantics not required for image publication, while an external lock adds mutable coordination state and another availability/security dependency when GitHub Actions already provides the required repository/ref queue primitive. + +## Consequences + +- Same-repository/same-ref release executions are serialized at the release-set boundary. +- Backend, combined, and frontend publications remain parallel within one admitted release. +- Up to 100 later same-group jobs/runs may wait; requests beyond the platform bound can be rejected and must not be described as guaranteed delivery. +- Different tag refs remain independent release groups and can run concurrently. +- Release implementation is split across an event-facing caller and a reusable publication workflow, so governance tests and operations documentation must read both files rather than assuming one workflow contains both PR validation and release publication. +- The decision is **Proposed** until the implementation is normally integrated into protected `develop` and exact-head workflow/review evidence is complete. A predecessor run or a source-only test is not release acceptance. + +## Verification and traceability + +- Reality RED: `86ea64021487c6b2c9897832d5645db7fbe3b7d7` — requires one whole-set caller lock and a reusable release boundary. +- Source repair: `c8b565ec7b38cd84bf980be9a6672fafe3bd126c` — introduces the caller lock and reusable release matrix. +- Governance contract repair: `e4ab2abd5e1108fbf2acfdd06e6ce42f61d30495` — moves release-only assertions to the reusable workflow and keeps caller+called composition checks. +- Focused regression: `backend/tests/test_docker_workflow_concurrency.py`. +- Broader release contract: `backend/tests/test_release_governance.py`. +- Operability description: `docs/operations/release-deployment-architecture.md`. +- Primary platform authority: GitHub, *Workflow syntax for GitHub Actions*, `jobs..concurrency`; GitHub, *Reusing workflow configurations*, supported keywords for reusable-workflow caller jobs. Accessed 2026-09-09. + +## Follow-up + +After #1592 and the applicable protected-base security prerequisite land normally, restack/retarget #1621 without dropping this boundary. Require one unchanged exact head with the then-live repository and central required checks terminal-success plus qualifying independent review. The first real tag publication after protected integration must retain digest, SBOM, provenance, and rollback evidence; failure of any component keeps deployment blocked. \ No newline at end of file diff --git a/docs/adr/README.md b/docs/adr/README.md index 4d461fff6..b957996fd 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -13,6 +13,7 @@ govern implementation. | [ADR-0002](0002-fitted-topic-artifact-consumption.md) | Conditionally consume only a versioned fitted topic artifact through a fail-closed adapter | Proposed | Target `PLANNED`; runtime `BLOCKED-UPSTREAM` | | [ADR-0003](0003-separate-topic-measurement-from-agenda-generation.md) | Keep statistical measurement separate from agenda generation | Proposed | Target and future capability `PLANNED`; no implementation authorization | | [ADR-0004](0004-status-weighted-calendar-conflicts.md) | Evaluate CalDAV VEVENT overlaps by occupying status; cancelled does not occupy | Accepted | `ACCEPTED-NARUON-POLICY`; advisory evaluate API only | +| [ADR-0005](0005-whole-release-publication-serialization.md) | Serialize each same-ref Docker release as one image set while retaining component parallelism inside the set | Proposed | PR #1621 `ACTIVE-PR`; no protected-release acceptance yet | The complete topic-intelligence requirements, architecture, contract, UML, conceptual ERD, security, test, and operability graph is indexed at @@ -22,7 +23,7 @@ is the single cross-document list for the planned adapter profile. ## Change rule -Create or update an ADR when a Naruon change adopts or declines an external service contract, introduces a new scientific/statistical inference contract, changes persistence or tenant authority, changes model/credential trust boundaries, or replaces a fail-closed product capability with a different production dependency. A Naruon ADR records Naruon's decision only; it cannot assign authority to, or accept a decision for, another service. +Create or update an ADR when a Naruon change adopts or declines an external service contract, introduces a new scientific/statistical inference contract, changes persistence or tenant authority, changes model/credential trust boundaries, or replaces a fail-closed product capability with a different production dependency. Cross-cutting release, provenance, or deployment ordering decisions that can change the meaning of a published version also require an ADR. A Naruon ADR records Naruon's decision only; it cannot assign authority to, or accept a decision for, another service. Every implementing PR must keep the corresponding source, tests, doctoring, architecture/operability contract, and CHANGELOG maturity truthful. An active PR, diff --git a/docs/operations/release-deployment-architecture.md b/docs/operations/release-deployment-architecture.md index 6feffcb61..0797385c8 100644 --- a/docs/operations/release-deployment-architecture.md +++ b/docs/operations/release-deployment-architecture.md @@ -3,31 +3,76 @@ ## 확인된 사실 / Confirmed - `ARCHITECTURE.md` defines the current runtime as Next.js frontend → FastAPI - backend → PostgreSQL with pgvector, with OpenAI and SMTP used only when - configured. + backend → PostgreSQL with pgvector, with the protected-document LLM provider + description tracked separately as governance drift until its canonical repair + lands. - `docker-compose.yml` is the local development stack for db/backend/frontend. - `.github/workflows/app-ci.yml` runs backend pytest and frontend test/lint/build checks on pull requests without release-branch push duplication. -- `.github/workflows/docker-publish.yml` validates backend/frontend Docker images - for PRs and publishes GHCR images only from `v*` tags whose value matches - `VERSION`. +- `.github/workflows/docker-publish.yml` is the event-facing Docker workflow. It + validates backend/frontend/combined images on supported PR bases and accepts + `v*` tag events only for release publication. +- PR #1621 proposes a release boundary in which the tag caller invokes + `.github/workflows/docker-release-images.yml`; that reusable workflow owns the + backend/naruon/frontend publication matrix, exact tag/`VERSION` equality check, + OCI metadata, multi-architecture GHCR publishing, SBOM, provenance, and digest + evidence. This is **active-PR evidence, not protected-branch authority** until + the stack is normally merged. +- Under the #1621 proposal, the caller `publish_images` job holds one + repository+ref concurrency group with `queue: max` and + `cancel-in-progress: false` for the full reusable-workflow execution. This + serializes same-ref release image sets while leaving the three component builds + parallel inside one admitted release. See + [`ADR-0005`](../adr/0005-whole-release-publication-serialization.md). +- `deploy_preflight` and `deploy_to_aks` depend on completion of the release + publication caller, so an AKS deployment cannot start until the whole called + image matrix succeeds. - `docker-compose.live-e2e.yml` is the live E2E stack: it uses pre-built images, seeds deterministic email data, scales backend replicas, and exposes the stack through nginx at `127.0.0.1:18080`. +## 플랫폼 제약 / Platform constraints + +GitHub Actions currently permits `queue: max` on concurrency groups, allowing up +to 100 pending jobs or workflow runs in one group; additional members can be +rejected when that bound is reached. `queue: max` cannot be combined with +`cancel-in-progress: true`. Reusable-workflow caller jobs support `concurrency` +and `permissions`. These platform facts are part of the release design rather +than an application-level guarantee: Naruon does not promise unbounded release +queueing. + +Primary references, accessed 2026-09-09: + +- GitHub. *Workflow syntax for GitHub Actions* — `jobs..concurrency`. +- GitHub. *Reusing workflow configurations* — supported keywords for jobs that + call reusable workflows. + ## 가설 / Hypothesis -- The first release candidate should use tag `v0.1.0`, then verify backend and - frontend GHCR manifests for `linux/amd64` and `linux/arm64` before production - promotion. +- The next protected release candidate should verify backend, combined `naruon`, + and frontend GHCR manifests for `linux/amd64` and `linux/arm64` before + production promotion. - Deployment promotion should use image digests rather than mutable tags after the tag workflow produces digest evidence. +- The first tag execution after ADR-0005 reaches protected `develop` is required + release evidence for the whole-set serialization path; source regression tests + or predecessor PR checks do not prove an actual publication run. ## 운영 절차 / Operating path -1. Build images locally or in CI from the release branch. -2. Run backend pytest and frontend test/lint/build checks. -3. Run live Docker E2E against built images. -4. Push `v$(cat VERSION)` only after checks and robot-review evidence are current. -5. Record GHCR digest, manifest platforms, and live E2E evidence in the PR or - release evidence comment. +1. Build images locally or in CI from the release branch and run the full + repository security/test gates on one exact head. +2. Run live Docker E2E against the candidate images and preserve the evidence. +3. Confirm the candidate `VERSION`, CHANGELOG, required review state, SBOM / + provenance expectations, and rollback target before creating the tag. +4. Push `v$(cat VERSION)` only after the protected source and review evidence are + current. Do not recreate or move a release tag to manufacture a rerun. +5. Let the release-set caller acquire the same-ref publication lock; backend, + combined, and frontend image jobs may run in parallel only inside that one + admitted set. +6. Require all component publications to succeed before deployment preflight. + A failed image publication blocks deployment rather than publishing a clean + release claim for a partial set. +7. Record GHCR digests, manifest platforms, SBOM/provenance receipts, live E2E + evidence, GitHub Release/tag identity, and rollback instructions in the + release evidence record. From d5b1e09d9c65174981734deb58780a67f6d40c27 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 15:04:18 +0900 Subject: [PATCH 14/16] test(ci): reject prerelease latest publication --- .../tests/test_docker_workflow_concurrency.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/backend/tests/test_docker_workflow_concurrency.py b/backend/tests/test_docker_workflow_concurrency.py index 83d825cad..e401f6b81 100644 --- a/backend/tests/test_docker_workflow_concurrency.py +++ b/backend/tests/test_docker_workflow_concurrency.py @@ -24,7 +24,7 @@ def test_docker_pr_concurrency_isolates_reruns_from_first_attempts() -> None: ) assert expected_group in header - assert bare_group not in header.splitlines() + assert bare_group not in {line.strip() for line in header.splitlines()} assert "cancel-in-progress: ${{ github.event_name == 'pull_request' }}" in header @@ -65,6 +65,20 @@ def test_docker_release_publication_serializes_whole_image_set_per_ref() -> None assert "matrix.component" not in publish_section +def test_release_latest_tag_is_stable_version_only() -> None: + """Fail closed before a prerelease VERSION can mutate the latest image tag.""" + release_workflow = ( + REPO_ROOT / ".github/workflows/docker-release-images.yml" + ).read_text(encoding="utf-8") + + assert 'if ! [[ "$VERSION" =~ ^[0-9]+\\.[0-9]+\\.[0-9]+$ ]]; then' in ( + release_workflow + ) + assert "must be a stable X.Y.Z release" in release_workflow + assert "flavor: |\n latest=false" in release_workflow + assert "type=raw,value=latest" in release_workflow + + def test_release_serialization_decision_and_operability_docs_match_active_pr() -> None: """Keep release-set semantics documented without claiming protected acceptance.""" adr = ( From 0d45b5324e5bd8b708212a9e729460a886230933 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 15:05:03 +0900 Subject: [PATCH 15/16] fix(ci): fail closed on prerelease latest tags --- .github/workflows/docker-release-images.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/docker-release-images.yml b/.github/workflows/docker-release-images.yml index fe389b2f6..4639d3e1e 100644 --- a/.github/workflows/docker-release-images.yml +++ b/.github/workflows/docker-release-images.yml @@ -55,6 +55,10 @@ jobs: id: version run: | VERSION="$(cat VERSION)" + if ! [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + printf 'VERSION %s must be a stable X.Y.Z release before publishing latest\n' "$VERSION" >&2 + exit 1 + fi TAG_VERSION="${GITHUB_REF_NAME#v}" if [ "$TAG_VERSION" != "$VERSION" ]; then printf 'Tag %s does not match VERSION %s\n' "$GITHUB_REF_NAME" "$VERSION" >&2 @@ -138,6 +142,8 @@ jobs: DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index with: images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ matrix.image }} + flavor: | + latest=false tags: | type=semver,pattern={{version}} type=raw,value=${{ steps.version.outputs.version }} From 6e1378c90d331d43e2a7df1dd49ba2e3d6d74841 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 9 Sep 2026 15:06:26 +0900 Subject: [PATCH 16/16] docs(ci): bind stable latest to release contract --- ...whole-release-publication-serialization.md | 20 ++++++++++++++--- .../release-deployment-architecture.md | 22 ++++++++++++++----- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/docs/adr/0005-whole-release-publication-serialization.md b/docs/adr/0005-whole-release-publication-serialization.md index 2537aa088..eb2a700d4 100644 --- a/docs/adr/0005-whole-release-publication-serialization.md +++ b/docs/adr/0005-whole-release-publication-serialization.md @@ -9,7 +9,7 @@ Naruon publishes backend, combined `naruon`, and frontend container images from one version tag. Per-component concurrency prevents two backend jobs, two frontend jobs, or two combined-image jobs for the same ref from evicting one another, but it does not make the three-image release a serialized set. Two workflow runs for the same tag/ref can otherwise interleave their component publications. A downstream deployment that depends only on completion of its own matrix can then run while another same-ref publication is also mutating release tags. -The release boundary therefore needs one lock whose lifetime covers all three image publications while preserving parallelism inside a single release. +The release boundary therefore needs one lock whose lifetime covers all three image publications while preserving parallelism inside a single release. The release workflow also writes the mutable `latest` tag, so a non-stable `VERSION` must fail before metadata or publication instead of moving `latest` to a prerelease image. ## Constraints @@ -17,6 +17,8 @@ The release boundary therefore needs one lock whose lifetime covers all three im - Tag publication must never cancel an in-progress release for the same repository/ref. - A queued same-ref release must not replace an earlier pending release merely because it arrived later. - Backend, combined, and frontend images should still build in parallel inside one release. +- Naruon's governed `VERSION` contract is a stable numeric `X.Y.Z`; prerelease/build suffixes are not release-publication inputs for this path. +- Automatic metadata-action `latest` generation must be disabled so the workflow has one explicit, reviewable `latest` source after stable-version validation. - Existing tag/`VERSION` equality checks, multi-architecture builds, OCI metadata, SBOM, provenance, digest evidence, and AKS deployment ordering must remain intact. - The design must use supported GitHub Actions primitives rather than a repository-local lock service or mutable external coordination record. @@ -35,8 +37,12 @@ concurrency: The called workflow owns the backend/naruon/frontend matrix and therefore keeps those three builds parallel after the caller acquires the release-set lock. `deploy_preflight` continues to need the caller job, so deployment remains downstream of completion of every matrix child in the called workflow. +Before it compares the tag with `VERSION`, the called workflow requires `VERSION` to match stable `X.Y.Z`. Any prerelease/build suffix fails closed before registry login or metadata generation. `docker/metadata-action` uses `flavor.latest=false`; the workflow then declares one explicit raw `latest` tag, which can only execute after the stable-version guard passes. This matches the existing repository version-governance test instead of creating a second prerelease policy in release YAML. + GitHub's current Actions contract documents `queue: max` for workflow/job concurrency, with up to 100 pending jobs or runs in one concurrency group, and disallows combining it with `cancel-in-progress: true`. GitHub also documents `jobs..concurrency` and `jobs..permissions` as supported keywords for jobs that call reusable workflows. +Docker metadata-action v6.2.0 documents `flavor.latest` as the control for automatic latest handling and shows that prerelease tag events can otherwise participate in latest-tag generation depending on tag strategy. Naruon therefore disables automatic handling and keeps its stable-only publication invariant explicit. + ## Alternatives considered ### Keep per-component release groups @@ -51,6 +57,10 @@ Rejected. PR validation intentionally cancels superseded first attempts while ta Rejected. It would establish a lock but unnecessarily removes safe component parallelism and lengthens release publication without improving the release-set invariant. +### Allow prereleases but conditionally omit `latest` + +Rejected for this release path. The repository's governed `VERSION` contract already requires stable `X.Y.Z`. Permitting a second version grammar only in Docker publication would create divergent release authority. A future prerelease channel requires its own explicit version/tag/channel contract rather than an implicit exception here. + ### Use an environment or external lock service Rejected for this boundary. Environments introduce deployment/protection semantics not required for image publication, while an external lock adds mutable coordination state and another availability/security dependency when GitHub Actions already provides the required repository/ref queue primitive. @@ -61,18 +71,22 @@ Rejected for this boundary. Environments introduce deployment/protection semanti - Backend, combined, and frontend publications remain parallel within one admitted release. - Up to 100 later same-group jobs/runs may wait; requests beyond the platform bound can be rejected and must not be described as guaranteed delivery. - Different tag refs remain independent release groups and can run concurrently. +- A prerelease/build-suffixed `VERSION` fails before publication and cannot move `latest` through this workflow. - Release implementation is split across an event-facing caller and a reusable publication workflow, so governance tests and operations documentation must read both files rather than assuming one workflow contains both PR validation and release publication. - The decision is **Proposed** until the implementation is normally integrated into protected `develop` and exact-head workflow/review evidence is complete. A predecessor run or a source-only test is not release acceptance. ## Verification and traceability -- Reality RED: `86ea64021487c6b2c9897832d5645db7fbe3b7d7` — requires one whole-set caller lock and a reusable release boundary. -- Source repair: `c8b565ec7b38cd84bf980be9a6672fafe3bd126c` — introduces the caller lock and reusable release matrix. +- Whole-set reality RED: `86ea64021487c6b2c9897832d5645db7fbe3b7d7` — requires one whole-set caller lock and a reusable release boundary. +- Whole-set source repair: `c8b565ec7b38cd84bf980be9a6672fafe3bd126c` — introduces the caller lock and reusable release matrix. - Governance contract repair: `e4ab2abd5e1108fbf2acfdd06e6ce42f61d30495` — moves release-only assertions to the reusable workflow and keeps caller+called composition checks. +- Release-channel RED: `d5b1e09d9c65174981734deb58780a67f6d40c27` — makes stable-only `latest` publication executable and repairs the previously vacuous bare-group assertion. +- Release-channel source repair: `0d45b5324e5bd8b708212a9e729460a886230933` — rejects non-`X.Y.Z` `VERSION` values and disables metadata-action automatic latest generation. - Focused regression: `backend/tests/test_docker_workflow_concurrency.py`. - Broader release contract: `backend/tests/test_release_governance.py`. - Operability description: `docs/operations/release-deployment-architecture.md`. - Primary platform authority: GitHub, *Workflow syntax for GitHub Actions*, `jobs..concurrency`; GitHub, *Reusing workflow configurations*, supported keywords for reusable-workflow caller jobs. Accessed 2026-09-09. +- Publication metadata authority: Docker, *metadata-action v6.2.0 README*, flavor/latest and semver guidance at exact action commit `dc802804100637a589fabce1cb79ff13a1411302`. Accessed 2026-09-09. ## Follow-up diff --git a/docs/operations/release-deployment-architecture.md b/docs/operations/release-deployment-architecture.md index 0797385c8..39687cca5 100644 --- a/docs/operations/release-deployment-architecture.md +++ b/docs/operations/release-deployment-architecture.md @@ -15,15 +15,19 @@ - PR #1621 proposes a release boundary in which the tag caller invokes `.github/workflows/docker-release-images.yml`; that reusable workflow owns the backend/naruon/frontend publication matrix, exact tag/`VERSION` equality check, - OCI metadata, multi-architecture GHCR publishing, SBOM, provenance, and digest - evidence. This is **active-PR evidence, not protected-branch authority** until - the stack is normally merged. + stable `X.Y.Z` version validation, OCI metadata, multi-architecture GHCR + publishing, SBOM, provenance, and digest evidence. This is **active-PR + evidence, not protected-branch authority** until the stack is normally merged. - Under the #1621 proposal, the caller `publish_images` job holds one repository+ref concurrency group with `queue: max` and `cancel-in-progress: false` for the full reusable-workflow execution. This serializes same-ref release image sets while leaving the three component builds parallel inside one admitted release. See [`ADR-0005`](../adr/0005-whole-release-publication-serialization.md). +- The release workflow disables docker/metadata-action automatic latest handling + and declares `latest` explicitly only after `VERSION` has passed the stable + `X.Y.Z` guard. A prerelease/build-suffixed `VERSION` fails before registry + publication rather than moving `latest` to an unstable image. - `deploy_preflight` and `deploy_to_aks` depend on completion of the release publication caller, so an AKS deployment cannot start until the whole called image matrix succeeds. @@ -41,11 +45,17 @@ and `permissions`. These platform facts are part of the release design rather than an application-level guarantee: Naruon does not promise unbounded release queueing. +Docker metadata-action v6.2.0 uses `flavor.latest` to control automatic latest +handling. Naruon sets `latest=false` there and keeps one explicit raw `latest` +tag after the stable-version guard, avoiding a second implicit latest source. + Primary references, accessed 2026-09-09: - GitHub. *Workflow syntax for GitHub Actions* — `jobs..concurrency`. - GitHub. *Reusing workflow configurations* — supported keywords for jobs that call reusable workflows. +- Docker. *metadata-action v6.2.0 README* — flavor/latest and semver guidance at + exact action commit `dc802804100637a589fabce1cb79ff13a1411302`. ## 가설 / Hypothesis @@ -63,8 +73,10 @@ Primary references, accessed 2026-09-09: 1. Build images locally or in CI from the release branch and run the full repository security/test gates on one exact head. 2. Run live Docker E2E against the candidate images and preserve the evidence. -3. Confirm the candidate `VERSION`, CHANGELOG, required review state, SBOM / - provenance expectations, and rollback target before creating the tag. +3. Confirm that `VERSION` is stable numeric `X.Y.Z`, the planned tag is exactly + `v$(cat VERSION)`, CHANGELOG and required review state are current, and SBOM / + provenance plus rollback expectations are recorded. Do not use this path for + prerelease/build-suffixed versions. 4. Push `v$(cat VERSION)` only after the protected source and review evidence are current. Do not recreate or move a release tag to manufacture a rerun. 5. Let the release-set caller acquire the same-ref publication lock; backend,