From d5a142c5a318f0be2834c54ad5bb966a7dc54f04 Mon Sep 17 00:00:00 2001 From: Lling0000 Date: Tue, 8 Sep 2026 00:31:53 +0800 Subject: [PATCH 1/3] feat: add account-scoped knowledge RAG with source citations --- .env.example | 13 + .gitattributes | 1 + .github/workflows/ci.yml | 8 + README.md | 6 + README_zh.md | 6 + docker-compose.yml | 19 + docs/rag-validation.md | 52 + docs/rag.md | 172 ++ frontend/src/App.tsx | 2 + frontend/src/api/chat.ts | 3 +- frontend/src/api/knowledge.ts | 42 + .../src/components/chat/ChatContainer.tsx | 8 + .../src/components/chat/MessageBubble.tsx | 6 +- .../knowledge/KnowledgeMarkdown.tsx | 57 + .../src/components/project/ProjectTabs.tsx | 1 + frontend/src/hooks/useChat.ts | 9 + frontend/src/i18n/knowledge.ts | 339 ++++ frontend/src/i18n/locales/en.ts | 3 + frontend/src/i18n/locales/es.ts | 3 + frontend/src/i18n/locales/ja.ts | 3 + frontend/src/i18n/locales/ko.ts | 3 + frontend/src/i18n/locales/zh.ts | 3 + frontend/src/pages/KnowledgePage.tsx | 198 +++ frontend/src/pages/ReportsPage.tsx | 2 +- frontend/src/types/index.ts | 8 + pyproject.toml | 3 +- scripts/qdrant-entrypoint.sh | 6 + src/opencmo/background/service.py | 2 +- src/opencmo/background/types.py | 2 +- src/opencmo/background/worker.py | 1 + src/opencmo/rag/__init__.py | 2 + src/opencmo/rag/capacity.py | 160 ++ src/opencmo/rag/config.py | 79 + src/opencmo/rag/evaluate.py | 171 ++ src/opencmo/rag/fetching.py | 52 + src/opencmo/rag/ingestion.py | 415 +++++ src/opencmo/rag/integration.py | 123 ++ src/opencmo/rag/middleware.py | 40 + src/opencmo/rag/parsing.py | 140 ++ src/opencmo/rag/providers.py | 246 +++ src/opencmo/rag/retrieval.py | 238 +++ src/opencmo/rag/schema.py | 70 + src/opencmo/rag/splitting.py | 71 + src/opencmo/rag/store.py | 166 ++ src/opencmo/rag/text.py | 49 + src/opencmo/rag/types.py | 102 ++ src/opencmo/report_pipeline.py | 4 + src/opencmo/reports.py | 19 + src/opencmo/scheduler.py | 2 + src/opencmo/services/blog_generation.py | 12 + src/opencmo/storage/_db.py | 2 + src/opencmo/web/app.py | 11 +- src/opencmo/web/chat_sessions.py | 6 +- src/opencmo/web/routers/chat.py | 54 +- src/opencmo/web/routers/knowledge.py | 271 ++++ src/opencmo/web/routers/tasks.py | 3 + tests/conftest.py | 11 + tests/fixtures/rag-image-only.pdf | Bin 0 -> 1697 bytes tests/fixtures/rag-text.pdf | Bin 0 -> 1863 bytes tests/fixtures/rag_evaluation.json | 1428 +++++++++++++++++ tests/test_background_worker.py | 10 +- tests/test_frontend_public_byok_paths.py | 2 +- tests/test_rag.py | 511 ++++++ tests/test_web.py | 8 +- 64 files changed, 5444 insertions(+), 15 deletions(-) create mode 100644 .gitattributes create mode 100644 docs/rag-validation.md create mode 100644 docs/rag.md create mode 100644 frontend/src/api/knowledge.ts create mode 100644 frontend/src/components/knowledge/KnowledgeMarkdown.tsx create mode 100644 frontend/src/i18n/knowledge.ts create mode 100644 frontend/src/pages/KnowledgePage.tsx create mode 100644 scripts/qdrant-entrypoint.sh create mode 100644 src/opencmo/rag/__init__.py create mode 100644 src/opencmo/rag/capacity.py create mode 100644 src/opencmo/rag/config.py create mode 100644 src/opencmo/rag/evaluate.py create mode 100644 src/opencmo/rag/fetching.py create mode 100644 src/opencmo/rag/ingestion.py create mode 100644 src/opencmo/rag/integration.py create mode 100644 src/opencmo/rag/middleware.py create mode 100644 src/opencmo/rag/parsing.py create mode 100644 src/opencmo/rag/providers.py create mode 100644 src/opencmo/rag/retrieval.py create mode 100644 src/opencmo/rag/schema.py create mode 100644 src/opencmo/rag/splitting.py create mode 100644 src/opencmo/rag/store.py create mode 100644 src/opencmo/rag/text.py create mode 100644 src/opencmo/rag/types.py create mode 100644 src/opencmo/web/routers/knowledge.py create mode 100644 tests/fixtures/rag-image-only.pdf create mode 100644 tests/fixtures/rag-text.pdf create mode 100644 tests/fixtures/rag_evaluation.json create mode 100644 tests/test_rag.py diff --git a/.env.example b/.env.example index 25e8f29..61d14ee 100644 --- a/.env.example +++ b/.env.example @@ -96,3 +96,16 @@ OPENAI_API_KEY=sk-your-api-key-here # OPENCMO_MODEL_DEFAULT=gpt-4o # OPENCMO_MODEL_CMO=gpt-4o # OPENCMO_MODEL_SEO=gpt-4o-mini + +# === Optional: account-scoped knowledge retrieval (RAG) === +# Install .[rag] or .[all]. Enable per account in Knowledge > Models and retrieval. +# Embedding/rerank URLs, model names and keys are saved per account in that UI. +# OPENCMO_RAG_ENABLED=1 # Master switch; 0 disables retrieval for every account +# OPENCMO_QDRANT_URL=http://127.0.0.1:6333 +# OPENCMO_QDRANT_API_KEY= # Shared by the app and the Compose Qdrant service +# OPENCMO_QDRANT_PREFIX=opencmo # Unique for each independent app/database deployment +# OPENCMO_RAG_STORAGE_PATH= # Defaults to knowledge/ next to OPENCMO_DB_PATH +# OPENCMO_RAG_MAX_DOCUMENTS=10000 +# OPENCMO_RAG_MAX_CHUNKS=500000 +# OPENCMO_KNOWLEDGE_CONCURRENCY=1 +# Docker: docker compose --profile rag up -d --build diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..898a073 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +/scripts/qdrant-entrypoint.sh text eol=lf diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6b9cb66..26b20d9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,13 @@ on: jobs: backend: runs-on: ubuntu-latest + services: + qdrant: + image: qdrant/qdrant:v1.19.1 + ports: + - 6333:6333 + env: + QDRANT__TELEMETRY_DISABLED: "true" strategy: matrix: python-version: ["3.11", "3.12"] @@ -29,6 +36,7 @@ jobs: DATAFORSEO_LOGIN: "" DATAFORSEO_PASSWORD: "" XUEQIU_COOKIE: "" + OPENCMO_RAG_TEST_QDRANT: http://127.0.0.1:6333 run: pytest tests/ -v --tb=short frontend: diff --git a/README.md b/README.md index 4950777..82b0dbe 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,12 @@ OpenCMO includes a report system inside each project workspace. Open the **Repor - **Multi-agent pipeline**: human-facing reports use a 6-phase pipeline instead of a single prompt. - **Graceful fallback**: if the deep pipeline fails, OpenCMO falls back to simpler generation paths so reports stay available. +## Knowledge library (RAG) + +Project documents and account-shared cases can now be indexed with recursive parent/child splitting, Qdrant dense retrieval, BM25/title recall, RRF fusion and model reranking. Chat, reports and content drafts share the retrieval service and persist source citations. Supported inputs include historical reports, text PDFs, DOCX, Markdown, TXT and public web pages. + +Enable the optional Qdrant service with `docker compose --profile rag up -d`, then configure your account's embedding and rerank APIs under the project's Knowledge library. See [RAG setup, permissions, APIs and evaluation](docs/rag.md). Real model quality must be evaluated with your configured providers. + ## Quick Start OpenCMO works with OpenAI-compatible APIs, including OpenAI, DeepSeek, NVIDIA NIM, Kimi-compatible gateways, and Ollama. diff --git a/README_zh.md b/README_zh.md index c5db04e..44f152a 100644 --- a/README_zh.md +++ b/README_zh.md @@ -121,6 +121,12 @@ OpenCMO 已经内置正式报告系统。你可以在项目中打开 **Reports** - **多智能体管线**：面向人的报告使用 6 阶段管线，而不是单次 prompt。 - **优雅降级**：深度管线失败时，会自动回退到更简单的生成路径，确保报告始终可用。 +## 知识库与 RAG + +支持把历史报告、客户资料和营销案例导入项目知识库：递归父子切分、Qdrant 语义召回、BM25 与标题召回、RRF 融合、模型重排及可定位原文的引用。聊天、报告和内容草稿共用检索服务；资料默认按项目隔离，可主动共享给同账号其他项目，对外内容另有用途开关。 + +支持文本 PDF、DOCX、Markdown、TXT、粘贴文本和指定网页。使用 `docker compose --profile rag up -d` 启动可选的 Qdrant 服务，再到项目的“知识库 → 模型与检索设置”配置 Embedding 和 Rerank API。详见 [部署、权限、接口与评测说明](docs/rag.md)。真实模型质量需使用实际配置的服务评测。 + ## 快速开始 OpenCMO 兼容 OpenAI 协议 API，包括 OpenAI、DeepSeek、NVIDIA NIM、Kimi 兼容网关、Ollama 等。 diff --git a/docker-compose.yml b/docker-compose.yml index e131d34..1363e71 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,26 @@ services: - opencmo_data:/data env_file: - .env + environment: + OPENCMO_QDRANT_URL: http://qdrant:6333 + OPENCMO_QDRANT_API_KEY: ${OPENCMO_QDRANT_API_KEY:-} + restart: unless-stopped + + qdrant: + image: qdrant/qdrant:v1.19.1 + profiles: [rag] + ports: + - "127.0.0.1:6333:6333" + environment: + OPENCMO_QDRANT_API_KEY: ${OPENCMO_QDRANT_API_KEY:-} + QDRANT__TELEMETRY_DISABLED: "true" + volumes: + - qdrant_data:/qdrant/storage + - ./scripts/qdrant-entrypoint.sh:/qdrant/opencmo-entrypoint.sh:ro + entrypoint: ["/bin/sh", "/qdrant/opencmo-entrypoint.sh"] + command: [] restart: unless-stopped volumes: opencmo_data: + qdrant_data: diff --git a/docs/rag-validation.md b/docs/rag-validation.md new file mode 100644 index 0000000..c2b0062 --- /dev/null +++ b/docs/rag-validation.md @@ -0,0 +1,52 @@ +# RAG validation + +Local verification performed on 2026-09-07/08. No production deployment was performed. + +## Automated checks + +- Full backend suite: **700 passed, 3 skipped**. The skipped cases are legacy Jinja2 routes. +- RAG suite: **37 passed**, including an actual Qdrant server test. +- Ruff: passed. +- Frontend TypeScript and Vite production build: passed; the existing large-bundle advisory remains. +- Browser: desktop and 390-pixel mobile layouts, document upload, retrieval, model configuration, source preview, centered citation dialog and Unicode/emoji source offsets passed with no page errors. + +Tests cover parent/child budgets and exact spans, PDF pages and image-only rejection, DOCX tables, HTML cleanup, URL restrictions, tenant/project isolation, sharing revocation, outbound-use restrictions, tombstones, shadow versions, profile migration failure, rebuilding a missing vector collection, account index leases, orphan reconciliation, persisted citations and non-reuse of revoked evidence. + +The broader suite needed test isolation fixes: asynchronous worker completion now waits for completion instead of a 50 ms sleep; route tests avoid unintended live scans; settings tests restore environment defaults. Windows file reads explicitly use UTF-8. + +Model calls in automated and browser tests are fixtures. An actual Qdrant engine test is not evidence that the configured embedding model meets semantic quality targets. + +## Capacity result + +An isolated native Qdrant 1.19.1 instance and SQLite were tested with synthetic documents/vectors. + +| Measurement | Result | +| --- | ---: | +| Documents | 10,000 | +| Child passages | 500,000 | +| Vector dimensions | 1,024 | +| Concurrent retrievals | 5 | +| Sample retrievals | 50 | +| Indexing time | 873.7 seconds | +| Ingestion throughput | 572.3 child passages/second | +| Retrieval P50 | 62 ms | +| Retrieval P95 | 2,828 ms | +| SQLite size | 3,867,803,648 bytes | +| Qdrant status | green | +| Indexed vectors | 499,200; remaining points are searchable in the smaller segment | +| Qdrant peak working set | Approximately 17.1 GiB | + +Environment: Windows host with 32 logical processors and approximately 31.3 GiB physical memory; Qdrant search threads configured to 4. The same Qdrant process also hosted small UI/evaluation collections, so the process memory figure is not a per-collection measurement. + +Latency combines dense, BM25 and title/entity retrieval, including cold initialization in the sample. It excludes embedding, query-rewrite and rerank API latency. It is not an end-to-end chat latency claim. + +An earlier Docker-backed run stopped at approximately 425,000 passages with an underlying storage I/O error. That run is not reported as a capacity pass. The completed run used a separate test disk with adequate free space. The capacity tool now checks estimated local disk requirements and documents the need to check the vector-storage disk separately. + +## Evaluation corpus + +The committed corpus contains **40 fictional documents and 120 Chinese/English questions**, split into development and holdout sets. The ablation runner completed BM25, dense, fusion and fusion-plus-rerank runs in synthetic wiring mode. + +These synthetic scores are deliberately not used to claim Recall@20 or nDCG@10 acceptance for real models. No actual Embedding/Rerank credentials were available during this implementation. **Real model quality remains unverified.** + +To evaluate it, configure the provider credentials and run live evaluation as described in [the RAG guide](rag.md). Live evaluation records degradation and fails its quality gate when holdout thresholds are not met. + diff --git a/docs/rag.md b/docs/rag.md new file mode 100644 index 0000000..9a81930 --- /dev/null +++ b/docs/rag.md @@ -0,0 +1,172 @@ +# Knowledge library and RAG + +OpenCMO can retrieve evidence from project documents, historical reports and account-shared case studies. The same service supplies chat, report sections and promotional content. + +## Start + +Install the optional RAG dependencies (included in the all extra), build the frontend, and run Qdrant: + + pip install -e ".[all]" + cd frontend + npm ci + npm run build + cd .. + docker compose --profile rag up -d qdrant + opencmo-web + +Copy .env.example to .env before using Compose. Alternatively, run the whole application with: + + docker compose --profile rag up -d --build + +The RAG Compose profile adds Qdrant 1.19.1 with a persistent volume. Its host port binds to localhost. Set OPENCMO_QDRANT_API_KEY in .env to configure the same authentication key on both the app and Qdrant. RAG remains disabled per account until enabled in the knowledge settings. Existing installations can run without the RAG profile. + +In a project, open **Knowledge library → Models and retrieval**: + +1. Configure Embedding and Rerank base URLs, model names and keys. +2. Test both connections, enable retrieval, and save. +3. Upload a document, paste text, import a public URL, or import historical reports. +4. Wait until documents show Ready. +5. Ask a question in chat or inspect results in Test retrieval. + +The preset uses https://api.siliconflow.cn/v1 with BAAI/bge-m3 and BAAI/bge-reranker-v2-m3. The protocol is OpenAI-compatible embeddings and a rerank endpoint returning results containing index and relevance_score. Model credentials are stored per account, not in the global model registry. Tenant-configured URLs must be HTTPS base URLs without user info, query parameters or fragments. + +A connection test makes model API requests. Saving an embedding/splitting change queues a rebuild. The old index remains active until the replacement is ready. + +## Scope and use + +- Documents are private to their project by default. +- Account sharing explicitly makes a source available to other projects in the same account. It never shares with another account. +- Public-facing use is a separate flag. Content generation only retrieves sources with that flag enabled. +- Private documents become unavailable when their project is deleted. Account-shared documents can survive their originating project. +- Revocation/deletion affects new retrieval and source access immediately. Existing generated conversation text is retained; revoked evidence is not reused as new chat context. +- AI-generated reports are labelled as generated analysis. They are not presented as independently verified observations. + +Chat modes: + +| Mode | Behavior | +| --- | --- | +| auto | Retrieve when enabled; clearly distinguish general advice when sources are missing. | +| only | Answer from knowledge evidence; no agent tools/handoffs; return an insufficient-evidence response when no passages are found. | +| off | Use the existing chat behavior without knowledge retrieval. | + +Quoted passages carry citation links. Clicking a link displays the authorized original text with the passage highlighted. Citation provenance persists across conversation reloads and report/draft storage. Source offsets refer to Unicode code points in normalized text; the frontend converts them for JavaScript string indexing. + +## Ingestion and indexing + +Supported inputs: text PDF, DOCX, Markdown, TXT, pasted text and individual public web pages. The file limit is 25 MiB; PDFs are limited to 500 pages. Image-only PDFs require OCR and are rejected with a clear status. OCR, PPT and Excel are not included. + +Original files, normalized text and version records are stored separately. PDF page numbers, heading paths, table context and exact text offsets are retained where available. Public URL imports validate resolved IP addresses and every redirect; fetching does not load arbitrary browser subresources. + +The splitter uses document structure followed by recursive paragraph/line/sentence/word boundaries: + +| Setting | Default | +| --- | ---: | +| Parent passage budget | 1,536 tokens | +| Child passage budget | 384 tokens | +| Child overlap | 64 tokens | +| Maximum parents returned | 6 | +| Knowledge context budget | 6,000 tokens | + +Token budgets use cl100k_base consistently. Provider input limits are checked separately. Children are indexed for retrieval; parents supply surrounding context. Source quotes always refer to original normalized spans, not synthetic heading prefixes. + +SQLite is authoritative for document ownership, active versions, passages and citations. FTS5 indexes normalized English terms and Jieba-segmented Chinese text. Qdrant stores dense child vectors with indexed account, project, scope, purpose and date/type metadata. + +Indexing runs in the existing worker with progress events, cancellation and retries. Account leases serialize index changes across worker processes. Duplicate uploads are idempotent. Manual document reindexing creates a shadow version so failed updates keep the old source available. + +Completed Human Reports are indexed automatically after RAG is enabled. Import historical reports queues their indexing; that batch finishing does not mean every document has finished embedding. Agent Briefs are not separately indexed. + +## Retrieval + +The service preserves the original query and can resolve references from up to three recent conversation rounds. It adds at most two alternative queries. Failure to rewrite falls back to the original question. + +| Lane | Default limit | Implementation | +| --- | ---: | --- | +| Dense | 30 | Qdrant; query variants merged within the lane | +| Lexical | 30 | SQLite FTS5/BM25 over passage text | +| Title/entity | 20 | Indexed matches restricted to titles, heading paths and tags | + +All lanes apply scope and use restrictions. SQL revalidates candidate ownership and active versions after remote calls. Title/entity retrieval uses an inverted index instead of scanning every passage. + +Candidates are deduplicated, fused with equal-weight reciprocal rank fusion, and sent to reranking: + + RRF score = sum(1 / (60 + rank)), with ranks starting at 1 + 50 fused candidates → rerank → up to 8 qualifying passages → up to 6 parents + +The initial rerank threshold is 0.2 and must be calibrated for the configured model. The default retrieval deadline is 10 seconds, with at most 2 seconds for rewriting and 4 seconds for reranking. Failed lanes remain visible in diagnostics; rerank failure uses the fused ranking and marks the result degraded. + +Chat and report integrations keep retrieved text separate from trusted instructions. Direct quotes and citation identifiers are checked after marketing rewriting. This structural validation does not replace human review of whether an inference is actually supported. + +## APIs + +All endpoints use existing authentication. Account identity is derived by the server. + +| Endpoint | Purpose | +| --- | --- | +| GET/POST /api/v1/knowledge/documents | List visible sources; upload multipart or import JSON text/URL | +| GET/PATCH/DELETE /api/v1/knowledge/documents/{id} | Inspect versions, update metadata or revoke a source | +| GET /api/v1/knowledge/documents/{id}/versions/{version} | Normalized source and chunk preview | +| GET /api/v1/knowledge/documents/{id}/versions/{version}/original | Authorized original-file download | +| POST /api/v1/knowledge/documents/{id}/reindex | Reparse/reindex through a new version | +| POST /api/v1/projects/{id}/knowledge/backfill-reports | Queue historical report imports | +| POST /api/v1/projects/{id}/knowledge/search | Retrieval diagnostics, ranked candidates and citations | +| GET /api/v1/knowledge/citations/{id} | Authorized citation and original text | +| GET/POST /api/v1/knowledge/settings | Read masked configuration or update account settings | +| POST /api/v1/knowledge/settings/test | Test embedding and rerank protocols | +| POST /api/v1/knowledge/rebuild | Queue a complete account rebuild | +| Existing task APIs | Inspect/cancel indexing jobs and stream progress | + +The chat endpoint accepts optional rag_mode. Its SSE stream adds retrieval progress and final citations, retrieval_id and rag_status. Existing requests remain valid. + +## Operations + +Environment controls: + +| Variable | Default | +| --- | --- | +| OPENCMO_RAG_ENABLED | 1: master switch; each account still opts in | +| OPENCMO_QDRANT_URL | http://127.0.0.1:6333; Compose uses http://qdrant:6333 | +| OPENCMO_QDRANT_API_KEY | Unset | +| OPENCMO_QDRANT_PREFIX | opencmo; use a unique prefix for each independent app/database deployment | +| OPENCMO_RAG_STORAGE_PATH | knowledge/ next to the SQLite database | +| OPENCMO_RAG_MAX_DOCUMENTS | 10,000 per account | +| OPENCMO_RAG_MAX_CHUNKS | 500,000 active child passages per account | +| OPENCMO_KNOWLEDGE_CONCURRENCY | 1 per worker; account leases also serialize across workers | + +Every six hours, the enabled scheduler queues cleanup for enabled accounts: deleted sources, stale incomplete generations and orphan points in known account collections. It also repairs missed report-enqueue events after the account enabled RAG. If scheduling is disabled, cleanup can be invoked through the ingestion maintenance function. + +Back up the SQLite database consistently (SQLite online backup or stop the app first), original-file directory and operator configuration. The SQLite backup includes account credentials and needs the same access protection as the live database. Qdrant snapshots are optional accelerators; vectors can be reconstructed from originals. After restoring a database without Qdrant data, use Rebuild the entire library. + +Query traces retain timing, ranks, warnings and query hashes rather than full query text. Embedding/rerank events store reported token usage when supplied and a separate local input-token estimate; these are not a billing ledger. Knowledge tasks do not serialize provider keys into their payloads. + +For 500,000 passages, reserve space on both the database disk and the vector-storage disk. The measured test run used about 3.60 GiB for SQLite and peaked at about 17.1 GiB Qdrant working set during indexing; allow memory and disk headroom beyond the steady-state footprint. See the validation report for the exact test scope. + +## Evaluation + +The committed fictional corpus contains 40 documents and 120 Chinese/English questions, split into development and holdout sets. + +Validate its structure: + + python -m opencmo.rag.evaluate --validate + +Run wiring-only ablations on an isolated Qdrant and a NEW database: + + python -m opencmo.rag.evaluate --synthetic --db output/eval-new.db --output output/evaluation.json + +For a real model evaluation, configure RAG_EMBEDDING_API_KEY and RAG_RERANK_API_KEY in the process environment. Optional RAG_EMBEDDING_BASE_URL, RAG_EMBEDDING_MODEL, RAG_RERANK_BASE_URL and RAG_RERANK_MODEL override the preset. Configure the normal chat model too if evaluating query rewriting. + + python -m opencmo.rag.evaluate --live --db output/eval-live-new.db --output output/evaluation-live.json + +Live mode exits unsuccessfully when the holdout gate is not met: Recall@20 ≥ 0.90, nDCG@10 ≥ 0.80, exact quote locations, and no degraded holdout requests. Synthetic execution is explicitly labelled and cannot pass that quality gate. Unanswerable questions also need answer-level review. + +Capacity testing generates synthetic vectors/documents and calls real SQLite/Qdrant, without model APIs: + + python -m opencmo.rag.capacity --db /large-test-disk/capacity-new.db --qdrant-url http://127.0.0.1:6333 --grpc-port 6334 --documents 10000 --chunks-per-doc 50 --dimensions 1024 --queries 50 --concurrency 5 --output output/capacity.json + +Use an isolated test server. The command checks estimated local free space; separately verify the Qdrant storage disk. It never overwrites an existing database. Model API latency is not included in its results. + +Run API/parser/retrieval tests with the optional native-server check: + + OPENCMO_RAG_TEST_QDRANT=http://127.0.0.1:6333 pytest tests/test_rag.py + +The native-server test uses unique collections and deletes only its own collection. Its model calls are still mocked; it verifies the database engine and authorization, not embedding quality. + diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 92de2d1..fc6e448 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -26,6 +26,7 @@ import { BrandKitPage } from "./pages/BrandKitPage"; import { ProjectMonitorsPage } from "./pages/ProjectMonitorsPage"; import { GitHubLeadsPage } from "./pages/GitHubLeadsPage"; import { ContentPage } from "./pages/ContentPage"; +import { KnowledgePage } from "./pages/KnowledgePage"; import { useAuth } from "./components/auth/useAuth"; // Heavy pages lazy-loaded: Three.js graph, react-markdown reports/chat, recharts performance @@ -152,6 +153,7 @@ function AppRoutes() { } /> } /> } /> + } /> } /> { export function getSessionMessages( sessionId: string, -): Promise<{ role: string; content: string }[]> { +): Promise<{ role: string; content: string; citations?: import("./knowledge").KnowledgeCitation[]; retrieval_id?: string; rag_status?: string }[]> { return apiJson(`/chat/sessions/${sessionId}/messages`); } @@ -48,6 +48,7 @@ export async function* streamChat( body: JSON.stringify({ session_id: sessionId, message, + rag_mode: localStorage.getItem("opencmo_rag_mode") || "auto", ...(projectId != null ? { project_id: projectId } : {}), ...(locale ? { locale } : {}), }), diff --git a/frontend/src/api/knowledge.ts b/frontend/src/api/knowledge.ts new file mode 100644 index 0000000..04fa888 --- /dev/null +++ b/frontend/src/api/knowledge.ts @@ -0,0 +1,42 @@ +import { apiJson } from "./client"; + +export interface KnowledgeDocument { + id: string; project_id: number | null; scope: "project" | "account"; + title: string; kind: string; status: string; error: string; external_use: number; + generated: number; version_count: number; created_at: string; updated_at: string; + versions?: { id: string; number: number; text_length: number }[]; +} +export interface KnowledgeCitation { + id: string; label: string; title: string; quote: string; heading: string; + start: number; end: number; page: number | null; generated: boolean; url: string; +} +export interface KnowledgeSearch { + retrieval_id: string; status: string; warnings: string[]; citations: KnowledgeCitation[]; + hits: { title: string; text: string; lanes: string[]; fusion_score: number; rerank_score: number | null }[]; + lane_results: Record; timings: Record; +} +export const listKnowledge = (project: number, offset = 0) => + apiJson(`/knowledge/documents?project_id=${project}&offset=${offset}`); +export const knowledgeDocument = (id: string) => apiJson(`/knowledge/documents/${id}`); +export const importKnowledge = (body: FormData | Record) => + apiJson<{ task_id?: string; document_id: string; duplicate: boolean }>("/knowledge/documents", + { method: "POST", body: body instanceof FormData ? body : JSON.stringify(body) }); +export const updateKnowledge = (id: string, body: Record) => + apiJson(`/knowledge/documents/${id}`, { method: "PATCH", body: JSON.stringify(body) }); +export const deleteKnowledge = (id: string) => apiJson(`/knowledge/documents/${id}`, { method: "DELETE" }); +export const reindexKnowledge = (id: string) => apiJson(`/knowledge/documents/${id}/reindex`, { method: "POST" }); +export const backfillKnowledge = (project: number) => + apiJson(`/projects/${project}/knowledge/backfill-reports`, { method: "POST" }); +export const searchKnowledge = (project: number, query: string, purpose: string) => + apiJson(`/projects/${project}/knowledge/search`, { method: "POST", body: JSON.stringify({ query, purpose }) }); +export const getKnowledgeSettings = () => apiJson>("/knowledge/settings"); +export const rebuildKnowledge = () => apiJson("/knowledge/rebuild", { method: "POST" }); +export const saveKnowledgeSettings = (body: Record) => + apiJson>("/knowledge/settings", { method: "POST", body: JSON.stringify(body) }); +export const testKnowledgeSettings = (body: Record) => + apiJson<{ ok: boolean; dimensions: number }>("/knowledge/settings/test", { method: "POST", body: JSON.stringify(body) }); +export const getCitation = (id: string) => + apiJson(`/knowledge/citations/${id}`); +export const getKnowledgeVersion = (id: string, version: string) => + apiJson<{ text: string; chunks: { id: string; level: string; start_offset: number; end_offset: number }[] }>( + `/knowledge/documents/${id}/versions/${version}`); diff --git a/frontend/src/components/chat/ChatContainer.tsx b/frontend/src/components/chat/ChatContainer.tsx index 84a5639..a2a4009 100644 --- a/frontend/src/components/chat/ChatContainer.tsx +++ b/frontend/src/components/chat/ChatContainer.tsx @@ -78,10 +78,18 @@ export function ChatContainer({ }) { const { t } = useI18n(); const projectName = projectContext?.project.brand_name ?? null; + const [ragMode, setRagMode] = useState(() => localStorage.getItem("opencmo_rag_mode") || "auto"); return (
{/* Agent badge */} +

{t("chat.agent", { name: currentAgent })} diff --git a/frontend/src/components/chat/MessageBubble.tsx b/frontend/src/components/chat/MessageBubble.tsx index 5edaff7..2d57844 100644 --- a/frontend/src/components/chat/MessageBubble.tsx +++ b/frontend/src/components/chat/MessageBubble.tsx @@ -1,8 +1,10 @@ -import ReactMarkdown from "react-markdown"; +import ReactMarkdown from "../knowledge/KnowledgeMarkdown"; +import { useI18n } from "../../i18n"; import type { ChatMessage } from "../../types"; import { Bot, User, CheckCircle, Loader2 } from "lucide-react"; export function MessageBubble({ message }: { message: ChatMessage }) { + const { t } = useI18n(); const isUser = message.role === "user"; const avatarClassName = isUser ? "bg-slate-200" : "bg-black"; const containerClassName = isUser ? "text-right" : ""; @@ -27,6 +29,8 @@ export function MessageBubble({ message }: { message: ChatMessage }) { {message.agent} )} + {message.rag_status === "running" &&

{t("knowledge.searching")}

} + {message.rag_status === "degraded" &&

{t("knowledge.degraded")}

} {message.tools && message.tools.length > 0 && (
{message.tools.map((tool, i) => ( diff --git a/frontend/src/components/knowledge/KnowledgeMarkdown.tsx b/frontend/src/components/knowledge/KnowledgeMarkdown.tsx new file mode 100644 index 0000000..9859e48 --- /dev/null +++ b/frontend/src/components/knowledge/KnowledgeMarkdown.tsx @@ -0,0 +1,57 @@ +import { useEffect, useRef, useState, type ComponentProps } from "react"; +import ReactMarkdown from "react-markdown"; +import { getCitation, type KnowledgeCitation } from "../../api/knowledge"; +import { useI18n } from "../../i18n"; + +function codePointOffset(text: string, index: number) { + let offset = 0; + for (const character of text) { + if (index-- <= 0) break; + offset += character.length; + } + return offset; +} + +export default function KnowledgeMarkdown(props: ComponentProps) { + const { t } = useI18n(); + const [source, setSource] = useState<(KnowledgeCitation & { source_text: string }) | null>(null); + const [opened, setOpened] = useState(false); + const [error, setError] = useState(false); + const [full, setFull] = useState(false); + const dialog = useRef(null); + useEffect(() => { + if (opened) dialog.current?.showModal(); + else dialog.current?.close(); + }, [opened]); + async function openSource(id: string) { + setSource(null); setError(false); setFull(false); setOpened(true); + try { setSource(await getCitation(id)); } catch { setError(true); } + } + return <> + { + const match = href?.match(/^\/api\/v1\/knowledge\/citations\/([a-f0-9-]+)$/); + return match + ? { event.preventDefault(); void openSource(match[1]!); }}>{children} + : {children}; + } }} /> + setOpened(false)} + className="m-auto w-[min(760px,94vw)] max-h-[85vh] overflow-y-auto rounded-2xl border border-slate-200 p-6 shadow-xl backdrop:bg-slate-900/30"> +
+ {source?.title ?? t("knowledge.sources")} + +
+ {error ?

{t("knowledge.sourceUnavailable")}

: !source + ?

{t("knowledge.searching")}

+ : <> +

{source.heading} {source.page != null && `${t("knowledge.page")} ${source.page}`}

+ {source.generated &&

{t("knowledge.generated")}

} + 
+            {source.source_text.slice(full ? 0 : codePointOffset(source.source_text, Math.max(0, source.start - 250)), codePointOffset(source.source_text, source.start))}
+            {source.quote}
+            {source.source_text.slice(codePointOffset(source.source_text, source.end), full ? undefined : codePointOffset(source.source_text, source.end + 250))}
+
+ + } + + ; +} diff --git a/frontend/src/components/project/ProjectTabs.tsx b/frontend/src/components/project/ProjectTabs.tsx index 0864cf3..d99bd6a 100644 --- a/frontend/src/components/project/ProjectTabs.tsx +++ b/frontend/src/components/project/ProjectTabs.tsx @@ -34,6 +34,7 @@ const TAB_GROUPS: Array<{ activeBg: "bg-amber-600 text-white shadow-sm shadow-amber-200", tabs: [ { path: "/reports", labelKey: "project.reports" }, + { path: "/knowledge", labelKey: "knowledge.title" }, { path: "/performance", labelKey: "project.performance" }, ], }, diff --git a/frontend/src/hooks/useChat.ts b/frontend/src/hooks/useChat.ts index 0e1a9ef..7ccabb7 100644 --- a/frontend/src/hooks/useChat.ts +++ b/frontend/src/hooks/useChat.ts @@ -132,6 +132,9 @@ export function useChat(initialProjectId: number | null = null) { id: nextId(), role: m.role as "user" | "assistant", content: m.content, + citations: m.citations, + retrieval_id: m.retrieval_id, + rag_status: m.rag_status, })), ); setCurrentAgent("CMO Agent"); @@ -195,6 +198,9 @@ export function useChat(initialProjectId: number | null = null) { const handleEvent = useCallback( (event: ChatEvent, msgId: string) => { switch (event.type) { + case "retrieval": + setMessages(prev => prev.map(m => m.id === msgId ? { ...m, rag_status: event.status } : m)); + break; case "delta": setMessages((prev) => prev.map((m) => @@ -252,6 +258,9 @@ export function useChat(initialProjectId: number | null = null) { ...m, agent: event.agent_name ?? m.agent, content: event.final_output ?? m.content, + citations: event.citations, + retrieval_id: event.retrieval_id, + rag_status: event.rag_status, } : m, ), diff --git a/frontend/src/i18n/knowledge.ts b/frontend/src/i18n/knowledge.ts new file mode 100644 index 0000000..79938b6 --- /dev/null +++ b/frontend/src/i18n/knowledge.ts @@ -0,0 +1,339 @@ +export const knowledgeEN = { + "knowledge.rebuildAll": "Rebuild the entire library", + "knowledge.searchComplete": "Search completed", + "knowledge.cancelled": "Cancelled", + "knowledge.newVersion": "Upload new version", + "knowledge.title": "Knowledge library", + "knowledge.description": "Import source documents, find evidence and cite the original.", + "knowledge.documents": "Documents", + "knowledge.settings": "Models and retrieval", + "knowledge.search": "Test retrieval", + "knowledge.file": "Upload file", + "knowledge.text": "Paste text", + "knowledge.url": "Web page", + "knowledge.name": "Document title", + "knowledge.import": "Import", + "knowledge.backfill": "Import historical reports", + "knowledge.shared": "Share with this account", + "knowledge.external": "Allowed in public-facing content", + "knowledge.private": "Project only", + "knowledge.ready": "Ready", + "knowledge.queued": "Queued", + "knowledge.indexing": "Indexing", + "knowledge.failed": "Failed", + "knowledge.retry": "Reindex", + "knowledge.remove": "Delete", + "knowledge.confirmDelete": "Delete this source? It will immediately stop appearing in retrieval.", + "knowledge.preview": "Versions and passages", + "knowledge.empty": "No documents yet.", + "knowledge.sources": "Sources", + "knowledge.sourceUnavailable": "This source is unavailable or access has been revoked.", + "knowledge.close": "Close", + "knowledge.page": "Page", + "knowledge.generated": "AI-generated report: historical analysis, not first-hand evidence.", + "knowledge.fullText": "Show full original text", + "knowledge.excerpt": "Show excerpt", + "knowledge.searching": "Finding source evidence…", + "knowledge.query": "Ask a question about your documents", + "knowledge.internal": "Internal analysis", + "knowledge.outbound": "Public-facing content", + "knowledge.noResults": "No relevant source evidence found.", + "knowledge.degraded": "Some retrieval services were unavailable; results may be incomplete.", + "knowledge.enabled": "Enable knowledge retrieval", + "knowledge.embeddingUrl": "Embedding API URL", + "knowledge.embeddingModel": "Embedding model", + "knowledge.embeddingKey": "Embedding API key", + "knowledge.rerankUrl": "Rerank API URL", + "knowledge.rerankModel": "Rerank model", + "knowledge.rerankKey": "Rerank API key", + "knowledge.parentTokens": "Parent passage tokens", + "knowledge.childTokens": "Child passage tokens", + "knowledge.overlap": "Overlap tokens", + "knowledge.threshold": "Minimum rerank score", + "knowledge.save": "Save settings", + "knowledge.test": "Test model connections", + "knowledge.saved": "Saved. Indexing tasks may continue in the background.", + "knowledge.success": "Connection successful", + "knowledge.failedAction": "The operation failed. Check configuration and source format.", + "knowledge.limits": "PDF, DOCX, MD or TXT · up to 25 MB · text PDFs only", + "knowledge.next": "Next page", + "knowledge.previous": "Previous page", + "knowledge.auto": "Knowledge when available", + "knowledge.only": "Knowledge sources only", + "knowledge.off": "Knowledge retrieval off", + "knowledge.mode": "Answer sources", + "knowledge.configured": "Configured; leave blank to keep", + "knowledge.details": "Retrieval diagnostics" +} as const; + +export const knowledgeZH = { + "knowledge.rebuildAll": "重建整个知识库", + "knowledge.searchComplete": "检索完成", + "knowledge.cancelled": "已取消", + "knowledge.newVersion": "上传新版本", + "knowledge.title": "知识库", + "knowledge.description": "导入资料，检索证据，并引用原文。", + "knowledge.documents": "资料", + "knowledge.settings": "模型与检索设置", + "knowledge.search": "检索试验", + "knowledge.file": "上传文件", + "knowledge.text": "粘贴文本", + "knowledge.url": "网页", + "knowledge.name": "资料标题", + "knowledge.import": "导入", + "knowledge.backfill": "导入历史报告", + "knowledge.shared": "同账号共享", + "knowledge.external": "可用于对外内容", + "knowledge.private": "仅当前项目", + "knowledge.ready": "可检索", + "knowledge.queued": "排队中", + "knowledge.indexing": "索引中", + "knowledge.failed": "失败", + "knowledge.retry": "重新索引", + "knowledge.remove": "删除", + "knowledge.confirmDelete": "删除此资料？删除后立即停止检索。", + "knowledge.preview": "版本与片段", + "knowledge.empty": "尚未导入资料。", + "knowledge.sources": "引用来源", + "knowledge.sourceUnavailable": "资料已删除或当前无权查看。", + "knowledge.close": "关闭", + "knowledge.page": "页码", + "knowledge.generated": "AI 生成报告：属于历史分析，不是一手证据。", + "knowledge.fullText": "查看完整原文", + "knowledge.excerpt": "查看摘录", + "knowledge.searching": "正在查找资料…", + "knowledge.query": "输入针对资料的问题", + "knowledge.internal": "内部分析", + "knowledge.outbound": "对外内容", + "knowledge.noResults": "未找到相关原文。", + "knowledge.degraded": "部分检索服务不可用，结果可能不完整。", + "knowledge.enabled": "启用知识库检索", + "knowledge.embeddingUrl": "Embedding API 地址", + "knowledge.embeddingModel": "向量模型", + "knowledge.embeddingKey": "Embedding API 密钥", + "knowledge.rerankUrl": "Rerank API 地址", + "knowledge.rerankModel": "重排模型", + "knowledge.rerankKey": "Rerank API 密钥", + "knowledge.parentTokens": "父块 token 上限", + "knowledge.childTokens": "子块 token 上限", + "knowledge.overlap": "重叠 token 数", + "knowledge.threshold": "最低重排分", + "knowledge.save": "保存设置", + "knowledge.test": "测试模型连接", + "knowledge.saved": "已保存，索引任务可能仍在后台运行。", + "knowledge.success": "连接成功", + "knowledge.failedAction": "操作失败，请检查配置和资料格式。", + "knowledge.limits": "PDF、DOCX、MD、TXT · 最大 25 MB · PDF 需包含文本", + "knowledge.next": "下一页", + "knowledge.previous": "上一页", + "knowledge.auto": "自动使用知识库", + "knowledge.only": "仅依据知识库", + "knowledge.off": "关闭知识库检索", + "knowledge.mode": "回答依据", + "knowledge.configured": "已配置，留空保留", + "knowledge.details": "检索详情" +} as const; + +export const knowledgeJA = { + "knowledge.rebuildAll": "ライブラリ全体を再構築", + "knowledge.searchComplete": "検索が完了しました", + "knowledge.cancelled": "キャンセル済み", + "knowledge.newVersion": "新しいバージョンをアップロード", + "knowledge.title": "ナレッジベース", + "knowledge.description": "資料を取り込み、根拠を検索して原文を引用します。", + "knowledge.documents": "資料", + "knowledge.settings": "モデルと検索", + "knowledge.search": "検索テスト", + "knowledge.file": "ファイルをアップロード", + "knowledge.text": "テキストを貼り付け", + "knowledge.url": "ウェブページ", + "knowledge.name": "資料のタイトル", + "knowledge.import": "取り込む", + "knowledge.backfill": "過去のレポートを取り込む", + "knowledge.shared": "このアカウントで共有", + "knowledge.external": "公開コンテンツでの使用を許可", + "knowledge.private": "このプロジェクトのみ", + "knowledge.ready": "検索可能", + "knowledge.queued": "待機中", + "knowledge.indexing": "索引作成中", + "knowledge.failed": "失敗", + "knowledge.retry": "再索引", + "knowledge.remove": "削除", + "knowledge.confirmDelete": "この資料を削除しますか？検索から直ちに除外されます。", + "knowledge.preview": "バージョンと段落", + "knowledge.empty": "資料はまだありません。", + "knowledge.sources": "引用元", + "knowledge.sourceUnavailable": "資料が利用できないか、アクセス権が取り消されています。", + "knowledge.close": "閉じる", + "knowledge.page": "ページ", + "knowledge.generated": "AI生成レポート：過去の分析であり、一次資料ではありません。", + "knowledge.fullText": "原文をすべて表示", + "knowledge.excerpt": "抜粋を表示", + "knowledge.searching": "資料を検索中…", + "knowledge.query": "資料について質問する", + "knowledge.internal": "内部分析", + "knowledge.outbound": "公開コンテンツ", + "knowledge.noResults": "関連する根拠が見つかりません。", + "knowledge.degraded": "一部の検索サービスが利用できず、結果が不完全な可能性があります。", + "knowledge.enabled": "ナレッジ検索を有効化", + "knowledge.embeddingUrl": "Embedding API URL", + "knowledge.embeddingModel": "埋め込みモデル", + "knowledge.embeddingKey": "Embedding APIキー", + "knowledge.rerankUrl": "Rerank API URL", + "knowledge.rerankModel": "再ランキングモデル", + "knowledge.rerankKey": "Rerank APIキー", + "knowledge.parentTokens": "親段落トークン数", + "knowledge.childTokens": "子段落トークン数", + "knowledge.overlap": "重複トークン数", + "knowledge.threshold": "最低再ランキングスコア", + "knowledge.save": "設定を保存", + "knowledge.test": "モデル接続をテスト", + "knowledge.saved": "保存しました。索引作成はバックグラウンドで続行されます。", + "knowledge.success": "接続成功", + "knowledge.failedAction": "操作に失敗しました。設定と資料形式を確認してください。", + "knowledge.limits": "PDF、DOCX、MD、TXT · 最大25 MB · テキストPDFのみ", + "knowledge.next": "次のページ", + "knowledge.previous": "前のページ", + "knowledge.auto": "利用可能な知識を使用", + "knowledge.only": "資料の根拠のみ", + "knowledge.off": "ナレッジ検索オフ", + "knowledge.mode": "回答の情報源", + "knowledge.configured": "設定済み。空欄で維持", + "knowledge.details": "検索診断" +} as const; + +export const knowledgeKO = { + "knowledge.rebuildAll": "전체 라이브러리 재구축", + "knowledge.searchComplete": "검색 완료", + "knowledge.cancelled": "취소됨", + "knowledge.newVersion": "새 버전 업로드", + "knowledge.title": "지식 라이브러리", + "knowledge.description": "자료를 가져와 근거를 검색하고 원문을 인용합니다.", + "knowledge.documents": "자료", + "knowledge.settings": "모델 및 검색", + "knowledge.search": "검색 테스트", + "knowledge.file": "파일 업로드", + "knowledge.text": "텍스트 붙여넣기", + "knowledge.url": "웹 페이지", + "knowledge.name": "자료 제목", + "knowledge.import": "가져오기", + "knowledge.backfill": "이전 보고서 가져오기", + "knowledge.shared": "이 계정에서 공유", + "knowledge.external": "외부 콘텐츠에 사용 허용", + "knowledge.private": "현재 프로젝트만", + "knowledge.ready": "검색 가능", + "knowledge.queued": "대기 중", + "knowledge.indexing": "색인 생성 중", + "knowledge.failed": "실패", + "knowledge.retry": "다시 색인", + "knowledge.remove": "삭제", + "knowledge.confirmDelete": "자료를 삭제할까요? 검색에서 즉시 제외됩니다.", + "knowledge.preview": "버전 및 구절", + "knowledge.empty": "아직 자료가 없습니다.", + "knowledge.sources": "출처", + "knowledge.sourceUnavailable": "자료를 사용할 수 없거나 접근 권한이 해제되었습니다.", + "knowledge.close": "닫기", + "knowledge.page": "페이지", + "knowledge.generated": "AI 생성 보고서: 과거 분석이며 1차 근거가 아닙니다.", + "knowledge.fullText": "전체 원문 보기", + "knowledge.excerpt": "발췌 보기", + "knowledge.searching": "근거 검색 중…", + "knowledge.query": "자료에 대한 질문 입력", + "knowledge.internal": "내부 분석", + "knowledge.outbound": "외부 콘텐츠", + "knowledge.noResults": "관련 원문을 찾지 못했습니다.", + "knowledge.degraded": "일부 검색 서비스를 사용할 수 없어 결과가 불완전할 수 있습니다.", + "knowledge.enabled": "지식 검색 사용", + "knowledge.embeddingUrl": "Embedding API 주소", + "knowledge.embeddingModel": "임베딩 모델", + "knowledge.embeddingKey": "Embedding API 키", + "knowledge.rerankUrl": "Rerank API 주소", + "knowledge.rerankModel": "재순위 모델", + "knowledge.rerankKey": "Rerank API 키", + "knowledge.parentTokens": "부모 구절 토큰 수", + "knowledge.childTokens": "자식 구절 토큰 수", + "knowledge.overlap": "겹침 토큰 수", + "knowledge.threshold": "최소 재순위 점수", + "knowledge.save": "설정 저장", + "knowledge.test": "모델 연결 테스트", + "knowledge.saved": "저장되었습니다. 색인 작업은 백그라운드에서 계속될 수 있습니다.", + "knowledge.success": "연결 성공", + "knowledge.failedAction": "작업에 실패했습니다. 설정과 자료 형식을 확인하세요.", + "knowledge.limits": "PDF, DOCX, MD, TXT · 최대 25 MB · 텍스트 PDF만", + "knowledge.next": "다음 페이지", + "knowledge.previous": "이전 페이지", + "knowledge.auto": "가능한 지식 사용", + "knowledge.only": "지식 근거만", + "knowledge.off": "지식 검색 끄기", + "knowledge.mode": "답변 출처", + "knowledge.configured": "설정됨; 비워 두면 유지", + "knowledge.details": "검색 진단" +} as const; + +export const knowledgeES = { + "knowledge.rebuildAll": "Reconstruir toda la biblioteca", + "knowledge.searchComplete": "Búsqueda completada", + "knowledge.cancelled": "Cancelado", + "knowledge.newVersion": "Subir nueva versión", + "knowledge.title": "Biblioteca de conocimiento", + "knowledge.description": "Importa documentos, busca evidencia y cita el original.", + "knowledge.documents": "Documentos", + "knowledge.settings": "Modelos y búsqueda", + "knowledge.search": "Probar búsqueda", + "knowledge.file": "Subir archivo", + "knowledge.text": "Pegar texto", + "knowledge.url": "Página web", + "knowledge.name": "Título del documento", + "knowledge.import": "Importar", + "knowledge.backfill": "Importar informes históricos", + "knowledge.shared": "Compartir con esta cuenta", + "knowledge.external": "Permitir en contenido público", + "knowledge.private": "Solo este proyecto", + "knowledge.ready": "Listo", + "knowledge.queued": "En cola", + "knowledge.indexing": "Indexando", + "knowledge.failed": "Error", + "knowledge.retry": "Reindexar", + "knowledge.remove": "Eliminar", + "knowledge.confirmDelete": "¿Eliminar esta fuente? Dejará de aparecer en las búsquedas.", + "knowledge.preview": "Versiones y fragmentos", + "knowledge.empty": "Aún no hay documentos.", + "knowledge.sources": "Fuentes", + "knowledge.sourceUnavailable": "Fuente no disponible o acceso revocado.", + "knowledge.close": "Cerrar", + "knowledge.page": "Página", + "knowledge.generated": "Informe generado por IA: análisis histórico, no evidencia primaria.", + "knowledge.fullText": "Ver texto original completo", + "knowledge.excerpt": "Ver extracto", + "knowledge.searching": "Buscando fuentes…", + "knowledge.query": "Haz una pregunta sobre tus documentos", + "knowledge.internal": "Análisis interno", + "knowledge.outbound": "Contenido público", + "knowledge.noResults": "No se encontró evidencia relevante.", + "knowledge.degraded": "Algunos servicios no están disponibles; los resultados pueden estar incompletos.", + "knowledge.enabled": "Activar búsqueda de conocimiento", + "knowledge.embeddingUrl": "URL de API Embedding", + "knowledge.embeddingModel": "Modelo de embeddings", + "knowledge.embeddingKey": "Clave API Embedding", + "knowledge.rerankUrl": "URL de API Rerank", + "knowledge.rerankModel": "Modelo de reranking", + "knowledge.rerankKey": "Clave API Rerank", + "knowledge.parentTokens": "Tokens del fragmento padre", + "knowledge.childTokens": "Tokens del fragmento hijo", + "knowledge.overlap": "Tokens de solapamiento", + "knowledge.threshold": "Puntuación mínima de reranking", + "knowledge.save": "Guardar ajustes", + "knowledge.test": "Probar conexiones", + "knowledge.saved": "Guardado. La indexación puede continuar en segundo plano.", + "knowledge.success": "Conexión correcta", + "knowledge.failedAction": "La operación falló. Revisa la configuración y el formato.", + "knowledge.limits": "PDF, DOCX, MD o TXT · hasta 25 MB · PDF con texto", + "knowledge.next": "Página siguiente", + "knowledge.previous": "Página anterior", + "knowledge.auto": "Usar conocimiento disponible", + "knowledge.only": "Solo fuentes de conocimiento", + "knowledge.off": "Búsqueda desactivada", + "knowledge.mode": "Fuentes de respuesta", + "knowledge.configured": "Configurado; dejar vacío para conservar", + "knowledge.details": "Diagnóstico de búsqueda" +} as const; diff --git a/frontend/src/i18n/locales/en.ts b/frontend/src/i18n/locales/en.ts index e6a4c6f..2bb51de 100644 --- a/frontend/src/i18n/locales/en.ts +++ b/frontend/src/i18n/locales/en.ts @@ -1,4 +1,7 @@ +import { knowledgeEN } from "../knowledge"; + export const en = { + ...knowledgeEN, // Common "common.cancel": "Cancel", "common.confirm": "Confirm", diff --git a/frontend/src/i18n/locales/es.ts b/frontend/src/i18n/locales/es.ts index 95e8c40..7a5c6f0 100644 --- a/frontend/src/i18n/locales/es.ts +++ b/frontend/src/i18n/locales/es.ts @@ -1,6 +1,9 @@ import type { TranslationKey } from "./en"; +import { knowledgeES } from "../knowledge"; + export const es: Partial> = { + ...knowledgeES, "common.cancel": "Cancelar", "common.confirm": "Confirmar", "common.logout": "Cerrar sesión", diff --git a/frontend/src/i18n/locales/ja.ts b/frontend/src/i18n/locales/ja.ts index 7fa2a2f..234982a 100644 --- a/frontend/src/i18n/locales/ja.ts +++ b/frontend/src/i18n/locales/ja.ts @@ -1,6 +1,9 @@ import type { TranslationKey } from "./en"; +import { knowledgeJA } from "../knowledge"; + export const ja: Partial> = { + ...knowledgeJA, // Common "common.cancel": "キャンセル", "common.confirm": "確認", diff --git a/frontend/src/i18n/locales/ko.ts b/frontend/src/i18n/locales/ko.ts index 84dcba0..3ac30cb 100644 --- a/frontend/src/i18n/locales/ko.ts +++ b/frontend/src/i18n/locales/ko.ts @@ -1,6 +1,9 @@ import type { TranslationKey } from "./en"; +import { knowledgeKO } from "../knowledge"; + export const ko: Partial> = { + ...knowledgeKO, "common.cancel": "취소", "common.confirm": "확인", "common.logout": "로그아웃", diff --git a/frontend/src/i18n/locales/zh.ts b/frontend/src/i18n/locales/zh.ts index 00545aa..bd2c5f3 100644 --- a/frontend/src/i18n/locales/zh.ts +++ b/frontend/src/i18n/locales/zh.ts @@ -1,6 +1,9 @@ import type { TranslationKey } from "./en"; +import { knowledgeZH } from "../knowledge"; + export const zh: Partial> = { + ...knowledgeZH, // Common "common.cancel": "取消", "common.confirm": "确认", diff --git a/frontend/src/pages/KnowledgePage.tsx b/frontend/src/pages/KnowledgePage.tsx new file mode 100644 index 0000000..4996055 --- /dev/null +++ b/frontend/src/pages/KnowledgePage.tsx @@ -0,0 +1,198 @@ +import { useState } from "react"; +import { useParams } from "react-router"; +import { useQuery, useQueryClient } from "@tanstack/react-query"; +import { BookOpen, FileText, Search, Settings, Upload } from "lucide-react"; +import { useI18n, type TranslationKey } from "../i18n"; +import { useProjectSummary } from "../hooks/useProject"; +import { ProjectHeader } from "../components/project/ProjectHeader"; +import { ProjectTabs } from "../components/project/ProjectTabs"; +import KnowledgeMarkdown from "../components/knowledge/KnowledgeMarkdown"; +import * as api from "../api/knowledge"; +import { utcDate } from "../utils/time"; + +const configuration: [string, TranslationKey, "text" | "password" | "number"][] = [ + ["embedding_base_url", "knowledge.embeddingUrl", "text"], + ["embedding_model", "knowledge.embeddingModel", "text"], + ["embedding_api_key", "knowledge.embeddingKey", "password"], + ["rerank_base_url", "knowledge.rerankUrl", "text"], + ["rerank_model", "knowledge.rerankModel", "text"], + ["rerank_api_key", "knowledge.rerankKey", "password"], + ["parent_tokens", "knowledge.parentTokens", "number"], + ["child_tokens", "knowledge.childTokens", "number"], + ["overlap_tokens", "knowledge.overlap", "number"], + ["rerank_min_score", "knowledge.threshold", "number"], +]; +const inputClass = "w-full rounded-xl border border-slate-200 bg-white px-3 py-2 text-sm outline-none focus:ring-2 focus:ring-blue-200"; +const buttonClass = "rounded-xl border border-slate-200 bg-white px-3 py-2 text-sm font-medium hover:bg-slate-50 disabled:opacity-50"; + +export function KnowledgePage() { + const projectId = Number(useParams().id); + const { t } = useI18n(); + const cache = useQueryClient(); + const project = useProjectSummary(projectId); + const [tab, setTab] = useState<"documents" | "search" | "settings">("documents"); + const [offset, setOffset] = useState(0); + const documents = useQuery({ queryKey: ["knowledge", projectId, offset], queryFn: () => api.listKnowledge(projectId, offset), refetchInterval: 4000 }); + const settings = useQuery({ queryKey: ["knowledge", "settings"], queryFn: api.getKnowledgeSettings, refetchInterval: 4000 }); + const [changes, setChanges] = useState>({}); + const config = { ...settings.data, ...changes }; + const [type, setType] = useState<"file" | "text" | "url">("file"); + const [file, setFile] = useState(null); + const [title, setTitle] = useState(""); + const [text, setText] = useState(""); + const [shared, setShared] = useState(false); + const [external, setExternal] = useState(false); + const [replaceId, setReplaceId] = useState(null); + const [busy, setBusy] = useState(false); + const [notice, setNotice] = useState(""); + const [error, setError] = useState(""); + const [query, setQuery] = useState(""); + const [purpose, setPurpose] = useState("internal"); + const [result, setResult] = useState(null); + const [previewId, setPreviewId] = useState(null); + const [versionId, setVersionId] = useState(""); + const detail = useQuery({ queryKey: ["knowledge", "detail", previewId], queryFn: () => api.knowledgeDocument(previewId!), enabled: Boolean(previewId) }); + const selectedVersion = versionId || detail.data?.versions?.[0]?.id || ""; + const preview = useQuery({ queryKey: ["knowledge", "version", previewId, selectedVersion], + queryFn: () => api.getKnowledgeVersion(previewId!, selectedVersion), enabled: Boolean(previewId && selectedVersion) }); + + async function action(fn: () => Promise, success: TranslationKey = "knowledge.saved") { + setBusy(true); setNotice(""); setError(""); + try { await fn(); setNotice(t(success)); await cache.invalidateQueries({ queryKey: ["knowledge"] }); } + catch (reason) { setError(t("knowledge.failedAction") + " " + (reason instanceof Error ? reason.message : "")); } + finally { setBusy(false); } + } + async function importSource() { + if (type === "file") { + if (!file) return; + const body = new FormData(); + body.set("file", file); body.set("project_id", String(projectId)); body.set("title", title || file.name); + body.set("scope", shared ? "account" : "project"); body.set("external_use", String(external)); + if (replaceId) body.set("document_id", replaceId); + await api.importKnowledge(body); + } else { + await api.importKnowledge({ project_id: projectId, title, [type === "url" ? "url" : "text"]: text, + scope: shared ? "account" : "project", external_use: external, ...(replaceId ? { document_id: replaceId } : {}) }); + } + setText(""); setFile(null); setTitle(""); setReplaceId(null); + } + const statusKey = (status: string): TranslationKey => + status === "ready" ? "knowledge.ready" : status === "failed" ? "knowledge.failed" : status === "cancelled" ? "knowledge.cancelled" : status === "indexing" ? "knowledge.indexing" : "knowledge.queued"; + + return
+ {project.data && <>} +
+
+

{t("knowledge.title")}

+

{t("knowledge.description")}

+
+ + {(error || notice) &&

{error || notice}

} + {settings.data && !config.enabled &&

+ {t("knowledge.enabled")} · +

} + {tab === "documents" &&
+
+
+ {(["file", "text", "url"] as const).map(key => + )} + +
+ {replaceId &&

{t("knowledge.newVersion")}

} + + {type === "file" ? :