diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e3826b1090..176a8268b8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,6 +41,33 @@ jobs: - name: Test run: make test + helm: + name: Helm chart + runs-on: ubuntu-latest + if: ${{ !startsWith(github.head_ref, 'chore/sync-develop-after-') }} + steps: + - uses: actions/checkout@v4 + + - name: Set up Helm + uses: azure/setup-helm@v4 + with: + version: v3.17.3 + + - name: Lint and render + run: | + set -euo pipefail + helm lint deploy/helm/octop + helm template octop deploy/helm/octop -n octop >/dev/null + helm template octop deploy/helm/octop -n octop \ + -f deploy/helm/octop/values.example.yaml >/dev/null + helm template octop deploy/helm/octop -n octop \ + --set postgresql.enabled=false \ + --set database.host=pg.example.com >/dev/null + if helm template octop deploy/helm/octop -n octop --set postgresql.enabled=false >/dev/null 2>/tmp/helm-err; then + echo "expected a missing database.host to fail" + exit 1 + fi + test-windows: name: Windows / Python 3.12 runs-on: windows-latest diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 2c4a6b3e5c..ea6aaa1407 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -150,14 +150,16 @@ jobs: uses: docker/build-push-action@v6 with: context: . - file: ./docker/Dockerfile + file: ./deploy/docker/Dockerfile push: false load: true provenance: false sbom: false platforms: ${{ matrix.platform }} + # NODE_MAX_OLD_SPACE_SIZE:Vite chunk 渲染峰值约 2GB,2048 会偶发退出码 134。 build-args: | OCTOP_IMAGE_VARIANT=${{ matrix.variant }} + NODE_MAX_OLD_SPACE_SIZE=4096 tags: | octop:ci-${{ matrix.variant }} ${{ steps.meta.outputs.tags }} diff --git a/.github/workflows/helm-publish.yml b/.github/workflows/helm-publish.yml new file mode 100644 index 0000000000..67a56d1c75 --- /dev/null +++ b/.github/workflows/helm-publish.yml @@ -0,0 +1,128 @@ +name: Helm Publish + +# 版本标签 v* 上把 deploy/helm/octop 打成包,推到与镜像相同的 GHCR。 +# 不另建 HTTP Helm 仓库。安装地址: +# oci://ghcr.io/tencentcloud/octop/charts/octop +# +# Chart 源码仍在仓库里。克隆后用目录安装;没有仓库的机器用上面的 OCI 地址。 +# 推送时改写版本,不把改写提交回 git: +# appVersion = 镜像 tag(v1.0.3b5 → 1.0.3b5) +# version = Helm SemVer(1.0.3b5 → 1.0.3-b5,1.0.3 保持 1.0.3) +# +# 触发与 docker-publish.yml 相同:推送 v*,或 workflow_dispatch 填 tag。 +on: + push: + tags: + - "v*" + workflow_dispatch: + inputs: + tag: + description: "v* tag to publish (e.g. v1.0.3b5). Required when not running on a tag." + required: false + type: string + +permissions: + contents: read + packages: write + +jobs: + publish: + name: Push chart to GHCR + runs-on: ubuntu-latest + if: >- + startsWith(github.ref, 'refs/tags/v') + || (github.event_name == 'workflow_dispatch' && startsWith(inputs.tag, 'v')) + steps: + - name: Resolve version from tag ref + id: ver + env: + DISPATCH_TAG: ${{ inputs.tag }} + run: | + set -euo pipefail + if [ -n "${DISPATCH_TAG:-}" ]; then + ref="$DISPATCH_TAG" + else + ref="${GITHUB_REF_NAME}" + fi + case "$ref" in + v*) + version="${ref#v}" + ;; + *) + echo "::error::Expected v* tag, got ref=$ref" + exit 1 + ;; + esac + echo "version=$version" >> "$GITHUB_OUTPUT" + echo "tag=$ref" >> "$GITHUB_OUTPUT" + + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ steps.ver.outputs.tag }} + + - name: Set up Helm + uses: azure/setup-helm@v4 + with: + version: v3.17.3 + + - name: Set chart version from the release tag + env: + OCTOP_VERSION: ${{ steps.ver.outputs.version }} + run: | + set -euo pipefail + python3 - << 'PY' + import os + import re + from pathlib import Path + image_tag = os.environ["OCTOP_VERSION"] + match = re.fullmatch(r"(\d+\.\d+\.\d+)([A-Za-z].*)?", image_tag) + if match is None: + raise SystemExit(f"unsupported version {image_tag}") + base, pre = match.group(1), match.group(2) + chart_version = f"{base}-{pre}" if pre else base + path = Path("deploy/helm/octop/Chart.yaml") + text = path.read_text() + text = re.sub(r"(?m)^version:.*$", f"version: {chart_version}", text, count=1) + text = re.sub(r'(?m)^appVersion:.*$', f'appVersion: "{image_tag}"', text, count=1) + path.write_text(text) + print(f"chart {chart_version}, image tag {image_tag}") + PY + + - name: Lint and package + run: | + set -euo pipefail + helm lint deploy/helm/octop + helm package deploy/helm/octop --destination /tmp/octop-chart + + - name: Log in to GHCR + run: | + echo "${{ secrets.GITHUB_TOKEN }}" | helm registry login ghcr.io \ + --username "${{ github.actor }}" --password-stdin + + - name: Push chart + env: + GHCR: ghcr.io/${{ github.repository }} + run: | + set -euo pipefail + registry="$(echo "$GHCR" | tr '[:upper:]' '[:lower:]')" + helm push /tmp/octop-chart/octop-*.tgz "oci://${registry}/charts" + + # gh api percent-encodes the path again, so a pre-encoded slash becomes %252F. + # Call the API directly. The chart lives at octop/charts/octop so it does + # not replace the container image package named octop. + - name: Mark the GHCR chart package public + continue-on-error: true + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + OWNER: ${{ github.repository_owner }} + run: | + set -euo pipefail + owner="$(echo "$OWNER" | tr '[:upper:]' '[:lower:]')" + package="$(python3 -c 'import urllib.parse; print(urllib.parse.quote("octop/charts/octop", safe=""))')" + curl -fsS -X PUT \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/orgs/${owner}/packages/container/${package}/visibility" \ + -d '{"visibility":"public"}' diff --git a/.github/workflows/octop-desktop.yml b/.github/workflows/octop-desktop.yml index d325a36732..c0536cbd79 100644 --- a/.github/workflows/octop-desktop.yml +++ b/.github/workflows/octop-desktop.yml @@ -74,7 +74,7 @@ jobs: - uses: actions/setup-node@v5 with: - # Dashboard stays on Node 20 (same as docker/Dockerfile + release.yml). + # Dashboard stays on Node 20 (same as deploy/docker/Dockerfile + release.yml). # Node 24 is a desktop/electron-only pin and nothing here builds it. node-version: "20" cache: npm diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a0e83dbd8..93a19fc243 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,9 +6,52 @@ ## [Unreleased] +## [1.0.3b6] - 2026-11-15 + +### 新增 +- 支持 Helm 集群部署。多实例共用一个 PostgreSQL +- 管理员可以限制创建专家时能选的存储后端 +- 新增托管专家,对话可以交给后端 Agent,如 OpenCode +- 知识库支持知识图谱和重排序 +- 新增本地模型供应商 Magpie +- 定时任务可以跟最新对话,也可以固定到某一场历史对话 +- 对话里列出本轮改过的文件、所在目录和增删行数 + +### 修复 +- 编辑托管专家时,不显示存储后端也能保存 + +### 安全 +- 不能再把宿主机目录挂进 Docker 专家 +- 腾讯云登录按用户隔离 + ## [1.0.3b5] - 2026-11-11 ### 新增 +- 自定义 MCP 可以填写 OAuth 应用凭证(Client ID / Secret、Scope、授权与令牌端点):授权服务器不支持自动注册客户端,或不提供发现信息时,也能在页面上完成授权并自动刷新令牌 +- 自定义 MCP 服务器的 OAuth 授权失效或尚未授权时,对话会提示"授权并重试",在弹窗中授权后自动重发原消息;授权被取消或失败时会提示原因 +- 新增 `octop migrate openclaw` / `octop migrate hermes`:把原有助手的模型供应商、人格与记忆文件、技能、MCP 服务器和通道迁移到一个专家,支持 `--dry-run` 预演;已存在的内容不会被覆盖,通道默认不启用 +- 知识库的文档和文件夹可以移动到同一知识库的其他文件夹或根目录:卡片与表格的操作栏新增「移动到…」,文件夹会连同其中的内容一起移动,无需重新上传和索引 +- 知识库的文档和文件夹也可以移动到其他知识库:移动弹窗按知识库列出目标文件夹,原文件和索引一起迁移,无需重新上传 +- 知识库检索支持重排序(Rerank):在「知识库设置」里选择提供重排序模型的供应商后,向量检索会先多取一批候选片段,再按与问题的相关度重排,只保留最相关的几条;重排序请求失败时自动沿用向量顺序 +- 专家市场默认打开「精选」,用书签标出接入市场前随安装包提供的 22 个专家模板 +- 通道卡片与 `/api/health` 展示每个通道的入站、出站、重连和丢弃计数;记忆存储页显示 checkpoint 保留窗口与 VACUUM 提示 +- 知识库可以构建知识图谱:用所选对话模型从文档片段中抽取实体和关系,在控制台查看关系图和出处,检索时沿图谱补充相关片段和关系 + +### 修复 +- 云端协同不再因 20 秒协议 ping 超时反复断开。保活改由应用层心跳负责,对端忙时不会被拆线后立刻重连 +- 服务端不再因协议 pong 迟到拆掉 WebSocket。演示站事件循环忙碌时,原先会因此挂断云端协同并立刻被重连 +- 回合进行中,「整理结果中」只出现在当前这条回复上。最终回答不再排到提问前面。手机上提问卡的提交按钮不再被输入框挡住 +- 关机时会等正在进行的数据库查询结束再关闭连接,避免 Windows 上测试进程被访问冲突打断 +- 模型池里的优先模型可以取消,并且不再显示为聊天框里的选中模型。聊天框只显示专家默认模型或这场对话的手选模型 +- 添加云端协同时,权限不足、连接被占用、连到本机或达到条数上限会立刻显示可读错误,不再重试原始关闭帧 +- 同一专家的不同会话不再互相取消正在等待的提问。侧栏展开时可以拖右缘改宽度 +- 重名的模型供应商返回冲突,不再报服务器错误。知识库引用只包含实际写进上下文的文档 +- 备份落到另一块磁盘时不再丢文件。两个账号在同一浏览器里不再共用主题 +- 未读数字可以打开对应会话。自定义 ACP 显示自己的名称,HTML 预览里的页内链接留在预览框内 +- 供应商的默认模型可以清空并保持为空。对话里的模型名与选择时一致,能区分不同厂家的同名模型 +- 服务端升级后,登录页和已登录页面都会提示刷新前端。PostgreSQL 备份缺少客户端工具时会说明原因,Docker 镜像带上该工具 +- 专家配置文件改为文本框编辑,打开时不再停在编辑器加载中 +- 飞书通道注册时改用持久化去重库,通道重建或服务重启后仍能识别重投事件 - 桌面客户端改为 Electron 壳,登录过期后自动续上 - 自定义 MCP 可填写 OAuth 应用凭证并完成授权 - 新增 `octop migrate openclaw` / `octop migrate hermes`,把原有助手迁到一个专家 diff --git a/README.md b/README.md index ba771bd325..a00f12febb 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@

Python 3.12+ License: MIT - Version + Version PyPI Code Style: Ruff GitHub stars @@ -278,13 +278,13 @@ Open **http://127.0.0.1:8088**. With Docker, a valid `OCTOP_DEFAULT_PASSWORD` cr ```bash # Build and start -docker compose -f docker/docker-compose.yml up -d +docker compose -f deploy/docker/docker-compose.yml up -d # China mirror when ghcr.io is unreachable (no local build) -docker compose -f docker/docker-compose.cn.yml up -d +docker compose -f deploy/docker/docker-compose.cn.yml up -d # Or build manually -bash docker/docker_build.sh +bash deploy/docker/docker_build.sh docker run -d \ -p 8088:8088 \ -v octop-data:/data/.octop \ @@ -295,6 +295,8 @@ docker run -d \ Open `http://localhost:8088`. A valid `OCTOP_DEFAULT_PASSWORD` creates the admin on first boot and writes credentials to `/data/.octop/credential.txt`. If it is unset or rejected by the password policy, no admin is created — complete the setup wizard (unlock password in `/data/.octop/octop-login.txt`). Override the username via `OCTOP_ADMIN_USERNAME`. +The Helm chart for a shared PostgreSQL and multiple replicas is documented in [deploy/helm/octop/README.md](deploy/helm/octop/README.md). Install it from `oci://ghcr.io/tencentcloud/octop/charts/octop`. + > **Password policy:** at least 8 characters with letters and digits. | Variable | Default | Description | @@ -351,7 +353,7 @@ To serve the dashboard under a reverse-proxy subpath (for example `https://examp | Local script | macOS / Linux | `bash scripts/install.sh` | | Local script | Windows | `scripts\install.bat` or `install.ps1` | | PyPI | Any | `pip install octop` (optional extras such as `local-embedding`) | -| Docker | Any | `docker/docker-compose.yml`; China mirror: `docker/docker-compose.cn.yml` | +| Docker | Any | `deploy/docker/docker-compose.yml`; China mirror: `deploy/docker/docker-compose.cn.yml` | All install scripts provision an isolated environment at `~/.octop/venv` and a `~/.octop/bin/octop` wrapper — they do not touch system Python. @@ -520,7 +522,7 @@ src/octop/ dashboard/ frontend source (Vite) — edit here, run make build-frontend -docker/ Docker Compose, entrypoint, build & deploy scripts +deploy/ Docker (deploy/docker) and the Kubernetes Helm chart (deploy/helm/octop) tests/ unit/ + integration/ ``` diff --git a/README_CN.md b/README_CN.md index 61b2c1c0ca..3dc81548c3 100644 --- a/README_CN.md +++ b/README_CN.md @@ -15,7 +15,7 @@

Python 3.12+ License: MIT - Version + Version PyPI Code Style: Ruff GitHub stars @@ -280,13 +280,13 @@ octop service start ```bash # 构建并启动 -docker compose -f docker/docker-compose.yml up -d +docker compose -f deploy/docker/docker-compose.yml up -d # 官方 GHCR 拉取失败时用国内源(不本地构建) -docker compose -f docker/docker-compose.cn.yml up -d +docker compose -f deploy/docker/docker-compose.cn.yml up -d # 或手动构建 -bash docker/docker_build.sh +bash deploy/docker/docker_build.sh docker run -d \ -p 8088:8088 \ -v octop-data:/data/.octop \ @@ -297,6 +297,8 @@ docker run -d \ 打开 `http://localhost:8088`。`OCTOP_DEFAULT_PASSWORD` 合格时首次启动会创建管理员,并把凭据写入 `/data/.octop/credential.txt`。未设置或密码被策略拒绝时不自动建账号,打开页面走设置向导(向导口令在 `/data/.octop/octop-login.txt`)。可通过 `OCTOP_ADMIN_USERNAME` 覆盖用户名。 +共用 PostgreSQL、多副本的 Kubernetes 安装说明在 [deploy/helm/octop/README_CN.md](deploy/helm/octop/README_CN.md)。安装地址是 `oci://ghcr.io/tencentcloud/octop/charts/octop`。 + > **密码策略:** 至少 8 位,且同时包含字母和数字。 | 变量 | 默认值 | 说明 | @@ -350,7 +352,7 @@ docker run -d \ | 本地脚本 | macOS / Linux | `bash scripts/install.sh` | | 本地脚本 | Windows | `scripts\install.bat` 或 `install.ps1` | | PyPI | 全平台 | `pip install octop`(可选 extras 如 `local-embedding`) | -| Docker | 全平台 | `docker/docker-compose.yml`;国内镜像 `docker/docker-compose.cn.yml` | +| Docker | 全平台 | `deploy/docker/docker-compose.yml`;国内镜像 `deploy/docker/docker-compose.cn.yml` | 所有安装脚本均在 `~/.octop/venv` 创建隔离环境,并通过 `~/.octop/bin/octop` 包装 CLI,不会影响系统 Python。 @@ -516,7 +518,7 @@ src/octop/ dashboard/ 前端源码(Vite)— 在此编辑,运行 make build-frontend -docker/ Docker Compose、入口脚本、构建与部署脚本 +deploy/ Docker(deploy/docker)与 Kubernetes Helm chart(deploy/helm/octop) tests/ unit/ + integration/ ``` diff --git a/dashboard/index.html b/dashboard/index.html index 359ab10edd..754398dfef 100644 --- a/dashboard/index.html +++ b/dashboard/index.html @@ -364,32 +364,32 @@

-
-
-
- - -

- - Opening… -

+
+
+
+ + +

+ + Opening… +

+
-
+

{escape(message)}

+""" + return HTMLResponse(html, status_code=400) + + def _resolve_custom_mcp_oauth_target( svc: ConnectorService, user_id: int, server_name: str ) -> tuple[str, dict[str, Any]]: @@ -1555,7 +1603,21 @@ async def oauth_callback( server: Any = Depends(get_server), ) -> HTMLResponse: """OAuth redirect target. Exchanges the code and stores credentials. No JWT required.""" + repo = server.services.repos.connector_repo if error: + # The user cancelled or the provider refused. Tell the custom MCP page + # that started the flow instead of leaving it waiting for a token. + failed_row = repo.consume_oauth_state(state) if state else None + if failed_row is not None and is_custom_mcp_kind(failed_row.kind): + delete_oauth_ctx(server.services.settings_repo, failed_row.state_id) + return _oauth_outcome_html( + request, + server, + failed_row, + status="cancelled" if error == "access_denied" else "error", + message_key="callback_auth_failed", + error=error, + ) return _oauth_callback_html( request, "callback_auth_failed", @@ -1569,7 +1631,6 @@ async def oauth_callback( status_code=400, ) - repo = server.services.repos.connector_repo row = repo.consume_oauth_state(state) if row is None: return _oauth_callback_html( @@ -1600,10 +1661,20 @@ async def oauth_callback( delete_oauth_ctx(server.services.settings_repo, row.state_id) except Exception as exc: logger.exception("oauth callback failed for %s", row.kind) - return _oauth_callback_html( + if not is_custom_mcp_kind(row.kind): + return _oauth_callback_html( + request, + "callback_token_exchange_failed", + status_code=400, + detail=str(exc), + ) + delete_oauth_ctx(server.services.settings_repo, row.state_id) + return _oauth_outcome_html( request, - "callback_token_exchange_failed", - status_code=400, + server, + row, + status="error", + message_key="callback_token_exchange_failed", detail=str(exc), ) @@ -1630,10 +1701,12 @@ async def oauth_callback( pending_payload["applied"] = True except Exception as exc: logger.exception("custom MCP oauth apply failed") - return _oauth_callback_html( + return _oauth_outcome_html( request, - "callback_save_failed", - status_code=400, + server, + row, + status="error", + message_key="callback_save_failed", detail=str(exc), ) @@ -1689,6 +1762,8 @@ async def oauth_pending( raise OctopError(ErrorCode.FORBIDDEN, "not your oauth session") server.services.settings_repo.delete(key) return { + "status": data.get("status") or "ok", + "message": data.get("message"), "kind": data.get("kind"), "tokens": data.get("tokens") or {}, "server_name": data.get("server_name"), diff --git a/src/octop/api/routers/experts.py b/src/octop/api/routers/experts.py index 94820c119c..b8ee653265 100644 --- a/src/octop/api/routers/experts.py +++ b/src/octop/api/routers/experts.py @@ -22,7 +22,7 @@ from pydantic import BaseModel, Field from octop.api.common.agent import apply_agent_audience, require_agent_owner_row, user_owns_agent -from octop.api.common.agent_runtime import AgentRuntimeFields, runtime_field_updates +from octop.api.common.agent_runtime import AgentRuntimeFields, persisted_agent_settings from octop.api.common.validators import assert_user_backend_root_dirs from octop.api.deps import current_user, get_server from octop.infra.agents.experts.avatar import ( @@ -800,7 +800,7 @@ async def install_published_expert( color=body.color, agent_id=body.agent_id, welcome_message=body.welcome_message, - runtime_config=runtime_field_updates(body, exclude_unset=True), + runtime_config=persisted_agent_settings(body, exclude_unset=True), enable_trajectory=body.enable_trajectory, conversation_mode=body.conversation_mode, workspace_patch=patch, @@ -954,7 +954,7 @@ async def install_expert_hub_item( composer_copies=copies, composer_report=report, quick_prompts=_quick_prompts_from_body(body), - **runtime_field_updates(body, exclude_unset=False), + **persisted_agent_settings(body, exclude_unset=False), ), ) except SkillHubMarketError as exc: @@ -1087,7 +1087,7 @@ async def create_agent_from_expert( locale=locale, default_model=body.default_model, config_extra=config_extra or None, - runtime_config=runtime_field_updates(body, exclude_unset=True), + runtime_config=persisted_agent_settings(body, exclude_unset=True), agent_id=body.agent_id, color=body.color, welcome_message=body.welcome_message, diff --git a/src/octop/api/routers/invites.py b/src/octop/api/routers/invites.py index 2834eb7e2f..9b618b7424 100644 --- a/src/octop/api/routers/invites.py +++ b/src/octop/api/routers/invites.py @@ -167,6 +167,7 @@ async def redeem_invite( from octop.infra.agents.experts.default_agent import try_bootstrap_default_agent await try_bootstrap_default_agent(server, user_id=user.id, locale=user.locale) + server.services.user_repo.touch_last_login(user.id) from octop.infra.auth.tokens import PURPOSE_ACCESS secret = server.services.secret_repo.get("jwt") diff --git a/src/octop/api/routers/knowledge_bases.py b/src/octop/api/routers/knowledge_bases.py index 097e60adec..e500cf7c9d 100644 --- a/src/octop/api/routers/knowledge_bases.py +++ b/src/octop/api/routers/knowledge_bases.py @@ -42,6 +42,12 @@ get_capability, set_feature_enabled, ) +from octop.infra.knowledge.graph import enqueue_base_graph +from octop.infra.knowledge.graph_extract import ( + get_graph_capability, + graph_models, + set_graph_settings, +) from octop.infra.knowledge.jobs import enqueue_index_document, reindex_all_documents from octop.infra.knowledge.ocr import ( ensure_ocr_deps_async, @@ -49,6 +55,11 @@ set_ocr_settings, validate_ocr_settings, ) +from octop.infra.knowledge.rerank import ( + get_rerank_capability, + rerank_models, + set_rerank_settings, +) from octop.infra.knowledge.service import ( MAX_BASES_PER_OWNER, MAX_DOCS_PER_KB, @@ -87,6 +98,96 @@ class FeatureBody(BaseModel): ocr_backend: str | None = Field(default=None, pattern="^(onnx|remote)$") ocr_model: str | None = None ocr_provider_id: str | None = None + rerank_enabled: bool | None = Field( + default=None, + description="Whether retrieved chunks are reordered by a rerank model.", + ) + rerank_provider_id: str | None = None + rerank_model: str | None = None + graph_enabled: bool | None = Field( + default=None, + description=( + "Whether knowledge graphs can be built and used in retrieval. " + "Enabling requires `graph_provider_id` and `graph_model`." + ), + ) + graph_provider_id: str | None = Field( + default=None, description="Provider whose chat model extracts entities and relations." + ) + graph_model: str | None = Field(default=None, description="Chat model ID used for extraction.") + + +class GraphStats(BaseModel): + built: bool = Field(description="Whether a graph was built for this knowledge base.") + entities: int + relations: int + documents_total: int = Field(description="Indexed documents in the knowledge base.") + documents_ready: int = Field(description="Documents whose passages are in the graph.") + documents_partial: int = Field( + description="Ready documents where some passages could not be extracted." + ) + documents_pending: int = Field(description="Documents queued or being extracted.") + documents_failed: int + + +class GraphNode(BaseModel): + id: str = Field(description="Normalized entity name; pass it as `name` to the entity route.") + name: str + type: str + mentions: int = Field(description="Number of passages that mention the entity.") + degree: int = Field(description="Number of returned edges touching the entity.") + + +class GraphEdge(BaseModel): + source: str + target: str + labels: list[str] = Field(description="Up to three relation names between the two entities.") + weight: float + + +class GraphResponse(BaseModel): + stats: GraphStats + nodes: list[GraphNode] + edges: list[GraphEdge] + + +class GraphEntityRef(BaseModel): + id: str + name: str + + +class GraphEntity(BaseModel): + id: str + name: str + type: str + description: str + mentions: int + + +class GraphEntityRelation(BaseModel): + source: GraphEntityRef + target: GraphEntityRef + label: str + description: str + weight: float + + +class GraphPassage(BaseModel): + doc_id: str + filename: str + path: str + ordinal: int + text: str + + +class GraphEntityResponse(BaseModel): + entity: GraphEntity + relations: list[GraphEntityRelation] + passages: list[GraphPassage] + + +class GraphBuildResponse(BaseModel): + enqueued: int = Field(description="Documents queued for extraction.") class OnnxDownloadBody(BaseModel): @@ -156,7 +257,15 @@ class MoveDocumentBody(BaseModel): target_folder: str = Field( default="", max_length=1024, - description="Path of the destination folder in the same knowledge base; empty for root.", + description="Path of the destination folder; empty for the root.", + ) + target_kb_id: str | None = Field( + default=None, + max_length=64, + description=( + "Knowledge base to move into; omit for the same base. The caller must be able " + "to write to both." + ), ) @@ -272,6 +381,8 @@ def _map_knowledge_error( if isinstance(exc, PermissionError): return OctopError.localized(ErrorCode.KNOWLEDGE_FORBIDDEN, locale) text = str(exc).lower() + if "knowledge graph" in text: + return OctopError.localized(ErrorCode.KNOWLEDGE_GRAPH_NOT_READY, locale) if isinstance(exc, RuntimeError): if "disabled" in text: return OctopError.localized(ErrorCode.KNOWLEDGE_FEATURE_DISABLED, locale) @@ -307,6 +418,8 @@ def _map_knowledge_error( return OctopError.localized(ErrorCode.KNOWLEDGE_BASE_LIMIT, locale) if "unsupported knowledge document content type" in text: return OctopError.localized(ErrorCode.KNOWLEDGE_UNSUPPORTED_TYPE, locale) + if "knowledge rerank" in text: + return OctopError.localized(ErrorCode.KNOWLEDGE_RERANK_NOT_READY, locale) if "knowledge ocr" in text: return OctopError.localized(ErrorCode.KNOWLEDGE_PREREQUISITES_FAILED, locale) if "unique constraint" in text or "duplicate key" in text or "already exists" in text: @@ -369,6 +482,12 @@ def _capability_payload(server: OctopServer) -> dict[str, Any]: payload["ocr"] = get_ocr_capability( server.services.settings_repo.get, server.services.provider_repo ) + payload["rerank"] = get_rerank_capability( + server.services.settings_repo.get, server.services.provider_repo + ) + payload["graph"] = get_graph_capability( + server.services.settings_repo.get, server.services.provider_repo + ) payload["limits"] = { "max_bases_per_owner": MAX_BASES_PER_OWNER, "max_docs_per_kb": MAX_DOCS_PER_KB, @@ -452,6 +571,39 @@ async def ocr_options( } +@router.get("/rerank-options", summary="List providers that offer a rerank model") +async def rerank_options( + server: OctopServer = Depends(get_server), + _admin: User = Depends(require_permission("knowledge_settings")), +) -> dict[str, Any]: + """Enabled providers with at least one rerank model, for the knowledge settings.""" + assert server.services is not None + providers = [] + for provider in server.services.provider_repo.list_all(): + models = rerank_models(provider) if provider.enabled else [] + if models: + providers.append( + {"provider_id": str(provider.id), "provider_name": provider.name, "models": models} + ) + return {"providers": providers} + + +@router.get("/graph-options", summary="List chat models usable for knowledge-graph extraction") +async def graph_options( + server: OctopServer = Depends(get_server), + _admin: User = Depends(require_permission("knowledge_settings")), +) -> dict[str, Any]: + assert server.services is not None + remote = [] + for provider in server.services.provider_repo.list_all(): + models = graph_models(provider) + if models: + remote.append( + {"provider_id": str(provider.id), "provider_name": provider.name, "models": models} + ) + return {"remote": remote} + + @router.put("/feature", summary="Enable or disable knowledge bases") async def put_feature( body: FeatureBody, @@ -491,6 +643,23 @@ async def put_feature( exc, locale=resolve_request_locale(request), server=server ) from exc try: + # First, so an unusable rerank choice is refused before anything is saved. + if body.rerank_enabled is not None: + set_rerank_settings( + server.services.settings_repo.set, + enabled=body.rerank_enabled, + provider_id=body.rerank_provider_id, + model=body.rerank_model, + provider_repo=server.services.provider_repo, + ) + if body.graph_enabled is not None: + set_graph_settings( + server.services.settings_repo.set, + enabled=body.graph_enabled, + provider_id=body.graph_provider_id, + model=body.graph_model, + provider_repo=server.services.provider_repo, + ) set_feature_enabled( server.services.settings_repo.get, server.services.settings_repo.set, @@ -1107,7 +1276,7 @@ async def rename_document( @router.post( "/{kb_id}/documents/{doc_id}/move", - summary="Move a document or folder into another folder", + summary="Move a document or folder into another folder or knowledge base", ) async def move_document( kb_id: str, @@ -1117,15 +1286,24 @@ async def move_document( server: OctopServer = Depends(get_server), user: User = Depends(require_permission("knowledge_bases")), ) -> dict[str, Any]: - """Move within one knowledge base. A folder takes its contents along; nothing is reindexed.""" + """Move a file or folder; a folder takes its contents along. + + Within one base nothing is reindexed. Into another base, originals and chunks + move along; files without reusable chunks are indexed again in the target. + """ try: - document = _knowledge_service(server).move_document( + service = _knowledge_service(server) + document = service.move_document( kb_id, doc_id, actor_user_id=user.id, target_folder=body.target_folder, is_admin=_is_admin(user), + target_kb_id=body.target_kb_id, ) + if document.kb_id != kb_id and server.services is not None: + for pending_id in service.pending_file_ids(document): + enqueue_index_document(server.services, document.kb_id, pending_id) return _row_payload(document, server=server) except Exception as exc: raise _map_knowledge_error( @@ -1209,3 +1387,106 @@ async def reindex_base( raise _map_knowledge_error( exc, locale=resolve_request_locale(request), server=server ) from exc + + +@router.get( + "/{kb_id}/graph", + response_model=GraphResponse, + summary="Get a knowledge base's entity graph", + description=( + "Returns build statistics plus the most-mentioned entities and the relations " + "between them. `stats.built` is false until the graph is built once." + ), +) +async def get_graph( + kb_id: str, + request: Request, + limit: int = Query(default=80, ge=1, le=300, description="Maximum entities to return."), + q: str = Query( + default="", max_length=80, description="Only entities whose name contains this." + ), + server: OctopServer = Depends(get_server), + user: User = Depends(current_user), +) -> dict[str, Any]: + try: + return _knowledge_service(server).get_graph( + kb_id, actor_user_id=user.id, is_admin=_is_admin(user), limit=limit, search=q + ) + except Exception as exc: + raise _map_knowledge_error( + exc, locale=resolve_request_locale(request), server=server + ) from exc + + +@router.get( + "/{kb_id}/graph/entity", + response_model=GraphEntityResponse, + summary="Get one graph entity with its relations and source passages", +) +async def get_graph_entity( + kb_id: str, + request: Request, + name: str = Query(min_length=1, max_length=200, description="Entity name or node id."), + server: OctopServer = Depends(get_server), + user: User = Depends(current_user), +) -> dict[str, Any]: + try: + return _knowledge_service(server).get_graph_entity( + kb_id, name, actor_user_id=user.id, is_admin=_is_admin(user) + ) + except Exception as exc: + raise _map_knowledge_error( + exc, locale=resolve_request_locale(request), server=server + ) from exc + + +@router.post( + "/{kb_id}/graph/build", + response_model=GraphBuildResponse, + summary="Build or rebuild a knowledge base's entity graph", + description=( + "Queues entity and relation extraction for every indexed document, using the " + "chat model chosen in knowledge settings. Extraction runs in the background and " + "sends each passage to that model. Once built, documents indexed later are " + "added automatically." + ), +) +async def build_graph( + kb_id: str, + request: Request, + server: OctopServer = Depends(get_server), + user: User = Depends(require_permission("knowledge_bases")), +) -> dict[str, int]: + try: + _require_usable(server, request) + doc_ids = _knowledge_service(server).graph_build_targets( + kb_id, actor_user_id=user.id, is_admin=_is_admin(user) + ) + assert server.services is not None + return {"enqueued": enqueue_base_graph(server.services, kb_id, doc_ids)} + except Exception as exc: + raise _map_knowledge_error( + exc, locale=resolve_request_locale(request), server=server + ) from exc + + +@router.delete( + "/{kb_id}/graph", + status_code=status.HTTP_204_NO_CONTENT, + summary="Delete a knowledge base's entity graph", + description="Documents and their vector index are untouched; later uploads are not extracted.", +) +async def delete_graph( + kb_id: str, + request: Request, + server: OctopServer = Depends(get_server), + user: User = Depends(require_permission("knowledge_bases")), +) -> None: + try: + _knowledge_service(server).clear_graph( + kb_id, actor_user_id=user.id, is_admin=_is_admin(user) + ) + except Exception as exc: + raise _map_knowledge_error( + exc, locale=resolve_request_locale(request), server=server + ) from exc diff --git a/src/octop/api/routers/managed.py b/src/octop/api/routers/managed.py new file mode 100644 index 0000000000..5658a8a965 --- /dev/null +++ b/src/octop/api/routers/managed.py @@ -0,0 +1,173 @@ +"""Create a managed agent bound to one ACP runner.""" + +from __future__ import annotations + +from typing import Any, Literal + +from fastapi import APIRouter, Depends +from pydantic import BaseModel, Field + +from octop.api.common.agent import apply_agent_audience +from octop.api.deps import current_user, get_server +from octop.api.routers.agents import _row_dict +from octop.infra.agents.experts.composer_files import ( + apply_composer_workspace_patch, + composer_patch_from_payload, +) +from octop.infra.agents.managed import ( + MANAGED_KIND, + apply_managed_backend, + managed_template_payload, + pin_managed_runtime, + seed_managed_workspace, +) +from octop.infra.agents.manager import AgentCreateSpec +from octop.infra.agents.workspace.dir import apply_explicit_workspace_dir +from octop.infra.errors import ErrorCode, OctopError +from octop.infra.history.trajectory.settings import apply_enable_trajectory +from octop.infra.users.groups import GroupAccess + +router = APIRouter() + + +class _LocalizedText(BaseModel): + zh: str = "" + en: str = "" + + +class _QuickPromptBody(BaseModel): + title: _LocalizedText + description: _LocalizedText = Field(default_factory=_LocalizedText) + prompt: _LocalizedText + color: str = "#e8f4ff" + icon_name: str | None = None + + +class _FileOverrideBody(BaseModel): + name: str + content: str = "" + + +class ManagedCreateBody(BaseModel): + name: str + runner: str = Field(description="Enabled ACP runner id") + workspace_dir: str = Field(description="Working directory passed to the ACP session") + description: str | None = None + welcome_message: str | None = None + agent_id: str | None = Field( + default=None, + max_length=64, + description="Optional public id; generated when omitted", + ) + color: str | None = None + icon_url: str | None = None + enable_trajectory: bool = True + visibility: Literal["owner", "all", "groups"] | None = None + group_ids: list[str] | None = None + quick_prompts: list[_QuickPromptBody] | None = None + file_overrides: list[_FileOverrideBody] | None = None + + +def _prompt_dict(prompt: _QuickPromptBody) -> dict[str, Any]: + return { + "title": {"zh": prompt.title.zh, "en": prompt.title.en}, + "description": {"zh": prompt.description.zh, "en": prompt.description.en}, + "prompt": {"zh": prompt.prompt.zh, "en": prompt.prompt.en}, + "color": prompt.color, + "icon_name": prompt.icon_name, + } + + +def _workspace_initializer(body: ManagedCreateBody) -> Any: + prompts = ( + [_prompt_dict(item) for item in body.quick_prompts] + if body.quick_prompts is not None + else None + ) + overrides = ( + [{"name": item.name, "content": item.content} for item in body.file_overrides] + if body.file_overrides + else None + ) + + async def _apply(_row: Any, workspace: Any) -> None: + agents_md = None + if overrides: + for item in overrides: + if item["name"].strip() == "AGENTS.md": + agents_md = item["content"] + await seed_managed_workspace(workspace, agents_md=agents_md, quick_prompts=prompts) + extra = [item for item in (overrides or []) if item["name"].strip() != "AGENTS.md"] + if extra: + patch = composer_patch_from_payload( + file_overrides=extra, + omit_files=None, + hub_skills=None, + ) + await apply_composer_workspace_patch(workspace, patch) + + return _apply + + +def _enabled_runner(server: Any, user_id: int, runner: str) -> None: + registry = server.app_runtime.agent_registry + runners = registry.acp_settings.load_runners(user_id) + spec = runners.get(runner) + if not isinstance(spec, dict) or not spec.get("enabled"): + raise OctopError(ErrorCode.NOT_FOUND, f"ACP runner {runner!r} is not enabled", status=404) + + +@router.get("/managed-agents/template", summary="Default managed workspace files") +async def managed_agent_template( + _user: Any = Depends(current_user), +) -> dict[str, Any]: + """Programming welcome cards and ``AGENTS.md`` copied into a new workspace.""" + return managed_template_payload() + + +@router.post("/managed-agents", summary="Create a managed ACP agent") +async def create_managed_agent( + body: ManagedCreateBody, + user: Any = Depends(current_user), + server: Any = Depends(get_server), +) -> dict[str, Any]: + """Create an agent that forwards each turn to one connected ACP runner. + + The backend is the local filesystem rooted at ``/``. ``workspace_dir`` is + the working directory for both the agent workspace and the ACP session. + """ + runner = body.runner.strip() + _enabled_runner(server, user.id, runner) + config: dict[str, Any] = {"managed_runner": runner} + apply_enable_trajectory(config, body.enable_trajectory) + apply_managed_backend(config) + pin_managed_runtime(config) + apply_explicit_workspace_dir(config, body.workspace_dir) + if body.visibility is not None: + GroupAccess(server.services.repos).validate_visibility(body.visibility, body.group_ids) + spec = AgentCreateSpec( + name=body.name.strip(), + agent_id=(body.agent_id or "").strip() or None, + user_id=user.id, + description=body.description, + welcome_message=body.welcome_message, + color=body.color, + icon_url=(body.icon_url or "").strip() or None, + kind=MANAGED_KIND, + config=config, + mcp_servers=[], + ) + assert server.app_runtime is not None + row = await server.app_runtime.agent_registry.create( + spec, + workspace_initializer=_workspace_initializer(body), + ) + if body.visibility is not None: + row = await apply_agent_audience( + server, + row.agent_id, + visibility=body.visibility, + group_ids=body.group_ids, + shared=body.visibility != "owner", + ) + return _row_dict(row, viewer_user_id=user.id, server=server) diff --git a/src/octop/api/routers/preferences.py b/src/octop/api/routers/preferences.py index 2459fd1ceb..c8be7839aa 100644 --- a/src/octop/api/routers/preferences.py +++ b/src/octop/api/routers/preferences.py @@ -16,6 +16,7 @@ PREFERENCES_KEY_TIMEZONE, ModelReasoningPreference, get_model_reasoning_from_json, + get_onboarding_completed_from_json, get_preferred_model_from_json, get_remote_browser_bookmarks_from_json, get_sidebar_nav_from_json, @@ -75,6 +76,10 @@ class PreferencesResponse(BaseModel): default=None, description="Custom sidebar layout. Null means the user has not customized navigation.", ) + onboarding_completed: bool = Field( + default=False, + description="True once the user finished or skipped the dashboard welcome guide.", + ) class PatchPreferencesBody(BaseModel): @@ -90,6 +95,10 @@ class PatchPreferencesBody(BaseModel): default=None, description="Replace the sidebar layout. Null clears it and restores the default nav.", ) + onboarding_completed: bool | None = Field( + default=None, + description="Mark the welcome guide as done (`true`) or show it again (`false`).", + ) @model_validator(mode="after") def at_least_one_field(self) -> Self: @@ -101,6 +110,7 @@ def at_least_one_field(self) -> Self: "model_reasoning", "timezone", "sidebar_nav", + "onboarding_completed", } ): raise ValueError("at least one preference field is required") @@ -134,6 +144,7 @@ def _response(row: Any) -> PreferencesResponse: }, timezone=parse_preferences_json(raw).get(PREFERENCES_KEY_TIMEZONE), sidebar_nav=_sidebar_nav_response(raw), + onboarding_completed=get_onboarding_completed_from_json(raw), ) @@ -222,5 +233,7 @@ async def patch_preferences( layout = None if body.sidebar_nav is None else body.sidebar_nav.model_dump() merged = merge_sidebar_nav_json(row.preferences_json if row else None, layout) server.services.user_repo.set_preferences_json(user.id, merged) + if body.onboarding_completed is not None: + await server.user_manager.set_onboarding_completed(user.username, body.onboarding_completed) row = server.services.user_repo.get(user.id) return _response(row) diff --git a/src/octop/api/routers/providers.py b/src/octop/api/routers/providers.py index aaadde8eb3..af5ecdb4d3 100644 --- a/src/octop/api/routers/providers.py +++ b/src/octop/api/routers/providers.py @@ -9,7 +9,7 @@ from typing import Any, Literal, cast from fastapi import APIRouter, Depends, Request -from pydantic import BaseModel +from pydantic import BaseModel, Field from octop.api.deps import current_user, get_server, require_permission from octop.infra.agents.providers.codex_apply import ( @@ -25,6 +25,7 @@ poll_device_token, request_device_code, ) +from octop.infra.agents.providers.magpie import MAGPIE_DEFAULT_BASE_URL, discover_local_magpie from octop.infra.agents.providers.model_flags import is_local_runtime_provider from octop.infra.agents.providers.presets import load_provider_presets from octop.infra.agents.providers.probe import ( @@ -285,7 +286,7 @@ def _row_to_dict(r: Any) -> dict[str, Any]: model["reasoning_config"] = capability models.append(model) extra = parse_provider_extra(r) - return { + payload = { "id": r.id, "name": r.name, "kind": r.kind, @@ -303,6 +304,10 @@ def _row_to_dict(r: Any) -> dict[str, Any]: models=models, ), } + integration = extra.get("integration") + if isinstance(integration, str) and integration.strip(): + payload["integration"] = integration.strip() + return payload @router.get("/presets") @@ -372,6 +377,48 @@ async def list_providers( admin_router = APIRouter() +class MagpieDiscoveryResponse(BaseModel): + available: bool = Field( + description="True when this Octop server reached a local Magpie hello response." + ) + base_url: str = Field(description="OpenAI-compatible base URL of the probed local Magpie.") + version: str | None = Field( + default=None, + description="Magpie version from /api/hello. Empty when discovery did not succeed.", + ) + + +def _extra_dict(raw: str | None) -> dict[str, Any]: + if not isinstance(raw, str) or not raw.strip(): + return {} + try: + parsed = json.loads(raw) + except json.JSONDecodeError: + return {} + return parsed if isinstance(parsed, dict) else {} + + +@admin_router.get( + "/magpie/discovery", + summary="Discover local Magpie", + description=( + "Read-only probe of Magpie on the Octop server host at the default " + "loopback address. Failures return available=false and do not include " + "the upstream body. This route has no URL parameter." + ), + response_model=MagpieDiscoveryResponse, +) +async def admin_discover_magpie( + _: Any = Depends(require_permission("providers")), +) -> MagpieDiscoveryResponse: + found = await discover_local_magpie() + return MagpieDiscoveryResponse( + available=found.available, + base_url=found.base_url or MAGPIE_DEFAULT_BASE_URL, + version=found.version, + ) + + @admin_router.get("") async def admin_list_providers( _: Any = Depends(require_permission("providers")), @@ -536,6 +583,7 @@ async def admin_fetch_provider_models( extra_headers=provider_headers(draft) or None, locale=resolve_request_locale(request), provider_name=(body.name or "").strip() or None, + extra=_extra_dict(body.extra_json), ) diff --git a/src/octop/api/routers/setup.py b/src/octop/api/routers/setup.py index e5e84d89db..f390cec278 100644 --- a/src/octop/api/routers/setup.py +++ b/src/octop/api/routers/setup.py @@ -457,6 +457,7 @@ async def initial_admin( locale=locale, role_name=admin_role.user_role_name if admin_role is not None else "管理员", ) + server.services.user_repo.touch_last_login(user.id) from octop.infra.auth.tokens import PURPOSE_ACCESS secret = server.services.secret_repo.get("jwt") diff --git a/src/octop/api/routers/update.py b/src/octop/api/routers/update.py index 1c3c905d72..7b109b97d4 100644 --- a/src/octop/api/routers/update.py +++ b/src/octop/api/routers/update.py @@ -12,7 +12,7 @@ from fastapi import APIRouter, BackgroundTasks, Body, Depends, Query, Request from pydantic import BaseModel, Field -from octop.api.deps import current_user, get_server, require_permission +from octop.api.deps import current_user, get_server, require_any_permission, require_permission from octop.api.routers.update_store import ( UpgradeTaskStatus, cache_status, @@ -451,7 +451,7 @@ def _restart_service_task(runtime: ServiceRuntime) -> None: async def restart_service_endpoint( background_tasks: BackgroundTasks, server: OctopServer = Depends(get_server), - _: Any = Depends(require_permission("update")), + _: Any = Depends(require_any_permission("update", "app_config")), ) -> dict[str, Any]: if desktop_shell_enabled(): raise OctopError(ErrorCode.FORBIDDEN, _SHELL_UPDATE_FORBIDDEN) diff --git a/src/octop/api/routers/user_roles.py b/src/octop/api/routers/user_roles.py index 45ebccc02b..20af2d3c0f 100644 --- a/src/octop/api/routers/user_roles.py +++ b/src/octop/api/routers/user_roles.py @@ -28,10 +28,13 @@ validate_permission_keys, ) from octop.infra.users.resource_policy import ( + POLICY_ALLOWED_BACKEND_TYPES, POLICY_MAX_AGENTS, POLICY_TOKEN_QUOTA, POLICY_WORKSPACE_ROOT_DIR, + allowed_backend_types_of, max_agents_of, + normalize_allowed_backend_types, normalize_max_agents, normalize_token_quota, normalize_workspace_root_dir, @@ -45,6 +48,7 @@ POLICY_WORKSPACE_ROOT_DIR, POLICY_TOKEN_QUOTA, POLICY_MAX_AGENTS, + POLICY_ALLOWED_BACKEND_TYPES, } @@ -135,6 +139,7 @@ def _policies_from_fields( workspace_root_dir: str | None, token_quota: int | None, max_agents: int | None, + allowed_backend_types: Any, ) -> list[tuple[str, str]]: items: list[tuple[str, str]] = [] root = normalize_workspace_root_dir(workspace_root_dir) @@ -146,6 +151,9 @@ def _policies_from_fields( agents = normalize_max_agents(max_agents) if agents is not None: items.append((POLICY_MAX_AGENTS, str(agents))) + backends = normalize_allowed_backend_types(allowed_backend_types) + if backends: + items.append((POLICY_ALLOWED_BACKEND_TYPES, backends)) return items @@ -162,6 +170,7 @@ def _policies_from_items(items: list[PolicyItem]) -> list[tuple[str, str]]: workspace_root_dir=by_name.get(POLICY_WORKSPACE_ROOT_DIR), token_quota=token_quota_of(by_name.get(POLICY_TOKEN_QUOTA)), max_agents=max_agents_of(by_name.get(POLICY_MAX_AGENTS)), + allowed_backend_types=by_name.get(POLICY_ALLOWED_BACKEND_TYPES), ) @@ -185,6 +194,7 @@ def public_role(row: UserRoleRow, *, avatar_url: str | None = None) -> dict[str, workspace_root_dir = None token_quota = None max_agents = None + allowed_backend_types = None else: permissions = expand_permission_grants(list(row.permissions)) system_role = "user" @@ -192,6 +202,9 @@ def public_role(row: UserRoleRow, *, avatar_url: str | None = None) -> dict[str, workspace_root_dir = workspace_root_dir_of(row.policy_value(POLICY_WORKSPACE_ROOT_DIR)) token_quota = token_quota_of(row.policy_value(POLICY_TOKEN_QUOTA)) max_agents = max_agents_of(row.policy_value(POLICY_MAX_AGENTS)) + allowed_backend_types = allowed_backend_types_of( + row.policy_value(POLICY_ALLOWED_BACKEND_TYPES) + ) return { "user_role_id": row.user_role_id, "user_role_name": row.user_role_name, @@ -204,6 +217,7 @@ def public_role(row: UserRoleRow, *, avatar_url: str | None = None) -> dict[str, "workspace_root_dir": workspace_root_dir, "token_quota": token_quota, "max_agents": max_agents, + "allowed_backend_types": allowed_backend_types, "created_at": row.created_at, "updated_at": row.updated_at, "avatar_icon": row.avatar_icon, diff --git a/src/octop/api/routers/users.py b/src/octop/api/routers/users.py index 5ebf5aa149..eddca699de 100644 --- a/src/octop/api/routers/users.py +++ b/src/octop/api/routers/users.py @@ -19,6 +19,7 @@ expand_permission_grants, ) from octop.infra.users.resource_policy import ( + normalize_allowed_backend_types, normalize_max_agents, normalize_token_quota, normalize_workspace_root_dir, @@ -53,6 +54,10 @@ class UserCreateBody(BaseModel): workspace_root_dir: str | None = None token_quota: int | None = Field(default=None, ge=0) max_agents: int | None = Field(default=None, ge=0) + allowed_backend_types: list[str] | None = Field( + default=None, + description="Backend types this user may select. null means unrestricted.", + ) create_default_expert: bool = Field( default=False, description="When true, create one expert from expert_template_id in this request.", @@ -78,6 +83,10 @@ class UserPatchBody(BaseModel): workspace_root_dir: str | None = None token_quota: int | None = Field(default=None, ge=0) max_agents: int | None = Field(default=None, ge=0) + allowed_backend_types: list[str] | None = Field( + default=None, + description="Backend types this user may select. null means unrestricted.", + ) class ResetPasswordBody(BaseModel): @@ -154,6 +163,7 @@ def _row_to_dict( "login_locked_until": locked_until if locked else 0, "login_retry_after_seconds": retry_after, "created_at": int(r.created_at), + "last_login_at": (int(r.last_login_at) if getattr(r, "last_login_at", None) else None), "permissions": expand_permission_grants(list(getattr(r, "permissions", None) or [])), "role_name": getattr(r, "role_name", None) or None, "avatar_icon": getattr(r, "avatar_icon", None) or None, @@ -208,9 +218,11 @@ def _policy_kwargs_replace_from_pairs(pairs: list[tuple[str, str]]) -> dict[str, linger after a switch (including to admin, which has no resource limits). """ from octop.infra.users.resource_policy import ( + POLICY_ALLOWED_BACKEND_TYPES, POLICY_MAX_AGENTS, POLICY_TOKEN_QUOTA, POLICY_WORKSPACE_ROOT_DIR, + allowed_backend_types_of, max_agents_of, token_quota_of, ) @@ -220,6 +232,9 @@ def _policy_kwargs_replace_from_pairs(pairs: list[tuple[str, str]]) -> dict[str, "workspace_root_dir": by_name.get(POLICY_WORKSPACE_ROOT_DIR), "token_quota": token_quota_of(by_name.get(POLICY_TOKEN_QUOTA)), "max_agents": max_agents_of(by_name.get(POLICY_MAX_AGENTS)), + "allowed_backend_types": allowed_backend_types_of( + by_name.get(POLICY_ALLOWED_BACKEND_TYPES) + ), } @@ -242,6 +257,8 @@ def _policy_kwargs_from_body(body: UserCreateBody | UserPatchBody) -> dict[str, policy_kwargs["token_quota"] = body.token_quota if "max_agents" in body.model_fields_set: policy_kwargs["max_agents"] = body.max_agents + if "allowed_backend_types" in body.model_fields_set: + policy_kwargs["allowed_backend_types"] = body.allowed_backend_types return policy_kwargs @@ -368,6 +385,8 @@ async def create_user( normalize_token_quota(policy_kwargs["token_quota"]) if "max_agents" in policy_kwargs: normalize_max_agents(policy_kwargs["max_agents"]) + if "allowed_backend_types" in policy_kwargs: + normalize_allowed_backend_types(policy_kwargs["allowed_backend_types"]) role_name = ( _clean_role_name(body.role_name) if "role_name" in body.model_fields_set else template_name ) diff --git a/src/octop/cli/commands/cloud.py b/src/octop/cli/commands/cloud.py index 0fb4a20a00..ba2d2a29f1 100644 --- a/src/octop/cli/commands/cloud.py +++ b/src/octop/cli/commands/cloud.py @@ -51,13 +51,17 @@ def _one_line(credential: Any) -> str: @cloud.command("import-tccli") -@click.option("--profile", default="default", show_default=True, help="tccli profile name.") +@click.option( + "--profile", + default=None, + help="tccli profile. Defaults to octop- for the acting user.", +) @click.option("--user", "as_user", default=None, help="Import for another user (admin only).") @click.option( "--json", "as_json", is_flag=True, default=False, help="Print machine-readable status." ) @click.option("--quiet", is_flag=True, default=False, help="Only print errors.") -def import_tccli(profile: str, as_user: str | None, as_json: bool, quiet: bool) -> None: +def import_tccli(profile: str | None, as_user: str | None, as_json: bool, quiet: bool) -> None: """Import ``~/.tccli/.credential`` into the Octop cloud store. Run right after an interactive tccli/OAuth login so the Workbench console @@ -73,9 +77,36 @@ def import_tccli(profile: str, as_user: str | None, as_json: bool, quiet: bool) credential_from_tccli, open_cloud_credential_store, refresh_credential, + tccli_profile_for_user, + tccli_profile_owner, ) from octop.infra.utils.paths import PathLayout + try: + acting = resolve_cli_acting_user_id(None, as_user) + except OctopError as exc: + acting = None + acting_error: OctopError | None = exc + else: + acting_error = None + + owner = tccli_profile_owner(profile) + if as_user is not None: + if acting is None: + raise SystemExit(f"error: {acting_error}") from acting_error + uid = acting + elif owner is not None: + uid = owner + elif acting is not None: + uid = acting + else: + fallback = _first_admin_id() + if fallback is None: + raise SystemExit(f"error: {acting_error}") from acting_error + uid = fallback + if not profile: + profile = tccli_profile_for_user(uid) + path = Path.home() / ".tccli" / f"{profile}.credential" if not path.is_file(): raise SystemExit(f"error: tccli 凭证不存在: {path}") @@ -93,16 +124,6 @@ def import_tccli(profile: str, as_user: str | None, as_json: bool, quiet: bool) except CloudError as exc: raise SystemExit(f"error: 本机凭证已失效: {exc}") from exc - try: - uid = resolve_cli_acting_user_id(None, as_user) - except OctopError as exc: - # Unattended call (skill script after an OAuth login): fall back to the - # first admin instead of failing the sync. - fallback = _first_admin_id() if as_user is None else None - if fallback is None: - raise SystemExit(f"error: {exc}") from exc - uid = fallback - paths = PathLayout.from_env() with open_cli_services() as services: store = open_cloud_credential_store( @@ -111,6 +132,13 @@ def import_tccli(profile: str, as_user: str | None, as_json: bool, quiet: bool) services.repos.secret_repo, ) store.save(uid, credential) + owner_row = services.user_repo.get(uid) + services.audit_repo.user_event( + owner_row.username if owner_row is not None and owner_row.username else str(uid), + "cloud.import_tccli", + target=f"user:{uid}", + payload=f"{profile}.credential", + ) payload = { "user_id": uid, diff --git a/src/octop/cli/commands/cron.py b/src/octop/cli/commands/cron.py index adfacca1f3..33ef273080 100644 --- a/src/octop/cli/commands/cron.py +++ b/src/octop/cli/commands/cron.py @@ -54,7 +54,21 @@ def list_jobs(agent_id: str | None, as_user: str | None) -> None: @click.option("--user", "as_user", default=None) @click.option("--trigger", required=True) @click.option("--prompt", required=True) -@click.option("--fresh-thread", is_flag=True, default=False) +@click.option( + "--fresh-thread", + is_flag=True, + default=False, + help="Start a new conversation on every run. Cannot be combined with --thread.", +) +@click.option( + "--thread", + "thread_id", + default=None, + help=( + "Pin the job to this historical conversation. " + "Omit to follow the latest conversation on the Dashboard channel." + ), +) @click.option("--task-type", default="agent") def create_job( agent_id: str | None, @@ -62,11 +76,14 @@ def create_job( trigger: str, prompt: str, fresh_thread: bool, + thread_id: str | None, task_type: str, ) -> None: """Create a cron job.""" from octop.cli.support.offline_ops import create_cron_offline, resolve_cron_user_id + if fresh_thread and (thread_id or "").strip(): + raise click.UsageError("--thread cannot be combined with --fresh-thread") aid = require_agent(agent_id) try: uid = resolve_cron_user_id(aid, as_user) @@ -77,6 +94,7 @@ def create_job( prompt=prompt, fresh_thread=fresh_thread, task_type=task_type, + thread_id=thread_id, ) except OctopError as exc: fail_octop(exc) diff --git a/src/octop/cli/support/offline_ops.py b/src/octop/cli/support/offline_ops.py index 2bbf96a5bb..2cc02ab561 100644 --- a/src/octop/cli/support/offline_ops.py +++ b/src/octop/cli/support/offline_ops.py @@ -14,6 +14,7 @@ ) from octop.infra.agents.experts.catalog import ExpertCatalog, default_library_root from octop.infra.agents.providers.model_flags import is_local_runtime_provider +from octop.infra.cron.session_target import thread_target_key from octop.infra.cron.task_type import normalize_cron_task_type, require_cron_prompt from octop.infra.cron.trigger import build_trigger from octop.infra.errors import ErrorCode, OctopError @@ -186,7 +187,12 @@ def delete_user_offline(username: str, *, home: Path | None = None) -> None: async def _ensure_dashboard_session( - services: Any, *, agent_id: str, user_id: int, session_key: str + services: Any, + *, + agent_id: str, + user_id: int, + session_key: str, + channel_type: str = ThreadRegistry.CHANNEL_DASHBOARD, ) -> None: registry = ThreadRegistry( session_repo=services.session_repo, @@ -196,7 +202,7 @@ async def _ensure_dashboard_session( session_key=session_key, agent_id=agent_id, user_id=user_id, - channel_type=ThreadRegistry.CHANNEL_DASHBOARD, + channel_type=channel_type, ) @@ -214,19 +220,38 @@ def create_cron_offline( prompt: str, fresh_thread: bool = False, task_type: str = "agent", + thread_id: str | None = None, home: Path | None = None, ) -> dict[str, Any]: build_trigger(trigger) cleaned_prompt = require_cron_prompt(prompt) + pinned = (thread_id or "").strip() session_key = ThreadRegistry.dashboard_key(agent_id=agent_id, user_id=user_id) + channel_type = ThreadRegistry.CHANNEL_DASHBOARD with open_cli_services(home) as svc: if svc.agent_repo.get(agent_id) is None: raise OctopError(ErrorCode.AGENT_NOT_FOUND, f"agent {agent_id!r} not found") - asyncio.run( - _ensure_dashboard_session( - svc, agent_id=agent_id, user_id=user_id, session_key=session_key + if pinned: + thread = svc.thread_repo.get(pinned) + if thread is None or thread.agent_id != agent_id: + raise OctopError(ErrorCode.NOT_FOUND, f"thread {pinned!r} not found") + if thread.user_id != user_id: + raise OctopError(ErrorCode.FORBIDDEN, "thread not owned by user") + session_key = thread_target_key(pinned) + channel_type = thread.channel_type + ensure_key = thread.session_key + else: + ensure_key = session_key + if svc.session_repo.get(ensure_key) is None: + asyncio.run( + _ensure_dashboard_session( + svc, + agent_id=agent_id, + user_id=user_id, + session_key=ensure_key, + channel_type=channel_type, + ) ) - ) cron_id = new_cron_id() svc.cron_repo.create( cron_id=cron_id, diff --git a/src/octop/config.py b/src/octop/config.py index 7e303201a9..814ed3b3d4 100644 --- a/src/octop/config.py +++ b/src/octop/config.py @@ -10,6 +10,7 @@ from pathlib import Path from typing import Any from urllib.parse import quote_plus, urlparse +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError logger = logging.getLogger(__name__) @@ -416,6 +417,116 @@ def env_bind_overrides() -> tuple[str | None, int | None]: return host, port +_LOG_LEVELS = frozenset({"debug", "info", "warning", "error"}) + + +def _optional_int(raw: dict[str, Any], key: str) -> int | None: + """Return an int field without echoing the raw value in errors.""" + if key not in raw or raw[key] is None: + return None + value = raw[key] + if isinstance(value, bool): + raise ValueError(f"{key} must be an integer") + if isinstance(value, int): + return value + if isinstance(value, str): + try: + return int(value.strip()) + except ValueError: + raise ValueError(f"{key} must be an integer") from None + raise ValueError(f"{key} must be an integer") + + +def _optional_bool(raw: dict[str, Any], key: str) -> None: + if key not in raw or raw[key] is None: + return + if not isinstance(raw[key], bool): + raise ValueError(f"{key} must be a boolean") + + +def _optional_str(raw: dict[str, Any], key: str) -> str | None: + if key not in raw or raw[key] is None: + return None + value = raw[key] + if not isinstance(value, str): + raise ValueError(f"{key} must be a string") + return value + + +def validate_persisted_config(raw: dict[str, Any]) -> None: + """Raise ``ValueError`` when ``raw`` cannot be stored as ``config.json``. + + Checks the on-disk document only. Environment overrides are not applied. + Messages name the field, never secret values. + """ + if not isinstance(raw, dict): + raise ValueError("config must be a JSON object") + + bind_host = _optional_str(raw, "bind_host") + if bind_host is not None and not bind_host.strip(): + raise ValueError("bind_host must not be empty") + port = _optional_int(raw, "port") + if port is not None and not 0 <= port <= 65535: + raise ValueError("port must be 0-65535") + level = _optional_str(raw, "log_level") + if level is not None and level.strip().lower() not in _LOG_LEVELS: + raise ValueError("log_level must be one of debug, info, warning, error") + ttl = _optional_int(raw, "access_token_ttl_seconds") + if ttl is not None and ttl < 1: + raise ValueError("access_token_ttl_seconds must be >= 1") + attempts = _optional_int(raw, "login_max_attempts") + if attempts is not None and attempts < 1: + raise ValueError("login_max_attempts must be >= 1") + lockout = _optional_int(raw, "login_lockout_seconds") + if lockout is not None and lockout < 0: + raise ValueError("login_lockout_seconds must be >= 0") + if "cors_origins" in raw and raw["cors_origins"] is not None: + origins = raw["cors_origins"] + if not isinstance(origins, list) or any(not isinstance(item, str) for item in origins): + raise ValueError("cors_origins must be a list of strings") + timezone = _optional_str(raw, "default_timezone") + if timezone is not None: + tz = timezone.strip() + if not tz: + raise ValueError("default_timezone must not be empty") + try: + ZoneInfo(tz) + except ZoneInfoNotFoundError: + raise ValueError("default_timezone must be an IANA timezone name") from None + for key in ("enable_dashboard", "enable_api_docs", "require_setup_password"): + _optional_bool(raw, key) + upload = _optional_int(raw, "max_upload_mb") + if upload is not None and not MIN_MAX_UPLOAD_MB <= upload <= MAX_MAX_UPLOAD_MB: + raise ValueError(f"max_upload_mb must be {MIN_MAX_UPLOAD_MB}-{MAX_MAX_UPLOAD_MB}") + idle = _optional_int(raw, "browser_idle_timeout_minutes") + if idle is not None and idle < 0: + raise ValueError("browser_idle_timeout_minutes must be >= 0") + + if "database" in raw: + section = _parse_database_section(raw.get("database")) + driver = str(section.get("driver", "sqlite")).strip().lower() + if driver == "postgresql": + db_port = _optional_int(section, "port") + if "port" in section and (db_port is None or not 1 <= db_port <= 65535): + raise ValueError("database.port must be 1-65535") + defaults = _defaults_for_file()["database"] + parse_database_config({**defaults, **section}) + + if "tls" in raw and raw.get("tls") is not None: + tls_raw = raw["tls"] + if not isinstance(tls_raw, dict): + raise ValueError("tls must be an object") + if "http_port" in tls_raw: + http_port = _optional_int(tls_raw, "http_port") + if http_port is None or not 1 <= http_port <= 65535: + raise ValueError("tls.http_port must be 1-65535") + _parse_tls_section(tls_raw) + if "backup" in raw: + _parse_backup_section(raw.get("backup")) + if "capabilities" in raw: + _parse_capabilities_section(raw.get("capabilities")) + + @cache def _warn_retired_history_config() -> None: logger.warning("history_v2_enabled and OCTOP_HISTORY_V2_ENABLED are retired and ignored") diff --git a/src/octop/i18n/domains/channel.py b/src/octop/i18n/domains/channel.py index 1c6b0518a5..e52fa2ccbd 100644 --- a/src/octop/i18n/domains/channel.py +++ b/src/octop/i18n/domains/channel.py @@ -11,6 +11,8 @@ "channel_runtime_reason", "channel_tool_hint_end", "channel_tool_hint_start", + "telegram_quote_prompt", + "telegram_reject_notice", ] @@ -39,3 +41,16 @@ def channel_runtime_reason(reason: str, locale: str | Locale = "en") -> str: """Localized label for a runtime-status reason code (``disabled``/``error``/…).""" label = lookup(f"channel.runtime.{reason}", locale) return label if isinstance(label, str) else reason + + +def telegram_reject_notice(reason: str, locale: str | Locale = "en") -> str: + """Short notice for a rejected Telegram button; unknown reasons use the default.""" + text = lookup(f"channel.telegram.reject.{reason}", locale) + return text if isinstance(text, str) else tr("channel.telegram.reject.default", locale) + + +def telegram_quote_prompt(quote: str, locale: str | Locale = "en") -> str: + """Prompt prefix that carries the quoted (replied-to) message body.""" + header = tr("channel.telegram.quote.header", locale) + current = tr("channel.telegram.quote.current", locale) + return f"{header}\n{quote}\n\n{current}\n" diff --git a/src/octop/i18n/domains/stream.py b/src/octop/i18n/domains/stream.py index 916dfcf19c..7afd627920 100644 --- a/src/octop/i18n/domains/stream.py +++ b/src/octop/i18n/domains/stream.py @@ -198,8 +198,12 @@ def classify_stream_error_message(message: str) -> str | None: "request timed out" in lower or "timed out or interrupted" in lower or "connection error" in lower + or "connection refused" in lower + or "all connection attempts failed" in lower or "apitimeouterror" in compact or "apiconnectionerror" in compact + or "connecterror" in compact + or "connecttimeout" in compact ): return TIMEOUT_NETWORK diff --git a/src/octop/i18n/domains/tools.py b/src/octop/i18n/domains/tools.py index ab5a8cf1fe..060235a926 100644 --- a/src/octop/i18n/domains/tools.py +++ b/src/octop/i18n/domains/tools.py @@ -38,6 +38,8 @@ "call_agent", "call_deferred_tool", "acp_runner", + # Managed ACP permission fallback. It is not a built-in tool users toggle. + "external-agent", } ) diff --git a/src/octop/i18n/en.json b/src/octop/i18n/en.json index 126c80dcd0..db75648bec 100644 --- a/src/octop/i18n/en.json +++ b/src/octop/i18n/en.json @@ -439,6 +439,7 @@ "STORAGE_BROWSE_FAILED": "Could not browse this storage backend.", "CHANNEL_KIND_UNSUPPORTED": "Unsupported channel type.", "CHANNEL_INVALID_CREDENTIALS": "Invalid channel credentials.", + "CHANNEL_OPTIONS_UNSUPPORTED": "Installed octop-gateway does not support these Telegram options. Install a compatible Gateway build before saving.", "CHANNEL_NAME_TAKEN": "This channel name is already in use. Please choose another.", "CONNECTOR_NOT_FOUND": "Connector not found.", "CONNECTOR_NAME_TAKEN": "This connector name is already in use. Please choose another.", @@ -450,6 +451,7 @@ "CONNECTOR_NOT_BOUND": "Connector is not bound to this agent.", "CONNECTOR_ALREADY_BOUND": "Connector is already bound.", "CONNECTOR_MCP_LOAD_FAILED": "Failed to load tools for MCP server(s): {servers}", + "CONNECTOR_AUTH_REQUIRED": "Connector(s) need authorization before use: {servers}", "BRIDGE_NOT_FOUND": "Cloud collab connection not found.", "BRIDGE_AUTH_FAILED": "Peer Octop rejected the credentials. Check the address, username, and password — or use Probe to verify the account first.", "BRIDGE_PEER_UNREACHABLE": "Could not reach the peer Octop instance. Confirm the URL is reachable, credentials are correct, and the peer has cloud collab support (needs /api/bridge/ws). If the peer is behind NAT, initiate the link from the side that can reach the other.", @@ -516,9 +518,12 @@ "KNOWLEDGE_BASE_LIMIT": "You can create at most 20 knowledge bases.", "KNOWLEDGE_UNSUPPORTED_TYPE": "This document type is not supported for knowledge bases.", "KNOWLEDGE_NAME_TAKEN": "You already have a knowledge base with this name.", + "KNOWLEDGE_GRAPH_NOT_READY": "Choose an available chat model for the knowledge graph in knowledge-base settings first.", "KNOWLEDGE_NAME_INVALID": "Invalid name.", "KNOWLEDGE_MOVE_INVALID": "A folder cannot be moved into itself or one of its subfolders.", "KNOWLEDGE_MOVE_NAME_TAKEN": "The target folder already has an item with this name.", + "KNOWLEDGE_RERANK_NOT_READY": "The selected rerank model is not available. Choose an enabled provider that offers a rerank model.", + "KNOWLEDGE_MOVE_BUSY": "Some of these documents are still being indexed. Move them to another knowledge base once indexing finishes.", "AVATAR_INVALID": "Avatar must be a PNG, JPEG, WebP, or GIF image.", "AVATAR_TOO_LARGE": "Avatar exceeds the 5 MB size limit.", "INVITE_INVALID": "This invite code is invalid.", @@ -536,9 +541,11 @@ "WORKSPACE_ROOT_CONTAINER_UNSUPPORTED": "Allowed storage root cannot be set while Octop is running in a container.", "TOKEN_QUOTA_EXCEEDED": "This account has reached its token quota. Please contact an administrator to raise the limit ({used}/{quota}).", "AGENT_QUOTA_EXCEEDED": "This account has reached its expert limit. Please contact an administrator to raise the limit ({used}/{quota}).", + "BACKEND_TYPE_NOT_ALLOWED": "This account cannot use backend type {types}.", "CAPTCHA_REQUIRED": "This server requires a captcha token to sign in.", "CAPTCHA_FAILED": "Captcha verification failed. Try again.", "CONFIG_FILE_CORRUPT": "Cannot update {path}: the file is not valid JSON ({detail}). Fix the syntax and retry — no settings were changed.", + "APP_CONFIG_INVALID": "Invalid configuration: {detail}.", "TEAM_NOT_FOUND": "Team not found.", "TEAM_MEMBERS_TOO_FEW": "A team needs at least two members.", "TEAM_MEMBER_INVALID": "Those experts cannot join a team on this instance. Peer experts, other teams, and deleted experts are not allowed.", @@ -661,6 +668,7 @@ "call_agent": "Call expert", "call_deferred_tool": "Call on demand", "acp_runner": "ACP external agent", + "external-agent": "Managed agent", "browser_use": "Use browser", "browser_control": "Browser control", "send_file_to_user": "Send file", @@ -738,6 +746,21 @@ "discord_connection_failed": "Discord connection failed. Check the network, proxy and bot configuration.", "discord_disconnected": "Discord is disconnected or reconnecting.", "discord_proxy_auth_invalid": "Proxy authentication must use user:password." + }, + "telegram": { + "reject": { + "pending_expired_or_resolved": "This card has expired. Please ask again.", + "question_index_changed": "The question has changed. Please use the new card.", + "user_mismatch": "This card does not belong to the current session.", + "owner_changed": "This card does not belong to the current session.", + "model_unavailable": "This model is no longer available. Please choose again.", + "thread_rebound": "The conversation has changed. Please try again.", + "default": "The action did not take effect. Please try again." + }, + "quote": { + "header": "[Quoted message]", + "current": "[Current message]" + } } }, "threads": { @@ -800,6 +823,7 @@ "knowledge": { "retrieval": { "citation": "[Knowledge base: {name} | Source: {filename} | Chunk: {ordinal}]\n", + "graph_preamble": "Relations between entities, extracted automatically from the same knowledge bases. Use them to connect the passages above; they are not sources to cite.", "preamble": "The following knowledge-base content is reference material. Cite the relevant source in your answer." } }, diff --git a/src/octop/i18n/zh.json b/src/octop/i18n/zh.json index 12eb7a3c22..ab7bc10d21 100644 --- a/src/octop/i18n/zh.json +++ b/src/octop/i18n/zh.json @@ -439,6 +439,7 @@ "STORAGE_BROWSE_FAILED": "无法浏览此存储后端。", "CHANNEL_KIND_UNSUPPORTED": "不支持的通道类型。", "CHANNEL_INVALID_CREDENTIALS": "通道凭证无效。", + "CHANNEL_OPTIONS_UNSUPPORTED": "当前安装的 octop-gateway 不支持这些 Telegram 选项。请先安装兼容的 Gateway 构建再保存。", "CHANNEL_NAME_TAKEN": "该通道名称已被使用,请换一个名称。", "CONNECTOR_NOT_FOUND": "未找到连接器。", "CONNECTOR_NAME_TAKEN": "该连接器名称已被使用,请换一个名称。", @@ -450,6 +451,7 @@ "CONNECTOR_NOT_BOUND": "连接器未绑定到此专家。", "CONNECTOR_ALREADY_BOUND": "连接器已绑定。", "CONNECTOR_MCP_LOAD_FAILED": "无法加载 MCP 工具:{servers}", + "CONNECTOR_AUTH_REQUIRED": "以下连接器需要授权后才能使用:{servers}", "BRIDGE_NOT_FOUND": "未找到云端协同连接。", "BRIDGE_AUTH_FAILED": "对端 Octop 拒绝了登录凭证。请核对地址、用户名和密码;也可先点「探测」验证账号是否可用。", "BRIDGE_PEER_UNREACHABLE": "无法连接对端 Octop 实例。请确认:地址可访问、账号密码正确,且对端已升级到支持云端协同的版本(需开放 /api/bridge/ws)。若对端在内网,需由可达的一侧主动发起连接。", @@ -516,9 +518,12 @@ "KNOWLEDGE_BASE_LIMIT": "每个用户最多可创建 20 个知识库。", "KNOWLEDGE_UNSUPPORTED_TYPE": "知识库不支持此文档类型。", "KNOWLEDGE_NAME_TAKEN": "您已有同名知识库。", + "KNOWLEDGE_GRAPH_NOT_READY": "请先在知识库设置中为知识图谱选择一个可用的对话模型。", "KNOWLEDGE_NAME_INVALID": "名称无效。", "KNOWLEDGE_MOVE_INVALID": "不能把文件夹移动到它自身或其子文件夹中。", "KNOWLEDGE_MOVE_NAME_TAKEN": "目标文件夹中已有同名项目。", + "KNOWLEDGE_RERANK_NOT_READY": "所选的重排序模型不可用,请选择已启用且提供重排序模型的供应商。", + "KNOWLEDGE_MOVE_BUSY": "其中有文档正在建立索引,完成后再移动到其他知识库。", "AVATAR_INVALID": "头像须为 PNG、JPEG、WebP 或 GIF 图片。", "AVATAR_TOO_LARGE": "头像超过 5 MB 大小限制。", "INVITE_INVALID": "邀请码无效。", @@ -536,9 +541,11 @@ "WORKSPACE_ROOT_CONTAINER_UNSUPPORTED": "容器环境下不能设置「允许的存储根目录」。", "TOKEN_QUOTA_EXCEEDED": "该账号已达到 token 配额上限,请联系管理员提升额度({used}/{quota})。", "AGENT_QUOTA_EXCEEDED": "该账号已达到专家数量上限,请联系管理员提升额度({used}/{quota})。", + "BACKEND_TYPE_NOT_ALLOWED": "该账号不能使用这些后端类型:{types}。", "CAPTCHA_REQUIRED": "此服务器登录需要验证码令牌。", "CAPTCHA_FAILED": "验证码校验失败,请重试。", "CONFIG_FILE_CORRUPT": "无法更新 {path}:该文件不是合法 JSON({detail})。请修正语法后重试;现有配置未被改动。", + "APP_CONFIG_INVALID": "配置无效:{detail}。", "TEAM_NOT_FOUND": "未找到该团队。", "TEAM_MEMBERS_TOO_FEW": "团队至少需要两名成员。", "TEAM_MEMBER_INVALID": "所选专家无法加入本机团队。对端专家、其他团队或已删除的专家不能作为成员。", @@ -661,6 +668,7 @@ "call_agent": "调用专家", "call_deferred_tool": "按需调用", "acp_runner": "ACP 外部 Agent", + "external-agent": "托管智能体", "browser_use": "使用浏览器", "browser_control": "浏览器控制", "send_file_to_user": "发送文件", @@ -738,6 +746,21 @@ "discord_connection_failed": "Discord 连接失败,请检查网络、代理和机器人配置。", "discord_disconnected": "Discord 尚未连接或正在重连。", "discord_proxy_auth_invalid": "代理认证格式应为 user:password。" + }, + "telegram": { + "reject": { + "pending_expired_or_resolved": "这张卡片已过期,请重新提问", + "question_index_changed": "问题已更新,请用新卡片", + "user_mismatch": "该卡片不属于当前会话", + "owner_changed": "该卡片不属于当前会话", + "model_unavailable": "该模型已不可用,请重新选择", + "thread_rebound": "会话已切换,请重新操作", + "default": "操作未生效,请重试" + }, + "quote": { + "header": "[引用消息]", + "current": "[当前消息]" + } } }, "threads": { @@ -800,6 +823,7 @@ "knowledge": { "retrieval": { "citation": "[知识库:{name}|来源:{filename}|片段:{ordinal}]\n", + "graph_preamble": "以下是从同一批知识库中自动抽取的实体关系,可用于串联上面的内容,但不要把它们当作可引用的来源。", "preamble": "以下知识库内容仅供参考;回答时请引用对应来源。" } }, diff --git a/src/octop/infra/agents/acp_permission_compat.py b/src/octop/infra/agents/acp_permission_compat.py new file mode 100644 index 0000000000..601103e07d --- /dev/null +++ b/src/octop/infra/agents/acp_permission_compat.py @@ -0,0 +1,46 @@ +"""Serialize ACP permission requests that octop-harness lets overlap. + +Cursor can ask for several approvals in one turn (two WebSearch calls, for +example). ``ACPHostedClient`` keeps a single future and a single pending +permission. A second ``request_permission`` replaces that future, the resume +resolves the replacement, and the first request waits forever. The agent then +never finishes the turn, so the chat stops on the preamble. +""" + +from __future__ import annotations + +import asyncio +from typing import Any + +_INSTALLED = False +_LOCK_ATTR = "_octop_permission_lock" + + +def install_acp_permission_queue() -> None: + """Make overlapping ``request_permission`` calls wait their turn.""" + global _INSTALLED + if _INSTALLED: + return + from octop_harness.acp.client import ACPHostedClient + + original = ACPHostedClient.request_permission + + async def request_permission( + self: Any, + options: list[Any], + session_id: str, + tool_call: Any, + **kwargs: Any, + ) -> Any: + lock = getattr(self, _LOCK_ATTR, None) + if lock is None: + lock = asyncio.Lock() + setattr(self, _LOCK_ATTR, lock) + async with lock: + return await original(self, options, session_id, tool_call, **kwargs) + + ACPHostedClient.request_permission = request_permission # type: ignore[method-assign] + _INSTALLED = True + + +__all__ = ["install_acp_permission_queue"] diff --git a/src/octop/infra/agents/acp_tool_name_compat.py b/src/octop/infra/agents/acp_tool_name_compat.py new file mode 100644 index 0000000000..a58d99b9cd --- /dev/null +++ b/src/octop/infra/agents/acp_tool_name_compat.py @@ -0,0 +1,80 @@ +"""Keep the ACP tool title when a later update omits it. + +Cursor sends ``tool_call`` with a title, then ``tool_call_update`` with only a +status. The harness fills a missing title with ``unknown``, and the chat +catalog translates that to "未知工具". Remember the first real title for each +call so the update keeps the original name. +""" + +from __future__ import annotations + +from typing import Any + +_INSTALLED = False +_TITLES = "_octop_tool_titles" +_PLACEHOLDER = "unknown" + + +def _raw_tool_name(update: Any) -> str: + title = getattr(update, "title", None) + if isinstance(title, str): + text = title.strip() + if text and text.lower() != _PLACEHOLDER: + return text + raw = getattr(update, "raw_input", None) + if isinstance(raw, dict): + for key in ("name", "toolName", "tool_name", "tool"): + value = raw.get(key) + if isinstance(value, str): + text = value.strip() + if text and text.lower() != _PLACEHOLDER: + return text + return "" + + +def install_acp_tool_titles() -> None: + """Reuse the first real ACP tool title when a later update has none.""" + global _INSTALLED + if _INSTALLED: + return + from octop_harness.acp.client import ACPHostedClient + + original_event = ACPHostedClient._tool_event_from_state + original_start = ACPHostedClient.start_prompt + + def start_prompt(self: Any, on_message: Any) -> None: + setattr(self, _TITLES, {}) + original_start(self, on_message) + + def _tool_event_from_state(self: Any, update: Any) -> dict[str, Any] | None: + event = original_event(self, update) + if not isinstance(event, dict): + return event + call_id = str(event.get("call_id") or "") + remembered = getattr(self, _TITLES, None) + if not isinstance(remembered, dict): + remembered = {} + setattr(self, _TITLES, remembered) + raw = _raw_tool_name(update) + current = str(event.get("name") or "") + if raw: + chosen = raw + elif current and current.lower() != _PLACEHOLDER: + chosen = current + elif call_id and isinstance(remembered.get(call_id), str): + chosen = str(remembered[call_id]) + else: + chosen = current + if call_id and chosen and chosen.lower() != _PLACEHOLDER: + remembered[call_id] = chosen + event["name"] = chosen + event["title"] = chosen + event["detail"] = chosen + return event + + ACPHostedClient.start_prompt = start_prompt # type: ignore[method-assign] + ACPHostedClient._tool_event_from_state = _tool_event_from_state # type: ignore[method-assign] + _INSTALLED = True + + +__all__ = ["install_acp_tool_titles"] diff --git a/src/octop/infra/agents/experts/catalog.py b/src/octop/infra/agents/experts/catalog.py index 660cc26f01..dfce7c2a64 100644 --- a/src/octop/infra/agents/experts/catalog.py +++ b/src/octop/infra/agents/experts/catalog.py @@ -35,6 +35,24 @@ # Prefer system path; keep root ``manifest.json`` readable for pre-migration agents. _WORKSPACE_MANIFEST_READ_PATHS = (WORKSPACE_MANIFEST_PATH, MANIFEST_FILENAME) + +def workspace_manifest_rel(workspace: Any) -> str: + """Manifest path under the agent's system directory, or the ``.octop`` default.""" + prefix = str(getattr(workspace, "system_files_path", "") or "").strip().strip("/") + if prefix: + return f"{prefix}/{MANIFEST_FILENAME}" + return WORKSPACE_MANIFEST_PATH + + +def _manifest_read_paths(workspace: Any) -> tuple[str, ...]: + primary = workspace_manifest_rel(workspace) + paths = [primary] + for rel in _WORKSPACE_MANIFEST_READ_PATHS: + if rel not in paths: + paths.append(rel) + return tuple(paths) + + # Bundled illustrated portraits live in the dashboard public assets tree. _BUILTIN_AVATAR_URL_TEMPLATE = "/experts/avatars/{expert_id}.svg" @@ -250,7 +268,7 @@ async def seed_expert_directory( async def read_workspace_manifest_text(workspace: BackendWorkspace) -> str | None: """Read welcome manifest text from the agent workspace (``.octop`` then legacy root).""" - for rel in _WORKSPACE_MANIFEST_READ_PATHS: + for rel in _manifest_read_paths(workspace): text = await workspace.aread_text(rel) if text is not None and str(text).strip(): return str(text) @@ -259,7 +277,7 @@ async def read_workspace_manifest_text(workspace: BackendWorkspace) -> str | Non async def read_workspace_manifest_bytes(workspace: BackendWorkspace) -> bytes | None: """Read welcome manifest bytes from the agent workspace (``.octop`` then legacy root).""" - for rel in _WORKSPACE_MANIFEST_READ_PATHS: + for rel in _manifest_read_paths(workspace): raw = await workspace.adownload_bytes(rel) if raw is not None and raw.strip(): return raw @@ -424,7 +442,7 @@ async def apply_workspace_quick_prompts( if _quick_prompts_have_content(prompt) ] payload = json.dumps(existing, ensure_ascii=False, indent=2).encode("utf-8") - await workspace.aupload_many([(WORKSPACE_MANIFEST_PATH, payload)]) + await workspace.aupload_many([(workspace_manifest_rel(workspace), payload)]) def read_text_file_contents(expert_dir: Path, paths: list[str]) -> list[dict[str, str]]: diff --git a/src/octop/infra/agents/experts/library/tencent-cvm-ops/MEMORY.md b/src/octop/infra/agents/experts/library/tencent-cvm-ops/MEMORY.md index f9df2ff86f..0bbe59acc3 100644 --- a/src/octop/infra/agents/experts/library/tencent-cvm-ops/MEMORY.md +++ b/src/octop/infra/agents/experts/library/tencent-cvm-ops/MEMORY.md @@ -16,7 +16,7 @@ | 工作台 OAuth | `~/.octop/secrets/cloud/.enc` | 工作台页面(实例列表、监控图表) | | tccli OAuth | `~/.tccli/.credential` | 对话里的我(走 tccli) | -工作台授权后会镜像一份到 tccli;`默认 profile=default`,若开启 `OCTOP_TCCLI_PROFILE_ISOLATION=1` 则为 `octop-`,真实值看 `~/.tccli/.octop-profile`。 +工作台授权后会镜像一份到 tccli,profile 固定为 `octop-`。专家进程里的 `OCTOP_TCCLI_PROFILE` 指向同一份,不要读 `default` 或 `~/.tccli/.octop-profile`。 工具路径和脚本用法在 `SOUL.md` 第二节。 diff --git a/src/octop/infra/agents/experts/library/tencent-cvm-ops/skills/tencentcloud-infra/scripts/tccli-oauth-helper.py b/src/octop/infra/agents/experts/library/tencent-cvm-ops/skills/tencentcloud-infra/scripts/tccli-oauth-helper.py index bb6093fb98..b088b5649b 100644 --- a/src/octop/infra/agents/experts/library/tencent-cvm-ops/skills/tencentcloud-infra/scripts/tccli-oauth-helper.py +++ b/src/octop/infra/agents/experts/library/tencent-cvm-ops/skills/tencentcloud-infra/scripts/tccli-oauth-helper.py @@ -82,27 +82,17 @@ def cred_path_of_profile(profile): return os.path.join(os.path.expanduser("~"), ".tccli", profile + ".credential") -_PROFILE_POINTER = os.path.expanduser("~/.tccli/.octop-profile") - - def resolve_profile(explicit=None): - """选 profile:--profile > OCTOP_TCCLI_PROFILE > 工作台写入的指针 > default。 + """选 profile:--profile > OCTOP_TCCLI_PROFILE > default。 - 多用户部署时工作台会把凭据写到 ``octop-`` 并更新指针文件, - 这样技能不用改调用方式也能读到正确的那份。 + 工作台和专家启动会把 ``OCTOP_TCCLI_PROFILE`` 设成当前用户的 + ``octop-``。不再读机器上共享的指针文件。 """ if explicit: return explicit env = os.environ.get("OCTOP_TCCLI_PROFILE", "").strip() if env: return env - try: - with open(_PROFILE_POINTER) as pointer: - name = pointer.read().strip() - if name: - return name - except OSError: - pass return "default" diff --git a/src/octop/infra/agents/experts/market_creation.py b/src/octop/infra/agents/experts/market_creation.py index 0df620c37d..e3c30c40cd 100644 --- a/src/octop/infra/agents/experts/market_creation.py +++ b/src/octop/infra/agents/experts/market_creation.py @@ -4,7 +4,7 @@ import asyncio import logging -from dataclasses import dataclass +from dataclasses import asdict, dataclass from typing import Any, Literal from octop.infra.agents.experts.avatar import materialize_remote_icon_url @@ -12,6 +12,7 @@ apply_workspace_quick_prompts, build_create_spec_from_expert, ) +from octop.infra.agents.settings.runtime_limits import AGENT_RUNTIME_CONFIG_KEYS from octop.infra.errors import ErrorCode, OctopError from octop.infra.history.trajectory.settings import apply_enable_trajectory from octop.infra.market import PortalMarketError, install_portal_expert @@ -42,6 +43,17 @@ class SkillHubMarketAgentCreateOptions: temperature: float | None = None top_p: float | None = None max_tokens: int | None = None + tool_call_limit_enabled: bool | None = None + tool_call_run_limit: int | None = None + model_retry_enabled: bool | None = None + model_retry_max_retries: int | None = None + model_retry_initial_delay: float | None = None + model_retry_max_delay: float | None = None + tool_retry_enabled: bool | None = None + tool_retry_max_retries: int | None = None + tool_retry_initial_delay: float | None = None + tool_retry_max_delay: float | None = None + compaction: dict[str, Any] | None = None enable_trajectory: bool = True conversation_mode: str | None = None workspace_patch: Any = None @@ -62,6 +74,16 @@ class SkillHubMarketAgentCreateResult: kind: str = "library" +def _runtime_config(options: SkillHubMarketAgentCreateOptions) -> dict[str, Any]: + """Runtime fields the market installer should persist. ``None`` means omit.""" + raw = asdict(options) + return { + key: raw[key] + for key in (*AGENT_RUNTIME_CONFIG_KEYS, "compaction") + if raw.get(key) is not None + } + + async def create_agent_from_skillhub_skillset( *, server: Any, @@ -149,17 +171,7 @@ async def create_agent_from_skillhub_skillset( default_model=options.default_model, config_extra=config_extra, agent_id=options.agent_id, - runtime_config={ - key: value - for key, value in { - "max_iters": options.max_iters, - "max_input_length": options.max_input_length, - "temperature": options.temperature, - "top_p": options.top_p, - "max_tokens": options.max_tokens, - }.items() - if value is not None - }, + runtime_config=_runtime_config(options), icon_url=portrait_url, color=options.color, welcome_message=(options.welcome_message if customized_welcome else None), diff --git a/src/octop/infra/agents/managed.py b/src/octop/infra/agents/managed.py new file mode 100644 index 0000000000..90608d7ae7 --- /dev/null +++ b/src/octop/infra/agents/managed.py @@ -0,0 +1,139 @@ +"""Managed agents: an ACP runner addressed like any other expert.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from octop.infra.errors import ErrorCode, OctopError + +_SEED_DIR = Path(__file__).with_name("managed_seed") + +MANAGED_KIND = "managed" + +MANAGED_BACKEND: dict[str, Any] = { + "type": "filesystem", + "root_dir": "/", + "virtual_mode": True, +} + + +def is_managed_agent(row: Any) -> bool: + return getattr(row, "kind", None) == MANAGED_KIND + + +def apply_managed_backend(config: dict[str, Any]) -> None: + """Pin storage to the local filesystem root. The user picks the workspace.""" + config["backend"] = dict(MANAGED_BACKEND) + + +def pin_managed_runtime(config: dict[str, Any]) -> None: + """Drop expert-only runtime knobs. The ACP runner owns the turn.""" + config.pop("conversation_mode", None) + memory = config.get("memory") + section = dict(memory) if isinstance(memory, dict) else {} + section["memory_enabled"] = False + section["memory_capture_enabled"] = False + section["memory_recall_inject_enabled"] = False + config["memory"] = section + + +def require_managed_fields(config: dict[str, Any]) -> None: + runner = config.get("managed_runner") + if not isinstance(runner, str) or not runner.strip(): + raise OctopError(ErrorCode.NOT_FOUND, "managed agent requires an ACP runner", status=400) + workspace = config.get("workspace_dir") + if not isinstance(workspace, str) or not workspace.strip(): + raise OctopError( + ErrorCode.WORKSPACE_OP_UNSUPPORTED, + "managed agent requires a working directory", + ) + + +def managed_default_workspace(agent_id: str) -> str: + """``{home}/{agent_id}`` on the machine that runs Octop.""" + return str(Path.home() / agent_id.strip()) + + +# Project directory each ACP agent already uses. Skills are ``{root}/skills``, +# subagents are ``{root}/agents`` — the same layout Octop uses under ``.octop``. +_RUNNER_PROJECT_ROOTS: dict[str, str] = { + "codebuddy": ".codebuddy", + "claude_code": ".claude", + "opencode": ".opencode", + "codex": ".codex", + "cursor_cli": ".cursor", + "kimi_code": ".kimi", + "pi": ".pi", +} +_DEFAULT_PROJECT_ROOT = ".octop" + + +def managed_project_root(runner: str) -> str: + """Workspace-relative directory that runner keeps its own project files in.""" + return _RUNNER_PROJECT_ROOTS.get(runner.strip(), _DEFAULT_PROJECT_ROOT) + + +def managed_skill_dir(runner: str) -> str: + return f"{managed_project_root(runner)}/skills" + + +def managed_subagent_dir(runner: str) -> str: + return f"{managed_project_root(runner)}/agents" + + +def managed_template_payload() -> dict[str, Any]: + """Welcome cards and prompt files copied into a new managed workspace.""" + manifest = json.loads((_SEED_DIR / "manifest.json").read_text(encoding="utf-8")) + prompts = manifest.get("quick_prompts") + files = [ + {"name": "AGENTS.md", "content": (_SEED_DIR / "AGENTS.md").read_text(encoding="utf-8")} + ] + return { + "files": files, + "quick_prompts": prompts if isinstance(prompts, list) else [], + "home": str(Path.home()), + } + + +async def seed_managed_workspace( + workspace: Any, + *, + agents_md: str | None = None, + quick_prompts: list[dict[str, Any]] | None = None, +) -> None: + """Copy ``AGENTS.md`` when the project does not already have one. + + An existing file is left in place unless the form sent a replacement. + Welcome cards merge into ``{system dir}/manifest.json``. Skills and subagents + stay in the ACP agent's own project directory and are not seeded here. + """ + from octop.infra.agents.experts.catalog import ( # noqa: PLC0415 + apply_workspace_quick_prompts, + workspace_manifest_rel, + ) + + if agents_md is not None: + await workspace.aupload_many([("AGENTS.md", agents_md.encode("utf-8"))]) + elif await workspace.aread_text("AGENTS.md") is None: + content = (_SEED_DIR / "AGENTS.md").read_bytes() + await workspace.aupload_many([("AGENTS.md", content)]) + + manifest_path = workspace_manifest_rel(workspace) + if await workspace.aread_text(manifest_path) is None: + raw = (_SEED_DIR / "manifest.json").read_bytes() + await workspace.aupload_many([(manifest_path, raw)]) + if quick_prompts is not None: + await apply_workspace_quick_prompts(workspace, quick_prompts) + + +def preserve_managed_config(config: dict[str, Any], previous: dict[str, Any]) -> None: + """Keep the ACP binding and filesystem root across later edits.""" + apply_managed_backend(config) + pin_managed_runtime(config) + runner = config.get("managed_runner") + if not isinstance(runner, str) or not runner.strip(): + previous_runner = previous.get("managed_runner") + if isinstance(previous_runner, str) and previous_runner.strip(): + config["managed_runner"] = previous_runner.strip() diff --git a/src/octop/infra/agents/managed_seed/AGENTS.md b/src/octop/infra/agents/managed_seed/AGENTS.md new file mode 100644 index 0000000000..88a6ab0b4e --- /dev/null +++ b/src/octop/infra/agents/managed_seed/AGENTS.md @@ -0,0 +1,17 @@ +# AGENTS.md + +你是这个代码仓库里的编程助手。按用户的目标改代码、查问题、补测试,并让改动可以复查。 + +## 原则 + +- 先读相关代码和测试,再改。不要凭猜测编造文件、接口或行为。 +- 改动尽量小,只动完成本次任务需要的地方。 +- 匹配仓库已有的风格、命名和目录结构。 +- 对外操作(提交、推送、发消息、改外部系统)拿不准就先确认。 +- 不把密钥、令牌或隐私内容写进代码、日志或回复。 + +## 交付 + +- 先说结论:改了什么、为什么、怎么验证。 +- 能跑的检查就跑(测试、类型检查、相关命令),并说明结果。 +- 失败时给出下一步,不要假装已经通过。 diff --git a/src/octop/infra/agents/managed_seed/manifest.json b/src/octop/infra/agents/managed_seed/manifest.json new file mode 100644 index 0000000000..4b52739c01 --- /dev/null +++ b/src/octop/infra/agents/managed_seed/manifest.json @@ -0,0 +1,71 @@ +{ + "id": "managed", + "label": { "zh": "托管专家", "en": "Managed expert" }, + "description": { + "zh": "把问题交给已连接的编程代理,并在工作区里协作。", + "en": "Hand each task to a connected coding agent and work in this directory." + }, + "prompt_files": ["AGENTS.md"], + "quick_prompts": [ + { + "title": { "zh": "修一个问题", "en": "Fix a bug" }, + "description": { "zh": "根据现象定位并改到通过", "en": "Find the cause and make it pass" }, + "prompt": { + "zh": "请在当前工作区修这个问题。先复现或定位原因,再做最小改动,并说明如何验证:\n\n", + "en": "Fix this in the current workspace. Find the cause first, then make the smallest change, and say how to verify it:\n\n" + }, + "color": "#fef3c7", + "icon_name": "wrench" + }, + { + "title": { "zh": "加一个功能", "en": "Add a feature" }, + "description": { "zh": "按现有结构把功能做进去", "en": "Fit the feature into the current structure" }, + "prompt": { + "zh": "请在当前工作区实现下面的功能。先看相关代码再改,保持现有风格,并说明改动范围:\n\n", + "en": "Implement this feature in the current workspace. Read the related code first, match the existing style, and summarize what changed:\n\n" + }, + "color": "#e8f4ff", + "icon_name": "zap" + }, + { + "title": { "zh": "看懂这段代码", "en": "Explain this code" }, + "description": { "zh": "讲清它做什么、数据怎么走", "en": "What it does and how data moves" }, + "prompt": { + "zh": "请阅读当前工作区里下面这些代码,说明它做什么、关键调用链,以及容易改错的地方:\n\n", + "en": "Read this code in the current workspace. Explain what it does, the main call path, and what is easy to get wrong:\n\n" + }, + "color": "#eef2ff", + "icon_name": "book-open" + }, + { + "title": { "zh": "审查改动", "en": "Review a change" }, + "description": { "zh": "看正确性、回归和遗漏", "en": "Correctness, regressions, and gaps" }, + "prompt": { + "zh": "请审查当前工作区里的这次改动。指出正确性、回归风险和遗漏的测试,不要泛泛而谈:\n\n", + "en": "Review the change in the current workspace. Call out correctness, regression risk, and missing tests. Be specific:\n\n" + }, + "color": "#dcfce7", + "icon_name": "activity" + }, + { + "title": { "zh": "补测试", "en": "Add tests" }, + "description": { "zh": "给关键路径补上能跑的测试", "en": "Cover the important path with a test that runs" }, + "prompt": { + "zh": "请为下面的行为补测试。先看现有测试怎么写,再补能失败也能通过的用例,并跑一下:\n\n", + "en": "Add tests for this behavior. Follow the existing tests, cover a case that can fail, and run them:\n\n" + }, + "color": "#f3e8ff", + "icon_name": "list-todo" + }, + { + "title": { "zh": "整理代码", "en": "Refactor" }, + "description": { "zh": "让结构更清楚,行为保持不变", "en": "Clarify the structure without changing behavior" }, + "prompt": { + "zh": "请整理下面这块代码,让它更好读,但不要改变对外行为。改完说明动了哪里,以及你怎么确认行为没变:\n\n", + "en": "Refactor this code so it is easier to follow, without changing observable behavior. Say what moved and how you checked:\n\n" + }, + "color": "#fff7ed", + "icon_name": "pen-tool" + } + ] +} diff --git a/src/octop/infra/agents/manager.py b/src/octop/infra/agents/manager.py index 3524e2b3bf..e62e216ff0 100644 --- a/src/octop/infra/agents/manager.py +++ b/src/octop/infra/agents/manager.py @@ -10,7 +10,7 @@ import threading import time from collections.abc import AsyncIterator, Awaitable, Callable, Sequence -from contextlib import asynccontextmanager, suppress +from contextlib import asynccontextmanager, nullcontext, suppress from dataclasses import dataclass, field, fields, replace from pathlib import Path from typing import TYPE_CHECKING, Any, cast @@ -24,6 +24,8 @@ NO_MODELS_CONFIGURED, format_agent_start_error, ) +from octop.infra.agents.acp_permission_compat import install_acp_permission_queue +from octop.infra.agents.acp_tool_name_compat import install_acp_tool_titles from octop.infra.agents.memory.backend import memory_backend_from_agent_config from octop.infra.agents.memory.slim import MemorySlimCoordinator from octop.infra.agents.providers import HostModelFactory, ProviderStore, sync_providers_to_harness @@ -36,6 +38,11 @@ thread_id_from_request, ) from octop.infra.agents.settings.acp import ACPSettingsStore +from octop.infra.agents.settings.compaction import ( + COMPACTION_CONFIG_KEY, + apply_compaction_update, + harness_summarization_kwargs, +) from octop.infra.agents.settings.decision import DecisionSettings, DecisionSettingsStore from octop.infra.agents.settings.langfuse import LangfuseSettings, LangfuseSettingsStore from octop.infra.agents.settings.media_generation import ( @@ -55,7 +62,9 @@ from octop.infra.agents.settings.runtime_limits import ( AGENT_RUNTIME_CONFIG_KEYS, apply_agent_runtime_to_stream_request, + call_guard_middleware, merge_agent_runtime_values, + resolve_model_retry, ) from octop.infra.agents.settings.runtime_limits import ( resolve_context_max_tokens as config_context_max_tokens, @@ -63,6 +72,7 @@ from octop.infra.backend.docker_spec import ( enrich_docker_backend_spec, inject_docker_global_environment, + omit_docker_volumes, ) from octop.infra.backend.resolver import ( default_agent_backend_spec, @@ -84,6 +94,7 @@ repair_workspace_skill_manifests, ) from octop.infra.utils.locale import Locale, resolve_user_locale +from octop.infra.utils.stream_close import close_event_stream from octop.infra.utils.turn_failure import turn_model_scope from octop.infra.utils.ulid import new_short_id @@ -99,6 +110,9 @@ logger = logging.getLogger(__name__) +install_acp_permission_queue() +install_acp_tool_titles() + def _model_retry_on_failure( exc: Exception, @@ -834,10 +848,12 @@ async def create( break else: raise RuntimeError("failed to allocate unique agent_id") - config = merge_agent_runtime_values( - spec.config, - spec.runtime_config, - ) + runtime_config = dict(spec.runtime_config) + compaction_set = COMPACTION_CONFIG_KEY in runtime_config + compaction_value = runtime_config.pop(COMPACTION_CONFIG_KEY, None) + config = merge_agent_runtime_values(spec.config, runtime_config) + if compaction_set: + config = apply_compaction_update(config, compaction_value) if spec.persona_mbti: config["persona"] = spec.persona_mbti.upper() from octop.infra.agents.workspace.dir import ( # noqa: PLC0415 @@ -848,12 +864,27 @@ async def create( apply_workspace_root_to_config, assert_agent_quota_available, raise_if_backend_outside_user_root, + raise_if_backend_types_disallowed, raise_if_workspace_dir_outside_user_root, ) + kind = spec.kind if spec.kind in {"expert", "team", "managed"} else "expert" + if kind == "managed": + from octop.infra.agents.managed import ( # noqa: PLC0415 + apply_managed_backend, + managed_default_workspace, + pin_managed_runtime, + require_managed_fields, + ) + + apply_managed_backend(config) + pin_managed_runtime(config) + raw_workspace = config.get("workspace_dir") + if not isinstance(raw_workspace, str) or not raw_workspace.strip(): + config["workspace_dir"] = managed_default_workspace(agent_id) + require_managed_fields(config) if spec.user_id is not None: - kind = spec.kind if spec.kind in {"expert", "team"} else "expert" - if kind == "expert": + if kind != "team": assert_agent_quota_available( self._repos.user_policy_repo, self._repos.agent_repo, @@ -876,6 +907,13 @@ async def create( config, ) self._assert_named_storage_backends(spec.user_id, config.get("backend")) + raise_if_backend_types_disallowed( + self._repos.user_policy_repo, + spec.user_id, + config.get("backend"), + named_kind=self._named_storage_kind, + require_present=True, + ) self._assert_model_ref(spec.user_id, spec.default_model) # Create-time: user-assigned workspace_dir wins; otherwise default+encode. @@ -885,6 +923,12 @@ async def create( # be user-configurable. New agents always use the default prefix. config.pop("system_files_path", None) config["system_files_path"] = DEFAULT_SYSTEM_FILES_PATH + if kind == "managed": + from octop.infra.agents.managed import managed_project_root # noqa: PLC0415 + + runner = config.get("managed_runner") + if isinstance(runner, str) and runner.strip(): + config["system_files_path"] = managed_project_root(runner) if spec.is_shared and spec.kind == "team": raise OctopError( ErrorCode.TEAM_NOT_SHAREABLE, @@ -941,7 +985,7 @@ async def create( ), knowledge_base_ids=knowledge_ids_json, mcp_servers=mcp_servers_json, - kind=spec.kind if spec.kind in {"expert", "team"} else "expert", + kind=kind, ) row = self._repos.agent_repo.get(agent_id) assert row is not None @@ -1085,7 +1129,9 @@ async def update(self, agent_id: str, **kwargs: Any) -> AgentRow: runtime_updates = { key: kwargs.pop(key) for key in AGENT_RUNTIME_CONFIG_KEYS if key in kwargs } - if runtime_updates: + compaction_set = COMPACTION_CONFIG_KEY in kwargs + compaction_value = kwargs.pop(COMPACTION_CONFIG_KEY, None) + if runtime_updates or compaction_set: raw_config = kwargs.get("config_json") if raw_config is None: current_row = self._repos.agent_repo.get(agent_id) @@ -1098,9 +1144,11 @@ async def update(self, agent_id: str, **kwargs: Any) -> AgentRow: parsed_config = {} if not isinstance(parsed_config, dict): parsed_config = {} - kwargs["config_json"] = json.dumps( - merge_agent_runtime_values(parsed_config, runtime_updates), - ) + if runtime_updates: + parsed_config = merge_agent_runtime_values(parsed_config, runtime_updates) + if compaction_set: + parsed_config = apply_compaction_update(parsed_config, compaction_value) + kwargs["config_json"] = json.dumps(parsed_config) if "config_json" in kwargs: parsed_profile_cfg = parse_config_json( kwargs["config_json"] if isinstance(kwargs["config_json"], str) else None @@ -1109,6 +1157,21 @@ async def update(self, agent_id: str, **kwargs: Any) -> AgentRow: agent_id, parsed_profile_cfg, pin_workspace_dir=True ) owner_row = self._repos.agent_repo.get(agent_id) + if owner_row is not None and getattr(owner_row, "kind", None) == "managed": + from octop.infra.agents.managed import preserve_managed_config # noqa: PLC0415 + + preserve_managed_config( + parsed_profile_cfg, + parse_config_json(owner_row.config_json), + ) + if ( + not compaction_set + and COMPACTION_CONFIG_KEY not in parsed_profile_cfg + and owner_row is not None + ): + previous = parse_config_json(owner_row.config_json).get(COMPACTION_CONFIG_KEY) + if isinstance(previous, dict): + parsed_profile_cfg[COMPACTION_CONFIG_KEY] = previous if owner_row is not None and owner_row.user_id is not None: from octop.infra.users.resource_policy import ( POLICY_WORKSPACE_ROOT_DIR, @@ -1116,6 +1179,7 @@ async def update(self, agent_id: str, **kwargs: Any) -> AgentRow: effective_workspace_root_dir, persisted_workspace_within_root, raise_if_backend_outside_user_root, + raise_if_backend_types_disallowed, ) allowed_root = effective_workspace_root_dir( @@ -1141,6 +1205,13 @@ async def update(self, agent_id: str, **kwargs: Any) -> AgentRow: owner_row.user_id, parsed_profile_cfg.get("backend"), ) + raise_if_backend_types_disallowed( + self._repos.user_policy_repo, + owner_row.user_id, + parsed_profile_cfg.get("backend"), + named_kind=self._named_storage_kind, + previous=parse_config_json(owner_row.config_json).get("backend"), + ) lifted = extract_profile_from_config(parsed_profile_cfg) kwargs["config_json"] = dumps_config(parsed_profile_cfg) for key, value in lifted.items(): @@ -2024,11 +2095,50 @@ async def stream(self, agent_id: str, request: dict[str, Any]) -> AsyncIterator[ with ( hitl_thread_scope(thread_id_from_request(req)), turn_model_scope(self._stream_model_label(agent_id, req, thread_id)), + self._turn_model_access_from_request(req), ): - async for chunk in self._harness_manager.stream(agent_id, cast(Any, req)): - yield chunk + chunk_stream = self._harness_manager.stream(agent_id, cast(Any, req)) + try: + async for chunk in chunk_stream: + yield chunk + finally: + await close_event_stream(chunk_stream) self._apply_pending_bootstrap_graph_refresh(agent_id) + def _turn_model_access_from_request(self, request: dict[str, Any]) -> Any: + raw = request.get("user") + if isinstance(raw, bool) or not isinstance(raw, int | str): + return nullcontext() + try: + user_id = int(raw) + except ValueError: + return nullcontext() + return self._turn_model_access_for_user(user_id) + + def _turn_model_access_for_thread(self, thread_id: str) -> Any: + repos = getattr(self, "_repos", None) + thread_repo = getattr(repos, "thread_repo", None) if repos is not None else None + if thread_repo is None: + return nullcontext() + row = thread_repo.get(thread_id) + user_id = row.user_id if row is not None else 0 + return self._turn_model_access_for_user(user_id) + + def _turn_model_access_for_user(self, user_id: int) -> Any: + from octop.infra.agents.providers.turn_access import turn_model_access + + if user_id <= 0: + return nullcontext() + return turn_model_access(user_id, is_admin=self._user_is_admin(user_id)) + + def _auto_route_candidates(self, *, needs_multimodal: bool) -> list[Any]: + from octop.infra.agents.providers.turn_access import filter_auto_route_candidates + + return filter_auto_route_candidates( + self._providers.list_auto_route_candidates(needs_multimodal=needs_multimodal), + repos=self._repos, + ) + async def call(self, agent_id: str, request: dict[str, Any]) -> dict[str, Any]: """Non-streaming harness invocation (one-shot agent call).""" if self._harness_manager is None: @@ -2040,6 +2150,7 @@ async def call(self, agent_id: str, request: dict[str, Any]) -> dict[str, Any]: with ( hitl_thread_scope(thread_id_from_request(req)), turn_model_scope(self._stream_model_label(agent_id, req, thread_id)), + self._turn_model_access_from_request(req), ): result = await self._harness_manager.call(agent_id, cast(Any, req)) self._apply_pending_bootstrap_graph_refresh(agent_id) @@ -2064,11 +2175,14 @@ async def resume_hitl( with ( hitl_thread_scope(thread_id), turn_model_scope(self._stream_model_label(agent_id, {}, thread_id)), + self._turn_model_access_for_thread(thread_id), ): - async for chunk in self._harness_manager.resume_hitl( - agent_id, thread_id, decisions - ): - yield chunk + chunk_stream = self._harness_manager.resume_hitl(agent_id, thread_id, decisions) + try: + async for chunk in chunk_stream: + yield chunk + finally: + await close_event_stream(chunk_stream) self._apply_pending_bootstrap_graph_refresh(agent_id) def cancel_stream(self, agent_id: str, thread_id: str) -> None: @@ -2538,8 +2652,43 @@ async def prepare_chat_mcp( custom_configs: dict[str, Any] = {} load_user: dict[str, int] = {} if uid is not None: + # Renew OAuth tokens that are about to expire, and flag the ones that + # cannot be renewed, before building the specs the tools load from. + for token_owner in {i for i in (uid, owner_id) if i is not None}: + try: + await self._connector_svc.ensure_fresh_custom_servers(token_owner) + except Exception: + logger.warning( + "prepare_chat_mcp agent=%s: custom MCP token refresh failed", + agent_id, + exc_info=True, + ) custom_configs = self._connector_svc.custom_harness_configs(uid) load_user = dict.fromkeys(custom_configs, uid) + # Tools stay cached on the agent after the first load, so a token that + # lapsed since then is only caught by checking the stored status. + awaiting_auth: set[str] = set() + if uid is not None: + try: + awaiting_auth = { + str(item["server"]) + for item in await self._connector_svc.auth_blocked_servers( + uid, requested, probe=False + ) + } + except Exception: + logger.warning( + "prepare_chat_mcp agent=%s: connector auth check failed", + agent_id, + exc_info=True, + ) + if awaiting_auth: + logger.info( + "prepare_chat_mcp agent=%s: waiting for authorization %s", + agent_id, + sorted(awaiting_auth), + ) + return sorted(awaiting_auth) if aliases and uid is not None and owner_id is not None: # Load the owner's spec when the turn asked for the shared server. # A viewer's own server with the same short name stays theirs unless @@ -2740,6 +2889,16 @@ async def prepare_chat_mcp( ] return still_missing + async def connector_auth_blocked( + self, user_id: int, server_names: list[str] + ) -> list[dict[str, Any]]: + """Of the MCP servers that failed to load, the ones waiting for authorization.""" + try: + return await self._connector_svc.auth_blocked_servers(user_id, server_names) + except Exception: + logger.warning("connector auth check failed for user=%s", user_id, exc_info=True) + return [] + async def _attach_gateway_tools( self, agent: HarnessAgent, @@ -3171,6 +3330,13 @@ def _assert_model_ref(self, user_id: int, model_ref: str | None) -> None: is_owner=False, ) + def _named_storage_kind(self, name: str) -> str | None: + row = self._repos.storage_backend_repo.get_by_name(name) + if row is None: + return None + kind = (row.kind or "").strip().lower() + return kind or None + def _assert_named_storage_backends(self, user_id: int, backend: Any) -> None: from octop.infra.backend.resolver import collect_named_storage_backend_refs from octop.infra.users.groups import RESOURCE_STORAGE_BACKEND, GroupAccess @@ -3620,8 +3786,13 @@ async def _start_agent( cfg, metadata, tags, user_display = self._agent_runtime_bundle(row) # Team hosts already have AGENTS/SOUL/MEMORY from seed_team_template. - # Never let harness copy builtin skills just to strip them later. - seed_workspace = init_workspace and not is_team_agent(row) + # Managed agents only keep AGENTS.md; the ACP runner owns skills + # and subagents under its own project directory. + seed_workspace = ( + init_workspace + and not is_team_agent(row) + and getattr(row, "kind", None) != "managed" + ) entry = await self._harness_manager.acreate_agent( cfg, agent_id=row.agent_id, @@ -3683,7 +3854,7 @@ async def _post_start_agent( len(tool_set), sorted(tool_set)[:8], ) - if not team_host: + if not team_host and getattr(row, "kind", None) != "managed": ws = agent.workspace try: from octop.infra.agents.builtin_skills import ( # noqa: PLC0415 @@ -3796,7 +3967,8 @@ def _backend_spec_for_row( ) # Windows: the dashboard defaults local backends to root_dir "/", which # resolves to a drive other than the workspace and breaks path checks. - return windows_neutralize_host_root(resolved, workspace_dir=workspace_dir) + resolved = windows_neutralize_host_root(resolved, workspace_dir=workspace_dir) + return omit_docker_volumes(resolved) def resolved_backend_spec(self, agent_id: str) -> Any: """Backend spec for *agent_id* with Octop docker enrichments applied.""" @@ -4078,7 +4250,14 @@ def _peer_manifest_metadata( host = self.resolve_workspace_dir(agent_id, persist_if_missing=False) except Exception: return extra - for rel in (Path(".octop") / "manifest.json", Path("manifest.json")): + from octop.infra.agents.workspace.dir import system_files_path_from_config # noqa: PLC0415 + + prefix = system_files_path_from_config(self.get_config(agent_id)) or ".octop" + for rel in ( + Path(prefix) / "manifest.json", + Path(".octop") / "manifest.json", + Path("manifest.json"), + ): path = host / rel if not path.is_file(): continue @@ -4147,6 +4326,68 @@ def _stream_model_label(self, agent_id: str, request: dict[str, Any], thread_id: return str(row.default_model).strip() return "" + def _managed_harness_config( + self, + row: AgentRow, + *, + cfg: dict[str, Any], + harness_workspace: Path, + backend: Any, + ) -> HarnessAgentConfig: + """Harness config for an ACP-backed agent. + + Chat, workspace files, skills, and subagents still go through the + same surfaces as an expert. Bootstrap, memory, and the model tool + stack stay off: the connected runner executes the turn. + """ + from octop_harness.acp.models import ACPConfig # noqa: PLC0415 + + from octop.infra.agents.managed import ( # noqa: PLC0415 + managed_skill_dir, + managed_subagent_dir, + ) + from octop.infra.agents.workspace.dir import ( # noqa: PLC0415 + system_files_path_from_config, + ) + + acp_user_id = row.user_id + if acp_user_id is None: + acp_user_id = self._connector_user_override.get(row.agent_id) + runners = self._acp_settings.load_runners(acp_user_id) if acp_user_id is not None else {} + acp_config = ACPConfig.from_dict({"runners": runners}) + runner = cfg.get("managed_runner") + runner_id = runner.strip() if isinstance(runner, str) else "" + off: dict[str, Any] = {} + for key in ( + "memory_capture_enabled", + "memory_recall_inject_enabled", + "memory_extract_on_session_end", + ): + if key in _HARNESS_AGENT_CONFIG_FIELDS: + off[key] = False + built = HarnessAgentConfig( + name=_memory_namespace(row.agent_id), + workspace_dir=harness_workspace, + system_files_path=system_files_path_from_config(cfg), + language=resolve_user_locale( + user_repo=self._repos.user_repo, + user_id=row.user_id or 0, + ), + backend=backend, + bootstrap_enabled=False, + memory_enabled=False, + acp_runners=acp_config.runners, + acp_delegate_enabled=False, + default_timezone=self._config.default_timezone, + **off, + ) + if runner_id and "managed_runner" in _HARNESS_AGENT_CONFIG_FIELDS: + built.managed_runner = runner_id + built.skills_dir = managed_skill_dir(runner_id) + if "subagents_path" in _HARNESS_AGENT_CONFIG_FIELDS: + built.subagents_path = managed_subagent_dir(runner_id) + return built + def _build_harness_config(self, row: AgentRow) -> HarnessAgentConfig: """Convert an AgentRow into a HarnessAgentConfig.""" from octop_harness.middleware.bootstrap import bootstrap_marker_exists # noqa: PLC0415 @@ -4187,6 +4428,15 @@ def _build_harness_config(self, row: AgentRow) -> HarnessAgentConfig: cfg = persisted backend = self._backend_spec_for_row(row, cfg=cfg, workspace_dir=workspace_dir) + if getattr(row, "kind", None) == "managed": + # The ACP runner is the agent. Reuse the workspace and runner + # binding; do not assemble the expert tool, memory, or bootstrap stack. + return self._managed_harness_config( + row, + cfg=cfg, + harness_workspace=harness_workspace, + backend=backend, + ) ws = self._backend_workspace_for_row( row, cfg=cfg, @@ -4299,6 +4549,7 @@ def _mobile_workspace_for(agent_id: str) -> Any: ) from octop.infra.agents.middleware.octop_ui_offload import OctopUiOffloadMiddleware from octop.infra.agents.middleware.reasoning import ReasoningRequestMiddleware + from octop.infra.agents.middleware.team_dispatch import TeamDispatchBriefingMiddleware from octop.infra.agents.middleware.thread_artifacts import ThreadArtifactsMiddleware from octop.infra.agents.middleware.token_quota import TokenQuotaMiddleware from octop.infra.agents.middleware.workspace_image import ( @@ -4320,6 +4571,7 @@ def _mobile_workspace_for(agent_id: str) -> Any: usage_repo=self._repos.usage_repo, ), ReasoningRequestMiddleware(), + TeamDispatchBriefingMiddleware(), KnowledgeSearchHintMiddleware(), BrowserProfileMiddleware(), BinaryReadGuardMiddleware(), @@ -4487,6 +4739,7 @@ def _mobile_workspace_for(agent_id: str) -> Any: **harness_fields, **_memory_extract_settings(cfg, is_ref_usable=self._providers.is_model_ref_usable), **_resolve_memory_backend_kwargs(cfg, workspace_dir=workspace_dir, config=self._config), + **harness_summarization_kwargs(cfg, supported_fields=_HARNESS_AGENT_CONFIG_FIELDS), ) if "tools_disabled" in _HARNESS_AGENT_CONFIG_FIELDS: from octop.infra.agents.settings.tool_catalog import effective_tools_disabled @@ -4508,7 +4761,9 @@ def _mobile_workspace_for(agent_id: str) -> Any: interrupt_on = apply_session_bypass(applied.interrupt_on, self._hitl_session_store) if interrupt_on is not applied.interrupt_on: applied = replace(applied, interrupt_on=interrupt_on) - if applied.model_retry_enabled: + model_retry = resolve_model_retry(cfg) + prefix: list[Any] = call_guard_middleware(cfg) + if model_retry.enabled and applied.model_retry_enabled: factory = ( self._harness_manager.shared_factory if self._harness_manager is not None else None ) @@ -4518,37 +4773,36 @@ def _mobile_workspace_for(agent_id: str) -> Any: SafeModelFailoverMiddleware, ) - retry_middleware: Any = SafeModelFailoverMiddleware( - health=self._model_health, - candidates=lambda _current_ref, needs_multimodal: ( - self._providers.list_auto_route_candidates( - needs_multimodal=needs_multimodal - ) - ), - model_for_ref=get_chat_model, - max_retries=applied.model_retry_max_retries, - initial_delay=applied.model_retry_initial_delay, - max_delay=applied.model_retry_max_delay, - on_failure="error", + prefix.append( + SafeModelFailoverMiddleware( + health=self._model_health, + candidates=lambda _current_ref, needs_multimodal: ( + self._auto_route_candidates(needs_multimodal=needs_multimodal) + ), + model_for_ref=get_chat_model, + max_retries=model_retry.max_retries, + initial_delay=model_retry.initial_delay, + max_delay=model_retry.max_delay, + on_failure="error", + ) ) else: from langchain.agents.middleware import ModelRetryMiddleware - retry_middleware = ModelRetryMiddleware( - max_retries=applied.model_retry_max_retries, - initial_delay=applied.model_retry_initial_delay, - max_delay=applied.model_retry_max_delay, - on_failure="error", + prefix.append( + ModelRetryMiddleware( + max_retries=model_retry.max_retries, + initial_delay=model_retry.initial_delay, + max_delay=model_retry.max_delay, + on_failure="error", + ) ) - - applied = replace( - applied, - model_retry_enabled=False, - middleware=[ - retry_middleware, - *(applied.middleware or []), - ], - ) + # The host owns model retry, so harness must not mount a second copy. + applied = replace( + applied, + model_retry_enabled=False, + middleware=[*prefix, *(applied.middleware or [])], + ) self._tool_guard_rules.ensure_seeded() return replace( applied, diff --git a/src/octop/infra/agents/middleware/team_dispatch.py b/src/octop/infra/agents/middleware/team_dispatch.py new file mode 100644 index 0000000000..2b9503098e --- /dev/null +++ b/src/octop/infra/agents/middleware/team_dispatch.py @@ -0,0 +1,60 @@ +"""Append a team-dispatch briefing to the leading system prompt. + +A team host hands each member a briefing before the member's turn. The briefing +travels in ``configurable`` and this middleware appends it at model-call time. +Nothing is written into the conversation: a ``SystemMessage`` in the turn input +lands mid-thread on later dispatches, and strict providers reject that shape +(Qwen chat templates return ``400 System message must be at the beginning``; +``langchain_anthropic`` raises on multiple non-consecutive system messages). +The harness bootstrap middleware also drops in-conversation system messages on +a fresh agent's first turn, which used to lose the briefing. +""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from typing import Any + +from langchain.agents.middleware import AgentMiddleware, ModelRequest, ModelResponse +from langchain_core.messages import SystemMessage +from langgraph.config import get_config + +CONFIG_KEY = "octop_team_dispatch_briefing" + + +def _with_briefing(request: ModelRequest[Any]) -> ModelRequest[Any]: + configurable = dict(get_config().get("configurable") or {}) + briefing = configurable.get(CONFIG_KEY) + if not isinstance(briefing, str): + return request + text = briefing.strip() + if not text: + return request + blocks: list[Any] = ( + list(request.system_message.content_blocks) if request.system_message else [] + ) + if blocks: + text = f"\n\n{text}" + blocks.append({"type": "text", "text": text}) + return request.override(system_message=SystemMessage(content_blocks=blocks)) + + +class TeamDispatchBriefingMiddleware(AgentMiddleware[Any, Any]): + """Append the host's dispatch briefing to the member's system prompt.""" + + def wrap_model_call( + self, + request: ModelRequest[Any], + handler: Callable[[ModelRequest[Any]], ModelResponse[Any]], + ) -> ModelResponse[Any]: + return handler(_with_briefing(request)) + + async def awrap_model_call( + self, + request: ModelRequest[Any], + handler: Callable[[ModelRequest[Any]], Awaitable[ModelResponse[Any]]], + ) -> ModelResponse[Any]: + return await handler(_with_briefing(request)) + + +__all__ = ["CONFIG_KEY", "TeamDispatchBriefingMiddleware"] diff --git a/src/octop/infra/agents/providers/headers.py b/src/octop/infra/agents/providers/headers.py new file mode 100644 index 0000000000..46a1eafcaf --- /dev/null +++ b/src/octop/infra/agents/providers/headers.py @@ -0,0 +1,61 @@ +"""Neutral provider HTTP header assembly (OpenCode + Magpie). + +Runtime chat session values are not stored here. Harness injects +``ProviderConfig.session_header`` from the current ``thread_id``. +Probe / list-models paths may stamp a throwaway UUID so gateways that +require a session header still accept the request. +""" + +from __future__ import annotations + +from typing import Any + +from octop.infra.agents.providers.magpie import ( + MAGPIE_SESSION_HEADER, + ensure_magpie_headers, + extra_dict, + is_magpie_provider, +) +from octop.infra.agents.providers.opencode_session import ( + ensure_opencode_session_header, +) +from octop.infra.agents.providers.opencode_session import ( + session_header_for_provider as opencode_session_header_for_provider, +) + + +def session_header_for_provider( + provider_name: Any, + base_url: Any = None, + extra: Any = None, +) -> str | None: + """Return the dynamic session header name for Magpie or OpenCode, else None.""" + if is_magpie_provider(provider_name=provider_name, base_url=base_url, extra=extra): + return MAGPIE_SESSION_HEADER + return opencode_session_header_for_provider(provider_name, base_url) + + +def assemble_provider_headers( + provider_name: Any, + headers: dict[str, str] | None, + *, + base_url: Any = None, + extra: Any = None, + probe: bool = False, +) -> dict[str, str]: + """Merge stored headers with Magpie UA / probe session or OpenCode session.""" + blob = extra_dict(extra) + out = dict(headers) if headers else {} + if is_magpie_provider(provider_name=provider_name, base_url=base_url, extra=blob): + return ensure_magpie_headers(out, probe=probe) + # Runtime configs leave the session value to harness. Probes and model + # listing still need a throwaway id because they never enter a turn. + if not probe: + return out + return ensure_opencode_session_header(provider_name, out, base_url=base_url) + + +__all__ = [ + "assemble_provider_headers", + "session_header_for_provider", +] diff --git a/src/octop/infra/agents/providers/health.py b/src/octop/infra/agents/providers/health.py index c115069f5d..0ee846a1dd 100644 --- a/src/octop/infra/agents/providers/health.py +++ b/src/octop/infra/agents/providers/health.py @@ -55,6 +55,43 @@ class _OpenCircuit: "does not exist", "no such model", ) +_TRANSPORT_TYPE_NAMES = frozenset( + { + "APIConnectionError", + "APITimeoutError", + "ConnectError", + "ConnectTimeout", + "ReadTimeout", + "WriteTimeout", + "PoolTimeout", + "NetworkError", + "RemoteProtocolError", + } +) + + +def _exception_chain(exc: BaseException) -> list[BaseException]: + seen: set[int] = set() + chain: list[BaseException] = [] + current: BaseException | None = exc + while current is not None and id(current) not in seen: + seen.add(id(current)) + chain.append(current) + nxt = current.__cause__ + if nxt is None and not current.__suppress_context__: + nxt = current.__context__ + current = nxt + return chain + + +def _is_transport_failure(exc: BaseException) -> bool: + """True for httpx/OpenAI connection wrappers, not only builtin ConnectionError.""" + for item in _exception_chain(exc): + if isinstance(item, TimeoutError | ConnectionError): + return True + if type(item).__name__ in _TRANSPORT_TYPE_NAMES: + return True + return False def classify_model_failure(exc: Exception) -> FailureDecision: @@ -79,7 +116,7 @@ def classify_model_failure(exc: Exception) -> FailureDecision: classified.rsplit(".", 1)[-1], FailureScope.MODEL, ) - if isinstance(exc, TimeoutError | ConnectionError): + if isinstance(exc, TimeoutError | ConnectionError) or _is_transport_failure(exc): return FailureDecision(True, type(exc).__name__.lower(), FailureScope.MODEL) return FailureDecision(False, "request_error", FailureScope.NONE) diff --git a/src/octop/infra/agents/providers/magpie.py b/src/octop/infra/agents/providers/magpie.py new file mode 100644 index 0000000000..22fec48e24 --- /dev/null +++ b/src/octop/infra/agents/providers/magpie.py @@ -0,0 +1,383 @@ +"""Magpie local-gateway identity, discovery, and catalog mapping. + +Magpie is an OpenAI-compatible provider. Discovery is a read-only probe of the +Octop host (not the dashboard browser). Session attribution uses +``X-Magpie-Session`` at turn time; this module never persists a static session +value into provider headers. +""" + +from __future__ import annotations + +import logging +import uuid +from dataclasses import dataclass +from typing import Any +from urllib.parse import urlsplit + +logger = logging.getLogger(__name__) + +MAGPIE_PRESET_ID = "magpie" +MAGPIE_PRESET_NAME = "Magpie" +MAGPIE_INTEGRATION = "magpie" +MAGPIE_DEFAULT_HOST = "127.0.0.1" +MAGPIE_DEFAULT_PORT = 3425 +MAGPIE_DEFAULT_SCHEME = "http" +MAGPIE_DEFAULT_API_PATH = "/v1" +MAGPIE_DEFAULT_BASE_URL = "http://127.0.0.1:3425/v1" +MAGPIE_HELLO_URL = "http://127.0.0.1:3425/api/hello" +MAGPIE_LOCAL_API_KEY = "magpie-octop" +MAGPIE_SESSION_HEADER = "X-Magpie-Session" +MAGPIE_HELLO_TIMEOUT_S = 1.0 + +_MAGPIE_PRESET_KEYS = frozenset({MAGPIE_PRESET_ID, MAGPIE_PRESET_NAME, MAGPIE_PRESET_NAME.lower()}) +_KNOWN_REASONING_ADAPTERS = frozenset( + { + "status_only", + "thinking", + "thinking_nested_effort", + "openai_reasoning_effort", + "anthropic_adaptive", + "anthropic_budget", + "dashscope", + "openrouter", + } +) +_CHAT_MODALITIES = frozenset({"text", "image", "audio"}) +_NON_CHAT_TYPES = frozenset({"embedding", "image", "video", "audio", "rerank"}) +_NON_CHAT_ID_HINTS = ( + "embed", + "bge-", + "e5-", + "gte-", + "rerank", + "dall-e", + "tts-", + "whisper", + "imagen", + "sora", +) + + +@dataclass(frozen=True) +class MagpieDiscovery: + """Result of the local Magpie hello probe.""" + + available: bool + base_url: str = MAGPIE_DEFAULT_BASE_URL + version: str | None = None + + +@dataclass(frozen=True) +class MagpieSyncPlan: + """Catalog diff applied only after the user confirms a refresh.""" + + models: list[dict[str, Any]] + added: list[str] + removed: list[str] + + +def magpie_user_agent() -> str: + """``User-Agent`` Magpie uses to attribute traffic to this Octop install.""" + from octop import __version__ + + return f"Octop/{__version__}" + + +def magpie_preset() -> dict[str, Any]: + """Octop-owned Magpie preset: empty catalog, local default URL, no models.""" + return { + "id": MAGPIE_PRESET_ID, + "name": MAGPIE_PRESET_NAME, + "base_url": MAGPIE_DEFAULT_BASE_URL, + "protocol": "openai", + "api_key_prefix": "", + "default_api_key": MAGPIE_LOCAL_API_KEY, + "integration": MAGPIE_INTEGRATION, + "models": [], + "logo_id": MAGPIE_PRESET_ID, + } + + +def extra_dict(extra: Any) -> dict[str, Any]: + """Normalize ``extra_json`` / extra dict input.""" + if isinstance(extra, dict): + return extra + return {} + + +def is_magpie_default_base_url(base_url: Any) -> bool: + """True only for the Magpie default OpenAI base (scheme/host/port/path).""" + if not base_url: + return False + try: + parsed = urlsplit(str(base_url).strip()) + port = parsed.port if parsed.port is not None else 80 + except ValueError: + return False + if parsed.scheme.lower() != MAGPIE_DEFAULT_SCHEME: + return False + host = (parsed.hostname or "").lower() + if host != MAGPIE_DEFAULT_HOST: + return False + if port != MAGPIE_DEFAULT_PORT: + return False + path = (parsed.path or "").rstrip("/") or "/" + return path == MAGPIE_DEFAULT_API_PATH + + +def is_magpie_provider( + *, + provider_name: Any = None, + base_url: Any = None, + extra: Any = None, +) -> bool: + """Identify Magpie: explicit integration, then preset name, then default URL.""" + blob = extra_dict(extra) + integration = str(blob.get("integration") or "").strip().lower() + if integration == MAGPIE_INTEGRATION: + return True + name = str(provider_name or "").strip() + if name.lower() in _MAGPIE_PRESET_KEYS: + return True + return is_magpie_default_base_url(base_url) + + +def ensure_magpie_headers( + headers: dict[str, str] | None, + *, + probe: bool = False, +) -> dict[str, str]: + """Add Magpie User-Agent; optionally a throwaway session for probes. + + Existing User-Agent / session header values win (case-insensitive). + Runtime chat sessions are injected by harness via ``session_header``. + """ + out = dict(headers) if headers else {} + if not any(key.lower() == "user-agent" for key in out): + out["User-Agent"] = magpie_user_agent() + if probe and not any(key.lower() == MAGPIE_SESSION_HEADER.lower() for key in out): + out[MAGPIE_SESSION_HEADER] = uuid.uuid4().hex + return out + + +def magpie_catalog_model(item: dict[str, Any]) -> dict[str, Any] | None: + """Map one Magpie ``/v1/models`` item to an Octop model dict, or skip it. + + IDs are kept verbatim (including nested ``/``). Embedding / image / video + entries are not imported as chat candidates. Missing capability fields stay + unknown — names such as ``group/*`` are not used to infer vision or reasoning. + """ + mid = item.get("id") + if not isinstance(mid, str) or not mid.strip(): + return None + mid = mid.strip() + if _is_non_chat_catalog_item(item, mid): + return None + + display = item.get("display_name") + name = display.strip() if isinstance(display, str) and display.strip() else mid + model: dict[str, Any] = {"id": mid, "name": name} + + context = _positive_int(item.get("context_window")) + if context is not None: + model["context_window"] = context + + max_tokens = _positive_int(item.get("max_output_tokens")) + if max_tokens is not None: + model["max_tokens"] = max_tokens + + inputs = _chat_input_modalities(item.get("modalities")) + if inputs: + model["input"] = inputs + + _apply_reasoning(model, item) + return model + + +def _positive_int(value: Any) -> int | None: + if isinstance(value, bool): + return None + if isinstance(value, int) and value > 0: + return value + if isinstance(value, float) and value > 0: + return int(value) + if isinstance(value, str) and value.strip().isdigit(): + parsed = int(value.strip()) + return parsed if parsed > 0 else None + return None + + +def _modality_list(raw: Any) -> list[Any]: + """Magpie sends ``{"input": [...]}``; older catalogs send a flat list.""" + if isinstance(raw, dict): + raw = raw.get("input") + return raw if isinstance(raw, list) else [] + + +def _chat_input_modalities(raw: Any) -> list[str]: + seen: list[str] = [] + for item in _modality_list(raw): + token = str(item).strip().lower() + if token in _CHAT_MODALITIES and token not in seen: + seen.append(token) + return seen + + +def _is_non_chat_catalog_item(item: dict[str, Any], model_id: str) -> bool: + kind = str(item.get("type") or item.get("task") or "").strip().lower() + if kind in _NON_CHAT_TYPES: + return True + lower_id = model_id.lower() + if any(hint in lower_id for hint in _NON_CHAT_ID_HINTS): + return True + modalities = _modality_list(item.get("modalities")) + if modalities: + tokens = {str(m).strip().lower() for m in modalities if str(m).strip()} + if tokens and "text" not in tokens: + return True + return False + + +def _model_id(item: dict[str, Any]) -> str: + mid = item.get("id") + if not isinstance(mid, str): + return "" + return mid.strip() + + +def initial_magpie_models(fetched: list[dict[str, Any]]) -> list[dict[str, Any]] | None: + """Models for a first add. ``None`` when the catalog has no chat models. + + Imported rows are enabled so the user can pick them. Nothing here selects + a preferred model or a route group. + """ + models: list[dict[str, Any]] = [] + seen: set[str] = set() + for item in fetched: + if not isinstance(item, dict): + continue + mid = _model_id(item) + if not mid or mid in seen: + continue + seen.add(mid) + row = dict(item) + row["id"] = mid + row["enabled"] = True + models.append(row) + return models or None + + +def plan_magpie_sync( + existing: list[dict[str, Any]], + fetched: list[dict[str, Any]], +) -> MagpieSyncPlan: + """Keep saved rows, append new ids, and list removals. + + Surviving rows keep enabled, capability overrides, and group visibility. + New rows stay disabled until the user enables them. An empty catalog + removes every saved id and must be confirmed by the caller. + """ + saved: dict[str, dict[str, Any]] = {} + for item in existing: + if not isinstance(item, dict): + continue + mid = _model_id(item) + if mid and mid not in saved: + saved[mid] = dict(item) + + incoming: dict[str, dict[str, Any]] = {} + order: list[str] = [] + for item in fetched: + if not isinstance(item, dict): + continue + mid = _model_id(item) + if not mid or mid in incoming: + continue + incoming[mid] = item + order.append(mid) + + added = [mid for mid in order if mid not in saved] + removed = [mid for mid in saved if mid not in incoming] + merged: list[dict[str, Any]] = [] + for mid in order: + if mid in saved: + merged.append(saved[mid]) + continue + fresh = dict(incoming[mid]) + fresh["id"] = mid + fresh["enabled"] = False + merged.append(fresh) + return MagpieSyncPlan(models=merged, added=added, removed=removed) + + +def _apply_reasoning(model: dict[str, Any], item: dict[str, Any]) -> None: + raw = item.get("reasoning_config") + if not isinstance(raw, dict): + raw = item.get("reasoning") + if isinstance(raw, dict): + adapter = str(raw.get("adapter") or "").strip() + if adapter and adapter not in _KNOWN_REASONING_ADAPTERS: + return + if raw.get("supported") is False: + return + model["reasoning"] = True + if adapter: + model["reasoning_config"] = dict(raw) + return + if raw is True: + model["reasoning"] = True + + +async def discover_local_magpie() -> MagpieDiscovery: + """Probe ``GET /api/hello`` on the Octop host. Failures are ``available=false``.""" + import httpx + + unavailable = MagpieDiscovery(available=False) + try: + async with httpx.AsyncClient( + timeout=MAGPIE_HELLO_TIMEOUT_S, + follow_redirects=False, + trust_env=False, + ) as client: + response = await client.get(MAGPIE_HELLO_URL) + except Exception as exc: + logger.info("magpie discovery failed: %r (%s)", exc, type(exc).__name__) + return unavailable + + if response.status_code < 200 or response.status_code >= 300: + return unavailable + try: + payload = response.json() + except Exception: + return unavailable + if not isinstance(payload, dict) or payload.get("name") != MAGPIE_INTEGRATION: + return unavailable + version = payload.get("version") + display = str(version).strip() if version is not None else "" + return MagpieDiscovery( + available=True, + base_url=MAGPIE_DEFAULT_BASE_URL, + version=display or None, + ) + + +__all__ = [ + "MAGPIE_DEFAULT_BASE_URL", + "MAGPIE_HELLO_URL", + "MAGPIE_INTEGRATION", + "MAGPIE_LOCAL_API_KEY", + "MAGPIE_PRESET_ID", + "MAGPIE_PRESET_NAME", + "MAGPIE_SESSION_HEADER", + "MagpieDiscovery", + "discover_local_magpie", + "ensure_magpie_headers", + "MagpieSyncPlan", + "initial_magpie_models", + "is_magpie_default_base_url", + "is_magpie_provider", + "magpie_catalog_model", + "magpie_preset", + "magpie_user_agent", + "plan_magpie_sync", +] diff --git a/src/octop/infra/agents/providers/presets.py b/src/octop/infra/agents/providers/presets.py index 64c8c23c55..b6ff0b41ed 100644 --- a/src/octop/infra/agents/providers/presets.py +++ b/src/octop/infra/agents/providers/presets.py @@ -347,6 +347,10 @@ def load_provider_presets() -> list[dict[str, Any]]: len(out), ) out.insert(insert_at, onnx_preset) + if not any(p.get("id") == "magpie" for p in out): + from octop.infra.agents.providers.magpie import magpie_preset + + out.insert(0, magpie_preset()) if not any(p.get("purpose") == "embedding" and p.get("id") != "onnx" for p in out): out.extend(_embedding_cloud_presets()) for preset in out: diff --git a/src/octop/infra/agents/providers/probe.py b/src/octop/infra/agents/providers/probe.py index 1b9d0614f7..5352941afc 100644 --- a/src/octop/infra/agents/providers/probe.py +++ b/src/octop/infra/agents/providers/probe.py @@ -12,7 +12,8 @@ import httpx from octop.infra.agents.providers import KIND_TO_PROTOCOL -from octop.infra.agents.providers.opencode_session import ensure_opencode_session_header +from octop.infra.agents.providers.headers import assemble_provider_headers +from octop.infra.agents.providers.magpie import is_magpie_provider, magpie_catalog_model from octop.infra.agents.providers.presets import load_provider_presets logger = logging.getLogger(__name__) @@ -52,7 +53,14 @@ def build_probe_chat_model( protocol = KIND_TO_PROTOCOL.get(row.kind, row.kind) base_url = row.base_url or "https://api.openai.com/v1" - headers = ensure_opencode_session_header(row.name, provider_headers(row), base_url=base_url) + extra = parse_provider_extra(row) + headers = assemble_provider_headers( + row.name, + provider_headers(row), + base_url=base_url, + extra=extra, + probe=True, + ) models = row.get_models() if hasattr(row, "get_models") else [] mid = model_id or (models[0]["id"] if models else "gpt-4o-mini") entry = next((m for m in models if m.get("id") == mid), None) @@ -78,7 +86,7 @@ def build_probe_chat_model( headers=headers, stream_usage=resolve_provider_stream_usage( provider_name=row.name, - extra=parse_provider_extra(row), + extra=extra, ), ) return build_chat_model(provider, model) @@ -179,10 +187,12 @@ async def _probe_embedding_endpoint( started = time.perf_counter() url = _embeddings_url(getattr(row, "base_url", None)) headers: dict[str, str] = {"Authorization": f"Bearer {getattr(row, 'api_key', None) or ''}"} - extra = ensure_opencode_session_header( + extra = assemble_provider_headers( getattr(row, "name", None), provider_headers(row), base_url=getattr(row, "base_url", None), + extra=parse_provider_extra(row), + probe=True, ) headers.update(extra) try: @@ -304,6 +314,7 @@ async def fetch_openai_compatible_models( extra_headers: dict[str, str] | None = None, locale: str = "en", provider_name: str | None = None, + extra: dict[str, Any] | None = None, ) -> dict[str, Any]: """Use Tencent plan catalogs; otherwise list models via ``GET {base}/models``.""" url = _models_list_url(base_url) @@ -322,9 +333,18 @@ async def fetch_openai_compatible_models( ): return {"ok": True, "models": preset["models"]} headers: dict[str, str] = {"Authorization": f"Bearer {api_key}"} - extra_headers = ensure_opencode_session_header(provider_name, extra_headers, base_url=base_url) - if extra_headers: - headers.update(extra_headers) + # Model listing is not a chat turn. OpenCode still needs a throwaway + # session; Magpie listing must not invent one. + magpie = is_magpie_provider(provider_name=provider_name, base_url=base_url, extra=extra) + merged_headers = assemble_provider_headers( + provider_name, + extra_headers, + base_url=base_url, + extra=extra, + probe=not magpie, + ) + if merged_headers: + headers.update(merged_headers) try: async with httpx.AsyncClient(timeout=_FETCH_MODELS_TIMEOUT_S) as client: response = await client.get(url, headers=headers) @@ -361,18 +381,28 @@ async def fetch_openai_compatible_models( "error": "response is not OpenAI-compatible (expected {data: [{id, …}]})", } - models: list[dict[str, str]] = [] + models: list[dict[str, Any]] = [] seen: set[str] = set() for item in data: if not isinstance(item, dict): continue - mid = item.get("id") - if not isinstance(mid, str) or not mid.strip(): + if magpie: + mapped = magpie_catalog_model(item) + if mapped is None: + continue + mid = str(mapped["id"]) + if mid in seen: + continue + seen.add(mid) + models.append(mapped) + continue + raw_id = item.get("id") + if not isinstance(raw_id, str) or not raw_id.strip(): continue - mid = mid.strip() + mid = raw_id.strip() if mid in seen: continue seen.add(mid) models.append({"id": mid, "name": mid}) - models.sort(key=lambda m: m["id"]) + models.sort(key=lambda m: str(m["id"])) return {"ok": True, "models": models} diff --git a/src/octop/infra/agents/providers/store.py b/src/octop/infra/agents/providers/store.py index 8ca03bb504..23854631a6 100644 --- a/src/octop/infra/agents/providers/store.py +++ b/src/octop/infra/agents/providers/store.py @@ -9,12 +9,15 @@ from octop_harness.config import ModelConfig, ProviderConfig +from octop.infra.agents.providers.headers import ( + assemble_provider_headers, + session_header_for_provider, +) from octop.infra.agents.providers.model_flags import ( is_chat_eligible_model, is_local_runtime_provider, is_vision_model, ) -from octop.infra.agents.providers.opencode_session import session_header_for_provider from octop.infra.agents.providers.reasoning import reasoning_capability if TYPE_CHECKING: @@ -341,11 +344,18 @@ def build_harness_configs(self) -> list[ProviderConfig]: continue extra = _provider_extra_dict(row) raw_headers = extra.get("headers") - headers = ( + stored_headers = ( {str(k): str(v) for k, v in raw_headers.items()} if isinstance(raw_headers, dict) else {} ) + headers = assemble_provider_headers( + row.name, + stored_headers, + base_url=row.base_url, + extra=extra, + probe=False, + ) out.append( ProviderConfig( id=row.name, @@ -355,7 +365,7 @@ def build_harness_configs(self) -> list[ProviderConfig]: name=row.name, models=models, headers=headers, - session_header=session_header_for_provider(row.name, row.base_url), + session_header=session_header_for_provider(row.name, row.base_url, extra=extra), stream_usage=resolve_provider_stream_usage( provider_name=row.name, extra=extra, diff --git a/src/octop/infra/agents/providers/turn_access.py b/src/octop/infra/agents/providers/turn_access.py new file mode 100644 index 0000000000..d1b0588f21 --- /dev/null +++ b/src/octop/infra/agents/providers/turn_access.py @@ -0,0 +1,51 @@ +"""Per-turn model access used to keep AUTO failover inside the caller's grants.""" + +from __future__ import annotations + +from collections.abc import Iterator, Sequence +from contextlib import contextmanager +from contextvars import ContextVar, Token +from typing import Any + +from octop.infra.users.groups import model_ref_allowed + +_TURN_ACCESS: ContextVar[tuple[int, bool] | None] = ContextVar( + "octop_turn_model_access", + default=None, +) + + +def current_turn_model_access() -> tuple[int, bool] | None: + """``(user_id, is_admin)`` for the active turn, or ``None`` when unset.""" + return _TURN_ACCESS.get() + + +@contextmanager +def turn_model_access(user_id: int, *, is_admin: bool) -> Iterator[None]: + """Bind the caller whose model grants constrain this turn's AUTO failover.""" + token: Token[tuple[int, bool] | None] = _TURN_ACCESS.set((user_id, is_admin)) + try: + yield + finally: + _TURN_ACCESS.reset(token) + + +def filter_auto_route_candidates[T](candidates: Sequence[T], *, repos: Any) -> list[T]: + """Drop candidates the current turn's user is not allowed to call. + + When no turn access is bound, the candidate list is unchanged. + """ + access = current_turn_model_access() + if access is None: + return list(candidates) + user_id, is_admin = access + return [ + candidate + for candidate in candidates + if model_ref_allowed( + repos, + user_id=user_id, + is_admin=is_admin, + model_ref=str(getattr(candidate, "ref", "")), + ) + ] diff --git a/src/octop/infra/agents/settings/acp_host.py b/src/octop/infra/agents/settings/acp_host.py new file mode 100644 index 0000000000..1c74a84156 --- /dev/null +++ b/src/octop/infra/agents/settings/acp_host.py @@ -0,0 +1,688 @@ +"""Detect and best-effort install ACP CLIs on the Octop host. + +Mirrors :mod:`octop.infra.utils.docker_env`: never raise for a missing binary. +Return a status payload the dashboard can show with an install script and an +agent prompt. A successful probe means the process answered ACP ``initialize``. +""" + +from __future__ import annotations + +import asyncio +import logging +import os +import platform +import shutil +import signal +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any, cast + +logger = logging.getLogger(__name__) + +_INSTALL_TIMEOUT_SEC = 180 +_PROBE_TIMEOUT_SEC = 20 +_NPX_PROBE_TIMEOUT_SEC = 50 +_ASK_TIMEOUT_SEC = 90 +_ASK_PROMPT = "who are you?" + +_PLATFORMS = ("linux", "darwin", "win32") + + +@dataclass(frozen=True) +class _PlatformScript: + script: str + auto: bool + + +@dataclass(frozen=True) +class _RunnerSpec: + command: str + args: tuple[str, ...] + requires: tuple[str, ...] + docs_url: str + label_zh: str + label_en: str + scripts: dict[str, _PlatformScript] + + +def _posix(script: str, *, auto: bool = True) -> _PlatformScript: + return _PlatformScript(script=script, auto=auto) + + +def _win(script: str, *, auto: bool = True) -> _PlatformScript: + return _PlatformScript(script=script, auto=auto) + + +_CATALOG: dict[str, _RunnerSpec] = { + "opencode": _RunnerSpec( + command="opencode", + args=("acp",), + requires=(), + docs_url="https://opencode.ai/docs", + label_zh="OpenCode", + label_en="OpenCode", + scripts={ + "linux": _posix("curl -fsSL https://opencode.ai/v2/install | bash"), + "darwin": _posix("curl -fsSL https://opencode.ai/v2/install | bash"), + "win32": _win( + "# OpenCode 2 has no PowerShell installer.\n" + "# Install inside WSL:\n" + "# curl -fsSL https://opencode.ai/v2/install | bash\n" + "# Or download the Windows zip from\n" + "# https://github.com/anomalyco/opencode/releases\n" + "# and put opencode.exe on PATH.", + auto=False, + ), + }, + ), + "codebuddy": _RunnerSpec( + command="codebuddy", + args=("--acp",), + requires=(), + docs_url="https://www.codebuddy.cn/docs/cli/installation", + label_zh="CodeBuddy", + label_en="CodeBuddy", + scripts={ + "linux": _posix("curl -fsSL https://www.codebuddy.cn/cli/install.sh | bash"), + "darwin": _posix("curl -fsSL https://www.codebuddy.cn/cli/install.sh | bash"), + "win32": _win("irm https://www.codebuddy.cn/cli/install.ps1 | iex"), + }, + ), + "claude_code": _RunnerSpec( + command="npx", + args=("-y", "@zed-industries/claude-agent-acp"), + requires=("claude",), + docs_url="https://code.claude.com/docs/en/setup", + label_zh="Claude Code", + label_en="Claude Code", + scripts={ + "linux": _posix( + "curl -fsSL https://claude.ai/install.sh | bash\n" + "# ACP starts with: npx -y @zed-industries/claude-agent-acp\n" + "# Install Node.js if npx is not on PATH." + ), + "darwin": _posix( + "curl -fsSL https://claude.ai/install.sh | bash\n" + "# ACP starts with: npx -y @zed-industries/claude-agent-acp\n" + "# Install Node.js if npx is not on PATH." + ), + "win32": _win( + "irm https://claude.ai/install.ps1 | iex\n" + "# ACP starts with: npx -y @zed-industries/claude-agent-acp\n" + "# Install Node.js if npx is not on PATH." + ), + }, + ), + "codex": _RunnerSpec( + command="npx", + args=("-y", "@zed-industries/codex-acp"), + requires=("codex",), + docs_url="https://developers.openai.com/codex/cli", + label_zh="Codex", + label_en="Codex", + scripts={ + "linux": _posix( + "curl -fsSL https://chatgpt.com/codex/install.sh | sh\n" + "# ACP starts with: npx -y @zed-industries/codex-acp\n" + "# Install Node.js if npx is not on PATH." + ), + "darwin": _posix( + "curl -fsSL https://chatgpt.com/codex/install.sh | sh\n" + "# ACP starts with: npx -y @zed-industries/codex-acp\n" + "# Install Node.js if npx is not on PATH." + ), + "win32": _win( + "irm https://chatgpt.com/codex/install.ps1 | iex\n" + "# ACP starts with: npx -y @zed-industries/codex-acp\n" + "# Install Node.js if npx is not on PATH." + ), + }, + ), + "kimi_code": _RunnerSpec( + command="kimi", + args=("acp",), + requires=(), + docs_url="https://www.kimi.com/code/docs/en/kimi-code-cli/guides/getting-started.html", + label_zh="Kimi Code", + label_en="Kimi Code", + scripts={ + "linux": _posix("curl -fsSL https://code.kimi.com/kimi-code/install.sh | bash"), + "darwin": _posix("curl -fsSL https://code.kimi.com/kimi-code/install.sh | bash"), + "win32": _win("irm https://code.kimi.com/kimi-code/install.ps1 | iex"), + }, + ), + "cursor_cli": _RunnerSpec( + command="agent", + args=("acp",), + requires=(), + docs_url="https://cursor.com/docs/cli/installation", + label_zh="Cursor CLI", + label_en="Cursor CLI", + scripts={ + "linux": _posix("curl https://cursor.com/install -fsS | bash"), + "darwin": _posix("curl https://cursor.com/install -fsS | bash"), + "win32": _win("irm 'https://cursor.com/install?win32=true' | iex"), + }, + ), + "pi": _RunnerSpec( + command="npx", + args=("-y", "pi-acp"), + requires=("pi",), + docs_url="https://pi.dev/docs/latest/quickstart", + label_zh="Pi", + label_en="Pi", + scripts={ + "linux": _posix( + "curl -fsSL https://pi.dev/install.sh | sh\n" + "# ACP starts with: npx -y pi-acp\n" + "# Install Node.js if npx is not on PATH." + ), + "darwin": _posix( + "curl -fsSL https://pi.dev/install.sh | sh\n" + "# ACP starts with: npx -y pi-acp\n" + "# Install Node.js if npx is not on PATH." + ), + "win32": _win( + "irm https://pi.dev/install.ps1 | iex\n" + "# ACP starts with: npx -y pi-acp\n" + "# Install Node.js if npx is not on PATH." + ), + }, + ), +} + + +def _platform_key() -> str: + if sys.platform.startswith("linux"): + return "linux" + if sys.platform == "darwin": + return "darwin" + if sys.platform.startswith("win"): + return "win32" + return sys.platform + + +def _host_env(extra: dict[str, str] | None) -> dict[str, str]: + env = {key: value for key, value in os.environ.items() if isinstance(value, str)} + if extra: + env.update({str(key): str(value) for key, value in extra.items()}) + home = Path.home() + candidates = ( + home / ".local" / "bin", + home / ".opencode" / "bin", + home / ".claude" / "bin", + home / ".codebuddy" / "bin", + home / ".kimi" / "bin", + home / ".pi" / "bin", + home / "bin", + home / ".cursor" / "bin", + home / "AppData" / "Local" / "codebuddy" / "bin", + home / "AppData" / "Local" / "cursor-agent", + ) + existing = [str(path) for path in candidates if path.is_dir()] + if existing: + env["PATH"] = os.pathsep.join(existing) + os.pathsep + env.get("PATH", "") + return env + + +def _which(name: str, env: dict[str, str]) -> str | None: + if not name: + return None + if os.path.isabs(name) or "/" in name or "\\" in name: + return name if os.path.isfile(name) else None + return shutil.which(name, path=env.get("PATH")) + + +def _shell_ok(plat: str) -> bool: + if plat == "win32": + return shutil.which("powershell") is not None or shutil.which("pwsh") is not None + return shutil.which("bash") is not None or shutil.which("sh") is not None + + +def _resolve_launch( + spec: _RunnerSpec | None, + *, + command: str, + args: list[str] | None, +) -> tuple[str, list[str]]: + if command.strip(): + return command.strip(), [str(item) for item in (args or [])] + if spec is None: + return "", [] + return spec.command, list(spec.args) + + +def _missing_binary( + *, + command: str, + spec: _RunnerSpec | None, + env: dict[str, str], +) -> str | None: + if not command: + return "command_missing" + if _which(command, env) is None: + leaf = Path(command).name.lower() + if leaf in {"npx", "npx.cmd", "npm", "npm.cmd"}: + return "node_missing" + return "command_missing" + if spec is not None: + for required in spec.requires: + if _which(required, env) is None: + return "agent_missing" + return None + + +def _run_install(script: str, plat: str) -> tuple[bool, str]: + line = "\n".join( + part for part in script.splitlines() if part.strip() and not part.strip().startswith("#") + ).strip() + if not line: + return False, "no_installer" + env = _host_env(None) + if plat == "win32": + exe = shutil.which("powershell") or shutil.which("pwsh") + if exe is None: + return False, "powershell_missing" + argv = [exe, "-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", line] + else: + exe = shutil.which("bash") or shutil.which("sh") + if exe is None: + return False, "shell_missing" + argv = [exe, "-c", line] + try: + result = subprocess.run( + argv, + check=False, + capture_output=True, + text=True, + timeout=_INSTALL_TIMEOUT_SEC, + env=env, + ) + except subprocess.TimeoutExpired: + logger.warning("ACP install timed out") + return False, "install_timeout" + except OSError as exc: + logger.warning("ACP install failed to start: %s", exc) + return False, "install_failed" + if result.returncode != 0: + stderr = (result.stderr or result.stdout or "").strip() + logger.warning("ACP install exited %s: %s", result.returncode, stderr[:300]) + return False, stderr[:300] or "install_failed" + return True, "" + + +def _kill_process(pid: int) -> None: + try: + import psutil + except ImportError: + os.kill(pid, signal.SIGTERM) + return + try: + parent = psutil.Process(pid) + except psutil.NoSuchProcess: + return + for child in parent.children(recursive=True): + try: + child.kill() + except psutil.NoSuchProcess: + continue + try: + parent.kill() + except psutil.NoSuchProcess: + return + + +class _ProbeClient: + """ACP client for initialize and a single probe prompt.""" + + def __init__(self) -> None: + self.text = "" + + def on_connect(self, conn: Any) -> None: + _ = conn + + async def request_permission(self, *_args: Any, **_kwargs: Any) -> Any: + from acp.schema import DeniedOutcome, RequestPermissionResponse + + return RequestPermissionResponse(outcome=DeniedOutcome(outcome="cancelled")) + + async def session_update(self, *_args: Any, **kwargs: Any) -> None: + update = kwargs.get("update") + if update is None and len(_args) >= 2: + update = _args[1] + from acp.schema import AgentMessageChunk + + if isinstance(update, AgentMessageChunk): + _merge_text(self, _chunk_text(getattr(update, "content", None))) + + async def write_text_file(self, *_args: Any, **_kwargs: Any) -> None: + return None + + async def read_text_file(self, *_args: Any, **_kwargs: Any) -> dict[str, str]: + return {"content": ""} + + async def create_terminal(self, *_args: Any, **_kwargs: Any) -> dict[str, str]: + return {"terminalId": ""} + + async def terminal_output(self, *_args: Any, **_kwargs: Any) -> dict[str, str]: + return {"output": "", "truncated": "false"} + + async def release_terminal(self, *_args: Any, **_kwargs: Any) -> None: + return None + + async def wait_for_terminal_exit(self, *_args: Any, **_kwargs: Any) -> dict[str, int]: + return {"exitCode": 0} + + async def kill_terminal(self, *_args: Any, **_kwargs: Any) -> None: + return None + + async def ext_method(self, method: str, params: dict[str, Any]) -> dict[str, Any]: + _ = method, params + return {} + + async def ext_notification(self, method: str, params: dict[str, Any]) -> None: + _ = method, params + + +def _chunk_text(content: Any) -> str: + if hasattr(content, "text") and isinstance(getattr(content, "text", None), str): + return str(content.text) + if isinstance(content, list): + return "".join(_chunk_text(item) for item in content) + if isinstance(content, dict) and content.get("type") == "text": + return str(content.get("text") or "") + return "" + + +def _merge_text(client: _ProbeClient, text: str) -> None: + if not text or text == client.text: + return + if not client.text or text.startswith(client.text): + client.text = text + return + client.text += text + + +async def _handshake(command: str, args: list[str], env: dict[str, str]) -> tuple[bool, str]: + from acp import PROTOCOL_VERSION, spawn_agent_process + from acp.schema import ClientCapabilities, Implementation + + timeout = ( + _NPX_PROBE_TIMEOUT_SEC + if Path(command).name.lower().startswith("npx") + else _PROBE_TIMEOUT_SEC + ) + try: + async with asyncio.timeout(timeout): + async with spawn_agent_process( + cast(Any, _ProbeClient()), + command, + *args, + cwd=str(Path.home()), + env=env, + ) as (conn, process): + try: + initialized = await conn.initialize( + protocol_version=PROTOCOL_VERSION, + capabilities=ClientCapabilities(), + client_info=Implementation(name="octop-acp-probe", version="0.1.0"), + ) + finally: + pid = getattr(process, "pid", None) + if isinstance(pid, int) and getattr(process, "returncode", None) is None: + _kill_process(pid) + version = getattr(initialized, "protocol_version", None) + if version != PROTOCOL_VERSION: + return False, f"protocol {version}" + return True, "" + except TimeoutError: + return False, "timed out waiting for ACP initialize" + except Exception as exc: + text = f"{type(exc).__name__}: {exc}" + return False, text[:400] + + +async def _ask(command: str, args: list[str], env: dict[str, str]) -> tuple[bool, str]: + """Open one ACP session, send a fixed prompt, and return the assistant text.""" + from acp import PROTOCOL_VERSION, spawn_agent_process + from acp.schema import ClientCapabilities, Implementation, TextContentBlock + + client = _ProbeClient() + try: + async with asyncio.timeout(_ASK_TIMEOUT_SEC): + async with spawn_agent_process( + cast(Any, client), + command, + *args, + cwd=str(Path.home()), + env=env, + ) as (conn, process): + try: + initialized = await conn.initialize( + protocol_version=PROTOCOL_VERSION, + capabilities=ClientCapabilities(), + client_info=Implementation(name="octop-acp-probe", version="0.1.0"), + ) + version = getattr(initialized, "protocol_version", None) + if version != PROTOCOL_VERSION: + return False, f"protocol {version}" + session = await conn.new_session(cwd=str(Path.home())) + await conn.prompt( + session_id=session.session_id, + prompt=[TextContentBlock(type="text", text=_ASK_PROMPT)], + ) + finally: + pid = getattr(process, "pid", None) + if isinstance(pid, int) and getattr(process, "returncode", None) is None: + _kill_process(pid) + text = client.text.strip() + if not text: + return False, "empty reply" + return True, text[:4000] + except TimeoutError: + partial = client.text.strip() + if partial: + return True, partial[:4000] + return False, "timed out waiting for ACP reply" + except Exception as exc: + text = f"{type(exc).__name__}: {exc}" + return False, text[:400] + + +def _label(spec: _RunnerSpec | None, runner: str, locale: str) -> str: + if spec is None: + return runner or "ACP" + if locale.startswith("en"): + return spec.label_en + return spec.label_zh + + +def _scripts(spec: _RunnerSpec | None) -> dict[str, str]: + if spec is None: + return {} + return {plat: spec.scripts[plat].script for plat in _PLATFORMS if plat in spec.scripts} + + +def _host_script(spec: _RunnerSpec | None, plat: str) -> str: + if spec is None: + return "" + item = spec.scripts.get(plat) + return "" if item is None else item.script + + +def _can_auto(spec: _RunnerSpec | None, plat: str, *, acp_ok: bool) -> bool: + if spec is None or acp_ok: + return False + item = spec.scripts.get(plat) + if item is None or not item.auto: + return False + return _shell_ok(plat) + + +def _agent_prompt( + *, + locale: str, + status: str, + plat: str, + label: str, + command: str, + script: str, + host: str, +) -> str: + en = locale.startswith("en") + plat_name = {"linux": "Linux", "darwin": "macOS", "win32": "Windows"}.get(plat, plat) + if status in {"ready", "installed"}: + if en: + return ( + f"{label} answered ACP initialize on this machine ({host}). " + "No further install is needed." + ) + return f"本机的 {label} 已能通过 ACP initialize(系统:{host})。无需再安装。" + if not script: + if en: + return ( + f"Install `{command or label}` on the machine that runs Octop ({host}, {plat_name}) " + "and confirm `ACP initialize` succeeds over stdio." + ) + return ( + f"请在运行 Octop 的这台机器上安装 `{command or label}`(系统:{host},{plat_name})," + "并确认它能通过 stdio 完成 ACP initialize。" + ) + if en: + return ( + f"Install {label} on this Octop host ({host}, {plat_name}) with:\n{script}\n" + "Then confirm the configured command answers ACP initialize. " + "Do not write tokens into the command." + ) + return ( + f"请在运行 Octop 的这台机器上安装 {label}(系统:{host},{plat_name})。" + f"使用下面的命令:\n{script}\n" + "装好后确认当前启动命令能通过 ACP initialize。不要把密钥写进命令。" + ) + + +async def acp_host_status( + *, + runner: str, + command: str = "", + args: list[str] | None = None, + env: dict[str, str] | None = None, + attempt_install: bool = False, + locale: str = "zh", +) -> dict[str, Any]: + """Probe one runner. When ``attempt_install`` is set, run the host installer first.""" + plat = _platform_key() + spec = _CATALOG.get(runner.strip()) + launch, launch_args = _resolve_launch(spec, command=command, args=args) + host_env = _host_env(env) + label = _label(spec, runner.strip(), locale) + script = _host_script(spec, plat) + installed_now = False + install_error = "" + + missing = _missing_binary(command=launch, spec=spec, env=host_env) + auto = _can_auto(spec, plat, acp_ok=False) and missing is not None + if attempt_install and auto and missing is not None: + ok, install_error = await asyncio.to_thread(_run_install, script, plat) + installed_now = ok + host_env = _host_env(env) + missing = _missing_binary(command=launch, spec=spec, env=host_env) + + acp_ok = False + acp_detail = "" + if missing is None and launch: + acp_ok, acp_detail = await _handshake(launch, launch_args, host_env) + + if acp_ok and installed_now: + status, reason, detail = "installed", "install_ok", "" + elif acp_ok: + status, reason, detail = "ready", "already_present", "" + elif missing == "agent_missing": + status, reason, detail = "missing", "agent_missing", "" + elif missing in {"command_missing", "node_missing"}: + if attempt_install and not auto: + status, reason, detail = "skipped", "manual_install_required", "" + elif attempt_install and installed_now: + status, reason, detail = "degraded", "install_failed", "command still missing" + elif attempt_install and install_error: + status, reason, detail = "degraded", "install_failed", install_error + else: + status, reason, detail = "missing", missing, "" + elif "timed out" in acp_detail: + status, reason, detail = "degraded", "handshake_timeout", acp_detail + else: + status, reason, detail = "degraded", "handshake_failed", acp_detail + + return { + "runner": runner.strip(), + "platform": plat, + "command": launch, + "args": launch_args, + "binary_found": missing is None, + "acp_ok": acp_ok, + "docs_url": "" if spec is None else spec.docs_url, + "install_script": script, + "scripts": _scripts(spec), + "agent_prompt": _agent_prompt( + locale=locale, + status=status, + plat=plat, + label=label, + command=launch, + script=script, + host=platform.platform(), + ), + "can_auto_install": _can_auto(spec, plat, acp_ok=acp_ok), + "status": status, + "reason": reason, + "detail": detail, + } + + +async def acp_host_ask( + *, + runner: str, + command: str = "", + args: list[str] | None = None, + env: dict[str, str] | None = None, +) -> dict[str, Any]: + """Send ``who are you?`` to the runner and return the assistant text.""" + spec = _CATALOG.get(runner.strip()) + launch, launch_args = _resolve_launch(spec, command=command, args=args) + host_env = _host_env(env) + missing = _missing_binary(command=launch, spec=spec, env=host_env) + if missing is not None or not launch: + return { + "ok": False, + "prompt": _ASK_PROMPT, + "text": "", + "reason": missing or "command_missing", + "detail": "", + "command": launch, + } + ok, text = await _ask(launch, launch_args, host_env) + if ok: + return { + "ok": True, + "prompt": _ASK_PROMPT, + "text": text, + "reason": "ok", + "detail": "", + "command": launch, + } + reason = "handshake_timeout" if "timed out" in text else "handshake_failed" + if text == "empty reply": + reason = "empty_reply" + return { + "ok": False, + "prompt": _ASK_PROMPT, + "text": "", + "reason": reason, + "detail": "" if text == "empty reply" else text, + "command": launch, + } + + +__all__ = ["acp_host_ask", "acp_host_status"] diff --git a/src/octop/infra/agents/settings/compaction.py b/src/octop/infra/agents/settings/compaction.py new file mode 100644 index 0000000000..7e41be6bd0 --- /dev/null +++ b/src/octop/infra/agents/settings/compaction.py @@ -0,0 +1,152 @@ +"""Per-expert automatic context compaction. + +Stored on the agent under ``config_json["compaction"]``. Empty means the +harness defaults: compact at 85% of the context window, keep 10%, and send at +most 8000 tokens of evicted history to the summary model. + +Forwarded to ``HarnessAgentConfig`` only when the installed octop-harness +accepts the fields: + +- ``summarization_enabled`` +- ``summarization_trigger`` — ``("fraction", float)`` +- ``summarization_keep`` — ``("fraction", float)`` +- ``summarization_trim_tokens`` +""" + +from __future__ import annotations + +import logging +from typing import Any + +logger = logging.getLogger(__name__) + +COMPACTION_CONFIG_KEY = "compaction" +DEFAULT_TRIGGER_FRACTION = 0.85 +DEFAULT_KEEP_FRACTION = 0.10 +DEFAULT_TRIM_TOKENS = 8000 +MIN_TRIM_TOKENS = 1000 + +_HARNESS_ENABLED = "summarization_enabled" +_HARNESS_TRIGGER = "summarization_trigger" +_HARNESS_KEEP = "summarization_keep" +_HARNESS_TRIM = "summarization_trim_tokens" + + +def _fraction(value: Any, *, upper: float) -> float | None: + if isinstance(value, bool) or not isinstance(value, int | float): + return None + parsed = float(value) + if 0 < parsed <= upper: + return parsed + return None + + +def _trim_tokens(value: Any) -> int | None: + if isinstance(value, bool) or not isinstance(value, int): + return None + if value < MIN_TRIM_TOKENS: + return None + return value + + +def normalize_compaction(value: Any) -> dict[str, Any] | None: + """Return the stored object, or ``None`` when it matches the harness default. + + Invalid entries are dropped. ``enabled: true`` alone is the default and is + not stored. ``None`` and non-objects clear the setting. + """ + if not isinstance(value, dict): + return None + cleaned: dict[str, Any] = {} + enabled = value.get("enabled") + if isinstance(enabled, bool) and enabled is False: + cleaned["enabled"] = False + trigger = _fraction(value.get("trigger_fraction"), upper=1) + if trigger is not None: + cleaned["trigger_fraction"] = trigger + keep_limit = trigger if trigger is not None else DEFAULT_TRIGGER_FRACTION + keep = _fraction(value.get("keep_fraction"), upper=1) + if keep is not None and keep < keep_limit: + cleaned["keep_fraction"] = keep + trim = _trim_tokens(value.get("trim_tokens")) + if trim is not None: + cleaned["trim_tokens"] = trim + if not cleaned: + return None + return cleaned + + +def apply_compaction_update(cfg: dict[str, Any], value: Any) -> dict[str, Any]: + """Merge one compaction update into an agent config dict. + + ``None`` and an empty or default-only object remove the key. + """ + merged = dict(cfg) + normalized = normalize_compaction(value) + if normalized is None: + merged.pop(COMPACTION_CONFIG_KEY, None) + else: + merged[COMPACTION_CONFIG_KEY] = normalized + return merged + + +def compaction_for_api(cfg: dict[str, Any]) -> dict[str, Any] | None: + """Return the stored compaction object for API responses.""" + raw = cfg.get(COMPACTION_CONFIG_KEY) + if not isinstance(raw, dict): + return None + return normalize_compaction(raw) + + +def harness_summarization_kwargs( + cfg: dict[str, Any], + *, + supported_fields: frozenset[str], +) -> dict[str, Any]: + """Map stored compaction onto harness config kwargs the install understands.""" + stored = compaction_for_api(cfg) + if not stored: + return {} + if stored.get("enabled") is False: + if _HARNESS_ENABLED in supported_fields: + return {_HARNESS_ENABLED: False} + logger.debug( + "agent compaction is disabled, but this octop-harness has no %s field", + _HARNESS_ENABLED, + ) + return {} + + out: dict[str, Any] = {} + trigger = stored.get("trigger_fraction") + if trigger is not None and _HARNESS_TRIGGER in supported_fields: + out[_HARNESS_TRIGGER] = ("fraction", float(trigger)) + keep = stored.get("keep_fraction") + if keep is not None and _HARNESS_KEEP in supported_fields: + out[_HARNESS_KEEP] = ("fraction", float(keep)) + trim = stored.get("trim_tokens") + if trim is not None and _HARNESS_TRIM in supported_fields: + out[_HARNESS_TRIM] = int(trim) + if ( + stored + and not out + and not supported_fields.intersection( + {_HARNESS_ENABLED, _HARNESS_TRIGGER, _HARNESS_KEEP, _HARNESS_TRIM} + ) + ): + logger.debug( + "installed octop-harness has no summarization config fields; compaction saved only" + ) + return out + + +__all__ = [ + "COMPACTION_CONFIG_KEY", + "DEFAULT_KEEP_FRACTION", + "DEFAULT_TRIGGER_FRACTION", + "DEFAULT_TRIM_TOKENS", + "MIN_TRIM_TOKENS", + "apply_compaction_update", + "compaction_for_api", + "harness_summarization_kwargs", + "normalize_compaction", +] diff --git a/src/octop/infra/agents/settings/runtime_limits.py b/src/octop/infra/agents/settings/runtime_limits.py index bc474b7ada..df215d30af 100644 --- a/src/octop/infra/agents/settings/runtime_limits.py +++ b/src/octop/infra/agents/settings/runtime_limits.py @@ -8,10 +8,18 @@ → ``model.bind(...)`` (LLM body) ``max_input_length`` → ``CONFIGURABLE_MAX_INPUT_TOKENS`` → ``model.profile["max_input_tokens"]`` + +Start-time middleware (read in ``AgentManager._build_harness_config``) +----------------------------------------------------------------------- +``tool_call_limit_enabled`` / ``tool_call_run_limit`` + → LangChain ``ToolCallLimitMiddleware`` (per run; default 40) +``model_retry_*`` → the host model-retry middleware (default on, 2 retries) +``tool_retry_*`` → LangChain ``ToolRetryMiddleware`` (default on, 5 retries) """ from __future__ import annotations +from dataclasses import dataclass from typing import Any from octop_harness.middleware.model_settings import ( @@ -25,23 +33,158 @@ "temperature", "top_p", "max_tokens", + "tool_call_limit_enabled", + "tool_call_run_limit", + "model_retry_enabled", + "model_retry_max_retries", + "model_retry_initial_delay", + "model_retry_max_delay", + "tool_retry_enabled", + "tool_retry_max_retries", + "tool_retry_initial_delay", + "tool_retry_max_delay", ) _DEFAULT_MAX_INPUT_TOKENS = 128_000 +DEFAULT_TOOL_CALL_RUN_LIMIT = 40 +DEFAULT_MODEL_RETRY_MAX_RETRIES = 2 +DEFAULT_TOOL_RETRY_MAX_RETRIES = 5 +DEFAULT_RETRY_INITIAL_DELAY = 1.0 +DEFAULT_RETRY_MAX_DELAY = 60.0 -def _positive_int(value: Any) -> int | None: +def _int_at_least(value: Any, minimum: int) -> int | None: if isinstance(value, bool): return None if isinstance(value, int): - return value if value > 0 else None - if isinstance(value, float) and value.is_integer(): - iv = int(value) - return iv if iv > 0 else None - if isinstance(value, str) and value.strip().isdigit(): - iv = int(value.strip()) - return iv if iv > 0 else None - return None + parsed = value + elif isinstance(value, float) and value.is_integer(): + parsed = int(value) + elif isinstance(value, str) and value.strip().isdigit(): + parsed = int(value.strip()) + else: + return None + return parsed if parsed >= minimum else None + + +def _positive_int(value: Any) -> int | None: + return _int_at_least(value, 1) + + +def _non_negative_int(value: Any) -> int | None: + return _int_at_least(value, 0) + + +def _positive_float(value: Any) -> float | None: + if isinstance(value, bool) or not isinstance(value, int | float | str): + return None + if isinstance(value, str): + stripped = value.strip() + if not stripped: + return None + try: + parsed = float(stripped) + except ValueError: + return None + else: + parsed = float(value) + return parsed if parsed > 0 else None + + +def _stored_bool(value: Any, *, default: bool) -> bool: + if isinstance(value, bool): + return value + return default + + +@dataclass(frozen=True) +class RetryBudget: + """Resolved model or tool retry settings for one expert.""" + + enabled: bool + max_retries: int + initial_delay: float + max_delay: float + + +def _retry_budget( + cfg: dict[str, Any], + *, + enabled_key: str, + retries_key: str, + initial_key: str, + max_key: str, + default_retries: int, +) -> RetryBudget: + initial = _positive_float(cfg.get(initial_key)) + maximum = _positive_float(cfg.get(max_key)) + retries = _non_negative_int(cfg.get(retries_key)) + initial_delay = initial if initial is not None else DEFAULT_RETRY_INITIAL_DELAY + max_delay = maximum if maximum is not None else DEFAULT_RETRY_MAX_DELAY + if max_delay < initial_delay: + max_delay = initial_delay + return RetryBudget( + enabled=_stored_bool(cfg.get(enabled_key), default=True), + max_retries=retries if retries is not None else default_retries, + initial_delay=initial_delay, + max_delay=max_delay, + ) + + +def resolve_model_retry(cfg: dict[str, Any]) -> RetryBudget: + """Model-call retry budget. Missing keys keep the host default (on, 2).""" + return _retry_budget( + cfg, + enabled_key="model_retry_enabled", + retries_key="model_retry_max_retries", + initial_key="model_retry_initial_delay", + max_key="model_retry_max_delay", + default_retries=DEFAULT_MODEL_RETRY_MAX_RETRIES, + ) + + +def resolve_tool_retry(cfg: dict[str, Any]) -> RetryBudget: + """Tool-exception retry budget. Missing keys keep the default (on, 5).""" + return _retry_budget( + cfg, + enabled_key="tool_retry_enabled", + retries_key="tool_retry_max_retries", + initial_key="tool_retry_initial_delay", + max_key="tool_retry_max_delay", + default_retries=DEFAULT_TOOL_RETRY_MAX_RETRIES, + ) + + +def resolve_tool_call_run_limit(cfg: dict[str, Any]) -> int | None: + """Per-run tool-call cap. ``None`` means the limiter is not mounted. + + An absent switch stays on. An absent count uses :data:`DEFAULT_TOOL_CALL_RUN_LIMIT`. + """ + if not _stored_bool(cfg.get("tool_call_limit_enabled"), default=True): + return None + limit = _positive_int(cfg.get("tool_call_run_limit")) + return limit if limit is not None else DEFAULT_TOOL_CALL_RUN_LIMIT + + +def call_guard_middleware(cfg: dict[str, Any]) -> list[Any]: + """Tool-call cap and tool-exception retry mounted for one expert.""" + from langchain.agents.middleware import ToolCallLimitMiddleware, ToolRetryMiddleware + + middleware: list[Any] = [] + run_limit = resolve_tool_call_run_limit(cfg) + if run_limit is not None: + middleware.append(ToolCallLimitMiddleware(run_limit=run_limit, exit_behavior="end")) + tool_retry = resolve_tool_retry(cfg) + if tool_retry.enabled: + middleware.append( + ToolRetryMiddleware( + max_retries=tool_retry.max_retries, + initial_delay=tool_retry.initial_delay, + max_delay=tool_retry.max_delay, + on_failure="continue", + ) + ) + return middleware def agent_recursion_limit(cfg: dict[str, Any]) -> int | None: @@ -172,11 +315,21 @@ def apply_agent_runtime_to_stream_request( "AGENT_RUNTIME_CONFIG_KEYS", "CONFIGURABLE_MAX_INPUT_TOKENS", "CONFIGURABLE_MODEL_SETTINGS", + "DEFAULT_MODEL_RETRY_MAX_RETRIES", + "DEFAULT_RETRY_INITIAL_DELAY", + "DEFAULT_RETRY_MAX_DELAY", + "DEFAULT_TOOL_CALL_RUN_LIMIT", + "DEFAULT_TOOL_RETRY_MAX_RETRIES", + "RetryBudget", "agent_max_input_tokens", "agent_model_settings", "agent_recursion_limit", "agent_runtime_values", "apply_agent_runtime_to_stream_request", + "call_guard_middleware", "merge_agent_runtime_values", "resolve_context_max_tokens", + "resolve_model_retry", + "resolve_tool_call_run_limit", + "resolve_tool_retry", ] diff --git a/src/octop/infra/agents/teams/team_manager.py b/src/octop/infra/agents/teams/team_manager.py index 0845f15298..b27bddcab0 100644 --- a/src/octop/infra/agents/teams/team_manager.py +++ b/src/octop/infra/agents/teams/team_manager.py @@ -37,7 +37,12 @@ from octop.infra.agents.teams.service import _user_may_use_member, is_team_agent from octop.infra.agents.threads.artifact import artifact_path_allowed, extract_artifact_paths from octop.infra.db.repos.agents import AgentRow -from octop.infra.gateway.media.tool_media import tool_name_base +from octop.infra.gateway.media.tool_media import ( + media_events_from_tool_result, + note_live_tool_call, + select_turn_tool_results, + tool_name_base, +) from octop.infra.history.projection import ( live_message_inputs, message_inputs, @@ -398,6 +403,8 @@ async def stream_peer_to_room( saw_tool_call = False decision_payload: dict[str, Any] | None = None emitted_tool_error_ids: set[str] = set() + live_tool_call_ids: set[str] = set() + emitted_media_ids: set[str] = set() hold_room = self._defer_member_wall(speaker_id, member_tid, room_thread_id) host_id = self._room_agent_id(room_thread_id) if hold_room else "" room_open = True @@ -479,6 +486,7 @@ async def emit(chunk: dict[str, Any]) -> None: kind = str(chunk.get("type") or "") if kind == "tool_call_chunk": saw_tool_call = True + note_live_tool_call(chunk, live_tool_call_ids) elif kind == "tool_result": log_failed_tool_results( chunk, @@ -487,6 +495,13 @@ async def emit(chunk: dict[str, Any]) -> None: seen=emitted_tool_error_ids, live=saw_tool_call, ) + await self._push_turn_tool_media( + chunk, + speaker_id=speaker_id, + room_thread_id=room_thread_id, + live_tool_call_ids=live_tool_call_ids, + emitted_media_ids=emitted_media_ids, + ) if kind == "token": piece = str(chunk.get("content") or "") if piece: @@ -1435,6 +1450,65 @@ async def _push_session_channel(self, session: SessionRow, text: str) -> None: text, ) + async def _push_turn_tool_media( + self, + chunk: dict[str, Any], + *, + speaker_id: str, + room_thread_id: str, + live_tool_call_ids: set[str], + emitted_media_ids: set[str], + ) -> None: + """Send this turn's delivery-tool files to IM sessions on the room. + + The room WebSocket still receives the raw ``tool_result``. Bytes come + from the speaker workspace (the member, or the host during wrap-up), + and the subject is the session that bound the channel to this room. + """ + if self._gateway is None or not room_thread_id: + return + sessions = self._thread_registry.im_sessions_for_thread(room_thread_id) + if not sessions: + return + from octop.infra.gateway.process.agent_resolve import ( # noqa: PLC0415 + harness_workspace_for_agent, + ) + + workspace = harness_workspace_for_agent(self._agent_manager, speaker_id) + if workspace is None: + logger.warning( + "team media skipped; workspace unavailable speaker=%s room=%s", + speaker_id, + room_thread_id, + ) + return + selected = select_turn_tool_results(chunk, live_tool_call_ids, emitted_media_ids) + if selected is None: + return + async for event in media_events_from_tool_result(selected, workspace=workspace): + parts = event.content + if not parts: + continue + for session in sessions: + channel_id = session.channel_id + if not channel_id: + continue + try: + await self._gateway.push_content( + session.channel_type, + channel_id, + session.to_channel_subject(), + parts, + ) + except Exception: + logger.warning( + "failed to push team media to channel thread=%s speaker=%s channel=%s", + room_thread_id, + speaker_id, + channel_id, + exc_info=True, + ) + async def _push_room_to_channels( self, thread_id: str, @@ -2447,21 +2521,27 @@ async def enrich(agent_id: str, req: Any) -> Any: pair = take_prompt(agent_id, getattr(req, "thread_id", None)) if not pair: return req - from langchain_core.messages import HumanMessage, SystemMessage + from langchain_core.messages import HumanMessage + + from octop.infra.agents.middleware.team_dispatch import ( + CONFIG_KEY as BRIEFING_KEY, + ) question, dispatch = pair - messages: list[Any] = [] if dispatch and dispatch != question: - messages.append(SystemMessage(content=dispatch)) + # Briefing belongs on the leading system prompt. A SystemMessage in + # the turn input lands mid-thread on the next dispatch, and strict + # providers reject it. TeamDispatchBriefingMiddleware appends it. + configurable = dict(getattr(req, "configurable", None) or {}) + configurable[BRIEFING_KEY] = dispatch + req.configurable = configurable if question: - messages.append( + req.messages = [ HumanMessage( content=question, id=f"team-peer:{getattr(req, 'thread_id', None)}:{new_ulid()}:human", ) - ) - if messages: - req.messages = messages + ] return req harness_team._enrich_request = enrich diff --git a/src/octop/infra/agents/workspace/execute_env.py b/src/octop/infra/agents/workspace/execute_env.py index fcb8d07bb1..298867e2d7 100644 --- a/src/octop/infra/agents/workspace/execute_env.py +++ b/src/octop/infra/agents/workspace/execute_env.py @@ -71,6 +71,7 @@ def agent_execute_env_defaults( agent_id: str, workspace_dir: Path, cfg: dict[str, Any] | None = None, + user_id: int | None = None, ) -> dict[str, str]: auth_dir = agent_auth_dir(workspace_dir, cfg) auth_dir.mkdir(parents=True, exist_ok=True) @@ -82,12 +83,15 @@ def agent_execute_env_defaults( auth_host=auth_dir, skills_host=skills_dir, ) - return { + env = { "OCTOP_AGENT_ID": agent_id, "OCTOP_AUTH_DIR": auth_env, "OCTOP_HOME": str(paths.root), "OCTOP_SKILLS_DIR": skills_env, } + if user_id is not None: + env["OCTOP_TCCLI_PROFILE"] = f"octop-{int(user_id)}" + return env def inject_agent_execute_env( @@ -128,6 +132,7 @@ def inject_agent_execute_env( agent_id=row.agent_id, workspace_dir=workspace_dir, cfg=cfg, + user_id=row.user_id, ) ) @@ -135,4 +140,13 @@ def inject_agent_execute_env( out["env"] = extra if kind == "local_shell": out.setdefault("inherit_env", True) + if kind == "docker" and "OCTOP_TCCLI_PROFILE" in extra: + # DockerSandbox reads ``environment``, not the local-shell ``env`` key. + # Set the profile after any caller-supplied values so a spec cannot + # point the skill at another user's tccli file. + environment = { + str(key): str(value) for key, value in dict(backend.get("environment") or {}).items() + } + environment["OCTOP_TCCLI_PROFILE"] = extra["OCTOP_TCCLI_PROFILE"] + out["environment"] = environment return out diff --git a/src/octop/infra/app_config.py b/src/octop/infra/app_config.py new file mode 100644 index 0000000000..6ffe720957 --- /dev/null +++ b/src/octop/infra/app_config.py @@ -0,0 +1,416 @@ +"""Read and write the process ``config.json`` without dropping unknown keys.""" + +from __future__ import annotations + +import json +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from octop.config import ( + OctopConfig, + _parse_database_url, + _parse_tls_section, + validate_persisted_config, +) +from octop.infra.desktop import desktop_shell_enabled +from octop.infra.errors import ErrorCode, OctopError, corrupt_config_error +from octop.infra.setup.self_update import green_packages_dir +from octop.infra.setup.service import detect_service_mode +from octop.infra.utils.json_file import JsonFileCorruptError, read_json_object, write_json_atomic + +_MAX_BYTES = 1_048_576 + +# Env vars that override the matching config.json field at process start. +_ENV_OVERRIDE_KEYS = ( + "OCTOP_BIND_HOST", + "OCTOP_PORT", + "OCTOP_LOG_LEVEL", + "OCTOP_ACCESS_TOKEN_TTL", + "OCTOP_LOGIN_MAX_ATTEMPTS", + "OCTOP_LOGIN_LOCKOUT_SECONDS", + "OCTOP_DEFAULT_TIMEZONE", + "OCTOP_CRON_TIMEZONE", + "OCTOP_CORS_ORIGINS", + "OCTOP_ENABLE_DASHBOARD", + "OCTOP_ENABLE_API_DOCS", + "OCTOP_REQUIRE_SETUP_PASSWORD", + "OCTOP_MAX_UPLOAD_MB", + "OCTOP_DATABASE_URL", + "OCTOP_DATABASE_DRIVER", + "OCTOP_DATABASE_SQLITE_PATH", + "OCTOP_DATABASE_HOST", + "OCTOP_DATABASE_PORT", + "OCTOP_DATABASE_NAME", + "OCTOP_DATABASE_USER", + "OCTOP_DATABASE_PASSWORD", +) + +_LOG_LEVELS = frozenset({"debug", "info", "warning", "error"}) + + +@dataclass(frozen=True) +class AppConfigSnapshot: + path: str + raw: str + valid: bool + error: str | None + config: dict[str, Any] | None + has_database_password: bool + overrides: list[str] + service_mode: str | None + restart_available: bool + + +def active_config_env_overrides() -> list[str]: + """Env vars that are set and therefore win over ``config.json`` after restart.""" + found: list[str] = [] + for key in _ENV_OVERRIDE_KEYS: + raw = os.environ.get(key) + if raw is not None and raw.strip() != "": + found.append(key) + return found + + +def restart_availability() -> tuple[str | None, bool]: + """Service mode and whether ``POST /api/update/restart`` can restart this process.""" + mode = detect_service_mode() + if desktop_shell_enabled(): + return mode, False + desktop_flag = (os.environ.get("OCTOP_DESKTOP") or "").strip().lower() + if green_packages_dir() is not None or desktop_flag in {"1", "true", "yes", "on"}: + return "desktop", True + return mode, mode is not None + + +def read_app_config(path: Path) -> AppConfigSnapshot: + """Load the on-disk file. The structured view omits ``database.password``.""" + service_mode, can_restart = restart_availability() + overrides = active_config_env_overrides() + if not path.is_file(): + config, has_password = _display_config({}) + raw = json.dumps(config, indent=2) + "\n" + return AppConfigSnapshot( + path=str(path), + raw=raw, + valid=True, + error=None, + config=config, + has_database_password=has_password, + overrides=overrides, + service_mode=service_mode, + restart_available=can_restart, + ) + try: + text = path.read_text(encoding="utf-8") + except UnicodeDecodeError: + return _invalid_snapshot( + path, + "", + "not valid UTF-8", + overrides, + service_mode, + can_restart, + ) + try: + parsed = json.loads(text) + except json.JSONDecodeError as exc: + return _invalid_snapshot( + path, + text, + f"line {exc.lineno}, column {exc.colno}", + overrides, + service_mode, + can_restart, + ) + if not isinstance(parsed, dict): + return _invalid_snapshot( + path, + text, + "expected a JSON object", + overrides, + service_mode, + can_restart, + ) + try: + config, has_password = _display_config(parsed) + except (TypeError, ValueError) as exc: + return _invalid_snapshot( + path, + text, + _safe_detail(exc), + overrides, + service_mode, + can_restart, + ) + return AppConfigSnapshot( + path=str(path), + raw=text if text.endswith("\n") else text + "\n", + valid=True, + error=None, + config=config, + has_database_password=has_password, + overrides=overrides, + service_mode=service_mode, + restart_available=can_restart, + ) + + +def save_structured_config(path: Path, incoming: dict[str, Any]) -> None: + """Merge the form payload into the existing file and keep unknown keys.""" + try: + existing = read_json_object(path) + except JsonFileCorruptError as exc: + raise corrupt_config_error(exc.path, exc.detail) from exc + if existing is None: + existing = {} + try: + merged = _apply_structured(existing, incoming) + except (KeyError, TypeError, ValueError) as exc: + raise _invalid(_safe_detail(exc)) from exc + _validate(merged) + write_json_atomic(path, merged) + + +def save_raw_config(path: Path, text: str) -> None: + """Replace the file with validated JSON. Formatting is normalized on write.""" + encoded = text.encode("utf-8") + if len(encoded) > _MAX_BYTES: + raise _invalid("file is larger than 1 MiB") + try: + parsed = json.loads(text) + except json.JSONDecodeError as exc: + raise _invalid(f"line {exc.lineno}, column {exc.colno}") from exc + if not isinstance(parsed, dict): + raise _invalid("config must be a JSON object") + _validate(parsed) + write_json_atomic(path, parsed) + + +def _validate(data: dict[str, Any]) -> None: + try: + validate_persisted_config(data) + except ValueError as exc: + raise _invalid(_safe_detail(exc)) from exc + + +def _invalid(detail: str) -> OctopError: + return OctopError(ErrorCode.APP_CONFIG_INVALID, detail, details={"detail": detail}) + + +def _safe_detail(exc: BaseException) -> str: + text = str(exc).strip() or "invalid configuration" + if len(text) > 240: + return "invalid configuration" + return text + + +def _invalid_snapshot( + path: Path, + raw: str, + detail: str, + overrides: list[str], + service_mode: str | None, + can_restart: bool, +) -> AppConfigSnapshot: + return AppConfigSnapshot( + path=str(path), + raw=raw, + valid=False, + error=detail, + config=None, + has_database_password=False, + overrides=overrides, + service_mode=service_mode, + restart_available=can_restart, + ) + + +def _display_config(data: dict[str, Any]) -> tuple[dict[str, Any], bool]: + base = OctopConfig() + if "tls" in data and data["tls"] is not None and not isinstance(data["tls"], dict): + raise ValueError("tls must be an object") + if ( + "database" in data + and data["database"] is not None + and not isinstance(data["database"], dict) + ): + raise ValueError("database must be an object") + db_in = data["database"] if isinstance(data.get("database"), dict) else {} + has_password = bool(db_in.get("password")) + driver = str(db_in.get("driver", base.database.driver) or "sqlite").strip().lower() + if driver not in {"sqlite", "postgresql"}: + driver = base.database.driver + url = db_in.get("url") + url_text = str(url).strip() if isinstance(url, str) and str(url).strip() else None + tls = _parse_tls_section(data.get("tls")) + level = str(data.get("log_level", base.log_level) or base.log_level).strip().lower() + if level not in _LOG_LEVELS: + level = base.log_level + origins = data.get("cors_origins", base.cors_origins) + if not isinstance(origins, list): + raise ValueError("cors_origins must be a list of strings") + config = { + "bind_host": str(data.get("bind_host", base.bind_host) or base.bind_host), + "port": _coerce_display_int(data.get("port", base.port), base.port), + "log_level": level, + "access_token_ttl_seconds": _coerce_display_int( + data.get("access_token_ttl_seconds", base.access_token_ttl_seconds), + base.access_token_ttl_seconds, + ), + "login_max_attempts": _coerce_display_int( + data.get("login_max_attempts", base.login_max_attempts), + base.login_max_attempts, + ), + "login_lockout_seconds": _coerce_display_int( + data.get("login_lockout_seconds", base.login_lockout_seconds), + base.login_lockout_seconds, + ), + "cors_origins": [str(item) for item in origins if str(item).strip()], + "default_timezone": str( + data.get("default_timezone", base.default_timezone) or base.default_timezone + ), + "enable_dashboard": _coerce_display_bool( + data.get("enable_dashboard"), base.enable_dashboard + ), + "enable_api_docs": _coerce_display_bool(data.get("enable_api_docs"), base.enable_api_docs), + "require_setup_password": _coerce_display_bool( + data.get("require_setup_password"), + base.require_setup_password, + ), + "max_upload_mb": _coerce_display_int( + data.get("max_upload_mb", base.max_upload_mb), base.max_upload_mb + ), + "database": { + "driver": driver, + "sqlite_path": str( + db_in.get("sqlite_path", base.database.sqlite_path) or base.database.sqlite_path + ), + "host": str(db_in.get("host", base.database.host) or base.database.host), + "port": _coerce_display_int(db_in.get("port", base.database.port), base.database.port), + "database": str( + db_in.get("database", base.database.database) or base.database.database + ), + "user": str(db_in.get("user", base.database.user) or base.database.user), + "password": None, + "url": url_text, + }, + "tls": { + "enabled": tls.enabled, + "mode": tls.mode, + "domains": list(tls.domains), + "cert_file": tls.cert_file, + "key_file": tls.key_file, + "issued_at": tls.issued_at, + "expires_at": tls.expires_at, + "acme_staging": tls.acme_staging, + "http_port": tls.http_port, + }, + } + return config, has_password + + +def _coerce_display_int(value: object, default: int) -> int: + if value is None: + return default + if isinstance(value, bool): + raise ValueError("expected an integer") + if isinstance(value, int): + return value + if isinstance(value, str) and value.strip(): + return int(value.strip()) + raise ValueError("expected an integer") + + +def _coerce_display_bool(value: object, default: bool) -> bool: + if value is None: + return default + if isinstance(value, bool): + return value + raise ValueError("expected a boolean") + + +def _apply_structured(existing: dict[str, Any], incoming: dict[str, Any]) -> dict[str, Any]: + out = dict(existing) + out["bind_host"] = str(incoming.get("bind_host", "")).strip() + out["port"] = int(incoming["port"]) + out["log_level"] = str(incoming.get("log_level", "info")).strip().lower() + out["access_token_ttl_seconds"] = int(incoming["access_token_ttl_seconds"]) + out["login_max_attempts"] = int(incoming["login_max_attempts"]) + out["login_lockout_seconds"] = int(incoming["login_lockout_seconds"]) + origins = incoming.get("cors_origins") or [] + out["cors_origins"] = [str(item).strip() for item in origins if str(item).strip()] + out["default_timezone"] = str(incoming.get("default_timezone", "")).strip() + out["enable_dashboard"] = bool(incoming.get("enable_dashboard")) + out["enable_api_docs"] = bool(incoming.get("enable_api_docs")) + out["require_setup_password"] = bool(incoming.get("require_setup_password")) + out["max_upload_mb"] = int(incoming["max_upload_mb"]) + out["database"] = _merge_database(existing.get("database"), incoming.get("database") or {}) + out["tls"] = _merge_tls(existing.get("tls"), incoming.get("tls") or {}) + return out + + +def _apply_password(section: dict[str, Any], password: object) -> None: + if password == "": + section.pop("password", None) + elif isinstance(password, str) and password: + section["password"] = password + + +def _merge_database(existing: object, incoming: dict[str, Any]) -> dict[str, Any]: + section = dict(existing) if isinstance(existing, dict) else {} + driver = str(incoming.get("driver", "sqlite")).strip().lower() + sqlite_path = str( + incoming.get("sqlite_path") or section.get("sqlite_path") or "octop.db" + ).strip() + section["sqlite_path"] = sqlite_path or "octop.db" + if driver == "sqlite": + section["driver"] = "sqlite" + section.pop("url", None) + _apply_password(section, incoming.get("password")) + return section + + url_raw = incoming.get("url") + url = str(url_raw).strip() if isinstance(url_raw, str) else "" + section["driver"] = "postgresql" + if url: + parsed = _parse_database_url(url) + section["host"] = parsed["host"] + section["port"] = int(parsed["port"]) + section["database"] = parsed["database"] + if parsed.get("user"): + section["user"] = parsed["user"] + section["url"] = url + if parsed.get("password"): + section["password"] = parsed["password"] + else: + section.pop("password", None) + return section + + section.pop("url", None) + section["host"] = str(incoming.get("host", "")).strip() + section["port"] = int(incoming.get("port", 5432)) + section["database"] = str(incoming.get("database", "")).strip() + section["user"] = str(incoming.get("user", "")).strip() + _apply_password(section, incoming.get("password")) + return section + + +def _merge_tls(existing: object, incoming: dict[str, Any]) -> dict[str, Any]: + section = dict(existing) if isinstance(existing, dict) else {} + domains = incoming.get("domains") or [] + section.update( + { + "enabled": bool(incoming.get("enabled")), + "mode": str(incoming.get("mode", "")).strip(), + "domains": [str(item).strip() for item in domains if str(item).strip()], + "cert_file": str(incoming.get("cert_file", "")).strip(), + "key_file": str(incoming.get("key_file", "")).strip(), + "issued_at": str(incoming.get("issued_at", "")).strip(), + "expires_at": str(incoming.get("expires_at", "")).strip(), + "acme_staging": bool(incoming.get("acme_staging")), + "http_port": int(incoming.get("http_port", 80)), + } + ) + return section diff --git a/src/octop/infra/auth/ldap/service.py b/src/octop/infra/auth/ldap/service.py index 2c29d3a85c..84e53b53cc 100644 --- a/src/octop/infra/auth/ldap/service.py +++ b/src/octop/infra/auth/ldap/service.py @@ -215,6 +215,7 @@ def _finish_login(self, account: Any, identity: LdapIdentity) -> User: # privileges on an existing account. refreshed = self._services.user_repo.get(account.id) or account user = self._cached_or_row(refreshed) + self._services.user_repo.touch_last_login(user.id) self._services.audit_repo.write( actor=user.username, action="auth.ldap_login", target=user.username ) @@ -277,6 +278,7 @@ def _provision(self, row: SsoProviderRow, config: LdapConfig, identity: LdapIden concurrent = self._services.user_repo.get_by_sso(row.id, identity.subject) if concurrent is None: raise + self._services.user_repo.touch_last_login(concurrent.id) return self._cached_or_row(concurrent) if policies: self._services.user_policy_repo.merge(uid, dict(policies)) @@ -291,6 +293,7 @@ def _provision(self, row: SsoProviderRow, config: LdapConfig, identity: LdapIden permissions=list(permissions), ) self._user_manager.register_cached_user(user) + self._services.user_repo.touch_last_login(uid) self._services.audit_repo.write( actor=username, action="user.ldap_create", diff --git a/src/octop/infra/auth/mfa/service.py b/src/octop/infra/auth/mfa/service.py index 06cd622869..8bac264ded 100644 --- a/src/octop/infra/auth/mfa/service.py +++ b/src/octop/infra/auth/mfa/service.py @@ -327,6 +327,7 @@ def passkey_login_verify( user = user_manager.get_by_id(user_id) if user is None: raise OctopError(ErrorCode.USER_DISABLED, "user not active") + self._s.user_repo.touch_last_login(user.id) self._s.audit_repo.write( actor=user.username, action="auth.passkey_login", target=str(user_id) ) diff --git a/src/octop/infra/auth/sso/service.py b/src/octop/infra/auth/sso/service.py index 8ef6f81cf8..8f1dcfdac8 100644 --- a/src/octop/infra/auth/sso/service.py +++ b/src/octop/infra/auth/sso/service.py @@ -345,6 +345,7 @@ async def exchange_login_code(self, code: str) -> User: user = self._user_manager.get_by_id(consumed["user_id"]) if user is None: raise ValueError("SSO user is unavailable") + self._services.user_repo.touch_last_login(user.id) self._services.audit_repo.write( actor=user.username, action="auth.oidc_login", target=user.username ) diff --git a/src/octop/infra/backend/docker_spec.py b/src/octop/infra/backend/docker_spec.py index 577ddba1b3..21941bc508 100644 --- a/src/octop/infra/backend/docker_spec.py +++ b/src/octop/infra/backend/docker_spec.py @@ -2,11 +2,16 @@ from __future__ import annotations +import logging from pathlib import Path -from typing import Any +from typing import Any, cast + +logger = logging.getLogger(__name__) DEFAULT_SANDBOX_PREFIX = "octop_sandbox" +# ``volumes`` is intentionally absent. Host bind mounts run as the Docker +# daemon and are not covered by the local-backend root policy. _DOCKER_PASSTHROUGH_KEYS = ( "allow_network", "memory", @@ -16,7 +21,6 @@ "max_output_bytes", "auto_remove", "workspace_path", - "volumes", "environment", "environment_file", "agent_id", @@ -36,6 +40,49 @@ def apply_docker_cfg_keys(spec: dict[str, Any], cfg: dict[str, Any]) -> None: spec[key] = cfg[key] +def omit_docker_volumes(spec: Any) -> Any: + """Drop ``volumes`` from docker specs, including composite children. + + A returned docker or composite dict is a shallow copy when a mount was + removed. Other specs are returned unchanged. + """ + if not isinstance(spec, dict): + return spec + kind = str(spec.get("type") or "").strip().lower() + if kind == "docker": + if "volumes" not in spec: + return spec + logger.warning("ignoring docker backend volumes; host mounts are not allowed") + out = dict(spec) + out.pop("volumes", None) + return out + if kind != "composite": + return spec + out = dict(spec) + changed = False + default = out.get("default") + if isinstance(default, dict): + stripped = omit_docker_volumes(default) + if stripped is not default: + out["default"] = stripped + changed = True + routes = out.get("routes") + if isinstance(routes, dict): + new_routes: dict[str, Any] = {} + routes_changed = False + for prefix, route in routes.items(): + if isinstance(route, dict): + stripped = omit_docker_volumes(route) + new_routes[str(prefix)] = stripped + routes_changed = routes_changed or stripped is not route + else: + new_routes[str(prefix)] = route + if routes_changed: + out["routes"] = new_routes + changed = True + return out if changed else spec + + def docker_spec_previewable(spec: Any) -> bool: """Whether Admin storage UI may browse this backend's files (``/admin/backend``). @@ -67,7 +114,9 @@ def enrich_docker_backend_spec( """ if str(spec.get("type") or "").lower() != "docker": return spec - out = dict(spec) + out = cast(dict[str, Any], omit_docker_volumes(spec)) + if out is spec: + out = dict(spec) out.setdefault("sandbox_prefix", DEFAULT_SANDBOX_PREFIX) scope = str(out.get("sandbox_scope") or "agent").strip().lower() or "agent" out.setdefault("sandbox_scope", scope) diff --git a/src/octop/infra/backend/resolve_backend.py b/src/octop/infra/backend/resolve_backend.py index 30450abcf1..2620a6ac18 100644 --- a/src/octop/infra/backend/resolve_backend.py +++ b/src/octop/infra/backend/resolve_backend.py @@ -11,6 +11,7 @@ from pathlib import Path from typing import Any +from octop.infra.backend.docker_spec import omit_docker_volumes from octop.infra.backend.opensandbox_mkdir import install_opensandbox_fs from octop.infra.backend.tos_backend import tos_backend_from_spec @@ -46,7 +47,7 @@ def resolve_octop_backend( from octop_harness.backends import resolve_backend return resolve_backend( - materialize_octop_specs(spec), + materialize_octop_specs(omit_docker_volumes(spec)), workspace_dir=workspace_dir, system_files_path=system_files_path, ) diff --git a/src/octop/infra/cloud/tencent.py b/src/octop/infra/cloud/tencent.py index 66d4de6e62..d937c0e182 100644 --- a/src/octop/infra/cloud/tencent.py +++ b/src/octop/infra/cloud/tencent.py @@ -332,7 +332,7 @@ def load(self, user_id: int) -> CloudCredential | None: def save(self, user_id: int, credential: CloudCredential) -> None: if self._secrets is not None: - self._secrets.put(_cred_key(user_id), _dump_credential(credential)) + self._secrets.put(_cred_key(user_id), self._seal(credential)) self._path(user_id).unlink(missing_ok=True) return path = self._path(user_id) @@ -411,7 +411,7 @@ def adopt_local(self) -> int: credential = self._credential_or_none(raw) if credential is None: continue - if secrets_repo.put_if_absent(_cred_key(user_id), _dump_credential(credential)): + if secrets_repo.put_if_absent(_cred_key(user_id), self._seal(credential)): imported += 1 if secrets_repo.get(_cred_key(user_id)) is not None: path.unlink(missing_ok=True) @@ -429,9 +429,23 @@ def _load_shared(self, user_id: int) -> CloudCredential | None: raw = secrets_repo.get(_cred_key(user_id)) if raw is None: return None + return self._credential_from_stored(raw) + + def _seal(self, credential: CloudCredential) -> bytes: + return self._fernet().encrypt(_dump_credential(credential)) + + def _credential_from_stored(self, raw: bytes) -> CloudCredential | None: + """Read a sealed blob, or a legacy plaintext JSON row.""" + try: + text = self._fernet().decrypt(raw).decode() + except Exception: # noqa: BLE001 - fall through to pre-seal rows + try: + text = raw.decode() + except UnicodeDecodeError: + return None try: - payload = json.loads(raw) - except (UnicodeDecodeError, json.JSONDecodeError): + payload = json.loads(text) + except json.JSONDecodeError: return None return self._credential_or_none(payload) @@ -1064,26 +1078,33 @@ def instance_action( return {"instance_id": instance_id, "action": action, "accepted": True} -_TCCLI_PROFILE_POINTER = ".octop-profile" -_TCCLI_ISOLATION_ENV = "OCTOP_TCCLI_PROFILE_ISOLATION" - - def tccli_profile_for_user(uid: int | None) -> str: - """tccli profile the mirror writes to. + """tccli profile for a known Octop account. - Multi-user isolation (``octop-``) stops two Octop accounts from - overwriting each other's machine-wide ``~/.tccli`` login. It stays opt-in - (``OCTOP_TCCLI_PROFILE_ISOLATION=1``) because plain ``tccli`` without - ``--profile`` always reads ``default``. + A user id always maps to ``octop-``. There is no switch back to the + machine-wide ``default`` profile. A missing user id (no account) stays on + ``default``. """ if uid is None: return "default" - enabled = os.environ.get(_TCCLI_ISOLATION_ENV, "").strip().lower() - if enabled not in ("1", "true", "yes", "on"): - return "default" return f"octop-{int(uid)}" +def tccli_profile_owner(profile: str | None) -> int | None: + """User id encoded in an ``octop-`` profile name.""" + if not profile or not profile.startswith("octop-"): + return None + raw = profile.removeprefix("octop-") + if not raw.isdigit(): + return None + return int(raw) + + +def tccli_credential_path_for_user(uid: int) -> Path: + """Host path of this user's tccli credential file.""" + return Path.home() / ".tccli" / f"{tccli_profile_for_user(uid)}.credential" + + def mirror_to_tccli( credential: CloudCredential, *, @@ -1120,12 +1141,10 @@ def mirror_to_tccli( } path = tccli_dir / f"{name}.credential" tmp = path.with_name(f".{path.name}.tmp") - pointer = tccli_dir / _TCCLI_PROFILE_POINTER try: _write_private(tmp, json.dumps(payload, indent=4).encode()) os.replace(tmp, path) os.chmod(path, 0o600) - _write_private(pointer, (name + "\n").encode()) except OSError: tmp.unlink(missing_ok=True) logger.warning("failed to mirror cloud credential into %s", path, exc_info=True) diff --git a/src/octop/infra/connectors/auth_status.py b/src/octop/infra/connectors/auth_status.py new file mode 100644 index 0000000000..fa5e49a27f --- /dev/null +++ b/src/octop/infra/connectors/auth_status.py @@ -0,0 +1,49 @@ +"""Whether a custom MCP server using OAuth can be used now or needs the user to authorize again. + +``ok`` a token is stored and either still valid or refreshable; +``required`` OAuth is needed and no token is stored; +``expired`` a stored token stopped working (expired without a refresh + token, refresh failed, or the MCP server rejected it). + +Servers that do not use OAuth have no status (``None``). +""" + +from __future__ import annotations + +import time +from typing import Any, Literal + +from octop.infra.connectors.custom_mcp import ( + oauth_configured, + oauth_required, + oauth_tokens_from_spec, +) + +AuthStatus = Literal["ok", "required", "expired"] + +AUTH_OK: AuthStatus = "ok" +AUTH_REQUIRED: AuthStatus = "required" +AUTH_EXPIRED: AuthStatus = "expired" +NEEDS_AUTH: frozenset[str] = frozenset({AUTH_REQUIRED, AUTH_EXPIRED}) + + +def _expired_without_refresh(tokens: dict[str, Any], now: int) -> bool: + raw = tokens.get("expires_at") + try: + expires_at = int(raw) if raw is not None else None + except (TypeError, ValueError): + expires_at = None + refresh = str(tokens.get("refresh_token") or "").strip() + return expires_at is not None and expires_at <= now and not refresh + + +def custom_auth_status(spec: dict[str, Any], *, now: int | None = None) -> AuthStatus | None: + """Status of a custom MCP server; ``None`` when it does not use OAuth.""" + if oauth_required(spec): + return AUTH_EXPIRED if oauth_tokens_from_spec(spec).get("expired") else AUTH_REQUIRED + if not oauth_configured(spec): + return None + tokens = oauth_tokens_from_spec(spec) + if _expired_without_refresh(tokens, int(time.time()) if now is None else now): + return AUTH_EXPIRED + return AUTH_OK diff --git a/src/octop/infra/connectors/custom_mcp.py b/src/octop/infra/connectors/custom_mcp.py index ab84976b95..0c1f5177ba 100644 --- a/src/octop/infra/connectors/custom_mcp.py +++ b/src/octop/infra/connectors/custom_mcp.py @@ -252,7 +252,7 @@ def normalize_server_spec(name: str, raw: Any) -> dict[str, Any]: if str(oauth.get("access_token") or "").strip(): spec[_OAUTH_KEY] = dict(oauth) elif oauth.get("required") is True: - spec[_OAUTH_KEY] = {"required": True} + spec[_OAUTH_KEY] = _required_marker(oauth) return spec @@ -312,10 +312,16 @@ def oauth_required(spec: dict[str, Any]) -> bool: return oauth.get("required") is True +def _required_marker(oauth: dict[str, Any]) -> dict[str, Any]: + """The stored "authorize first" marker, keeping whether a token had lapsed.""" + return {"required": True, **({"expired": True} if oauth.get("expired") is True else {})} + + def mark_oauth_reauth_required(spec: dict[str, Any]) -> dict[str, Any]: """Drop stale OAuth tokens and flag the dashboard to prompt re-authorization.""" out = {k: v for k, v in spec.items() if k != _OAUTH_KEY} - out[_OAUTH_KEY] = {"required": True} + # ``expired`` tells "authorize again" apart from "never authorized". + out[_OAUTH_KEY] = {"required": True, "expired": True} return out @@ -424,7 +430,7 @@ def merge_preserved_oauth( if isinstance(old_oauth, dict) and str(old_oauth.get("access_token") or "").strip(): spec[_OAUTH_KEY] = {"required": True} if client_changed else dict(old_oauth) elif isinstance(old_oauth, dict) and old_oauth.get("required") is True: - spec[_OAUTH_KEY] = {"required": True} + spec[_OAUTH_KEY] = _required_marker(old_oauth) out[name] = spec return out diff --git a/src/octop/infra/connectors/service.py b/src/octop/infra/connectors/service.py index 67d1d542f6..de2dba4cde 100644 --- a/src/octop/infra/connectors/service.py +++ b/src/octop/infra/connectors/service.py @@ -11,6 +11,12 @@ from octop.config import OctopConfig from octop.infra.connectors import qcc +from octop.infra.connectors.auth_status import ( + AUTH_EXPIRED, + AUTH_OK, + NEEDS_AUTH, + custom_auth_status, +) from octop.infra.connectors.builder import build_http_mcp_spec, mcp_server_name, new_internal_token from octop.infra.connectors.catalog import get_catalog_entry, uses_internal_http_mcp from octop.infra.connectors.crypto import decrypt_credentials, encrypt_credentials @@ -21,12 +27,14 @@ enabled_harness_configs, expand_custom_instances, extract_servers, + harness_spec_for_server, is_custom_mcp_kind, mark_oauth_reauth_required, merge_preserved_oauth, oauth_configured, oauth_tokens_from_spec, redact_servers_for_api, + server_display_name, server_enabled, set_oauth_required_in_spec, shared_mcp_server_name, @@ -468,6 +476,7 @@ def list_instances_for_api(self, user_id: int) -> list[dict[str, Any]]: out: list[dict[str, Any]] = [] visible = self._repo.list_visible(user_id) custom_row = self._repo.get_by_user_kind(user_id, CUSTOM_MCP_KIND) + custom_specs: dict[str, dict[str, Any]] = {} for inst in visible: if is_custom_mcp_kind(inst.kind): continue @@ -489,30 +498,88 @@ def list_instances_for_api(self, user_id: int) -> list[dict[str, Any]]: } ) if custom_row is not None and custom_row.has_credentials: - out.extend( - expand_custom_instances( - parent=custom_row, - servers=extract_servers(self.decrypt(custom_row.instance_id)), - ) - ) + own = extract_servers(self.decrypt(custom_row.instance_id)) + custom_specs.update(own) + out.extend(expand_custom_instances(parent=custom_row, servers=own)) # Per-server ``shared`` lives in the encrypted bag, not on the parent row. for parent in self._repo.list_by_kind(CUSTOM_MCP_KIND): if parent.user_id == user_id or not parent.has_credentials: continue - out.extend( - expand_custom_instances( - parent=parent, - servers=extract_servers(self.decrypt(parent.instance_id)), - shared_view=True, - ) + servers = extract_servers(self.decrypt(parent.instance_id)) + custom_specs.update( + { + shared_mcp_server_name(parent.instance_id, name): spec + for name, spec in servers.items() + if isinstance(spec, dict) + } ) + out.extend(expand_custom_instances(parent=parent, servers=servers, shared_view=True)) for item in out: + spec = custom_specs.get(str(item.get("mcp_server_name") or "")) + if is_custom_mcp_kind(str(item.get("kind") or "")) and isinstance(spec, dict): + status = custom_auth_status(spec) + if status: + item["auth_status"] = status item.setdefault("owner_user_id", user_id) item.setdefault("shared", False) # list_visible already applied the per-instance group layer. item["usable"] = True return out + async def auth_blocked_servers( + self, user_id: int, server_names: list[str], *, probe: bool = True + ) -> list[dict[str, Any]]: + """Which custom MCP servers in *server_names* need the user to authorize. + + Called after MCP tools could not be loaded. A stored token whose status + still looks fine is probed once: a 401/403 from the MCP server means it + was revoked, and the server is marked so the dashboard prompts too. + """ + wanted = set(server_names) + blocked: list[dict[str, Any]] = [] + own = self.get_custom_servers(user_id) + candidates: list[tuple[str, int, str, dict[str, Any]]] = [ + (name, user_id, name, spec) + for name, spec in own.items() + if name in wanted and isinstance(spec, dict) + ] + for parent in self._repo.list_by_kind(CUSTOM_MCP_KIND): + if parent.user_id == user_id or not parent.has_credentials: + continue + for name, spec in extract_servers(self.decrypt(parent.instance_id)).items(): + shown = shared_mcp_server_name(parent.instance_id, name) + if shown in wanted and isinstance(spec, dict) and spec.get("shared") is True: + candidates.append((shown, parent.user_id, name, spec)) + for shown, owner_id, name, spec in candidates: + status = custom_auth_status(spec) + if ( + status == AUTH_OK + and probe + and str(spec.get("transport") or "") in ("streamable_http", "http") + ): + harness = harness_spec_for_server(spec) + if await _token_rejected( + str(harness.get("url") or ""), + dict(harness.get("headers") or {}), + kind=CUSTOM_MCP_KIND, + ): + servers = dict(self.get_custom_servers(owner_id)) + if isinstance(servers.get(name), dict): + servers[name] = mark_oauth_reauth_required(dict(servers[name])) + self._save_custom_servers(owner_id, servers) + status = AUTH_EXPIRED + if status in NEEDS_AUTH: + blocked.append( + { + "server": shown, + "display_name": server_display_name(name, spec), + "auth_status": status, + "can_authorize": owner_id == user_id, + "target": {"type": "custom_mcp", "server_name": name}, + } + ) + return blocked + def shared_custom_harness_aliases(self, owner_user_id: int) -> dict[str, str]: """Map a viewer's collision-safe name to the owner's harness server name. @@ -928,3 +995,19 @@ async def complete_feishu_user_auth_for_instance( creds["cli_config_key"] = cli_key self.encrypt_and_store(instance_id=instance_id, payload=creds) return {**result, "cli_config_key": cli_key, "default_as": "user"} + + +async def _token_rejected(url: str, headers: dict[str, str], *, kind: str) -> bool: + """True only when the MCP server answers 401/403; anything else is not an auth problem.""" + if not url: + return False + from octop.infra.connectors.probe import probe_streamable_http_mcp + + try: + result = await asyncio.wait_for( + probe_streamable_http_mcp(url, headers, kind=kind), timeout=30 + ) + except Exception: + logger.warning("auth probe failed for %s", kind, exc_info=True) + return False + return result.get("error_type") == "auth" diff --git a/src/octop/infra/cron/delivery.py b/src/octop/infra/cron/delivery.py index a06fe6630f..580eb81b03 100644 --- a/src/octop/infra/cron/delivery.py +++ b/src/octop/infra/cron/delivery.py @@ -9,7 +9,9 @@ from langchain_core.messages import AIMessage, HumanMessage from octop.i18n import tr +from octop.infra.cron.session_target import pinned_thread_id from octop.infra.cron.task_type import CronTaskType, normalize_cron_task_type +from octop.infra.db.repos.threads import ThreadRow from octop.infra.gateway.process import build_harness_request from octop.infra.gateway.process.message_keys import COMPOSER_CTX_KEY, build_composer_context from octop.infra.gateway.process.usage_record import UsageTracker, record_turn_usage @@ -67,11 +69,16 @@ def replace_repos(self, repos: RepoBundle) -> None: async def deliver(self, command: CronDeliveryCommand) -> None: """Run one delivery under the target channel session lock.""" + session_key, pinned = self._target_session(command) async def _locked() -> None: if command.fresh_thread: - await self._gateway.thread_registry.reset_by_session_key(command.session_key) - session = self._gateway.require_session(command.agent_id, command.session_key) + await self._gateway.thread_registry.reset_by_session_key(session_key) + elif pinned: + await self._bind_pinned_thread(session_key, pinned) + else: + await self._bind_latest_thread(session_key) + session = self._gateway.require_session(command.agent_id, session_key) if session.user_id != command.user_id: raise ValueError( f"session {command.session_key!r} does not belong to user {command.user_id!r}" @@ -83,7 +90,7 @@ async def _locked() -> None: await self._gateway.run_in_session( command.agent_id, - command.session_key, + session_key, _locked, ) @@ -288,6 +295,58 @@ def _attach_turn_knowledge_config( locale=locale, ) + def _target_session(self, command: CronDeliveryCommand) -> tuple[str, str | None]: + """Return the channel session and, for a history pick, its thread id.""" + pinned = pinned_thread_id(command.session_key) + if not pinned: + return command.session_key, None + row = self._gateway.thread_registry.get_thread(pinned) + if ( + not isinstance(row, ThreadRow) + or row.agent_id != command.agent_id + or row.user_id != command.user_id + ): + raise ValueError(f"thread {pinned!r} is not available for this job") + return row.session_key, pinned + + async def _bind_pinned_thread(self, session_key: str, thread_id: str) -> None: + """Point the channel at the historical conversation chosen for this job.""" + registry = self._gateway.thread_registry + if registry.get_bound_thread_id(session_key) == thread_id: + return + session = registry.get_session(session_key) + if session is None: + return + await registry.rebind( + session_key=session_key, + thread_id=thread_id, + agent_id=session.agent_id, + acting_user_id=session.user_id, + ) + + async def _bind_latest_thread(self, session_key: str) -> None: + """Point this channel at its latest conversation before the run. + + The stored ``session_key`` names the channel. Several threads can share + it; delivery follows the one the user last spoke in, not whichever + thread happens to be open. + """ + registry = self._gateway.thread_registry + latest = registry.latest_active_thread(session_key=session_key) + if not isinstance(latest, ThreadRow): + return + if registry.get_bound_thread_id(session_key) == latest.thread_id: + return + session = registry.get_session(session_key) + if session is None: + return + await registry.rebind( + session_key=session_key, + thread_id=latest.thread_id, + agent_id=session.agent_id, + acting_user_id=session.user_id, + ) + def _remember_thread_title(self, session: SessionRow, command: CronDeliveryCommand) -> None: """Name the conversation when the scheduled run writes it. diff --git a/src/octop/infra/cron/manager.py b/src/octop/infra/cron/manager.py index 90a3118c25..98b66ad601 100644 --- a/src/octop/infra/cron/manager.py +++ b/src/octop/infra/cron/manager.py @@ -22,6 +22,7 @@ from octop.infra.connectors.gateway.agently_watch import AgentlyWatchManager from octop.infra.cron.delivery import CronDeliveryService from octop.infra.cron.job import CronJob +from octop.infra.cron.session_target import LATEST_TARGET, pinned_thread_id from octop.infra.cron.trigger import ( ONE_SHOT_MISFIRE_GRACE_SECONDS, RECURRING_MISFIRE_GRACE_SECONDS, @@ -176,12 +177,8 @@ async def create(self, spec: CronCreateSpec | None = None, **kwargs: Any) -> Cro if spec is None: spec = CronCreateSpec(**kwargs) self._mail_source(spec.trigger, spec.user_id) - session_key = spec.session_key or ThreadRegistry.dashboard_key( - agent_id=spec.agent_id, - user_id=spec.user_id, - ) - await self._ensure_session( - session_key, + session_key = await self._prepare_session_key( + spec.session_key, agent_id=spec.agent_id, user_id=spec.user_id, ) @@ -253,6 +250,12 @@ async def update( existing = self._repos.cron_repo.get(cron_id) if existing is None: raise OctopError(ErrorCode.NOT_FOUND, f"cron job {cron_id!r} not found") + if session_key is not None: + session_key = await self._prepare_session_key( + session_key, + agent_id=existing.agent_id, + user_id=existing.user_id, + ) trigger_changed = trigger is not None and trigger != existing.trigger enabled_changed = enabled_bool is not None and enabled_bool != bool(existing.enabled) # Validate a new trigger, or a mailbox, before persisting. Echoing the @@ -624,6 +627,33 @@ async def stop_mail_watch(self, instance_id: str) -> None: async def resume_mail_watch(self, instance_id: str) -> None: await self.sync_mail_watch(instance_id) + async def _prepare_session_key( + self, + raw: str | None, + *, + agent_id: str, + user_id: int, + ) -> str: + """Store a channel key, or ``thread:{id}`` for one historical conversation. + + Empty and ``latest`` mean the owner's Dashboard channel. Each run then + follows that channel's most recent conversation unless a thread is pinned. + """ + text = (raw or "").strip() + pinned = pinned_thread_id(text) + if pinned: + row = self._gateway.thread_registry.get_thread(pinned) + if row is None or row.agent_id != agent_id: + raise OctopError(ErrorCode.NOT_FOUND, f"thread {pinned!r} not found") + if row.user_id != user_id: + raise OctopError(ErrorCode.FORBIDDEN, "thread not owned by user") + await self._ensure_session(row.session_key, agent_id=agent_id, user_id=user_id) + return text + if not text or text == LATEST_TARGET: + text = ThreadRegistry.dashboard_key(agent_id=agent_id, user_id=user_id) + await self._ensure_session(text, agent_id=agent_id, user_id=user_id) + return text + async def _ensure_session(self, session_key: str, *, agent_id: str, user_id: int) -> None: registry = self._gateway.thread_registry existing = registry.get_session(session_key) diff --git a/src/octop/infra/cron/session_target.py b/src/octop/infra/cron/session_target.py new file mode 100644 index 0000000000..1b1daf9b66 --- /dev/null +++ b/src/octop/infra/cron/session_target.py @@ -0,0 +1,19 @@ +"""Cron session_key values that name either a channel or one historical thread.""" + +from __future__ import annotations + +THREAD_TARGET_PREFIX = "thread:" +LATEST_TARGET = "latest" + + +def pinned_thread_id(session_key: str) -> str | None: + """Return the thread id when ``session_key`` pins a historical conversation.""" + text = session_key.strip() + if not text.startswith(THREAD_TARGET_PREFIX): + return None + thread_id = text[len(THREAD_TARGET_PREFIX) :].strip() + return thread_id or None + + +def thread_target_key(thread_id: str) -> str: + return f"{THREAD_TARGET_PREFIX}{thread_id.strip()}" diff --git a/src/octop/infra/cron/tools.py b/src/octop/infra/cron/tools.py index 752f1910f8..cf9160bad6 100644 --- a/src/octop/infra/cron/tools.py +++ b/src/octop/infra/cron/tools.py @@ -10,6 +10,7 @@ from pydantic import Field from octop.infra.cron.manager import CronCreateSpec +from octop.infra.cron.session_target import thread_target_key from octop.infra.cron.task_type import ( CRON_NAME_MAX_LEN, require_cron_name, @@ -38,19 +39,28 @@ "Use 'agent' only when each run must query, analyze, or summarize fresh data." ) +_SESSION_TARGET_HELP = ( + "Where the run lands when fresh_thread is false. " + "'latest' (default): each run uses the most recent conversation on the current channel. " + "'history': stay on one conversation. Pass thread_id, or omit it to pin the current conversation. " + "Ignored when fresh_thread is true, which starts a new conversation every run." +) + _CRONJOB_CREATE_DESC = ( - "Create a scheduled cron job bound to the current conversation session. " - "Results are delivered to the same channel (QQ/WeChat/dashboard/…). " + "Create a scheduled cron job on the current channel (QQ/WeChat/dashboard/…). " "name is required: a short label for the job list. " + f"session_target: {_SESSION_TARGET_HELP} " f"task_type: {_TASK_TYPE_HELP} " "Examples: 'remind me to drink water at 14:00 daily' → " - "name='Daily water reminder', task_type=text, prompt='Time to drink water'; " - "'summarize my inbox every morning' → name='Morning inbox summary', task_type=agent. " + "name='Daily water reminder', task_type=text, prompt='Time to drink water', session_target=latest; " + "'summarize my inbox every morning into this chat' → " + "name='Morning inbox summary', task_type=agent, session_target=history. " f"trigger: {_TRIGGER_HELP}" ) _CRONJOB_UPDATE_DESC = ( - "Update an existing cron job (trigger, prompt, enabled, task_type, …). " + "Update an existing cron job (trigger, prompt, enabled, task_type, session_target, …). " + f"session_target: {_SESSION_TARGET_HELP} " f"task_type: {_TASK_TYPE_HELP} " f"trigger: {_TRIGGER_HELP}" ) @@ -89,6 +99,28 @@ def _get_owned(mgr: CronManager, cron_id: str, agent_id: str, user_id: int) -> C return row +def _stored_session_key( + *, + channel_session_key: str, + fresh_thread: bool, + session_target: str | None, + thread_id: str | None, +) -> str: + """Channel key for latest runs, or ``thread:{id}`` for a history pin.""" + target = (session_target or "latest").strip().lower() + if fresh_thread or target == "latest": + return channel_session_key + if target != "history": + raise ValueError("session_target must be 'latest' or 'history'") + pinned = (thread_id or "").strip() + if not pinned: + cfg = get_config().get("configurable") or {} + pinned = str(cfg.get("thread_id") or "").strip() + if not pinned: + raise ValueError("history session_target requires thread_id") + return thread_target_key(pinned) + + def _ok(data: Any) -> str: return json.dumps(data, ensure_ascii=False, indent=2) @@ -145,8 +177,23 @@ async def cronjob_create( ], fresh_thread: Annotated[ bool, - Field(description="If true, reset conversation context before each agent run."), + Field( + description="If true, start a new conversation before each run. Ignores session_target." + ), ] = False, + session_target: Annotated[ + str, + Field(description=_SESSION_TARGET_HELP), + ] = "latest", + thread_id: Annotated[ + str | None, + Field( + description=( + "Historical conversation to pin when session_target is 'history'. " + "Omit to pin the current conversation." + ), + ), + ] = None, enabled: Annotated[ bool, Field(description="Whether the job is active immediately after creation."), @@ -157,7 +204,7 @@ async def cronjob_create( ] = "text", ) -> str: try: - agent_id, user_id, session_key = _tool_ctx() + agent_id, user_id, channel_key = _tool_ctx() _require_agent_owner(mgr, agent_id, user_id) cron_id = new_cron_id() cleaned_prompt = require_cron_prompt(prompt) @@ -169,7 +216,12 @@ async def cronjob_create( trigger=trigger, prompt=cleaned_prompt, fresh_thread=fresh_thread, - session_key=session_key, + session_key=_stored_session_key( + channel_session_key=channel_key, + fresh_thread=fresh_thread, + session_target=session_target, + thread_id=thread_id, + ), enabled=enabled, task_type=require_cron_task_type(task_type), ) @@ -187,13 +239,30 @@ async def cronjob_update( Field(description="New prompt / message text."), ] = None, fresh_thread: bool | None = None, + session_target: Annotated[ + str | None, + Field(description=_SESSION_TARGET_HELP), + ] = None, + thread_id: Annotated[ + str | None, + Field(description="Historical conversation to pin when session_target is 'history'."), + ] = None, enabled: bool | None = None, task_type: Annotated[str | None, Field(description=_TASK_TYPE_HELP)] = None, ) -> str: try: - agent_id, user_id, _session_key = _tool_ctx() + agent_id, user_id, channel_key = _tool_ctx() existing = _get_owned(mgr, cron_id, agent_id, user_id) cleaned_prompt = require_cron_prompt(prompt) if prompt is not None else None + stored_session: str | None = None + if fresh_thread or session_target is not None or (thread_id or "").strip(): + stored_session = _stored_session_key( + channel_session_key=channel_key, + fresh_thread=bool(fresh_thread), + session_target=session_target + or ("history" if (thread_id or "").strip() else "latest"), + thread_id=thread_id, + ) row = await mgr.update( cron_id, name=( @@ -207,6 +276,7 @@ async def cronjob_update( ), trigger=trigger, prompt=cleaned_prompt, + session_key=stored_session, fresh_thread=fresh_thread, enabled=int(enabled) if enabled is not None else None, task_type=require_cron_task_type(task_type) if task_type is not None else None, diff --git a/src/octop/infra/db/migrate.py b/src/octop/infra/db/migrate.py index f5b815bc0d..ac576ceff9 100644 --- a/src/octop/infra/db/migrate.py +++ b/src/octop/infra/db/migrate.py @@ -1683,6 +1683,33 @@ def _ensure_thread_contributed_schema(db: DatabasePool) -> None: _ensure_column(db, "threads", "contributed", "INTEGER NOT NULL DEFAULT 0") +_LOGIN_AUDIT_ACTIONS_SQL = ( + "'auth.login', 'auth.ldap_login', 'auth.oidc_login', 'auth.passkey_login'" +) + + +def _ensure_user_last_login_schema(db: DatabasePool) -> None: + """``users.last_login_at`` (v25). Idempotent; fills blanks from login audit rows.""" + if not _table_exists(db, "users"): + return + _ensure_column(db, "users", "last_login_at", "INTEGER") + if not _table_exists(db, "audit_log"): + return + actions = _LOGIN_AUDIT_ACTIONS_SQL + with db.connect() as conn: + conn.execute( + "UPDATE users SET last_login_at = (" + "SELECT MAX(ts) FROM audit_log " + "WHERE actor = users.username " + f"AND action IN ({actions})" + ") WHERE last_login_at IS NULL AND EXISTS (" + "SELECT 1 FROM audit_log " + "WHERE actor = users.username " + f"AND action IN ({actions})" + ")" + ) + + def _ensure_user_mfa_schema(db: DatabasePool) -> None: """MFA tables + users.mfa_* / password_version columns (v25). Idempotent.""" if not _table_exists(db, "users"): @@ -2487,9 +2514,10 @@ def run_migrations(db: DatabasePool) -> None: _ensure_knowledge_base_icon_url_schema(db) _ensure_bridge_icon_url_schema(db) _ensure_bridge_shared_schema(db) - # Always repair: version may already be 25 while MFA tables or lease - # columns were never created. + # Always repair: version may already be 25 while MFA tables, lease + # columns, or last_login_at were never created. _ensure_user_mfa_schema(db) + _ensure_user_last_login_schema(db) _ensure_instance_lease_schema(db) _ensure_control_blobs_schema(db) _ensure_thread_contributed_schema(db) diff --git a/src/octop/infra/db/migrations/025_user_mfa.pg.sql b/src/octop/infra/db/migrations/025_user_mfa.pg.sql index 23cea7d46e..2c38db7bb4 100644 --- a/src/octop/infra/db/migrations/025_user_mfa.pg.sql +++ b/src/octop/infra/db/migrations/025_user_mfa.pg.sql @@ -1,10 +1,26 @@ -- Schema v25: GitHub-style MFA (TOTP, WebAuthn/passkeys, recovery codes), --- DB-backed auth challenges, and same-row cron/channel leases. +-- DB-backed auth challenges, same-row cron/channel leases, and +-- users.last_login_at. ALTER TABLE users ADD COLUMN IF NOT EXISTS mfa_version INTEGER NOT NULL DEFAULT 0; ALTER TABLE users ADD COLUMN IF NOT EXISTS mfa_failed_count INTEGER NOT NULL DEFAULT 0; ALTER TABLE users ADD COLUMN IF NOT EXISTS mfa_locked_until INTEGER NOT NULL DEFAULT 0; ALTER TABLE users ADD COLUMN IF NOT EXISTS password_version INTEGER NOT NULL DEFAULT 0; +ALTER TABLE users ADD COLUMN IF NOT EXISTS last_login_at INTEGER; + +-- Fill blanks from existing sign-in audit rows. Later logins update the column. +UPDATE users +SET last_login_at = ( + SELECT MAX(ts) FROM audit_log + WHERE actor = users.username + AND action IN ('auth.login', 'auth.ldap_login', 'auth.oidc_login', 'auth.passkey_login') +) +WHERE last_login_at IS NULL + AND EXISTS ( + SELECT 1 FROM audit_log + WHERE actor = users.username + AND action IN ('auth.login', 'auth.ldap_login', 'auth.oidc_login', 'auth.passkey_login') + ); CREATE TABLE IF NOT EXISTS user_totp ( id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, diff --git a/src/octop/infra/db/migrations/025_user_mfa.sql b/src/octop/infra/db/migrations/025_user_mfa.sql index 5707b10425..bdce279d1d 100644 --- a/src/octop/infra/db/migrations/025_user_mfa.sql +++ b/src/octop/infra/db/migrations/025_user_mfa.sql @@ -1,11 +1,27 @@ -- Schema v25: GitHub-style MFA (TOTP, WebAuthn/passkeys, recovery codes), --- DB-backed auth challenges, and same-row cron/channel leases. SQLite applies --- this through idempotent ensure helpers so a replay does not fail. +-- DB-backed auth challenges, same-row cron/channel leases, and +-- users.last_login_at. SQLite applies this through idempotent ensure helpers +-- so a replay does not fail. ALTER TABLE users ADD COLUMN mfa_version INTEGER NOT NULL DEFAULT 0; ALTER TABLE users ADD COLUMN mfa_failed_count INTEGER NOT NULL DEFAULT 0; ALTER TABLE users ADD COLUMN mfa_locked_until INTEGER NOT NULL DEFAULT 0; ALTER TABLE users ADD COLUMN password_version INTEGER NOT NULL DEFAULT 0; +ALTER TABLE users ADD COLUMN last_login_at INTEGER; + +-- Fill blanks from existing sign-in audit rows. Later logins update the column. +UPDATE users +SET last_login_at = ( + SELECT MAX(ts) FROM audit_log + WHERE actor = users.username + AND action IN ('auth.login', 'auth.ldap_login', 'auth.oidc_login', 'auth.passkey_login') +) +WHERE last_login_at IS NULL + AND EXISTS ( + SELECT 1 FROM audit_log + WHERE actor = users.username + AND action IN ('auth.login', 'auth.ldap_login', 'auth.oidc_login', 'auth.passkey_login') + ); CREATE TABLE IF NOT EXISTS user_totp ( id INTEGER PRIMARY KEY AUTOINCREMENT, diff --git a/src/octop/infra/db/repos/agents.py b/src/octop/infra/db/repos/agents.py index 113705e925..bfa402633f 100644 --- a/src/octop/infra/db/repos/agents.py +++ b/src/octop/infra/db/repos/agents.py @@ -88,7 +88,7 @@ def from_row(cls, r: DbRow) -> AgentRow: except (KeyError, IndexError): kind_raw = None kind = str(kind_raw).strip() if kind_raw else "expert" - if kind not in {"expert", "team"}: + if kind not in {"expert", "team", "managed"}: kind = "expert" return cls( id=r["id"], @@ -148,7 +148,7 @@ def create( kind: str = "expert", ) -> str: ts = now_ts() - agent_kind = kind if kind in {"expert", "team"} else "expert" + agent_kind = kind if kind in {"expert", "team", "managed"} else "expert" with self._db.transaction() as conn: conn.execute( "INSERT INTO agents(agent_id, user_id, name, description, " diff --git a/src/octop/infra/db/repos/knowledge.py b/src/octop/infra/db/repos/knowledge.py index 2f1860562f..b3c8780415 100644 --- a/src/octop/infra/db/repos/knowledge.py +++ b/src/octop/infra/db/repos/knowledge.py @@ -2,6 +2,7 @@ from __future__ import annotations +from collections.abc import Sequence from dataclasses import dataclass from octop.infra.db.pool import DatabasePool @@ -499,6 +500,93 @@ def move_document( document, normalize_kb_path(f"{target_folder}/{path_basename(document.path)}") ) + def list_subtree(self, document: KnowledgeDocumentRow) -> list[KnowledgeDocumentRow]: + """The document itself plus, for a folder, everything below it.""" + if not document.is_dir: + return [document] + prefix = f"{document.path}/" + with self._db.connect() as conn: + rows = conn.execute( + "SELECT * FROM knowledge_documents WHERE kb_id = ? AND " + "(path = ? OR substr(path, 1, ?) = ?) ORDER BY is_dir DESC, path", + (document.kb_id, document.path, len(prefix), prefix), + ).fetchall() + return map_rows(rows, KnowledgeDocumentRow) + + def move_to_base( + self, + document: KnowledgeDocumentRow, + *, + target_kb_id: str, + target_folder: str, + max_documents: int, + reindex_ids: Sequence[str] = (), + ) -> KnowledgeDocumentRow | None: + """Move a document (and a folder's subtree) into ``target_folder`` of another base. + + Document counts move with the files. ``max_documents`` > 0 caps the target; + exceeding it raises ``ValueError`` and changes nothing. Files in + ``reindex_ids`` go back to ``pending`` for the target to index again. + """ + subtree = self.list_subtree(document) + file_count = sum(1 for row in subtree if not row.is_dir) + new_path = normalize_kb_path(f"{target_folder}/{path_basename(document.path)}") + ts = now_ts() + with self._db.transaction() as conn: + if file_count: + if max_documents > 0: + cursor = conn.execute( + "UPDATE knowledge_bases SET doc_count = doc_count + ?, updated_at = ? " + "WHERE knowledge_base_id = ? AND doc_count + ? <= ?", + (file_count, ts, target_kb_id, file_count, max_documents), + ) + if cursor.rowcount != 1: + raise ValueError( + f"knowledge bases support at most {max_documents} documents" + ) + else: + conn.execute( + "UPDATE knowledge_bases SET doc_count = doc_count + ?, updated_at = ? " + "WHERE knowledge_base_id = ?", + (file_count, ts, target_kb_id), + ) + conn.execute( + "UPDATE knowledge_bases SET doc_count = CASE " + "WHEN doc_count > ? THEN doc_count - ? ELSE 0 END, " + "updated_at = ? WHERE knowledge_base_id = ?", + (file_count, file_count, ts, document.kb_id), + ) + if document.is_dir: + prefix = f"{document.path}/" + conn.execute( + "UPDATE knowledge_documents SET " + "kb_id = ?, path = ? || substr(path, ?), updated_at = ? " + "WHERE kb_id = ? AND substr(path, 1, ?) = ?", + ( + target_kb_id, + new_path, + len(document.path) + 1, + ts, + document.kb_id, + len(prefix), + prefix, + ), + ) + conn.execute( + "UPDATE knowledge_documents SET " + "kb_id = ?, filename = ?, path = ?, updated_at = ? " + "WHERE document_id = ?", + (target_kb_id, path_basename(new_path), new_path, ts, document.id), + ) + for doc_id in reindex_ids: + conn.execute( + "UPDATE knowledge_documents SET " + "status = 'pending', error_message = '', chunk_count = 0, updated_at = ? " + "WHERE document_id = ?", + (ts, doc_id), + ) + return self.get_document(document.id) + def _relocate_document( self, document: KnowledgeDocumentRow, new_path: str ) -> KnowledgeDocumentRow | None: diff --git a/src/octop/infra/db/repos/threads.py b/src/octop/infra/db/repos/threads.py index d76aa31479..f7f79d1b6e 100644 --- a/src/octop/infra/db/repos/threads.py +++ b/src/octop/infra/db/repos/threads.py @@ -7,7 +7,13 @@ from dataclasses import dataclass, field from octop.infra.db.pool import DatabasePool -from octop.infra.db.repos._base import DbRow, bool_int, map_rows, now_ts +from octop.infra.db.repos._base import ( + DbRow, + bool_int, + map_rows, + now_ts, + sql_in_placeholders, +) from octop.infra.utils.thread_artifact import ( MAX_THREAD_ARTIFACTS, ThreadArtifact, @@ -230,6 +236,24 @@ def latest_user_activity(self, agent_id: str) -> int | None: return None return int(row[0]) + def latest_activity_by_agent(self, user_id: int, agent_ids: list[str]) -> dict[str, int]: + """Latest own-thread touch per agent. + + ``last_active`` of 0 means the thread has no turns yet, so those rows + are ignored. Another user's threads on a shared expert do not count. + """ + if not agent_ids: + return {} + placeholders = sql_in_placeholders(len(agent_ids)) + with self._db.connect() as conn: + rows = conn.execute( + "SELECT agent_id, MAX(last_active) AS last_active FROM threads " + f"WHERE user_id = ? AND last_active > 0 AND agent_id IN ({placeholders}) " + "GROUP BY agent_id", + (user_id, *agent_ids), + ).fetchall() + return {str(row["agent_id"]): int(row["last_active"]) for row in rows} + def list_by_agent(self, *, agent_id: str, limit: int = 50) -> list[ThreadRow]: # last_active=0 is "no turns yet" (has_messages sentinel). Fall back to # created_at so brand-new empty threads sort to the top of the sidebar @@ -288,6 +312,20 @@ def list_by_session(self, *, session_key: str, limit: int = 50) -> list[ThreadRo ).fetchall() return map_rows(rows, ThreadRow) + def latest_active_for_session(self, *, session_key: str) -> ThreadRow | None: + """Newest conversation on this channel session that already has a turn. + + ``last_active`` of 0 is an empty thread. A blank chat created later must + not hide the conversation the user last spoke in. + """ + with self._db.connect() as conn: + row = conn.execute( + "SELECT * FROM threads WHERE session_key = ? AND last_active > 0 " + "ORDER BY last_active DESC, thread_id DESC LIMIT 1", + (session_key,), + ).fetchone() + return ThreadRow.from_row(row) if row else None + def set_title_if_null(self, thread_id: str, title: str) -> None: with self._db.transaction() as conn: conn.execute( diff --git a/src/octop/infra/db/repos/users.py b/src/octop/infra/db/repos/users.py index e6f5983560..bb4ab89692 100644 --- a/src/octop/infra/db/repos/users.py +++ b/src/octop/infra/db/repos/users.py @@ -68,6 +68,7 @@ class UserRow: permissions: builtins.list[str] = field(default_factory=list) role_name: str | None = None avatar_icon: str | None = None + last_login_at: int | None = None @classmethod def from_row(cls, r: DbRow) -> UserRow: @@ -98,6 +99,9 @@ def from_row(cls, r: DbRow) -> UserRow: permissions=_parse_permissions(r["permissions"] if "permissions" in keys else None), role_name=role_name or None, avatar_icon=avatar_icon or None, + last_login_at=( + int(r["last_login_at"]) if "last_login_at" in keys and r["last_login_at"] else None + ), ) @@ -392,6 +396,15 @@ def clear_login_lockout(self, user_id: int) -> None: (user_id,), ) + def touch_last_login(self, user_id: int, *, now: int | None = None) -> None: + """Record a successful sign-in. ``now`` is unix seconds.""" + ts = now if now is not None else now_ts() + with self._db.transaction() as conn: + conn.execute( + "UPDATE users SET last_login_at = ? WHERE id = ?", + (ts, user_id), + ) + def bump_mfa_version(self, user_id: int) -> int: """Increment ``mfa_version`` and clear MFA throttle. Returns the new version.""" with self._db.transaction() as conn: diff --git a/src/octop/infra/errors.py b/src/octop/infra/errors.py index a6480a134f..46fc3a4355 100644 --- a/src/octop/infra/errors.py +++ b/src/octop/infra/errors.py @@ -57,6 +57,7 @@ class ErrorCode(StrEnum): STORAGE_BROWSE_FAILED = "STORAGE_BROWSE_FAILED" CHANNEL_KIND_UNSUPPORTED = "CHANNEL_KIND_UNSUPPORTED" CHANNEL_INVALID_CREDENTIALS = "CHANNEL_INVALID_CREDENTIALS" + CHANNEL_OPTIONS_UNSUPPORTED = "CHANNEL_OPTIONS_UNSUPPORTED" CHANNEL_NAME_TAKEN = "CHANNEL_NAME_TAKEN" CONNECTOR_NOT_FOUND = "CONNECTOR_NOT_FOUND" CONNECTOR_NAME_TAKEN = "CONNECTOR_NAME_TAKEN" @@ -68,6 +69,7 @@ class ErrorCode(StrEnum): CONNECTOR_NOT_BOUND = "CONNECTOR_NOT_BOUND" CONNECTOR_ALREADY_BOUND = "CONNECTOR_ALREADY_BOUND" CONNECTOR_MCP_LOAD_FAILED = "CONNECTOR_MCP_LOAD_FAILED" + CONNECTOR_AUTH_REQUIRED = "CONNECTOR_AUTH_REQUIRED" BRIDGE_NOT_FOUND = "BRIDGE_NOT_FOUND" BRIDGE_AUTH_FAILED = "BRIDGE_AUTH_FAILED" BRIDGE_PEER_UNREACHABLE = "BRIDGE_PEER_UNREACHABLE" @@ -139,7 +141,10 @@ class ErrorCode(StrEnum): KNOWLEDGE_NAME_TAKEN = "KNOWLEDGE_NAME_TAKEN" KNOWLEDGE_NAME_INVALID = "KNOWLEDGE_NAME_INVALID" KNOWLEDGE_MOVE_INVALID = "KNOWLEDGE_MOVE_INVALID" + KNOWLEDGE_MOVE_BUSY = "KNOWLEDGE_MOVE_BUSY" KNOWLEDGE_MOVE_NAME_TAKEN = "KNOWLEDGE_MOVE_NAME_TAKEN" + KNOWLEDGE_RERANK_NOT_READY = "KNOWLEDGE_RERANK_NOT_READY" + KNOWLEDGE_GRAPH_NOT_READY = "KNOWLEDGE_GRAPH_NOT_READY" AVATAR_INVALID = "AVATAR_INVALID" AVATAR_TOO_LARGE = "AVATAR_TOO_LARGE" INVITE_INVALID = "INVITE_INVALID" @@ -157,6 +162,7 @@ class ErrorCode(StrEnum): WORKSPACE_ROOT_CONTAINER_UNSUPPORTED = "WORKSPACE_ROOT_CONTAINER_UNSUPPORTED" TOKEN_QUOTA_EXCEEDED = "TOKEN_QUOTA_EXCEEDED" AGENT_QUOTA_EXCEEDED = "AGENT_QUOTA_EXCEEDED" + BACKEND_TYPE_NOT_ALLOWED = "BACKEND_TYPE_NOT_ALLOWED" CAPTCHA_REQUIRED = "CAPTCHA_REQUIRED" CAPTCHA_FAILED = "CAPTCHA_FAILED" MFA_INVALID = "MFA_INVALID" @@ -167,6 +173,7 @@ class ErrorCode(StrEnum): WEBAUTHN_INVALID_DOMAIN = "WEBAUTHN_INVALID_DOMAIN" WEBAUTHN_LOGIN_DISABLED = "WEBAUTHN_LOGIN_DISABLED" CONFIG_FILE_CORRUPT = "CONFIG_FILE_CORRUPT" + APP_CONFIG_INVALID = "APP_CONFIG_INVALID" TEAM_NOT_FOUND = "TEAM_NOT_FOUND" TEAM_MEMBERS_TOO_FEW = "TEAM_MEMBERS_TOO_FEW" TEAM_MEMBER_INVALID = "TEAM_MEMBER_INVALID" @@ -222,6 +229,7 @@ class ErrorCode(StrEnum): ErrorCode.STORAGE_BROWSE_FAILED: 400, ErrorCode.CHANNEL_KIND_UNSUPPORTED: 400, ErrorCode.CHANNEL_INVALID_CREDENTIALS: 400, + ErrorCode.CHANNEL_OPTIONS_UNSUPPORTED: 400, ErrorCode.CHANNEL_NAME_TAKEN: 409, ErrorCode.CONNECTOR_NOT_FOUND: 404, ErrorCode.CONNECTOR_NAME_TAKEN: 409, @@ -233,6 +241,7 @@ class ErrorCode(StrEnum): ErrorCode.CONNECTOR_NOT_BOUND: 400, ErrorCode.CONNECTOR_ALREADY_BOUND: 409, ErrorCode.CONNECTOR_MCP_LOAD_FAILED: 502, + ErrorCode.CONNECTOR_AUTH_REQUIRED: 409, ErrorCode.BRIDGE_NOT_FOUND: 404, # Peer credential failure. 400 matches other invalid-credential codes; # 401 would make the dashboard treat it as a dead local session. @@ -306,7 +315,10 @@ class ErrorCode(StrEnum): ErrorCode.KNOWLEDGE_NAME_TAKEN: 409, ErrorCode.KNOWLEDGE_NAME_INVALID: 400, ErrorCode.KNOWLEDGE_MOVE_INVALID: 400, + ErrorCode.KNOWLEDGE_MOVE_BUSY: 409, ErrorCode.KNOWLEDGE_MOVE_NAME_TAKEN: 409, + ErrorCode.KNOWLEDGE_RERANK_NOT_READY: 400, + ErrorCode.KNOWLEDGE_GRAPH_NOT_READY: 409, ErrorCode.AVATAR_INVALID: 400, ErrorCode.AVATAR_TOO_LARGE: 413, ErrorCode.INVITE_INVALID: 400, @@ -324,6 +336,7 @@ class ErrorCode(StrEnum): ErrorCode.WORKSPACE_ROOT_CONTAINER_UNSUPPORTED: 400, ErrorCode.TOKEN_QUOTA_EXCEEDED: 403, ErrorCode.AGENT_QUOTA_EXCEEDED: 403, + ErrorCode.BACKEND_TYPE_NOT_ALLOWED: 403, ErrorCode.CAPTCHA_REQUIRED: 400, ErrorCode.CAPTCHA_FAILED: 400, ErrorCode.MFA_INVALID: 401, @@ -334,6 +347,7 @@ class ErrorCode(StrEnum): ErrorCode.WEBAUTHN_INVALID_DOMAIN: 400, ErrorCode.WEBAUTHN_LOGIN_DISABLED: 403, ErrorCode.CONFIG_FILE_CORRUPT: 400, + ErrorCode.APP_CONFIG_INVALID: 400, ErrorCode.TEAM_NOT_FOUND: 404, ErrorCode.TEAM_MEMBERS_TOO_FEW: 400, ErrorCode.TEAM_MEMBER_INVALID: 400, diff --git a/src/octop/infra/gateway/channels/telegram_bridge.py b/src/octop/infra/gateway/channels/telegram_bridge.py new file mode 100644 index 0000000000..15d4c1e89d --- /dev/null +++ b/src/octop/infra/gateway/channels/telegram_bridge.py @@ -0,0 +1,376 @@ +"""Host-owned Telegram choices; Gateway owns transport and callback identity checks.""" + +from __future__ import annotations + +import asyncio +import logging +from collections.abc import AsyncIterator, Awaitable, Callable +from dataclasses import dataclass, fields +from typing import Any + +from octop_gateway.models import ( + ChannelSubject, + InboundMessage, + MessageEvent, + MessageEventType, + TextContent, +) + +from octop.i18n.domains.channel import telegram_reject_notice +from octop.infra.errors import ErrorCode, OctopError +from octop.infra.gateway.hitl.format import extract_questions +from octop.infra.gateway.process.message_keys import ( + resolve_user_id_for_message, + session_key_from_message, +) +from octop.infra.gateway.slash.parser import parse_slash +from octop.infra.utils.locale import Locale, resolve_user_locale +from octop.infra.utils.stream_close import close_event_stream + +logger = logging.getLogger(__name__) + +# HITL pending 记录存活 30 分钟,按钮回调不能比它先过期(gateway 默认 300s)。 +_BUTTON_TTL = 1800.0 + + +def validate_telegram_config(kind: str, config: dict[str, Any]) -> None: + """Reject options silently ignored by an older installed Gateway.""" + if kind != "telegram": + return + from octop_gateway.channels.telegram import TelegramConfig + + supported = {item.name for item in fields(TelegramConfig)} + # These options belong to Octop rather than TelegramConfig. + unsupported = set(config) - supported - {"response_mode", "enabled"} + if unsupported: + raise OctopError( + ErrorCode.CHANNEL_OPTIONS_UNSUPPORTED, + "Installed octop-gateway does not support Telegram options: " + + ", ".join(sorted(unsupported)) + + ". Install a Gateway build supporting these options before saving.", + details={"unsupported_options": sorted(unsupported)}, + ) + try: + for item in fields(TelegramConfig): + if ( + isinstance(item.default, bool) + and item.name in config + and not isinstance(config[item.name], bool) + ): + raise ValueError(f"Telegram {item.name} must be a boolean") + factory = getattr(TelegramConfig, "from_dict", None) + if callable(factory): + factory(config) + else: + TelegramConfig(**{key: value for key, value in config.items() if key in supported}) + except (TypeError, ValueError) as exc: + raise OctopError(ErrorCode.CHANNEL_INVALID_CREDENTIALS, str(exc)) from exc + + +@dataclass(frozen=True) +class _ButtonState: + owner_id: int | None + thread_id: str | None + pending_id: str | None + question_index: int + model_page: int | None + text: str + + +class _ButtonReply(InboundMessage): + """Host-created queue item; platform dictionaries cannot construct this type.""" + + button_state: _ButtonState + + +def _is_ask_card(event: MessageEvent) -> bool: + """HITL card text in this event lists the same options the buttons offer. + + Telegram renders those options natively, so the card is only a fallback + for channels without button support or when sending buttons fails. + """ + return event.type == MessageEventType.MESSAGE and bool( + event.metadata.get("octop_hitl_pending_id") + ) + + +def is_telegram_button_reply(msg: InboundMessage) -> bool: + """Keep trusted button replies out of manager-level text batching.""" + return isinstance(msg, _ButtonReply) + + +class TelegramBridge: + def __init__(self, gateway: Any, processor: Any, channel_id: str) -> None: + self.gateway = gateway + self.processor = processor + self.channel_id = channel_id + + async def __call__(self, msg: InboundMessage) -> AsyncIterator[MessageEvent]: + if isinstance(msg, _ButtonReply): + # The normal worker holds the session lock for validation and execution. + reason = self._button_reject_reason_call(msg) + if reason is not None: + await self._notify_button_rejected(msg, reason) + return + if msg.button_state.model_page is not None: + if msg.channel_subject is not None: + await self._send_models(msg, msg.channel_subject, msg.button_state.model_page) + return + msg = InboundMessage(**msg.model_dump(exclude={"button_state"})) + + agent_id = msg.tenant_id or "" + session_key = session_key_from_message(msg, agent_id=agent_id) + pending_id = None + failed = False + # 问询卡片先扣住:Telegram 有原生按钮时只发按钮版,文本卡片会重复。 + held_card = None + stream = self.processor(msg) + try: + async for event in stream: + failed = failed or event.type == MessageEventType.ERROR + event_pending = event.metadata.get("octop_hitl_pending_id") + if not failed and event_pending: + pending_id = event_pending + if _is_ask_card(event): + held_card = event + continue + if event.type == MessageEventType.COMPLETED and not failed: + if held_card is not None: + try: + sent = await self._send_choices(msg, session_key, pending_id) + except Exception: # noqa: BLE001 按钮发送失败时文本卡片是唯一入口 + logger.warning("telegram choice buttons failed; using text card") + sent = False + if not sent: + yield held_card + held_card = None + else: + await self._send_choices(msg, session_key, pending_id) + elif held_card is not None and event.type in ( + MessageEventType.ERROR, + MessageEventType.MESSAGE, + ): + yield held_card + held_card = None + yield event + if held_card is not None: + yield held_card + except asyncio.CancelledError: + # Harness execution can outlive consumption of its stream iterator. + thread_id = self.gateway.thread_registry.get_bound_thread_id(session_key) + if thread_id: + self.gateway._agent_manager.cancel_stream(agent_id, thread_id) + raise + finally: + # GeneratorExit 穿过 async-for 不会关闭内层生成器;不显式关闭 + # 会让整条 processor 链交给 GC 在异源 Context 里清理(曾致 + # ContextVar reset 崩溃、resume 中途丢失)。 + await close_event_stream(stream) + + async def _send_choices( + self, msg: InboundMessage, session_key: str, pending_id: str | None + ) -> bool: + """Render pending choices as native buttons; False when text must carry them.""" + agent_id = msg.tenant_id or "" + manager = self.gateway.channel_manager + channel = manager.get_channel(self.channel_id) if manager else None + if channel is None or not callable(getattr(channel, "send_buttons", None)): + return False + subject = msg.channel_subject + if subject is None or not msg.metadata.get("user_id"): + return False + subject = ChannelSubject( + subject_id=subject.subject_id, + display_name=subject.display_name, + chat_type=subject.chat_type, + metadata={**subject.metadata, **msg.metadata}, + ) + owner = self.gateway._repos.agent_repo.get(agent_id) + user_id = resolve_user_id_for_message(msg, agent_owner_id=owner.user_id if owner else None) + hitl = self.gateway.processor.hitl_coordinator + record = hitl.resolve_ask_pending(session_key, agent_id=agent_id, user_id=user_id) + actions = [] + text = "" + if record is not None and record.pending_id == pending_id: + questions = extract_questions(record.action_requests) + index = record.ask_question_index + if index >= len(questions): + return False + question = questions[index] + # Multi-select continues to use the existing text answer parser. + if question.get("multiSelect") or question.get("multi_select"): + return False + text = str(question.get("question") or question.get("header") or "?") + for number, option in enumerate(question.get("options") or [], start=1): + if isinstance(option, dict) and option.get("label"): + actions.append( + ( + str(option["label"]), + self._action(msg, subject, str(number), record.pending_id, index), + ) + ) + else: + cmd = parse_slash(msg.text) + if cmd is None or cmd.name not in ("model", "models") or cmd.args.strip(): + return False + if owner is None: + return False + await self._send_models(msg, subject) + return False + if actions: + await channel.send_buttons(subject, text, actions, ttl=_BUTTON_TTL) + return True + return False + + def _locale(self, msg: InboundMessage) -> Locale: + """Per-user locale for host-generated copy (stored preference, then channel hint).""" + agent_id = msg.tenant_id or "" + owner = self.gateway._repos.agent_repo.get(agent_id) + user_id = resolve_user_id_for_message( + msg, agent_owner_id=owner.user_id if owner is not None else None + ) + return resolve_user_locale( + user_repo=getattr(self.gateway._repos, "user_repo", None), + user_id=user_id if isinstance(user_id, int) else 0, + channel_type=msg.channel_type or "telegram", + metadata=msg.metadata, + ) + + async def _notify_button_rejected(self, msg: _ButtonReply, reason: str) -> None: + """按钮失效不再静默:按 reason 给一句短提示。 + + gateway 已在 invoke 前应答过 callback query,无法再用 toast,只能发一条消息。 + 不先发 typing:单 chat 1 req/s 限流下它只会拖慢提示本身。 + """ + manager = self.gateway.channel_manager + channel = manager.get_channel(self.channel_id) if manager else None + subject = msg.channel_subject + if channel is None or subject is None: + return + send_text = getattr(channel, "push_text", None) + if callable(send_text): + try: + await send_text(subject, telegram_reject_notice(reason, self._locale(msg))) + except Exception: # noqa: BLE001 反馈失败不应再抛 + logger.warning("reject notice send failed reason=%s", reason) + + def _model_refs(self) -> list[str]: + from octop.infra.agents.providers.store import enabled_model_refs + + refs: set[str] = set() + for provider in self.gateway._repos.provider_repo.list_all(): + if provider.enabled: + refs.update( + enabled_model_refs( + provider.name, provider.get_models(), provider_api_key=provider.api_key + ) + ) + return sorted(refs) + + async def _send_models( + self, msg: InboundMessage, subject: ChannelSubject, page: int = 0 + ) -> None: + manager = self.gateway.channel_manager + channel = manager.get_channel(self.channel_id) if manager else None + if channel is None: + return + refs = self._model_refs() + size = 48 # Leave room for both navigation actions within 50 buttons. + page = min(page, max(0, (len(refs) - 1) // size)) + actions = [ + (ref, self._action(msg, subject, f"/model {ref}")) + for ref in refs[page * size : (page + 1) * size] + ] + for target, label in ((page - 1, "←"), (page + 1, "→")): + if target >= 0 and target * size < len(refs): + actions.append((label, self._action(msg, subject, "", model_page=target))) + if actions: + await channel.send_buttons(subject, f"/model ({page + 1})", actions, ttl=_BUTTON_TTL) + + def _action( + self, + msg: InboundMessage, + subject: ChannelSubject, + text: str, + pending_id: str | None = None, + question_index: int = 0, + model_page: int | None = None, + ) -> Callable[[], Awaitable[None]]: + agent_id = msg.tenant_id or "" + session_key = session_key_from_message(msg, agent_id=agent_id) + thread_id = self.gateway.thread_registry.get_bound_thread_id(session_key) + owner = self.gateway._repos.agent_repo.get(agent_id) + owner_id = owner.user_id if owner else None + + state = _ButtonState(owner_id, thread_id, pending_id, question_index, model_page, text) + + async def action() -> None: + manager = self.gateway.channel_manager + channel = manager.get_channel(self.channel_id) if manager else None + if channel is None: + return + reply = _ButtonReply( + **{ + **msg.model_dump(exclude={"button_state"}), + "content": [TextContent(text=text)], + "channel_subject": subject, + }, + button_state=state, + ) + channel.enqueue(reply) + + return action + + def _button_reject_reason_call(self, msg: _ButtonReply) -> str | None: + state = msg.button_state + agent_id = msg.tenant_id or "" + session_key = session_key_from_message(msg, agent_id=agent_id) + reason = self._button_reject_reason(msg, state, agent_id, session_key) + if reason is not None: + # 按钮点击被丢弃必须留下线索,同时把 reason 交给上层做用户反馈。 + logger.warning( + "telegram button reply dropped: agent=%s session=%s reason=%s text=%r", + agent_id, + session_key, + reason, + state.text, + ) + return reason + + def _button_reject_reason( + self, msg: _ButtonReply, state: _ButtonState, agent_id: str, session_key: str + ) -> str | None: + current_owner = self.gateway._repos.agent_repo.get(agent_id) + if current_owner is None or current_owner.user_id != state.owner_id: + return "owner_changed" + # 群聊里任何人的新消息都会重绑 thread,thread 硬校验会让仍然有效的按钮在几秒内 + # 失效(点击后毫无反应);此时归属由 gateway 的 user+chat 校验、owner/model 校验 + # 以及 HITL 的 session+user+question index 兜住。私聊保持 thread 严格校验。 + strict_thread = not msg.metadata.get("is_group") + if ( + strict_thread + and self.gateway.thread_registry.get_bound_thread_id(session_key) != state.thread_id + ): + return "thread_rebound" + if ( + state.text.startswith("/model ") + and state.text.removeprefix("/model ") not in self._model_refs() + ): + return "model_unavailable" + if state.pending_id is not None: + record = self.gateway.processor.hitl_coordinator.store.get_pending( + state.pending_id, session_key=session_key, agent_id=agent_id + ) + if record is None: + return "pending_expired_or_resolved" + if record.ask_question_index != state.question_index: + return "question_index_changed" + if record.user_id != resolve_user_id_for_message(msg, agent_owner_id=state.owner_id): + return "user_mismatch" + if ( + strict_thread + and self.gateway.thread_registry.get_bound_thread_id(record.session_key) + != record.thread_id + ): + return "thread_rebound" + return None diff --git a/src/octop/infra/gateway/gateway.py b/src/octop/infra/gateway/gateway.py index caad3dd3f0..3579325745 100644 --- a/src/octop/infra/gateway/gateway.py +++ b/src/octop/infra/gateway/gateway.py @@ -16,7 +16,7 @@ from octop_gateway.channel import ChannelCredentialsError from octop_gateway.channels import ChannelKind from octop_gateway.manager import ChannelManager -from octop_gateway.models import ChannelSubject +from octop_gateway.models import ChannelSubject, ContentPart from octop.i18n import channel_probe_incomplete, channel_runtime_reason, tr from octop.infra.db.lease import LEASE_SECONDS, RECONCILE_SECONDS @@ -290,7 +290,11 @@ async def boot(self, *, serve_background: bool = True) -> None: trajectory_service=self._trajectory_service, ) - self._channel_manager = ChannelManager(channels={}) + from octop.infra.gateway.channels.telegram_bridge import is_telegram_button_reply + + self._channel_manager = ChannelManager( + channels={}, control_message_predicate=is_telegram_button_reply + ) self._channel_manager.set_pre_lock_handler(self._preempt_cancel_on_stop) await self._channel_manager.start() @@ -403,6 +407,9 @@ def get_channel(self, channel_id: str) -> ChannelRow | None: return self._repos.channel_repo.get(channel_id) async def create_channel(self, spec: ChannelCreateSpec) -> ChannelRow: + from octop.infra.gateway.channels.telegram_bridge import validate_telegram_config + + validate_telegram_config(str(spec.kind), spec.config) config_json = json.dumps(spec.config) existing = self._repos.channel_repo.get_by_agent_and_name(spec.agent_id, spec.name) if existing is not None: @@ -446,6 +453,17 @@ async def update_channel( config_json: str | None = None, enabled: int | None = None, ) -> ChannelRow | None: + from octop.infra.gateway.channels.telegram_bridge import validate_telegram_config + + existing = self._repos.channel_repo.get(channel_id) + effective_kind = kind or (existing.kind if existing else "") + if effective_kind == "telegram": + config = ( + json.loads(config_json) + if config_json is not None + else (self._config_from_row(existing) if existing else {}) + ) + validate_telegram_config(effective_kind, config) self._repos.channel_repo.update( channel_id, kind=kind, @@ -653,6 +671,25 @@ async def push_text( return await manager.push_text(channel_id, subject, cleaned) + async def push_content( + self, + channel_type: str, + channel_id: str, + subject: ChannelSubject, + parts: list[ContentPart], + ) -> None: + """Proactively push files or images to an IM user via ChannelManager. + + Team member turns finish outside the inbound reply loop, so media cannot + ride a ``MessageEvent``. ``channel_type`` matches :meth:`push_text`; + routing uses ``channel_id``. + """ + del channel_type + if not parts: + return + manager = self._require_channel_manager() + await manager.push_content(channel_id, subject, parts) + async def probe_channel( self, channel_id: str, *, locale: Locale = DEFAULT_LOCALE ) -> dict[str, Any]: @@ -705,6 +742,9 @@ async def _probe_row( raise RuntimeError("gateway not booted") try: + from octop.infra.gateway.channels.telegram_bridge import validate_telegram_config + + validate_telegram_config(row.kind, raw_cfg) await manager.probe_channel( row.kind, raw_cfg, @@ -855,6 +895,17 @@ async def _register_channel(self, row: ChannelRow) -> None: processor = processor_for_response_mode(self._processor, response_mode) if row.kind == "weixin" and response_mode == "stream": processor = wrap_weixin_stream_processor(processor) + if row.kind == "telegram": + from octop.infra.gateway.channels.telegram_bridge import ( + TelegramBridge, + validate_telegram_config, + ) + + validate_telegram_config(row.kind, config) + # Wrap once: the bridge sends choice menus on COMPLETED, so a second layer + # would push every menu twice. + if not isinstance(processor, TelegramBridge): + processor = TelegramBridge(self, processor, row.channel_id) manager = self._require_channel_manager() await manager.add_channel( row.kind, diff --git a/src/octop/infra/gateway/hitl/coordinator.py b/src/octop/infra/gateway/hitl/coordinator.py index b680517549..84a8eb44f1 100644 --- a/src/octop/infra/gateway/hitl/coordinator.py +++ b/src/octop/infra/gateway/hitl/coordinator.py @@ -29,6 +29,7 @@ from octop.infra.gateway.slash.ctx import ensure_thread_id from octop.infra.gateway.slash.formatting import markdown_bullets from octop.infra.utils.locale import Locale, normalize_locale +from octop.infra.utils.stream_close import close_event_stream if TYPE_CHECKING: from octop.infra.agents.manager import AgentManager @@ -167,16 +168,19 @@ async def iter_with_session_auto_approve( held = None decisions = None try: - async for chunk in source: - if isinstance(chunk, dict) and chunk.get("type") == "hitl_required": - allowed = approve_decisions_if_allowed(policies, thread_id, chunk) - if allowed is not None: - held = chunk - decisions = [dict(item) for item in allowed] - continue - held = None - decisions = None - yield chunk + try: + async for chunk in source: + if isinstance(chunk, dict) and chunk.get("type") == "hitl_required": + allowed = approve_decisions_if_allowed(policies, thread_id, chunk) + if allowed is not None: + held = chunk + decisions = [dict(item) for item in allowed] + continue + held = None + decisions = None + yield chunk + finally: + await close_event_stream(source) except Exception: card = held if held is not None else carried if card is None: @@ -487,7 +491,10 @@ async def iter_answer_resolution( if next_card is not None: if outcome is not None: outcome.awaiting_more = True - yield MessageEvent.text(f"{tr('ask.answer_recorded', lang)}\n\n{next_card}") + yield MessageEvent.text( + f"{tr('ask.answer_recorded', lang)}\n\n{next_card}", + metadata={"octop_hitl_pending_id": record.pending_id}, + ) return if resume_message is None: yield MessageEvent.error_event(tr("hitl.expired", lang)) @@ -518,23 +525,26 @@ async def iter_answer_resolution( # Resolve before streaming: a follow-up question registered mid-stream # must not be clobbered by a late mark_resolved on the old record. self._store.mark_resolved(record.pending_id, "approved") + resume_stream = project_resume_stream( + agent_manager, + record.agent_id, + record.thread_id, + decisions, + usage_tracker=usage_tracker or UsageTracker(), + history_tracker=history_tracker, + locale=lang, + projection_state=state, + hitl_coordinator=self, + hitl_ctx=hitl_ctx, + ) try: - async for ev in project_resume_stream( - agent_manager, - record.agent_id, - record.thread_id, - decisions, - usage_tracker=usage_tracker or UsageTracker(), - history_tracker=history_tracker, - locale=lang, - projection_state=state, - hitl_coordinator=self, - hitl_ctx=hitl_ctx, - ): + async for ev in resume_stream: yield ev except Exception as exc: yield MessageEvent.error_event(tr("hitl.resume_failed", lang, error=str(exc))) return + finally: + await close_event_stream(resume_stream) if outcome is not None: outcome.completed_turn = True @@ -678,7 +688,7 @@ async def _iter_approval_decision( "approved" if approve else "rejected" ) try: - async for ev in project_resume_stream( + resume_stream = project_resume_stream( agent_manager, ctx.agent_id, thread_id, @@ -689,15 +699,19 @@ async def _iter_approval_decision( projection_state=projection_state, hitl_coordinator=self, hitl_ctx=hitl_ctx, - ): - if record is not None and not ack_sent: - yield MessageEvent.text( - self._decision_ack(lang, approve=approve, sticky=effective) - ) - ack_sent = True - had_output = True - yield ev - history_completed = True + ) + try: + async for ev in resume_stream: + if record is not None and not ack_sent: + yield MessageEvent.text( + self._decision_ack(lang, approve=approve, sticky=effective) + ) + ack_sent = True + had_output = True + yield ev + history_completed = True + finally: + await close_event_stream(resume_stream) except Exception as exc: self._restore_sticky(stamps) if record is None and not had_output: diff --git a/src/octop/infra/gateway/process/harness_request.py b/src/octop/infra/gateway/process/harness_request.py index 6dcaec811f..4c120058aa 100644 --- a/src/octop/infra/gateway/process/harness_request.py +++ b/src/octop/infra/gateway/process/harness_request.py @@ -13,6 +13,7 @@ attachment_empty_message, attachment_extracted_body, ) +from octop.i18n.domains.channel import telegram_quote_prompt from octop.infra.gateway.media.attachment_hints import ( VISION_MAX_COUNT, content_blocks_need_vision, @@ -190,6 +191,14 @@ async def build_content_from_message( extracted = await _extracted_document_texts(msg, media_backend=media_backend, locale=locale) current_text = "\n\n".join(part for part in [msg.text or "", *file_hints, *extracted] if part) + # Commands and HITL answers are consumed before request assembly. Keep their + # control text intact and add quoted context only to the model's user turn. + if msg.channel_type in ("telegram", "qq"): + quote = msg.metadata.get("reply_quote_text") or ( + None if msg.metadata.get("reply_to_is_own_bot") else msg.metadata.get("reply_to_text") + ) + if isinstance(quote, str) and quote: + current_text = telegram_quote_prompt(quote, locale) + current_text combined_text = _group_turn_text(msg, current_text, workspace=workspace, locale=locale) if not image_blocks: diff --git a/src/octop/infra/gateway/process/message_keys.py b/src/octop/infra/gateway/process/message_keys.py index 6a3bc5e68f..f0a25caa5c 100644 --- a/src/octop/infra/gateway/process/message_keys.py +++ b/src/octop/infra/gateway/process/message_keys.py @@ -4,6 +4,7 @@ import time from typing import Any +from urllib.parse import quote from octop_gateway.models import ImageContent, InboundMessage @@ -131,10 +132,24 @@ def session_key_from_message(msg: InboundMessage, *, agent_id: str) -> str: explicit = meta.get("session_key") if isinstance(explicit, str) and explicit.strip(): return explicit.strip() + subject_id = subject_id_from_message(msg) + if ( + msg.channel_type == "telegram" + and meta.get("topic_isolation") is True + and meta.get("message_thread_id") is not None + ): + # Keep the four-part key shape used by peer-session routing. Escaping + # prevents separators in a channel instance ID from creating collisions. + chat_id = meta.get("chat_id", subject_id) + topic_id = meta.get("message_thread_id") + subject_id = "/".join( + quote(str(part), safe="") + for part in (msg.channel_id, chat_id, topic_id if topic_id is not None else 0) + ) return ThreadRegistry.make_key( agent_id=agent_id, channel_type=msg.channel_type or "unknown", - channel_subject_id=subject_id_from_message(msg), + channel_subject_id=subject_id, channel_chat_type=chat_type_from_message(msg), ) @@ -150,6 +165,8 @@ def images_from_message(msg: InboundMessage) -> list[ImageContent]: "message_type", "chat_type", "chat_id", + "message_thread_id", + "topic_isolation", "chatid", "conversation_id", "user_openid", diff --git a/src/octop/infra/gateway/process/processor.py b/src/octop/infra/gateway/process/processor.py index 617a8e7fd6..5a100cba2b 100644 --- a/src/octop/infra/gateway/process/processor.py +++ b/src/octop/infra/gateway/process/processor.py @@ -89,6 +89,7 @@ get_preferred_model_from_json, ) from octop.infra.utils.locale import resolve_user_locale +from octop.infra.utils.stream_close import close_event_stream from octop.infra.utils.turn_failure import log_failed_tool_results, log_turn_failure from octop.infra.utils.ulid import new_ulid @@ -602,8 +603,11 @@ async def _finalize( history_finalize=_finalize, member_hitl=self.teams.deliver_member_hitl, ) - async for ev in stream: - yield ev + try: + async for ev in stream: + yield ev + finally: + await close_event_stream(stream) if not slash_outcome.completed_turn: return thread_id = self._thread_registry.get_bound_thread_id(session_key) @@ -909,7 +913,7 @@ async def __call__(self, msg: InboundMessage) -> AsyncIterator[MessageEvent]: usage_tracker = UsageTracker() history_tracker = TurnHistoryTracker.from_request({}) answer_outcome = HitlAnswerOutcome() - async for ev in self._hitl.iter_answer_resolution( + answer_stream = self._hitl.iter_answer_resolution( ask_record, msg.text, agent_manager=self._agent_manager, @@ -918,8 +922,12 @@ async def __call__(self, msg: InboundMessage) -> AsyncIterator[MessageEvent]: history_tracker=history_tracker, outcome=answer_outcome, member_hitl=self.teams.deliver_member_hitl, - ): - yield ev + ) + try: + async for ev in answer_stream: + yield ev + finally: + await close_event_stream(answer_stream) if answer_outcome.completed_turn: self._touch_thread_after_turn(ask_record.thread_id, msg.text) if usage_tracker.usage: @@ -1072,7 +1080,7 @@ async def __call__(self, msg: InboundMessage) -> AsyncIterator[MessageEvent]: history_tracker = TurnHistoryTracker.from_request(request) projection_state = StreamProjectionState() try: - async for ev in project_stream( + turn_stream = project_stream( self._agent_manager, agent_id, request, @@ -1089,8 +1097,12 @@ async def __call__(self, msg: InboundMessage) -> AsyncIterator[MessageEvent]: session_key=session_key, channel_type=channel_type, ), - ): - yield ev + ) + try: + async for ev in turn_stream: + yield ev + finally: + await close_event_stream(turn_stream) stream_ok = True hitl_paused = projection_state.hitl_paused except Exception as exc: @@ -1187,7 +1199,7 @@ async def iter_turn_chunks(self, msg: InboundMessage) -> AsyncIterator[dict[str, if cmd is not None and cmd.name in HITL_APPROVAL_COMMANDS: usage_tracker = UsageTracker() slash_outcome = HitlSlashOutcome() - async for ev in self._hitl.iter_slash_resolution( + slash_stream = self._hitl.iter_slash_resolution( cmd, self._slash_ctx( agent_id=agent_id, @@ -1202,9 +1214,13 @@ async def iter_turn_chunks(self, msg: InboundMessage) -> AsyncIterator[dict[str, outcome=slash_outcome, history_finalize=self._complete_resumed_history, member_hitl=self.teams.deliver_member_hitl, - ): - for frame in message_event_to_cli_chunks(ev): - yield _maybe_stamp_team_host(frame, agent_id, team_host) + ) + try: + async for ev in slash_stream: + for frame in message_event_to_cli_chunks(ev): + yield _maybe_stamp_team_host(frame, agent_id, team_host) + finally: + await close_event_stream(slash_stream) thread_id = self._thread_registry.get_bound_thread_id(session_key) if thread_id is None: thread_id = await self._thread_registry.get_or_create_by_key( @@ -1483,6 +1499,9 @@ async def iter_turn_chunks(self, msg: InboundMessage) -> AsyncIterator[dict[str, payload: dict[str, Any] = {"type": "error", "message": message} if error_code: payload["error_code"] = error_code + if error_code == "CONNECTOR_AUTH_REQUIRED" and isinstance(exc, OctopError): + # Tells the chat which connectors to authorize before retrying. + payload["error_details"] = exc.details history_tracker.observe(payload) await self._persist_incomplete_turn(thread_id, history_tracker, title_source=msg.text) failure_saved = True @@ -1577,6 +1596,9 @@ async def iter_hitl_resume_chunks( payload: dict[str, Any] = {"type": "error", "message": message} if error_code: payload["error_code"] = error_code + if error_code == "CONNECTOR_AUTH_REQUIRED" and isinstance(exc, OctopError): + # Tells the chat which connectors to authorize before retrying. + payload["error_details"] = exc.details history_tracker.observe(payload) await self._persist_incomplete_turn(thread_id, history_tracker, title_source=None) failure_saved = True @@ -1908,6 +1930,14 @@ async def _resolve_turn_mcp_servers( if failed: detail = f"mcp load failed: {', '.join(failed)}" if raise_on_failure: + blocked = await self._agent_manager.connector_auth_blocked(user_id, failed) + if blocked: + names = ", ".join(str(item["display_name"]) for item in blocked) + raise OctopError( + ErrorCode.CONNECTOR_AUTH_REQUIRED, + f"connector authorization required: {names}", + details={"connectors": blocked, "servers": names}, + ) raise OctopError(ErrorCode.CONNECTOR_MCP_LOAD_FAILED, detail) logger.warning( "skipping turn MCP for agent=%s user=%s (%s)", diff --git a/src/octop/infra/gateway/process/response_mode.py b/src/octop/infra/gateway/process/response_mode.py index 17da1f5fb4..0a98269272 100644 --- a/src/octop/infra/gateway/process/response_mode.py +++ b/src/octop/infra/gateway/process/response_mode.py @@ -56,6 +56,7 @@ async def collapse_to_invoke_response( """ text_buffer = "" media_buffer: list[ContentPart] = [] + hitl_pending_id: str | None = None try: async for event in events: @@ -66,6 +67,7 @@ async def collapse_to_invoke_response( continue if event.type == MessageEventType.MESSAGE: + hitl_pending_id = event.metadata.get("octop_hitl_pending_id") or hitl_pending_id for part in event.content: if isinstance(part, TextContent): text = part.text.strip() @@ -109,7 +111,13 @@ async def collapse_to_invoke_response( content.append(TextContent(text=final_text)) content.extend(media_buffer) if content: - yield MessageEvent(type=MessageEventType.MESSAGE, content=content) + yield MessageEvent( + type=MessageEventType.MESSAGE, + content=content, + metadata=( + {"octop_hitl_pending_id": hitl_pending_id} if hitl_pending_id else {} + ), + ) yield event return diff --git a/src/octop/infra/gateway/process/stream_project.py b/src/octop/infra/gateway/process/stream_project.py index eb13e0a66c..578efd40e3 100644 --- a/src/octop/infra/gateway/process/stream_project.py +++ b/src/octop/infra/gateway/process/stream_project.py @@ -21,6 +21,7 @@ from octop.infra.gateway.process.usage_record import UsageTracker from octop.infra.history.projection import TurnHistoryTracker from octop.infra.utils.locale import DEFAULT_LOCALE, Locale, normalize_locale +from octop.infra.utils.stream_close import close_event_stream from octop.infra.utils.turn_failure import log_failed_tool_results if TYPE_CHECKING: @@ -208,7 +209,7 @@ def _tool_end(raw: str) -> MessageEvent: if projection_state is not None: projection_state.hitl_paused = True projection_state.hitl_pending_id = record.pending_id - yield MessageEvent.text(card) + yield MessageEvent.text(card, metadata={"octop_hitl_pending_id": record.pending_id}) return elif ctype == "state_update": @@ -253,14 +254,16 @@ async def project_stream( ) -> AsyncIterator[MessageEvent]: del media_backend # IM tool media uses agent.backend directly thread_id = hitl_ctx.thread_id if hitl_ctx is not None else "" - async for ev in _project_chunks( - _with_session_auto_approve( - agent_manager.stream(agent_id, request), - agent_manager=agent_manager, - agent_id=agent_id, - thread_id=thread_id, - hitl_coordinator=hitl_coordinator, - ), + source = agent_manager.stream(agent_id, request) + chunks = _with_session_auto_approve( + source, + agent_manager=agent_manager, + agent_id=agent_id, + thread_id=thread_id, + hitl_coordinator=hitl_coordinator, + ) + chunk_stream = _project_chunks( + chunks, agent_manager=agent_manager, agent_id=agent_id, locale=locale, @@ -269,8 +272,15 @@ async def project_stream( projection_state=projection_state, hitl_coordinator=hitl_coordinator, hitl_ctx=hitl_ctx, - ): - yield ev + ) + try: + async for ev in chunk_stream: + yield ev + finally: + await close_event_stream(chunk_stream) + if chunks is not source: + await close_event_stream(chunks) + await close_event_stream(source) async def project_resume_stream( @@ -286,14 +296,16 @@ async def project_resume_stream( hitl_coordinator: HitlChannelCoordinator | None = None, hitl_ctx: HitlStreamContext | None = None, ) -> AsyncIterator[MessageEvent]: - async for ev in _project_chunks( - _with_session_auto_approve( - agent_manager.resume_hitl(agent_id, thread_id, decisions), - agent_manager=agent_manager, - agent_id=agent_id, - thread_id=thread_id, - hitl_coordinator=hitl_coordinator, - ), + source = agent_manager.resume_hitl(agent_id, thread_id, decisions) + chunks = _with_session_auto_approve( + source, + agent_manager=agent_manager, + agent_id=agent_id, + thread_id=thread_id, + hitl_coordinator=hitl_coordinator, + ) + chunk_stream = _project_chunks( + chunks, agent_manager=agent_manager, agent_id=agent_id, locale=locale, @@ -302,5 +314,12 @@ async def project_resume_stream( projection_state=projection_state, hitl_coordinator=hitl_coordinator, hitl_ctx=hitl_ctx, - ): - yield ev + ) + try: + async for ev in chunk_stream: + yield ev + finally: + await close_event_stream(chunk_stream) + if chunks is not source: + await close_event_stream(chunks) + await close_event_stream(source) diff --git a/src/octop/infra/gateway/threads.py b/src/octop/infra/gateway/threads.py index cc70d46e35..c82bb6297c 100644 --- a/src/octop/infra/gateway/threads.py +++ b/src/octop/infra/gateway/threads.py @@ -459,6 +459,10 @@ def set_contributed(self, thread_id: str, contributed: bool) -> None: def list_threads_for_session(self, *, session_key: str, limit: int = 50) -> list[ThreadRow]: return self._threads.list_by_session(session_key=session_key, limit=limit) + def latest_active_thread(self, *, session_key: str) -> ThreadRow | None: + """Most recent conversation on this channel, ignoring empty threads.""" + return self._threads.latest_active_for_session(session_key=session_key) + def create_thread( self, *, diff --git a/src/octop/infra/knowledge/graph.py b/src/octop/infra/knowledge/graph.py new file mode 100644 index 0000000000..919067869f --- /dev/null +++ b/src/octop/infra/knowledge/graph.py @@ -0,0 +1,391 @@ +"""Build a knowledge base's entity graph and use it at retrieval time. + +A graph is opt-in per knowledge base: it exists once its owner builds it, +and from then on newly indexed documents are added to it. Building needs +the instance-wide extraction model (see :mod:`graph_extract`); reading an +existing graph does not. +""" + +from __future__ import annotations + +import asyncio +import logging +import re +from collections.abc import Callable, Sequence +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from typing import Any + +from octop.infra.knowledge.graph_extract import GraphExtractor, get_graph_capability +from octop.infra.knowledge.graph_store import ( + STATUS_FAILED, + STATUS_PENDING, + STATUS_PROCESSING, + STATUS_READY, + ChunkGraph, + GraphRelation, + KnowledgeGraphStore, + PgKnowledgeGraphStore, + SqliteKnowledgeGraphStore, + entity_key, +) +from octop.infra.knowledge.plane import open_knowledge_index, postgres_database + +logger = logging.getLogger(__name__) + +# One document at a time; its chunks are extracted by a few parallel model calls. +GRAPH_CONCURRENCY = 1 +_CHUNK_WORKERS = 4 +_graph_semaphore: asyncio.Semaphore | None = None + +_MAX_SEEDS = 8 +_MAX_NEIGHBORS = 20 +MAX_FACTS = 8 +_MAX_ENTITY_RELATIONS = 50 +_MAX_ENTITY_PASSAGES = 10 +_MAX_PASSAGE_CHARS = 600 +_ASCII_WORD_RE = re.compile(r"[a-z0-9_]") + +ChunkExtractor = Callable[..., ChunkGraph] + + +def reset_graph_semaphore_for_tests() -> None: + global _graph_semaphore + _graph_semaphore = None + + +def _get_graph_semaphore() -> asyncio.Semaphore: + global _graph_semaphore + if _graph_semaphore is None: + _graph_semaphore = asyncio.Semaphore(GRAPH_CONCURRENCY) + return _graph_semaphore + + +def open_knowledge_graph(services: Any, kb_id: str) -> KnowledgeGraphStore: + """SQLite sidecar graph, or PostgreSQL tables on a PostgreSQL control plane.""" + database = postgres_database(services) + if database is None: + return SqliteKnowledgeGraphStore(kb_id) + return PgKnowledgeGraphStore(kb_id, database) + + +def graph_enabled(services: Any) -> bool: + return bool(get_graph_capability(services.settings_repo.get)["enabled"]) + + +# -- building ---------------------------------------------------------------- + + +def build_document_graph( + services: Any, + kb_id: str, + doc_id: str, + *, + extractor: ChunkExtractor | None = None, +) -> None: + """Extract one indexed document's chunks and replace its part of the graph.""" + store = open_knowledge_graph(services, kb_id) + document = services.knowledge_repo.get_document(doc_id) + if document is None or document.kb_id != kb_id or document.is_dir: + store.delete_document(doc_id) + return + store.mark_document(doc_id, STATUS_PROCESSING) + try: + if document.status != "ready": + raise RuntimeError("knowledge document is not indexed yet") + chunks = open_knowledge_index(services, kb_id).list_doc_chunks(doc_id) + extract = extractor or GraphExtractor(services) + + def run(item: tuple[int, str]) -> ChunkGraph | None: + ordinal, text = item + try: + return extract(filename=document.filename, ordinal=ordinal, text=text) + except Exception: + logger.warning( + "knowledge graph extraction failed for %s chunk %s", + doc_id, + ordinal, + exc_info=True, + ) + return None + + with ThreadPoolExecutor(max_workers=_CHUNK_WORKERS) as pool: + results = list(pool.map(run, chunks)) + extracted = [result for result in results if result is not None] + failed = len(results) - len(extracted) + if chunks and not extracted: + raise RuntimeError("knowledge graph extraction failed for every passage") + note = f"{failed} of {len(results)} passages could not be extracted" if failed else "" + store.replace_document(doc_id, extracted, error=note) + except Exception as exc: + store.mark_document(doc_id, STATUS_FAILED, error=str(exc)) + raise + + +def enqueue_document_graph(services: Any, kb_id: str, doc_id: str) -> asyncio.Task[None]: + """Schedule one document's extraction outside the event loop.""" + loop = asyncio.get_running_loop() + sem = _get_graph_semaphore() + + async def _run() -> None: + async with sem: + try: + await loop.run_in_executor(None, build_document_graph, services, kb_id, doc_id) + except Exception: + logger.warning("knowledge graph build failed for %s", doc_id, exc_info=True) + + return asyncio.create_task(_run()) + + +def mark_document_reindexed(services: Any, kb_id: str, doc_id: str) -> bool: + """After (re)indexing: return True when the document should be extracted again. + + Knowledge bases without a graph are left alone. Otherwise the document's + old rows are dropped, since they describe text that was just replaced, + and it waits as pending until extraction runs (now, or on the next build + when no extraction model is available). + """ + store = open_knowledge_graph(services, kb_id) + if not store.exists(): + return False + store.mark_document(doc_id, STATUS_PENDING) + capability = get_graph_capability( + services.settings_repo.get, getattr(services, "provider_repo", None) + ) + return bool(capability["usable"]) + + +def enqueue_base_graph(services: Any, kb_id: str, doc_ids: Sequence[str]) -> int: + """Queue extraction for *doc_ids* and forget documents that no longer exist.""" + store = open_knowledge_graph(services, kb_id) + wanted = set(doc_ids) + for stale in set(store.document_states()) - wanted: + store.delete_document(stale) + for doc_id in doc_ids: + store.mark_document(doc_id, STATUS_PENDING) + enqueue_document_graph(services, kb_id, doc_id) + return len(doc_ids) + + +def resume_pending_graph_jobs(services: Any) -> None: + """Re-queue extraction that a previous process left queued or half done.""" + try: + capability = get_graph_capability( + services.settings_repo.get, getattr(services, "provider_repo", None) + ) + if not capability["usable"]: + return + for base in services.knowledge_repo.list_all(): + states = open_knowledge_graph(services, base.id).document_states() + for doc_id, state in states.items(): + if state.status in (STATUS_PENDING, STATUS_PROCESSING): + enqueue_document_graph(services, base.id, doc_id) + except Exception: + # The graph is optional; trouble reading it must not stop the server from starting. + logger.warning("could not resume knowledge graph jobs", exc_info=True) + + +# -- retrieval --------------------------------------------------------------- + + +@dataclass(frozen=True) +class GraphExpansion: + """What the graph adds to one query against one knowledge base.""" + + facts: tuple[str, ...] = () + chunks: tuple[tuple[str, int], ...] = () + + +def _mentions(query: str, key: str) -> bool: + """True when *key* occurs in *query* and is not just part of a longer ASCII word.""" + start = query.find(key) + while start >= 0: + end = start + len(key) + before = query[start - 1] if start else "" + after = query[end] if end < len(query) else "" + inside_word = (_ASCII_WORD_RE.match(key[0]) and _ASCII_WORD_RE.match(before or " ")) or ( + _ASCII_WORD_RE.match(key[-1]) and _ASCII_WORD_RE.match(after or " ") + ) + if not inside_word: + return True + start = query.find(key, start + 1) + return False + + +def expand_query( + store: KnowledgeGraphStore, query: str, *, max_facts: int = MAX_FACTS +) -> GraphExpansion: + """Find entities named in *query* and collect their relations and passages. + + Relations of the named entities come first, then relations one more hop + away, so a question about A can still reach what is said about B when the + documents only connect them through a shared entity. + Passages are only taken one hop out: a relation costs a line of context, + while a passage displaces a vector hit, and two hops away is often off topic. + """ + folded = " ".join(query.split()).casefold() + seeds = [key for key in store.match_keys(folded) if _mentions(folded, key)] + if not seeds: + return GraphExpansion() + seeds = sorted(seeds, key=lambda key: (-len(key), key))[:_MAX_SEEDS] + direct = store.relations_of(seeds) + neighbors: dict[str, None] = {} + for relation in direct: + for end in (relation.source, relation.target): + if end not in seeds: + neighbors.setdefault(end) + near = list(neighbors)[:_MAX_NEIGHBORS] + known = {(item.source, item.target, item.label) for item in direct} + indirect = [ + relation + for relation in (store.relations_of(near) if near else []) + if (relation.source, relation.target, relation.label) not in known + ] + # Each list is already sorted by weight. + stated: list[GraphRelation] = [*direct, *indirect][:max_facts] + names = store.entities([end for item in stated for end in (item.source, item.target)]) + facts: list[str] = [] + for item in stated: + source = names[item.source].name if item.source in names else item.source + target = names[item.target].name if item.target in names else item.target + line = f"{source} -> {item.label} -> {target}" + facts.append(f"{line}: {item.description}" if item.description else line) + + chunks: dict[tuple[str, int], None] = {} + for item in direct: + for ref in item.chunks: + chunks.setdefault(ref) + for seed in seeds: + for ref in store.entity_chunks(seed): + chunks.setdefault(ref) + return GraphExpansion(facts=tuple(facts), chunks=tuple(chunks)) + + +# -- views ------------------------------------------------------------------- + + +def graph_stats(store: KnowledgeGraphStore, doc_ids: Sequence[str]) -> dict[str, Any]: + """Counts for the dashboard: graph size and how many documents are in it.""" + states = store.document_states() + entities, relations = store.counts() + tally = {STATUS_PENDING: 0, STATUS_PROCESSING: 0, STATUS_READY: 0, STATUS_FAILED: 0} + partial = 0 + for doc_id in doc_ids: + state = states.get(doc_id) + if state is None: + continue + tally[state.status] = tally.get(state.status, 0) + 1 + if state.status == STATUS_READY and state.error: + partial += 1 + return { + "built": bool(states), + "entities": entities, + "relations": relations, + "documents_total": len(doc_ids), + "documents_ready": tally[STATUS_READY], + "documents_partial": partial, + "documents_pending": tally[STATUS_PENDING] + tally[STATUS_PROCESSING], + "documents_failed": tally[STATUS_FAILED], + } + + +def graph_view(store: KnowledgeGraphStore, *, limit: int, search: str = "") -> dict[str, Any]: + """The most-mentioned entities and the relations between them.""" + keys = store.top_entity_keys(limit, search=search) + entities = store.entities(keys) + edges: dict[tuple[str, str], dict[str, Any]] = {} + for relation in store.relations_among(keys): + # One drawn edge per pair, whichever way the relations point. + pair = (relation.source, relation.target) + edge = edges.get(pair) or edges.get((pair[1], pair[0])) + if edge is None: + edges[pair] = { + "source": relation.source, + "target": relation.target, + "labels": [relation.label], + "weight": relation.weight, + } + continue + edge["weight"] += relation.weight + if relation.label not in edge["labels"] and len(edge["labels"]) < 3: + edge["labels"].append(relation.label) + degrees: dict[str, int] = {} + for edge in edges.values(): + for end in (edge["source"], edge["target"]): + degrees[end] = degrees.get(end, 0) + 1 + return { + "nodes": [ + { + "id": key, + "name": entities[key].name, + "type": entities[key].type, + "mentions": entities[key].mentions, + "degree": degrees.get(key, 0), + } + for key in keys + if key in entities + ], + "edges": list(edges.values()), + } + + +def entity_view(services: Any, kb_id: str, name: str) -> dict[str, Any]: + """One entity with its relations and the passages that mention it.""" + store = open_knowledge_graph(services, kb_id) + key = entity_key(name) + entity = store.entities([key]).get(key) + if entity is None: + raise LookupError("knowledge graph entity not found") + relations = store.relations_of([key])[:_MAX_ENTITY_RELATIONS] + others = store.entities([end for item in relations for end in (item.source, item.target)]) + + def end(other: str) -> dict[str, str]: + found = others.get(other) + return {"id": other, "name": found.name if found else other} + + documents = { + document.id: document + for document in services.knowledge_repo.list_documents(kb_id) + if document.status == "ready" and not document.is_dir + } + index = open_knowledge_index(services, kb_id) + texts: dict[str, dict[int, str]] = {} + passages: list[dict[str, Any]] = [] + for doc_id, ordinal in store.entity_chunks(key, limit=_MAX_ENTITY_PASSAGES): + document = documents.get(doc_id) + if document is None: + continue + if doc_id not in texts: + texts[doc_id] = dict(index.list_doc_chunks(doc_id)) + text = texts[doc_id].get(ordinal) + if text is None: + continue + passages.append( + { + "doc_id": doc_id, + "filename": document.filename, + "path": document.path, + "ordinal": ordinal, + "text": text[:_MAX_PASSAGE_CHARS], + } + ) + return { + "entity": { + "id": entity.key, + "name": entity.name, + "type": entity.type, + "description": entity.description, + "mentions": entity.mentions, + }, + "relations": [ + { + "source": end(item.source), + "target": end(item.target), + "label": item.label, + "description": item.description, + "weight": item.weight, + } + for item in relations + ], + "passages": passages, + } diff --git a/src/octop/infra/knowledge/graph_extract.py b/src/octop/infra/knowledge/graph_extract.py new file mode 100644 index 0000000000..7a72b71a85 --- /dev/null +++ b/src/octop/infra/knowledge/graph_extract.py @@ -0,0 +1,221 @@ +"""Knowledge-graph settings and LLM extraction of entities and relations.""" + +from __future__ import annotations + +import json +from typing import Any + +from langchain_core.messages import HumanMessage + +from octop.infra.agents.providers.model_flags import is_chat_eligible_model +from octop.infra.agents.providers.probe import build_probe_chat_model +from octop.infra.knowledge.graph_store import ChunkEntity, ChunkGraph, ChunkRelation, entity_key +from octop.infra.utils.llm_text import llm_text_content + +_ENABLED_KEY = "knowledge_graph_enabled" +_PROVIDER_ID_KEY = "knowledge_graph_provider_id" +_MODEL_KEY = "knowledge_graph_model" + +ENTITY_TYPES = ( + "person", + "organization", + "location", + "product", + "technology", + "event", + "document", + "concept", + "time", + "other", +) +MAX_ENTITIES_PER_CHUNK = 15 +MAX_RELATIONS_PER_CHUNK = 20 +_MAX_NAME_CHARS = 80 +_MAX_LABEL_CHARS = 60 +_MAX_DESCRIPTION_CHARS = 300 + +_PROMPT = """You extract a knowledge graph from one passage of a document. + +Return ONLY a JSON object, with no prose and no code fence: +{{"entities":[{{"name":"...","type":"...","description":"..."}}], +"relations":[{{"source":"...","target":"...","relation":"...","description":"...","strength":7}}]}} + +Rules: +- Entities are the people, organizations, places, products, systems, events, documents, + concepts and dates the passage is actually about. Skip pronouns and generic words. +- "type" is one of: {types}. +- Use each entity's full name as written in the passage, and reuse exactly that spelling + in "source" and "target". Both ends of a relation must be listed entities. +- "relation" is a short phrase naming how source relates to target; "description" is one + sentence grounded in the passage. +- "strength" is an integer from 1 to 10: how explicitly the passage states the relation. +- Write names, relations and descriptions in the language of the passage. +- At most {max_entities} entities and {max_relations} relations. +- If nothing qualifies, return {{"entities":[],"relations":[]}}. + +Document: {filename} +Passage: +\"\"\" +{text} +\"\"\" +""" + + +def _as_bool(value: str | None) -> bool: + return str(value or "").strip().lower() == "true" + + +def _provider(provider_repo: Any, provider_id: str) -> Any | None: + if provider_repo is None or not provider_id.isdigit(): + return None + return provider_repo.get(int(provider_id)) + + +def graph_models(provider: Any) -> list[dict[str, str]]: + """Chat models of *provider* that can run extraction.""" + if not (provider and provider.enabled and provider.api_key and provider.base_url): + return [] + return [ + {"id": str(model["id"]), "name": str(model.get("name") or model["id"])} + for model in provider.get_models() + if str(model.get("id") or "").strip() + and is_chat_eligible_model( + model, provider_name=provider.name, provider_api_key=provider.api_key + ) + ] + + +def _ready(provider: Any | None, model_id: str) -> bool: + return bool(model_id) and any(model["id"] == model_id for model in graph_models(provider)) + + +def get_graph_capability(settings_get: Any, provider_repo: Any = None) -> dict[str, Any]: + """Return the stored extraction model and whether graphs can be built now.""" + enabled = _as_bool(settings_get(_ENABLED_KEY)) + provider_id = (settings_get(_PROVIDER_ID_KEY) or "").strip() + model = (settings_get(_MODEL_KEY) or "").strip() + ready = _ready(_provider(provider_repo, provider_id), model) + return { + "enabled": enabled, + "provider_id": provider_id, + "model": model, + "ready": ready, + "usable": enabled and ready, + } + + +def set_graph_settings( + settings_set: Any, + *, + enabled: bool, + provider_id: str | None, + model: str | None, + provider_repo: Any = None, +) -> None: + """Persist the extraction model; enabling requires a ready chat model.""" + selected_provider = (provider_id or "").strip() + selected_model = (model or "").strip() + if enabled and not _ready(_provider(provider_repo, selected_provider), selected_model): + raise ValueError("knowledge graph model is not available on the selected provider") + settings_set(_ENABLED_KEY, "true" if enabled else "false") + settings_set(_PROVIDER_ID_KEY, selected_provider) + settings_set(_MODEL_KEY, selected_model) + + +def _clip(value: object, limit: int) -> str: + return " ".join(str(value or "").split())[:limit].strip() + + +def _strength(value: object) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float, str)): + return 5.0 + try: + number = float(value) + except ValueError: + return 5.0 + return min(max(number, 1.0), 10.0) + + +def _decode(reply: str) -> dict[str, Any]: + # Models often wrap the object in a code fence or a sentence; take the object itself. + text = reply.strip() + start, end = text.find("{"), text.rfind("}") + if start < 0 or end <= start: + raise ValueError("knowledge graph extraction returned no JSON object") + try: + decoded = json.loads(text[start : end + 1]) + except json.JSONDecodeError as exc: + raise ValueError("knowledge graph extraction returned invalid JSON") from exc + if not isinstance(decoded, dict): + raise ValueError("knowledge graph extraction returned no JSON object") + return decoded + + +def parse_extraction(reply: str, *, ordinal: int) -> ChunkGraph: + """Turn one model reply into validated entities and relations.""" + decoded = _decode(reply) + entities: dict[str, ChunkEntity] = {} + raw_entities = decoded.get("entities") + for item in raw_entities if isinstance(raw_entities, list) else []: + if not isinstance(item, dict): + continue + name = _clip(item.get("name"), _MAX_NAME_CHARS) + key = entity_key(name) + if len(key) < 2 or key in entities or len(entities) >= MAX_ENTITIES_PER_CHUNK: + continue + kind = _clip(item.get("type"), 30).lower() + entities[key] = ChunkEntity( + name=name, + type=kind if kind in ENTITY_TYPES else "other", + description=_clip(item.get("description"), _MAX_DESCRIPTION_CHARS), + ) + relations: dict[tuple[str, str, str], ChunkRelation] = {} + raw_relations = decoded.get("relations") + for item in raw_relations if isinstance(raw_relations, list) else []: + if not isinstance(item, dict) or len(relations) >= MAX_RELATIONS_PER_CHUNK: + continue + source = entity_key(_clip(item.get("source"), _MAX_NAME_CHARS)) + target = entity_key(_clip(item.get("target"), _MAX_NAME_CHARS)) + label = _clip(item.get("relation"), _MAX_LABEL_CHARS) + # A relation to something the model did not list as an entity is a guess; drop it. + if source not in entities or target not in entities or source == target or not label: + continue + relations.setdefault( + (source, target, label.casefold()), + ChunkRelation( + source=entities[source].name, + target=entities[target].name, + label=label, + description=_clip(item.get("description"), _MAX_DESCRIPTION_CHARS), + strength=_strength(item.get("strength")), + ), + ) + return ChunkGraph( + ordinal=ordinal, + entities=tuple(entities.values()), + relations=tuple(relations.values()), + ) + + +class GraphExtractor: + """Extract one chunk's entities and relations with the configured chat model.""" + + def __init__(self, services: Any) -> None: + capability = get_graph_capability( + services.settings_repo.get, getattr(services, "provider_repo", None) + ) + if not capability["usable"]: + raise RuntimeError("knowledge graph extraction is not ready") + provider = _provider(services.provider_repo, capability["provider_id"]) + self._model = build_probe_chat_model(provider, model_id=capability["model"]) + + def __call__(self, *, filename: str, ordinal: int, text: str) -> ChunkGraph: + prompt = _PROMPT.format( + types=", ".join(ENTITY_TYPES), + max_entities=MAX_ENTITIES_PER_CHUNK, + max_relations=MAX_RELATIONS_PER_CHUNK, + filename=filename, + text=text, + ) + reply = llm_text_content(self._model.invoke([HumanMessage(content=prompt)])) + return parse_extraction(reply, ordinal=ordinal) diff --git a/src/octop/infra/knowledge/graph_store.py b/src/octop/infra/knowledge/graph_store.py new file mode 100644 index 0000000000..945ee69374 --- /dev/null +++ b/src/octop/infra/knowledge/graph_store.py @@ -0,0 +1,499 @@ +"""Entity and relation rows extracted from knowledge-base chunks. + +The graph is derived index data, stored next to the vector index: a +``graph.sqlite`` sidecar per knowledge base, or ``public.knowledge_graph_*`` +tables when the control plane is PostgreSQL. Rows are kept per chunk and +merged by normalized entity name when read, so replacing or deleting one +document never leaves orphans behind. +""" + +from __future__ import annotations + +import sqlite3 +from collections import Counter +from collections.abc import Iterator, Sequence +from contextlib import contextmanager +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from octop.config import DatabaseConfig +from octop.infra.utils.paths import PathLayout + +STATUS_PENDING = "pending" +STATUS_PROCESSING = "processing" +STATUS_READY = "ready" +STATUS_FAILED = "failed" + +_IN_BATCH = 400 + + +def entity_key(name: str) -> str: + """Normalized identity of an entity name (case and spacing insensitive).""" + return " ".join(str(name or "").split()).casefold() + + +@dataclass(frozen=True) +class ChunkEntity: + name: str + type: str = "" + description: str = "" + + +@dataclass(frozen=True) +class ChunkRelation: + source: str + target: str + label: str + description: str = "" + strength: float = 5.0 + + +@dataclass(frozen=True) +class ChunkGraph: + """What one chunk contributed.""" + + ordinal: int + entities: tuple[ChunkEntity, ...] = () + relations: tuple[ChunkRelation, ...] = () + + +@dataclass(frozen=True) +class GraphEntity: + key: str + name: str + type: str + description: str + mentions: int + + +@dataclass(frozen=True) +class GraphRelation: + source: str + target: str + label: str + description: str + weight: float + chunks: tuple[tuple[str, int], ...] = field(default=()) + + +@dataclass(frozen=True) +class GraphDocumentState: + status: str + error: str = "" + + +class KnowledgeGraphStore: + """Graph rows of one knowledge base. Subclasses supply the connection.""" + + _entities = "knowledge_graph_entities" + _relations = "knowledge_graph_relations" + _documents = "knowledge_graph_documents" + _mark = "?" + + def __init__(self, kb_id: str) -> None: + self._kb_id = kb_id + + # -- connection plumbing ------------------------------------------------- + + def _connect(self) -> Any: + raise NotImplementedError + + def _present(self) -> bool: + """False when nothing was ever written, so reads can skip the storage.""" + return True + + def _prepare(self, conn: Any) -> None: + for statement in ( + f""" + CREATE TABLE IF NOT EXISTS {self._entities} ( + kb_id TEXT NOT NULL, + doc_id TEXT NOT NULL, + ordinal INTEGER NOT NULL, + name_key TEXT NOT NULL, + name TEXT NOT NULL, + type TEXT NOT NULL DEFAULT '', + description TEXT NOT NULL DEFAULT '', + PRIMARY KEY (kb_id, doc_id, ordinal, name_key) + ) + """, + f""" + CREATE INDEX IF NOT EXISTS idx_knowledge_graph_entities_key + ON {self._entities} (kb_id, name_key) + """, + f""" + CREATE TABLE IF NOT EXISTS {self._relations} ( + kb_id TEXT NOT NULL, + doc_id TEXT NOT NULL, + ordinal INTEGER NOT NULL, + source_key TEXT NOT NULL, + target_key TEXT NOT NULL, + label TEXT NOT NULL, + description TEXT NOT NULL DEFAULT '', + strength REAL NOT NULL DEFAULT 5, + PRIMARY KEY (kb_id, doc_id, ordinal, source_key, target_key, label) + ) + """, + f""" + CREATE INDEX IF NOT EXISTS idx_knowledge_graph_relations_source + ON {self._relations} (kb_id, source_key) + """, + f""" + CREATE INDEX IF NOT EXISTS idx_knowledge_graph_relations_target + ON {self._relations} (kb_id, target_key) + """, + f""" + CREATE TABLE IF NOT EXISTS {self._documents} ( + kb_id TEXT NOT NULL, + doc_id TEXT NOT NULL, + status TEXT NOT NULL, + error TEXT NOT NULL DEFAULT '', + PRIMARY KEY (kb_id, doc_id) + ) + """, + ): + conn.execute(statement) + + @contextmanager + def _session(self) -> Iterator[Any]: + conn = self._connect() + try: + yield conn + conn.commit() + finally: + conn.close() + + def _sql(self, statement: str) -> str: + if self._mark == "?": + return statement + # psycopg reads a bare % as a placeholder, so literal ones are doubled. + return statement.replace("%", "%%").replace("?", self._mark) + + def _fetch(self, statement: str, params: Sequence[object] = ()) -> list[tuple[Any, ...]]: + if not self._present(): + return [] + with self._session() as conn: + rows = conn.execute(self._sql(statement), tuple(params)).fetchall() + return [tuple(row) for row in rows] + + def _fetch_in( + self, statement: str, keys: Sequence[str], *, repeat: int = 1 + ) -> list[tuple[Any, ...]]: + """Run *statement* once per batch of keys; ``{keys}`` expands to placeholders.""" + rows: list[tuple[Any, ...]] = [] + unique = list(dict.fromkeys(keys)) + for start in range(0, len(unique), _IN_BATCH): + batch = unique[start : start + _IN_BATCH] + marks = ",".join("?" for _ in batch) + rows.extend( + self._fetch( + statement.replace("{keys}", marks), + [self._kb_id, *(batch * repeat)], + ) + ) + return rows + + # -- writes -------------------------------------------------------------- + + def _delete_rows(self, conn: Any, doc_id: str) -> None: + for table in (self._entities, self._relations): + conn.execute( + self._sql(f"DELETE FROM {table} WHERE kb_id = ? AND doc_id = ?"), + (self._kb_id, doc_id), + ) + + def _set_state(self, conn: Any, doc_id: str, status: str, error: str) -> None: + conn.execute( + self._sql(f"DELETE FROM {self._documents} WHERE kb_id = ? AND doc_id = ?"), + (self._kb_id, doc_id), + ) + conn.execute( + self._sql( + f"INSERT INTO {self._documents}(kb_id, doc_id, status, error) VALUES (?, ?, ?, ?)" + ), + (self._kb_id, doc_id, status, error), + ) + + def replace_document( + self, doc_id: str, chunks: Sequence[ChunkGraph], *, error: str = "" + ) -> None: + """Atomically replace everything one document contributed and mark it ready.""" + entity_rows: dict[tuple[int, str], tuple[object, ...]] = {} + relation_rows: dict[tuple[int, str, str, str], tuple[object, ...]] = {} + for chunk in chunks: + for entity in chunk.entities: + key = entity_key(entity.name) + if key: + entity_rows.setdefault( + (chunk.ordinal, key), + ( + self._kb_id, + doc_id, + chunk.ordinal, + key, + " ".join(entity.name.split()), + entity.type, + entity.description, + ), + ) + for relation in chunk.relations: + source, target = entity_key(relation.source), entity_key(relation.target) + if not source or not target or source == target or not relation.label: + continue + relation_rows.setdefault( + (chunk.ordinal, source, target, relation.label), + ( + self._kb_id, + doc_id, + chunk.ordinal, + source, + target, + relation.label, + relation.description, + float(relation.strength), + ), + ) + with self._session() as conn: + self._delete_rows(conn, doc_id) + cursor = conn.cursor() + if entity_rows: + cursor.executemany( + self._sql( + f"INSERT INTO {self._entities}" + "(kb_id, doc_id, ordinal, name_key, name, type, description) " + "VALUES (?, ?, ?, ?, ?, ?, ?)" + ), + list(entity_rows.values()), + ) + if relation_rows: + cursor.executemany( + self._sql( + f"INSERT INTO {self._relations}" + "(kb_id, doc_id, ordinal, source_key, target_key, label, " + "description, strength) VALUES (?, ?, ?, ?, ?, ?, ?, ?)" + ), + list(relation_rows.values()), + ) + self._set_state(conn, doc_id, STATUS_READY, error) + + def mark_document(self, doc_id: str, status: str, *, error: str = "") -> None: + """Record a document's graph state; anything but ``ready`` drops its rows.""" + with self._session() as conn: + if status != STATUS_READY: + self._delete_rows(conn, doc_id) + self._set_state(conn, doc_id, status, error) + + def delete_document(self, doc_id: str) -> None: + if not self._present(): + return + with self._session() as conn: + self._delete_rows(conn, doc_id) + conn.execute( + self._sql(f"DELETE FROM {self._documents} WHERE kb_id = ? AND doc_id = ?"), + (self._kb_id, doc_id), + ) + + def clear(self) -> None: + """Drop the whole graph of this knowledge base.""" + if not self._present(): + return + with self._session() as conn: + for table in (self._entities, self._relations, self._documents): + conn.execute(self._sql(f"DELETE FROM {table} WHERE kb_id = ?"), (self._kb_id,)) + + # -- reads --------------------------------------------------------------- + + def document_states(self) -> dict[str, GraphDocumentState]: + rows = self._fetch( + f"SELECT doc_id, status, error FROM {self._documents} WHERE kb_id = ?", + (self._kb_id,), + ) + return { + str(doc_id): GraphDocumentState(str(status), str(error or "")) + for doc_id, status, error in rows + } + + def exists(self) -> bool: + """True once a graph was built here (even if every document later left it).""" + rows = self._fetch( + f"SELECT 1 FROM {self._documents} WHERE kb_id = ? LIMIT 1", (self._kb_id,) + ) + return bool(rows) + + def counts(self) -> tuple[int, int]: + """Return ``(entities, relations)`` after merging by name.""" + entities = self._fetch( + f"SELECT COUNT(DISTINCT name_key) FROM {self._entities} WHERE kb_id = ?", + (self._kb_id,), + ) + relations = self._fetch( + "SELECT COUNT(*) FROM (SELECT DISTINCT source_key, target_key, label " + f"FROM {self._relations} WHERE kb_id = ?) AS merged", + (self._kb_id,), + ) + return ( + int(entities[0][0]) if entities else 0, + int(relations[0][0]) if relations else 0, + ) + + def top_entity_keys(self, limit: int, *, search: str = "") -> list[str]: + """Most-mentioned entity keys, optionally filtered by a name fragment.""" + if limit <= 0: + return [] + params: list[object] = [self._kb_id] + where = "" + fragment = entity_key(search) + if fragment: + escaped = fragment.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + where = " AND name_key LIKE ? ESCAPE '\\'" + params.append(f"%{escaped}%") + params.append(limit) + rows = self._fetch( + f"SELECT name_key, COUNT(*) AS mentions FROM {self._entities} " + f"WHERE kb_id = ?{where} GROUP BY name_key " + "ORDER BY mentions DESC, name_key LIMIT ?", + params, + ) + return [str(row[0]) for row in rows] + + def match_keys(self, text: str) -> list[str]: + """Entity keys whose name occurs inside *text* (a user query).""" + haystack = entity_key(text) + if not haystack: + return [] + rows = self._fetch( + f"SELECT DISTINCT name_key FROM {self._entities} " + "WHERE kb_id = ? AND ? LIKE '%' || name_key || '%'", + (self._kb_id, haystack), + ) + # LIKE treats % and _ inside a name as wildcards; confirm the literal match here. + return [str(row[0]) for row in rows if str(row[0]) in haystack] + + def entities(self, keys: Sequence[str]) -> dict[str, GraphEntity]: + """Merge per-chunk rows into one entity per key.""" + rows = self._fetch_in( + f"SELECT name_key, name, type, description FROM {self._entities} " + "WHERE kb_id = ? AND name_key IN ({keys})", + keys, + ) + names: dict[str, Counter[str]] = {} + types: dict[str, Counter[str]] = {} + descriptions: dict[str, str] = {} + mentions: Counter[str] = Counter() + for key, name, kind, description in rows: + key = str(key) + mentions[key] += 1 + names.setdefault(key, Counter())[str(name)] += 1 + if kind: + types.setdefault(key, Counter())[str(kind)] += 1 + if len(str(description or "")) > len(descriptions.get(key, "")): + descriptions[key] = str(description) + return { + key: GraphEntity( + key=key, + name=_most_common(names[key]), + type=_most_common(types[key]) if key in types else "", + description=descriptions.get(key, ""), + mentions=count, + ) + for key, count in mentions.items() + } + + def relations_among(self, keys: Sequence[str]) -> list[GraphRelation]: + """Relations whose two ends are both in *keys*.""" + wanted = set(keys) + return [ + relation + for relation in self.relations_of(keys) + if relation.source in wanted and relation.target in wanted + ] + + def relations_of(self, keys: Sequence[str]) -> list[GraphRelation]: + """Relations touching any of *keys*, merged and sorted by weight.""" + rows = self._fetch_in( + "SELECT source_key, target_key, label, description, strength, doc_id, ordinal " + f"FROM {self._relations} " + "WHERE kb_id = ? AND (source_key IN ({keys}) OR target_key IN ({keys}))", + keys, + repeat=2, + ) + weights: dict[tuple[str, str, str], float] = {} + descriptions: dict[tuple[str, str, str], str] = {} + chunks: dict[tuple[str, str, str], dict[tuple[str, int], None]] = {} + for source, target, label, description, strength, doc_id, ordinal in rows: + ident = (str(source), str(target), str(label)) + ref = (str(doc_id), int(ordinal)) + seen = chunks.setdefault(ident, {}) + if ref in seen: + continue # the same row can come back from two key batches + seen[ref] = None + weights[ident] = weights.get(ident, 0.0) + float(strength) + if len(str(description or "")) > len(descriptions.get(ident, "")): + descriptions[ident] = str(description) + merged = [ + GraphRelation( + source=ident[0], + target=ident[1], + label=ident[2], + description=descriptions.get(ident, ""), + weight=weight, + chunks=tuple(chunks[ident]), + ) + for ident, weight in weights.items() + ] + return sorted(merged, key=lambda item: (-item.weight, item.source, item.target, item.label)) + + def entity_chunks(self, key: str, *, limit: int = 20) -> list[tuple[str, int]]: + """Chunks that mention one entity, in document order.""" + rows = self._fetch( + f"SELECT doc_id, ordinal FROM {self._entities} " + "WHERE kb_id = ? AND name_key = ? ORDER BY doc_id, ordinal LIMIT ?", + (self._kb_id, key, limit), + ) + return [(str(doc_id), int(ordinal)) for doc_id, ordinal in rows] + + +def _most_common(counter: Counter[str]) -> str: + # Ties resolve by name so the picked spelling does not depend on row order. + return min(counter.items(), key=lambda item: (-item[1], item[0]))[0] + + +class SqliteKnowledgeGraphStore(KnowledgeGraphStore): + """``graph.sqlite`` beside the knowledge base's vector index.""" + + def __init__(self, kb_id: str) -> None: + super().__init__(kb_id) + self._path = PathLayout.from_env().knowledge_dir / kb_id / "graph.sqlite" + + @property + def path(self) -> Path: + return self._path + + def _present(self) -> bool: + return self._path.is_file() + + def _connect(self) -> Any: + self._path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(self._path) + self._prepare(conn) + return conn + + +class PgKnowledgeGraphStore(KnowledgeGraphStore): + """``public.knowledge_graph_*`` tables shared by every knowledge base.""" + + _entities = "public.knowledge_graph_entities" + _relations = "public.knowledge_graph_relations" + _documents = "public.knowledge_graph_documents" + _mark = "%s" + + def __init__(self, kb_id: str, database: DatabaseConfig) -> None: + super().__init__(kb_id) + self._conninfo = database.postgresql_conninfo() + with self._session() as conn: + self._prepare(conn) + + def _connect(self) -> Any: + import psycopg + + return psycopg.connect(self._conninfo) diff --git a/src/octop/infra/knowledge/index.py b/src/octop/infra/knowledge/index.py index 1945e339f7..73921ab3bd 100644 --- a/src/octop/infra/knowledge/index.py +++ b/src/octop/infra/knowledge/index.py @@ -100,6 +100,25 @@ def list_doc_chunks(self, doc_id: str) -> list[tuple[int, str]]: ).fetchall() return [(int(ordinal), str(text)) for ordinal, text in rows] + def read_doc_chunks(self, doc_id: str) -> list[tuple[str, list[float], dict[str, object]]]: + """Return one document's chunks as ``(text, embedding, metadata)``, in order.""" + with self._connect() as conn: + rows = conn.execute( + "SELECT text, embedding, meta_json FROM chunks WHERE doc_id = ? ORDER BY ordinal", + (doc_id,), + ).fetchall() + chunks: list[tuple[str, list[float], dict[str, object]]] = [] + for text, blob, meta_json in rows: + decoded = json.loads(meta_json) + chunks.append( + ( + str(text), + list(struct.unpack(f"<{len(blob) // 4}f", blob)), + decoded if isinstance(decoded, dict) else {}, + ) + ) + return chunks + def delete_doc(self, doc_id: str) -> None: with self._connect() as conn: conn.execute("DELETE FROM chunks WHERE doc_id = ?", (doc_id,)) diff --git a/src/octop/infra/knowledge/jobs.py b/src/octop/infra/knowledge/jobs.py index beb4bc49a0..8173827bd3 100644 --- a/src/octop/infra/knowledge/jobs.py +++ b/src/octop/infra/knowledge/jobs.py @@ -3,17 +3,21 @@ from __future__ import annotations import asyncio +import logging from typing import Any from octop.infra.knowledge.blobs import KnowledgeBlobs from octop.infra.knowledge.chunk import chunk_text from octop.infra.knowledge.embed import embed_knowledge_texts from octop.infra.knowledge.gate import assert_knowledge_usable +from octop.infra.knowledge.graph import enqueue_document_graph, mark_document_reindexed from octop.infra.knowledge.ocr import optional_ocr_extractor from octop.infra.knowledge.params import get_advanced_settings from octop.infra.knowledge.parse import parse_document from octop.infra.knowledge.plane import open_knowledge_index +logger = logging.getLogger(__name__) + # Parsing/OCR also consumes memory; only one document should be in flight. INDEX_CONCURRENCY = 1 _index_semaphore: asyncio.Semaphore | None = None @@ -38,9 +42,10 @@ def process_document(services: Any, kb_id: str, doc_id: str) -> None: assert_knowledge_usable(services.settings_repo.get, getattr(services, "provider_repo", None)) document = repo.get_document(doc_id) base = repo.get_base(kb_id) - if document is None or document.kb_id != kb_id or base is None: + if document is None or base is None: raise LookupError("knowledge document or base not found") - if document.is_dir: + if document.kb_id != kb_id or document.is_dir: + # Moved to another base after this job was queued; the move queued it there. return repo.update_document(doc_id, status="processing", error_message="") try: @@ -69,6 +74,16 @@ def enqueue_index_document(services: Any, kb_id: str, doc_id: str) -> asyncio.Ta async def _run() -> None: async with sem: await loop.run_in_executor(None, process_document, services, kb_id, doc_id) + # The document is searchable now; its graph is refreshed outside the index slot. + try: + stale = await loop.run_in_executor( + None, mark_document_reindexed, services, kb_id, doc_id + ) + except Exception: + logger.warning("knowledge graph refresh skipped for %s", doc_id, exc_info=True) + return + if stale: + enqueue_document_graph(services, kb_id, doc_id) return asyncio.create_task(_run()) diff --git a/src/octop/infra/knowledge/pg_index.py b/src/octop/infra/knowledge/pg_index.py index e2d7a118fa..75d85862e9 100644 --- a/src/octop/infra/knowledge/pg_index.py +++ b/src/octop/infra/knowledge/pg_index.py @@ -1,7 +1,7 @@ """pgvector chunk index for one knowledge base. ``CREATE EXTENSION vector`` is an operator step (see -``docker/postgres/init-vector.sql`` and ADR 002). This module only creates +``deploy/docker/postgres/init-vector.sql`` and ADR 002). This module only creates ``public.knowledge_chunks`` once the ``vector`` type exists. """ @@ -42,7 +42,7 @@ def _ensure_schema(self) -> None: if present is None: raise RuntimeError( "pgvector is not installed on this PostgreSQL. " - "Run CREATE EXTENSION vector (docker/postgres/init-vector.sql) " + "Run CREATE EXTENSION vector (deploy/docker/postgres/init-vector.sql) " "before using knowledge search with database.driver=postgresql." ) adopt_legacy_content_tables(self._database) @@ -129,6 +129,29 @@ def list_doc_chunks(self, doc_id: str) -> list[tuple[int, str]]: ).fetchall() return [(int(ordinal), str(text)) for ordinal, text in rows] + def read_doc_chunks(self, doc_id: str) -> list[tuple[str, list[float], dict[str, object]]]: + """Return one document's chunks as ``(text, embedding, metadata)``, in order.""" + with self._connect() as conn: + rows = conn.execute( + f""" + SELECT text, embedding::text, meta_json FROM {_TABLE} + WHERE kb_id = %s AND doc_id = %s + ORDER BY ordinal + """, + (self._kb_id, doc_id), + ).fetchall() + chunks: list[tuple[str, list[float], dict[str, object]]] = [] + for text, embedding, meta_json in rows: + decoded = json.loads(meta_json) if isinstance(meta_json, str) else meta_json + chunks.append( + ( + str(text), + [float(value) for value in json.loads(embedding)], + decoded if isinstance(decoded, dict) else {}, + ) + ) + return chunks + def delete_doc(self, doc_id: str) -> None: with self._connect() as conn: conn.execute( diff --git a/src/octop/infra/knowledge/rerank.py b/src/octop/infra/knowledge/rerank.py new file mode 100644 index 0000000000..df1d64ae63 --- /dev/null +++ b/src/octop/infra/knowledge/rerank.py @@ -0,0 +1,165 @@ +"""Optional second-stage ranking of retrieved knowledge chunks. + +Vector search is recall-oriented: it brings back passages that look like the +question. A reranker reads the question together with each passage and scores +how well the passage answers it, so retrieval fetches a wider candidate set and +keeps the reranker's top ``k``. + +The model is a ``/rerank`` endpoint of a configured provider (the Jina / Cohere +request shape, which SiliconFlow, vLLM, Xinference and TEI also serve). +""" + +from __future__ import annotations + +import json +from typing import Any + +import httpx + +from octop.infra.agents.providers.model_flags import is_rerank_model +from octop.infra.agents.providers.opencode_session import ensure_opencode_session_header +from octop.infra.agents.providers.probe import provider_headers + +_ENABLED_KEY = "knowledge_rerank_enabled" +_PROVIDER_ID_KEY = "knowledge_rerank_provider_id" +_MODEL_KEY = "knowledge_rerank_model" + +# How many vector hits the reranker gets to choose from, relative to ``k``. +_CANDIDATE_MULTIPLIER = 4 +_MIN_CANDIDATES = 20 +_MAX_CANDIDATES = 50 +_TIMEOUT_SEC = 30.0 + + +def _as_bool(value: str | None) -> bool: + if value is None: + return False + try: + decoded: Any = json.loads(value) + except json.JSONDecodeError: + decoded = value + if isinstance(decoded, bool): + return decoded + return str(decoded).strip().lower() == "true" + + +def rerank_models(provider: Any) -> list[dict[str, str]]: + """Rerank models a provider offers, as ``{id, name}``.""" + return [ + {"id": str(model["id"]), "name": str(model.get("name") or model["id"])} + for model in provider.get_models() + if str(model.get("id") or "").strip() and is_rerank_model(model) + ] + + +def _provider(provider_repo: Any, provider_id: str) -> Any | None: + if provider_repo is None or not provider_id.isdigit(): + return None + return provider_repo.get(int(provider_id)) + + +def _ready(provider: Any | None, model: str) -> bool: + return bool( + provider + and provider.enabled + and provider.api_key + and provider.base_url + and any(candidate["id"] == model for candidate in rerank_models(provider)) + ) + + +def get_rerank_capability(settings_get: Any, provider_repo: Any = None) -> dict[str, Any]: + """Stored rerank choice and whether it can be used right now.""" + provider_id = (settings_get(_PROVIDER_ID_KEY) or "").strip() + model = (settings_get(_MODEL_KEY) or "").strip() + enabled = _as_bool(settings_get(_ENABLED_KEY)) + ready = _ready(_provider(provider_repo, provider_id), model) + return { + "enabled": enabled, + "provider_id": provider_id, + "model": model, + "ready": ready, + "usable": enabled and ready, + } + + +def set_rerank_settings( + settings_set: Any, + *, + enabled: bool, + provider_id: str | None, + model: str | None, + provider_repo: Any, +) -> None: + """Persist the rerank choice; enabling requires a provider that serves the model.""" + provider_id = (provider_id or "").strip() + model = (model or "").strip() + if enabled and not _ready(_provider(provider_repo, provider_id), model): + raise ValueError("knowledge rerank model is not available on the selected provider") + settings_set(_ENABLED_KEY, "true" if enabled else "false") + if provider_id or model or not enabled: + settings_set(_PROVIDER_ID_KEY, provider_id) + settings_set(_MODEL_KEY, model) + + +def candidate_count(k: int) -> int: + """Vector hits to fetch so the reranker has something to choose from.""" + return max(k, min(max(k * _CANDIDATE_MULTIPLIER, _MIN_CANDIDATES), _MAX_CANDIDATES)) + + +def rerank_scores(services: Any, query: str, texts: list[str]) -> list[float] | None: + """Relevance score per text, aligned with ``texts``; None when reranking is off. + + Raises when the provider call fails, so the caller can fall back to the + vector order and say why. + """ + capability = get_rerank_capability(services.settings_repo.get, services.provider_repo) + if not capability["usable"] or not texts: + return None + provider = services.provider_repo.get(int(capability["provider_id"])) + headers: dict[str, str] = {"Authorization": f"Bearer {provider.api_key}"} + extra = ensure_opencode_session_header( + getattr(provider, "name", None), + provider_headers(provider), + base_url=provider.base_url, + ) + if extra: + headers.update(extra) + with httpx.Client(timeout=_TIMEOUT_SEC) as client: + response = client.post( + f"{provider.base_url.rstrip('/')}/rerank", + headers=headers, + json={ + "model": capability["model"], + "query": query, + "documents": texts, + "top_n": len(texts), + }, + ) + response.raise_for_status() + return _parse_scores(response.json(), len(texts)) + + +def _parse_scores(payload: Any, count: int) -> list[float]: + """Read ``results[].{index, relevance_score}`` (also ``output.results`` and ``score``).""" + results = payload + if isinstance(payload, dict): + output = payload.get("output") + results = payload.get("results") or ( + output.get("results") if isinstance(output, dict) else None + ) + if not isinstance(results, list): + raise RuntimeError("knowledge rerank response has no results") + # Texts a provider leaves out (it may ignore ``top_n``) rank below every scored one. + scores = [float("-inf")] * count + for item in results: + if not isinstance(item, dict): + raise RuntimeError("knowledge rerank response has an invalid result") + index = item.get("index") + score = item.get("relevance_score", item.get("score")) + if not isinstance(index, int) or isinstance(index, bool) or not 0 <= index < count: + raise RuntimeError("knowledge rerank response has an invalid index") + if not isinstance(score, (int, float)) or isinstance(score, bool): + raise RuntimeError("knowledge rerank response has an invalid score") + scores[index] = float(score) + return scores diff --git a/src/octop/infra/knowledge/retrieve.py b/src/octop/infra/knowledge/retrieve.py index 81b1286f6a..9e88fa55a6 100644 --- a/src/octop/infra/knowledge/retrieve.py +++ b/src/octop/infra/knowledge/retrieve.py @@ -11,11 +11,22 @@ from octop.infra.knowledge.citations import append_citations_marker, citations_from_ranked from octop.infra.knowledge.embed import embed_knowledge_texts from octop.infra.knowledge.gate import assert_knowledge_usable +from octop.infra.knowledge.graph import ( + MAX_FACTS, + GraphExpansion, + expand_query, + graph_enabled, + open_knowledge_graph, +) from octop.infra.knowledge.index import Hit from octop.infra.knowledge.plane import open_knowledge_index +from octop.infra.knowledge.rerank import candidate_count, get_rerank_capability, rerank_scores DEFAULT_RETRIEVAL_K = 8 DEFAULT_CONTEXT_CHAR_BUDGET = 6000 +# Share of the returned passages, and of the character budget, the graph may claim. +_GRAPH_PASSAGE_DIVISOR = 4 +_GRAPH_FACT_BUDGET_DIVISOR = 5 logger = logging.getLogger(__name__) @@ -82,8 +93,16 @@ def _retrieve_context_sync( ) visible_by_id = {base.id: base for base in visible} selected_ids = _unique_ids(knowledge_base_ids) + rerank = get_rerank_capability( + services.settings_repo.get, getattr(services, "provider_repo", None) + )["usable"] + # A reranker needs more than ``k`` candidates to be worth asking. + fetch_k = candidate_count(k) if rerank else k + use_graph = graph_enabled(services) ranked: list[tuple[Any, Hit, Any]] = [] + facts: list[str] = [] + related: list[tuple[Any, Any, int]] = [] for kb_id in selected_ids: base = visible_by_id.get(kb_id) if base is None: @@ -93,13 +112,85 @@ def _retrieve_context_sync( for document in services.knowledge_repo.list_documents(kb_id) if document.status == "ready" and not document.is_dir } - for hit in open_knowledge_index(services, kb_id).search(query_vectors[0], k=k): + for hit in open_knowledge_index(services, kb_id).search(query_vectors[0], k=fetch_k): document = ready_documents.get(hit.doc_id) if document is not None: ranked.append((base, hit, document)) + if use_graph: + expansion = _expand(services, kb_id, query or "") + facts.extend(expansion.facts) + related.extend( + (base, ready_documents[doc_id], ordinal) + for doc_id, ordinal in expansion.chunks + if doc_id in ready_documents + ) ranked.sort(key=lambda item: item[1].score, reverse=True) - return _format_context(ranked[:k], char_budget=char_budget, locale=locale) + if rerank: + ranked = _rerank(services, (query or "").strip(), ranked[:fetch_k]) + top = _with_graph_passages(services, ranked[:k], related, k=k) + return _format_context(top, char_budget=char_budget, locale=locale, facts=facts[:MAX_FACTS]) + + +def _expand(services: Any, kb_id: str, query: str) -> GraphExpansion: + """Graph lookups are an addition; a failure leaves plain vector retrieval.""" + try: + return expand_query(open_knowledge_graph(services, kb_id), query) + except Exception: + logger.warning("knowledge graph lookup skipped for %s", kb_id, exc_info=True) + return GraphExpansion() + + +def _with_graph_passages( + services: Any, + top: list[tuple[Any, Hit, Any]], + related: Sequence[tuple[Any, Any, int]], + *, + k: int, +) -> list[tuple[Any, Hit, Any]]: + """Give the last few slots to graph-linked passages the vector search missed. + + These passages are connected to the question through an entity rather + than by wording, so they would never win on similarity; they take at + most a quarter of the result instead of competing on score. + """ + slots = k // _GRAPH_PASSAGE_DIVISOR + if slots <= 0 or not related: + return top + present = {hit.chunk_id for _, hit, _ in top} + texts: dict[str, dict[int, str]] = {} + extra: list[tuple[Any, Hit, Any]] = [] + for base, document, ordinal in related: + chunk_id = f"{document.id}:{ordinal}" + if chunk_id in present: + continue + if document.id not in texts: + texts[document.id] = dict( + open_knowledge_index(services, base.id).list_doc_chunks(document.id) + ) + text = texts[document.id].get(ordinal) + if text is None: + continue + present.add(chunk_id) + extra.append((base, Hit(chunk_id, document.id, ordinal, text, 0.0, {}), document)) + if len(extra) == slots: + break + return top[: max(k - len(extra), 0)] + extra + + +def _rerank( + services: Any, query: str, ranked: list[tuple[Any, Hit, Any]] +) -> list[tuple[Any, Hit, Any]]: + """Reorder candidates by reranker relevance; keep the vector order if it fails.""" + try: + scores = rerank_scores(services, query, [hit.text for _, hit, _ in ranked]) + except Exception: + logger.warning("knowledge rerank failed; using vector order", exc_info=True) + return ranked + if scores is None: + return ranked + order = sorted(range(len(ranked)), key=lambda index: scores[index], reverse=True) + return [ranked[index] for index in order] def _unique_ids(ids: Sequence[str]) -> list[str]: @@ -107,11 +198,18 @@ def _unique_ids(ids: Sequence[str]) -> list[str]: def _format_context( - ranked: Sequence[tuple[Any, Hit, Any]], *, char_budget: int, locale: str + ranked: Sequence[tuple[Any, Hit, Any]], + *, + char_budget: int, + locale: str, + facts: Sequence[str] = (), ) -> str: sections: list[str] = [] included: list[tuple[Any, Hit, Any]] = [] - remaining = char_budget + fact_block = _format_facts( + facts, limit=char_budget // _GRAPH_FACT_BUDGET_DIVISOR, locale=locale + ) + remaining = char_budget - len(fact_block) for base, hit, document in ranked: citation = tr( "knowledge.retrieval.citation", @@ -132,4 +230,19 @@ def _format_context( if not sections: return "" body = tr("knowledge.retrieval.preamble", locale) + "\n\n" + "\n\n".join(sections) - return append_citations_marker(body, citations_from_ranked(included)) + return append_citations_marker(body + fact_block, citations_from_ranked(included)) + + +def _format_facts(facts: Sequence[str], *, limit: int, locale: str) -> str: + """Relations from the knowledge graph as a short list after the passages.""" + lines: list[str] = [] + used = 0 + for fact in dict.fromkeys(facts): + line = f"- {fact}" + if used + len(line) > limit: + break + lines.append(line) + used += len(line) + 1 + if not lines: + return "" + return "\n\n" + tr("knowledge.retrieval.graph_preamble", locale) + "\n" + "\n".join(lines) diff --git a/src/octop/infra/knowledge/service.py b/src/octop/infra/knowledge/service.py index 6ab1968103..947b429d5c 100644 --- a/src/octop/infra/knowledge/service.py +++ b/src/octop/infra/knowledge/service.py @@ -5,6 +5,7 @@ from __future__ import annotations +import logging from pathlib import Path from typing import Any, cast @@ -13,6 +14,13 @@ from octop.infra.errors import ErrorCode, OctopError from octop.infra.knowledge.blobs import KnowledgeBlobs from octop.infra.knowledge.gate import assert_knowledge_usable +from octop.infra.knowledge.graph import ( + entity_view, + graph_stats, + graph_view, + open_knowledge_graph, +) +from octop.infra.knowledge.graph_extract import get_graph_capability from octop.infra.knowledge.ocr import ( OCR_IMAGE_SUFFIXES, load_ocr_config, @@ -22,6 +30,8 @@ from octop.infra.knowledge.plane import open_knowledge_index, postgres_database from octop.infra.knowledge.relpath import normalize_kb_path, path_basename, path_parent +logger = logging.getLogger(__name__) + MAX_DOCS_PER_KB = 100 MAX_BASES_PER_OWNER = 20 MAX_DOCUMENT_BYTES = upload_mb_to_bytes(DEFAULT_MAX_UPLOAD_MB) @@ -433,6 +443,7 @@ def delete_document( if row.is_dir: continue open_knowledge_index(self._services, kb_id).delete_doc(row.id) + open_knowledge_graph(self._services, kb_id).delete_document(row.id) self._blobs().delete(kb_id, row.id, row.filename) def reindex_document( @@ -480,6 +491,109 @@ def rename_document( raise LookupError("knowledge document not found") return cast(KnowledgeDocumentRow, result) + def _move_across_bases( + self, + document: KnowledgeDocumentRow, + *, + source: KnowledgeBaseRow, + dest: KnowledgeBaseRow, + target: str, + ) -> KnowledgeDocumentRow: + """Move a file or folder into ``target`` of another base. + + Originals and chunks are copied to the target before the rows switch + base, and removed from the source after, so a failure leaves the source + intact. Chunks are reused when both bases embed with the same model; + otherwise (or when a file has none yet) the file goes back to pending + and the caller schedules indexing in the target. + """ + new_path = normalize_kb_path(f"{target}/{path_basename(document.path)}") + if self._repo.get_document_by_path(dest.id, new_path) is not None: + raise OctopError( + ErrorCode.KNOWLEDGE_MOVE_NAME_TAKEN, + "the target folder already has an item with this name", + ) + files = [row for row in self._repo.list_subtree(document) if not row.is_dir] + if any(row.status == "processing" for row in files): + raise OctopError( + ErrorCode.KNOWLEDGE_MOVE_BUSY, + "documents still being indexed cannot move to another knowledge base", + ) + if dest.max_documents > 0 and dest.doc_count + len(files) > dest.max_documents: + raise OctopError( + ErrorCode.KNOWLEDGE_DOC_LIMIT, + f"knowledge bases support at most {dest.max_documents} documents", + ) + blobs = self._blobs() + source_index = open_knowledge_index(self._services, source.id) + dest_index = open_knowledge_index(self._services, dest.id) + same_model = source.embedding_model == dest.embedding_model + reindex: list[str] = [] + copied: list[KnowledgeDocumentRow] = [] + try: + for row in files: + copied.append(row) + if blobs.exists(source.id, row.id, row.filename): + content = blobs.read_bytes(source.id, row.id, row.filename) + blobs.write(dest.id, row.id, row.filename, content) + chunks = ( + source_index.read_doc_chunks(row.id) + if same_model and row.status == "ready" + else [] + ) + if chunks: + dest_index.replace_doc_chunks( + row.id, + [text for text, _, _ in chunks], + [embedding for _, embedding, _ in chunks], + metadata=[meta for _, _, meta in chunks], + ) + elif row.status != "failed": + reindex.append(row.id) + result = self._repo.move_to_base( + document, + target_kb_id=dest.id, + target_folder=target, + max_documents=dest.max_documents, + reindex_ids=reindex, + ) + except ValueError as exc: + self._discard_copies(dest.id, copied) + if "at most" in str(exc): + raise OctopError(ErrorCode.KNOWLEDGE_DOC_LIMIT, str(exc)) from exc + raise + except Exception: + self._discard_copies(dest.id, copied) + raise + if result is None: + raise LookupError("knowledge document not found") + self._discard_copies(source.id, files) + dimension = source.embedding_dim if same_model else 0 + if dimension and not dest.embedding_dim: + self._repo.update_base(dest.id, embedding_dim=dimension) + return cast(KnowledgeDocumentRow, result) + + def _discard_copies(self, kb_id: str, rows: list[KnowledgeDocumentRow]) -> None: + """Best-effort removal of originals and chunks a base no longer owns.""" + index = open_knowledge_index(self._services, kb_id) + blobs = self._blobs() + for row in rows: + try: + index.delete_doc(row.id) + blobs.delete(kb_id, row.id, row.filename) + except Exception: + logger.warning( + "could not clean up knowledge document %s in %s", row.id, kb_id, exc_info=True + ) + + def pending_file_ids(self, document: KnowledgeDocumentRow) -> list[str]: + """Files of a document or folder still waiting to be indexed.""" + return [ + row.id + for row in self._repo.list_subtree(document) + if not row.is_dir and row.status == "pending" + ] + def move_document( self, kb_id: str, @@ -488,25 +602,35 @@ def move_document( target_folder: str, actor_user_id: int, is_admin: bool = False, + target_kb_id: str | None = None, ) -> KnowledgeDocumentRow: - """Move a document (file or folder) into another folder of the same base. + """Move a document (file or folder) into a folder of this or another base. ``target_folder`` is a folder path; ``""`` is the root. Moving a folder - takes its whole subtree. The index is untouched: chunks belong to the - document id, not to its path. + takes its whole subtree. Within one base the index is untouched: chunks + belong to the document id, not to its path. ``target_kb_id`` names + another base the actor can write to; see ``_move_across_bases``. """ - self.get_writable_base(kb_id, actor_user_id=actor_user_id, is_admin=is_admin) + source = self.get_writable_base(kb_id, actor_user_id=actor_user_id, is_admin=is_admin) document = self._repo.get_document(doc_id) if document is None or document.kb_id != kb_id: raise LookupError("knowledge document not found") + dest_kb_id = (target_kb_id or "").strip() or kb_id + dest = ( + source + if dest_kb_id == kb_id + else self.get_writable_base(dest_kb_id, actor_user_id=actor_user_id, is_admin=is_admin) + ) try: target = normalize_kb_path(target_folder) except ValueError as exc: raise LookupError("knowledge folder not found") from exc if target: - folder = self._repo.get_document_by_path(kb_id, target) + folder = self._repo.get_document_by_path(dest.id, target) if folder is None or not folder.is_dir: raise LookupError("knowledge folder not found") + if dest.id != kb_id: + return self._move_across_bases(document, source=source, dest=dest, target=target) if document.is_dir and (target == document.path or target.startswith(f"{document.path}/")): raise OctopError( ErrorCode.KNOWLEDGE_MOVE_INVALID, @@ -525,6 +649,54 @@ def move_document( raise LookupError("knowledge document not found") return cast(KnowledgeDocumentRow, result) + def get_graph( + self, + kb_id: str, + *, + actor_user_id: int, + is_admin: bool = False, + limit: int = 80, + search: str = "", + ) -> dict[str, Any]: + """Return graph statistics plus the most-mentioned entities and their relations.""" + self.get_readable_base(kb_id, actor_user_id=actor_user_id, is_admin=is_admin) + store = open_knowledge_graph(self._services, kb_id) + return { + "stats": graph_stats(store, self._graph_document_ids(kb_id)), + **graph_view(store, limit=limit, search=search), + } + + def get_graph_entity( + self, kb_id: str, name: str, *, actor_user_id: int, is_admin: bool = False + ) -> dict[str, Any]: + """Return one entity with its relations and source passages.""" + self.get_readable_base(kb_id, actor_user_id=actor_user_id, is_admin=is_admin) + return entity_view(self._services, kb_id, name) + + def graph_build_targets( + self, kb_id: str, *, actor_user_id: int, is_admin: bool = False + ) -> list[str]: + """Check that a graph can be built and return the documents to extract.""" + self.get_writable_base(kb_id, actor_user_id=actor_user_id, is_admin=is_admin) + capability = get_graph_capability( + self._services.settings_repo.get, getattr(self._services, "provider_repo", None) + ) + if not capability["usable"]: + raise RuntimeError("knowledge graph extraction is not ready") + return self._graph_document_ids(kb_id) + + def clear_graph(self, kb_id: str, *, actor_user_id: int, is_admin: bool = False) -> None: + """Drop the graph; documents indexed later are no longer extracted.""" + self.get_writable_base(kb_id, actor_user_id=actor_user_id, is_admin=is_admin) + open_knowledge_graph(self._services, kb_id).clear() + + def _graph_document_ids(self, kb_id: str) -> list[str]: + return [ + document.id + for document in self._repo.list_documents(kb_id) + if document.status == "ready" and not document.is_dir + ] + def delete_base(self, kb_id: str, *, actor_user_id: int, is_admin: bool = False) -> None: self.require_owner(kb_id, actor_user_id=actor_user_id, is_admin=is_admin) database = postgres_database(self._services) @@ -532,6 +704,7 @@ def delete_base(self, kb_id: str, *, actor_user_id: int, is_admin: bool = False) from octop.infra.knowledge.pg_index import PgVectorKnowledgeIndex PgVectorKnowledgeIndex(kb_id, database).delete_kb() + open_knowledge_graph(self._services, kb_id).clear() self._repo.delete_base(kb_id) from octop.infra.users.groups import RESOURCE_KNOWLEDGE_BASE, GroupAccess diff --git a/src/octop/infra/proactive/scheduler.py b/src/octop/infra/proactive/scheduler.py index 3b3350b89c..03ba112fa7 100644 --- a/src/octop/infra/proactive/scheduler.py +++ b/src/octop/infra/proactive/scheduler.py @@ -333,12 +333,22 @@ def suspend(self) -> None: """Stop creating new loops. Tests call this so leftover sleeps do not hang pytest.""" self._suspended = True + def _agent_kind(self, agent_id: str) -> str | None: + repo = self._agent_repo + if repo is None: + return None + row = repo.get(agent_id) + kind = getattr(row, "kind", None) if row is not None else None + return kind if isinstance(kind, str) else None + def ensure_scheduled(self, agent_id: str) -> None: """Start the loop if this agent is enabled and not already scheduled.""" if self._suspended: return if agent_id in self._tasks: return + if self._agent_kind(agent_id) == "managed": + return cfg = self._config_repo.get(agent_id) if cfg.enabled: self._schedule(agent_id) diff --git a/src/octop/infra/server.py b/src/octop/infra/server.py index 65c851b432..11f8dd139d 100644 --- a/src/octop/infra/server.py +++ b/src/octop/infra/server.py @@ -682,6 +682,7 @@ def _sign_user_token(user: Any) -> str: from octop.infra.knowledge.gate import ( # noqa: PLC0415 warn_if_local_embedding_deps_missing, ) + from octop.infra.knowledge.graph import resume_pending_graph_jobs # noqa: PLC0415 from octop.infra.knowledge.jobs import resume_pending_index_jobs # noqa: PLC0415 from octop.infra.knowledge.ocr import warn_if_local_ocr_deps_missing # noqa: PLC0415 @@ -694,6 +695,7 @@ def _sign_user_token(user: Any) -> str: self.services.provider_repo, ) resume_pending_index_jobs(self.services) + resume_pending_graph_jobs(self.services) # Resume Bridge links that opted into auto-reconnect (best-effort). async def _resume_bridges() -> None: diff --git a/src/octop/infra/users/manager.py b/src/octop/infra/users/manager.py index a46351691f..ca427c18d1 100644 --- a/src/octop/infra/users/manager.py +++ b/src/octop/infra/users/manager.py @@ -32,6 +32,7 @@ RemoteBrowserBookmark, get_remote_browser_bookmarks_from_json, merge_model_preferences_json, + merge_onboarding_completed_json, merge_preferences_json, validate_remote_browser_bookmarks, ) @@ -517,6 +518,7 @@ async def authenticate(self, username: str, password: str) -> User | None: permissions=list(row.permissions), ) self._users[row.username] = user + self._services.user_repo.touch_last_login(row.id) self._services.audit_repo.write(actor=row.username, action="auth.login") return user @@ -579,11 +581,14 @@ async def set_resource_policy( workspace_root_dir: Any = UNSET, token_quota: Any = UNSET, max_agents: Any = UNSET, + allowed_backend_types: Any = UNSET, ) -> None: from octop.infra.users.resource_policy import ( + POLICY_ALLOWED_BACKEND_TYPES, POLICY_MAX_AGENTS, POLICY_TOKEN_QUOTA, POLICY_WORKSPACE_ROOT_DIR, + normalize_allowed_backend_types, normalize_max_agents, normalize_token_quota, normalize_workspace_root_dir, @@ -596,6 +601,7 @@ async def set_resource_policy( root_arg: Any = UNSET quota_arg: Any = UNSET max_agents_arg: Any = UNSET + backend_types_arg: Any = UNSET if workspace_root_dir is not UNSET: root_arg = normalize_workspace_root_dir(workspace_root_dir) updates[POLICY_WORKSPACE_ROOT_DIR] = root_arg @@ -605,6 +611,9 @@ async def set_resource_policy( if max_agents is not UNSET: max_agents_arg = normalize_max_agents(max_agents) updates[POLICY_MAX_AGENTS] = None if max_agents_arg is None else str(max_agents_arg) + if allowed_backend_types is not UNSET: + backend_types_arg = normalize_allowed_backend_types(allowed_backend_types) + updates[POLICY_ALLOWED_BACKEND_TYPES] = backend_types_arg if not updates: return self._services.user_policy_repo.merge(row.id, updates) @@ -629,6 +638,13 @@ async def set_resource_policy( target=username, payload=str(max_agents_arg) if max_agents_arg is not None else "", ) + if allowed_backend_types is not UNSET: + self._services.audit_repo.write( + actor=ACTOR_ADMIN, + action="user.set_allowed_backend_types", + target=username, + payload=backend_types_arg or "", + ) async def set_workspace_root_dir(self, username: str, workspace_root_dir: str | None) -> None: await self.set_resource_policy(username, workspace_root_dir=workspace_root_dir) @@ -740,6 +756,13 @@ async def set_model_preferences( ) self._services.user_repo.set_preferences_json(row.id, merged) + async def set_onboarding_completed(self, username: str, completed: bool) -> None: + row = self._services.user_repo.get_by_username(username) + if row is None: + raise OctopError(ErrorCode.NOT_FOUND, "user not found") + merged = merge_onboarding_completed_json(row.preferences_json, completed) + self._services.user_repo.set_preferences_json(row.id, merged) + async def disable(self, username: str) -> None: row = self._services.user_repo.get_by_username(username) if row is None: diff --git a/src/octop/infra/users/permissions.py b/src/octop/infra/users/permissions.py index 50f9a09edb..e8be123d24 100644 --- a/src/octop/infra/users/permissions.py +++ b/src/octop/infra/users/permissions.py @@ -145,6 +145,7 @@ def _p( # --- control (nav.control) — page/tab labels --- "terminal": _p("terminal", "control", "工作台/终端", "Workbench / Terminal"), "browser": _p("browser", "control", "工作台/浏览器", "Workbench / Browser"), + "cloud": _p("cloud", "control", "工作台/云服务器", "Workbench / Cloud"), "desktop": _p("desktop", "control", "远程桌面", "Remote Desktop"), "mobile": _p("mobile", "control", "远程手机", "Remote Phone"), # --- admin: grouped by page, chip = tab title --- @@ -231,6 +232,15 @@ def _p( page_zh="用户", page_en="Users", ), + "app_config": _p( + "app_config", + "admin", + "应用配置", + "App config", + page="advanced", + page_zh="应用设置", + page_en="App settings", + ), "providers": _p( "providers", "admin", diff --git a/src/octop/infra/users/preferences.py b/src/octop/infra/users/preferences.py index 43a9cac141..886b6452e7 100644 --- a/src/octop/infra/users/preferences.py +++ b/src/octop/infra/users/preferences.py @@ -17,6 +17,7 @@ PREFERENCES_KEY_MODEL_REASONING = "model_reasoning" PREFERENCES_KEY_TIMEZONE = "timezone" PREFERENCES_KEY_SIDEBAR_NAV = "sidebar_nav" +PREFERENCES_KEY_ONBOARDING_COMPLETED = "onboarding_completed" # Keep in sync with dashboard/src/layouts/sidebarNav.tsx SIDEBAR_NAV_KEYS. SIDEBAR_NAV_KEYS = frozenset( @@ -310,3 +311,17 @@ def merge_sidebar_nav_json(current_raw: str | None, layout: Any | None) -> str: groups, items = validate_sidebar_nav(layout) data[PREFERENCES_KEY_SIDEBAR_NAV] = sidebar_nav_to_dict(groups, items) return json.dumps(data, ensure_ascii=False, separators=(",", ":")) + + +def get_onboarding_completed_from_json(raw: str | None) -> bool: + """True once the user has finished or skipped the dashboard welcome guide.""" + return parse_preferences_json(raw).get(PREFERENCES_KEY_ONBOARDING_COMPLETED) is True + + +def merge_onboarding_completed_json(current_raw: str | None, completed: bool) -> str: + data = parse_preferences_json(current_raw) + if completed: + data[PREFERENCES_KEY_ONBOARDING_COMPLETED] = True + else: + data.pop(PREFERENCES_KEY_ONBOARDING_COMPLETED, None) + return json.dumps(data, ensure_ascii=False, separators=(",", ":")) diff --git a/src/octop/infra/users/resource_policy.py b/src/octop/infra/users/resource_policy.py index 3b55273db7..634780c304 100644 --- a/src/octop/infra/users/resource_policy.py +++ b/src/octop/infra/users/resource_policy.py @@ -1,9 +1,10 @@ -"""Per-user named policies: workspace root, token quota, and future rows.""" +"""Per-user named policies: workspace root, token quota, backend types, and future rows.""" from __future__ import annotations +import json import os -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from typing import Any from octop.infra.errors import ErrorCode, OctopError @@ -21,6 +22,26 @@ POLICY_WORKSPACE_ROOT_DIR = "workspace_root_dir" POLICY_TOKEN_QUOTA = "token_quota" POLICY_MAX_AGENTS = "max_agents" +POLICY_ALLOWED_BACKEND_TYPES = "allowed_backend_types" + +# Expert-picker types. Storage kind ``shell`` is the same capability as builtin +# ``local_shell``. Builtin filesystem and storage kind ``filesystem`` share one +# type. ``composite`` is a wrapper, not a policy value. +BACKEND_POLICY_TYPES: tuple[str, ...] = ( + "local_shell", + "filesystem", + "cos", + "s3", + "oss", + "obs", + "tos", + "custom", + "postgres", + "docker", + "opensandbox", +) +_BACKEND_POLICY_TYPE_SET = frozenset(BACKEND_POLICY_TYPES) +_POLICY_TYPE_ALIASES = {"shell": "local_shell"} def active_policy_value(row: Any) -> str | None: @@ -107,6 +128,9 @@ def public_policy_fields(rows: Sequence[Any] | Mapping[str, str] | None) -> dict "workspace_root_dir": workspace_root_dir_of(by_name), "token_quota": token_quota_of(by_name.get(POLICY_TOKEN_QUOTA)), "max_agents": max_agents_of(by_name.get(POLICY_MAX_AGENTS)), + "allowed_backend_types": allowed_backend_types_of( + by_name.get(POLICY_ALLOWED_BACKEND_TYPES) + ), } @@ -160,6 +184,171 @@ def normalize_max_agents(raw: int | None) -> int | None: return limit +def _backend_type_tokens(raw: Any) -> list[str]: + if raw is None: + return [] + if isinstance(raw, str): + text = raw.strip() + if not text: + return [] + if text.startswith("["): + try: + parsed = json.loads(text) + except json.JSONDecodeError as exc: + raise OctopError( + ErrorCode.FORBIDDEN, + "invalid backend type list", + status=400, + ) from exc + raw = parsed + else: + raw = [part.strip() for part in text.split(",")] + if isinstance(raw, (list, tuple, set)): + tokens: list[str] = [] + seen: set[str] = set() + for item in raw: + token = str(item).strip().lower() + if not token or token in seen: + continue + seen.add(token) + tokens.append(token) + return tokens + raise OctopError(ErrorCode.FORBIDDEN, "invalid backend type list", status=400) + + +def _canonical_policy_type(token: str) -> str: + return _POLICY_TYPE_ALIASES.get(token, token) + + +def normalize_allowed_backend_types(raw: Any) -> str | None: + """Return catalog-ordered types, or ``None`` when the user is unrestricted. + + Empty input clears the policy. Unknown names are rejected. ``shell`` is + stored as ``local_shell``. + """ + tokens = _backend_type_tokens(raw) + if not tokens: + return None + selected = {_canonical_policy_type(token) for token in tokens} + unknown = [ + token for token in tokens if _canonical_policy_type(token) not in _BACKEND_POLICY_TYPE_SET + ] + if unknown: + raise OctopError( + ErrorCode.FORBIDDEN, + f"unknown backend types: {unknown}", + status=400, + ) + return ",".join(token for token in BACKEND_POLICY_TYPES if token in selected) + + +def allowed_backend_types_of(raw: Any) -> list[str] | None: + """Enabled allow-list, or ``None`` when this policy does not apply.""" + if isinstance(raw, (list, tuple, set)): + stored = normalize_allowed_backend_types(raw) + return stored.split(",") if stored else None + if isinstance(raw, Mapping) and "value" not in raw and POLICY_ALLOWED_BACKEND_TYPES in raw: + raw = raw.get(POLICY_ALLOWED_BACKEND_TYPES) + text = active_policy_value(raw) + if text is None: + return None + selected = [ + _canonical_policy_type(token) + for token in (part.strip().lower() for part in text.split(",")) + if _canonical_policy_type(token) in _BACKEND_POLICY_TYPE_SET + ] + ordered = [token for token in BACKEND_POLICY_TYPES if token in selected] + return ordered or None + + +def backend_policy_types_in( + spec: Any, + *, + named_kind: Callable[[str], str | None], +) -> set[str]: + """Policy types referenced by an agent backend spec, including composites.""" + found: set[str] = set() + + def add_named(name: str) -> None: + kind = named_kind(name) + if kind: + found.add(_canonical_policy_type(kind.strip().lower())) + + def walk(node: Any) -> None: + if node is None: + return + if isinstance(node, str): + text = node.strip() + if not text: + return + if text.startswith("named:"): + add_named(text.split(":", 1)[1]) + return + found.add(_canonical_policy_type(text.lower())) + return + if not isinstance(node, dict): + return + kind = node.get("type") + if not isinstance(kind, str) or not kind.strip(): + return + kind = kind.strip().lower() + if kind == "named": + name = node.get("name") + if isinstance(name, str) and name.strip(): + add_named(name.strip()) + return + if kind == "composite": + walk(node.get("default")) + routes = node.get("routes") + if isinstance(routes, dict): + for sub in routes.values(): + walk(sub) + return + found.add(_canonical_policy_type(kind)) + + walk(spec) + return found + + +def raise_if_backend_types_disallowed( + policy_repo: Any, + user_id: int, + backend: Any, + *, + named_kind: Callable[[str], str | None], + previous: Any = None, + require_present: bool = False, +) -> None: + """Reject backend types outside the user's allow-list. + + A missing policy is unrestricted. On update, types already stored on the + agent stay valid so an unrelated edit can still save. A create that omits + the backend is treated as the default ``local_shell``. + """ + allowed = allowed_backend_types_of(policy_repo.get(user_id, POLICY_ALLOWED_BACKEND_TYPES)) + if not allowed: + return + allowed_set = set(allowed) + if backend in (None, {}) and require_present: + used = {"local_shell"} + else: + used = backend_policy_types_in(backend, named_kind=named_kind) + if previous is not None: + used -= backend_policy_types_in(previous, named_kind=named_kind) + blocked = [ + token for token in BACKEND_POLICY_TYPES if token in used and token not in allowed_set + ] + blocked.extend(sorted(used - allowed_set - _BACKEND_POLICY_TYPE_SET)) + if not blocked: + return + types = ", ".join(blocked) + raise OctopError( + ErrorCode.BACKEND_TYPE_NOT_ALLOWED, + f"backend types not allowed: {types}", + details={"types": types}, + ) + + def assert_backend_within_user_root( backend: Any, allowed_root: str | None, diff --git a/src/octop/infra/utils/stream_close.py b/src/octop/infra/utils/stream_close.py new file mode 100644 index 0000000000..beb9dc236f --- /dev/null +++ b/src/octop/infra/utils/stream_close.py @@ -0,0 +1,7 @@ +"""Use Gateway's public helper for deterministic nested event-stream cleanup.""" + +from __future__ import annotations + +from octop_gateway.utils import close_event_stream + +__all__ = ["close_event_stream"] diff --git a/tests/integration/test_agents_api.py b/tests/integration/test_agents_api.py index 75071f9a17..5482c51796 100644 --- a/tests/integration/test_agents_api.py +++ b/tests/integration/test_agents_api.py @@ -85,6 +85,113 @@ async def test_agent_runtime_fields_are_first_class_api_fields(env): assert "temperature" not in updated["config"] +async def test_agent_call_guard_fields_round_trip(env): + c, _, auth = env + r = await c.post( + "/api/agents", + headers=auth, + json={ + "name": "call-guard", + "tool_call_run_limit": 12, + "model_retry_max_retries": 4, + "tool_retry_enabled": False, + }, + ) + assert r.status_code == 201, r.text + created = r.json() + agent_id = created["agent_id"] + assert created["tool_call_run_limit"] == 12 + assert created["model_retry_max_retries"] == 4 + assert created["tool_retry_enabled"] is False + assert created["tool_call_limit_enabled"] is None + assert created["tool_retry_max_retries"] is None + assert "tool_call_run_limit" not in created["config"] + assert "tool_retry_enabled" not in created["config"] + + r = await c.patch( + f"/api/agents/{agent_id}", + headers=auth, + json={ + "tool_call_limit_enabled": False, + "tool_call_run_limit": None, + "tool_retry_enabled": None, + }, + ) + assert r.status_code == 200, r.text + updated = r.json() + assert updated["tool_call_limit_enabled"] is False + assert updated["tool_call_run_limit"] is None + assert updated["tool_retry_enabled"] is None + assert updated["model_retry_max_retries"] == 4 + + r = await c.post( + "/api/agents", + headers=auth, + json={ + "name": "call-guard-invalid", + "model_retry_initial_delay": 30, + "model_retry_max_delay": 2, + }, + ) + assert r.status_code == 422 + + +async def test_agent_compaction_is_per_expert_and_preserved(env): + c, _, auth = env + r = await c.post( + "/api/agents", + headers=auth, + json={ + "name": "compaction-config", + "compaction": { + "trigger_fraction": 0.7, + "keep_fraction": 0.2, + "trim_tokens": 4000, + }, + }, + ) + assert r.status_code == 201, r.text + created = r.json() + agent_id = created["agent_id"] + assert created["compaction"] == { + "trigger_fraction": 0.7, + "keep_fraction": 0.2, + "trim_tokens": 4000, + } + assert "compaction" not in created["config"] + + r = await c.patch( + f"/api/agents/{agent_id}", + headers=auth, + json={"config": {"plugins": {}}}, + ) + assert r.status_code == 200, r.text + assert r.json()["compaction"]["trigger_fraction"] == 0.7 + + r = await c.patch( + f"/api/agents/{agent_id}", + headers=auth, + json={"compaction": {"enabled": False}}, + ) + assert r.status_code == 200, r.text + assert r.json()["compaction"] == {"enabled": False} + + r = await c.patch( + f"/api/agents/{agent_id}", + headers=auth, + json={"compaction": None}, + ) + assert r.status_code == 200, r.text + assert r.json()["compaction"] is None + + r = await c.post( + "/api/agents", + headers=auth, + json={"name": "compaction-invalid", "compaction": {"keep_fraction": 0.9}}, + ) + assert r.status_code == 422 + + async def test_create_keeps_legacy_runtime_values_from_config(env): c, _, auth = env r = await c.post( diff --git a/tests/integration/test_agents_list_scope.py b/tests/integration/test_agents_list_scope.py index 1feff0799f..2022bb659a 100644 --- a/tests/integration/test_agents_list_scope.py +++ b/tests/integration/test_agents_list_scope.py @@ -57,6 +57,37 @@ async def test_admin_scope_all_lists_every_user(env) -> None: assert guest_agent_id in ids +async def test_admin_can_update_another_users_agent(env) -> None: + c, _srv, admin_auth = env + guest_auth = await create_user(c, admin_auth, username="guest_manage") + other_auth = await create_user(c, admin_auth, username="peer_manage") + created = await c.post( + "/api/agents/from-expert/default", + headers=guest_auth, + json={"name": "guest-manage-bot"}, + ) + assert created.status_code == 201, created.text + agent_id = created.json()["agent_id"] + owner_id = created.json()["user_id"] + + denied = await c.patch( + f"/api/agents/{agent_id}", + headers=other_auth, + json={"description": "nope"}, + ) + assert denied.status_code == 403 + + renamed = await c.patch( + f"/api/agents/{agent_id}", + headers=admin_auth, + json={"name": "renamed-by-admin", "description": "admin edit"}, + ) + assert renamed.status_code == 200, renamed.text + assert renamed.json()["name"] == "renamed-by-admin" + assert renamed.json()["description"] == "admin edit" + assert renamed.json()["user_id"] == owner_id + + async def test_non_admin_cannot_use_scope_all(env) -> None: c, _srv, admin_auth = env user_auth = await create_user(c, admin_auth, username="scope_user") diff --git a/tests/integration/test_cloud_api.py b/tests/integration/test_cloud_api.py new file mode 100644 index 0000000000..8097641653 --- /dev/null +++ b/tests/integration/test_cloud_api.py @@ -0,0 +1,72 @@ +"""Cloud workbench authorization: panel permission and per-user tccli import.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from tests.support.auth import create_user, resolve_user_id + + +def _oauth_file(path: Path, *, secret_id: str, expires_at: float) -> None: + path.write_text( + json.dumps( + { + "type": "oauth", + "secretId": secret_id, + "secretKey": "secret-key-value", + "token": "sts-token", + "expiresAt": expires_at, + "oauth": { + "openId": "open-id", + "accessToken": "access-token", + "expiresAt": expires_at, + "refreshToken": "refresh-token", + "site": "cn", + }, + } + ) + ) + + +async def test_cloud_routes_require_workbench_permission(env, tmp_path: Path, monkeypatch): + client, _, admin = env + plain = await create_user(client, admin, username="cloud-plain", permissions=[]) + browser_only = await create_user( + client, admin, username="cloud-browser", permissions=["browser"] + ) + workbench = await create_user(client, admin, username="cloud-user", permissions=["cloud"]) + denied_status = await client.get("/api/cloud/status", headers=plain) + assert denied_status.status_code == 403 + denied_browser = await client.get("/api/cloud/status", headers=browser_only) + assert denied_browser.status_code == 403 + denied_import = await client.post("/api/cloud/import-from-tccli", headers=plain) + assert denied_import.status_code == 403 + denied_action = await client.post( + "/api/cloud/action", + headers=plain, + json={"region": "ap-guangzhou", "instance_id": "ins-test", "action": "stop"}, + ) + assert denied_action.status_code == 403 + + tccli = tmp_path / ".tccli" + tccli.mkdir() + _oauth_file(tccli / "default.credential", secret_id="AKIDshared", expires_at=111.0) + uid = await resolve_user_id(client, admin, "cloud-user") + own_expires = 1893456000.0 + _oauth_file( + tccli / f"octop-{uid}.credential", + secret_id="AKIDown", + expires_at=own_expires, + ) + monkeypatch.setattr("octop.infra.cloud.tencent.Path.home", lambda: tmp_path) + + missing = await client.get("/api/cloud/status", headers=workbench) + assert missing.status_code == 200 + assert missing.json()["authorized"] is False + + imported = await client.post("/api/cloud/import-from-tccli", headers=workbench) + assert imported.status_code == 200, imported.text + body = imported.json() + assert body["authorized"] is True + assert body["expires_at"] == own_expires diff --git a/tests/integration/test_connectors_api.py b/tests/integration/test_connectors_api.py index cbfb83c259..3bec141fc2 100644 --- a/tests/integration/test_connectors_api.py +++ b/tests/integration/test_connectors_api.py @@ -1060,7 +1060,7 @@ async def test_oauth_callback_escapes_error_html(env, exchange_error): state_id=state_id, state=state_id, user_id=user_id, - kind="notion", + kind="custom-mcp", code_verifier="verifier", redirect_after="/connectors", ) @@ -1076,3 +1076,94 @@ async def test_oauth_callback_escapes_error_html(env, exchange_error): assert response.status_code == 400 assert payload not in response.text assert escape(payload) in response.text + + +@pytest.mark.parametrize( + ("error", "status"), [("access_denied", "cancelled"), ("server_error", "error")] +) +async def test_a_cancelled_or_refused_authorization_is_reported_to_the_opener(env, error, status): + c, srv, auth, _ = env + user_id = await resolve_user_id(c, auth, "admin") + state_id = new_ulid() + srv.services.repos.connector_repo.create_oauth_state( + state_id=state_id, + state="opaque-state", + user_id=user_id, + kind="custom-mcp", + code_verifier="verifier", + redirect_after="/chat/agent-1?x=1", + ) + + response = await c.get( + "/api/connectors/oauth/callback", params={"error": error, "state": "opaque-state"} + ) + + assert response.status_code == 400 + assert f'"status": "{status}"' in response.text + assignment = re.search(r"window.location.href = (.*);", response.text) + assert assignment is not None + target = urlsplit(json.loads(assignment.group(1))) + assert target.path == "/chat/agent-1" + assert parse_qs(target.query) == { + "x": ["1"], + "oauth_state": [state_id], + "oauth_error": [status], + } + # The state is used up, and the outcome is waiting for a page that polls. + assert srv.services.repos.connector_repo.consume_oauth_state("opaque-state") is None + pending = await c.get(f"/api/connectors/oauth/pending/{state_id}", headers=auth) + assert pending.status_code == 200 + assert pending.json()["status"] == status + assert pending.json()["tokens"] == {} + + +async def test_a_cancelled_catalog_authorization_keeps_the_plain_callback_page(env): + c, srv, auth, _ = env + user_id = await resolve_user_id(c, auth, "admin") + state_id = new_ulid() + srv.services.repos.connector_repo.create_oauth_state( + state_id=state_id, + state="catalog-state", + user_id=user_id, + kind="notion", + code_verifier="verifier", + redirect_after="/connectors", + ) + + response = await c.get( + "/api/connectors/oauth/callback", + params={"error": "access_denied", "state": "catalog-state"}, + ) + + assert response.status_code == 400 + assert "octop:connector-oauth" not in response.text + pending = await c.get(f"/api/connectors/oauth/pending/{state_id}", headers=auth) + assert pending.status_code == 404 + + +async def test_a_failed_token_exchange_is_reported_as_an_error(env): + c, srv, auth, _ = env + user_id = await resolve_user_id(c, auth, "admin") + state_id = new_ulid() + srv.services.repos.connector_repo.create_oauth_state( + state_id=state_id, + state=state_id, + user_id=user_id, + kind="custom-mcp", + code_verifier="verifier", + redirect_after="/connectors", + ) + with patch( + "octop.api.routers.connectors.exchange_oauth_code", + new_callable=AsyncMock, + side_effect=ValueError("invalid_grant"), + ): + response = await c.get( + "/api/connectors/oauth/callback", params={"code": "c", "state": state_id} + ) + + assert response.status_code == 400 + assert '"status": "error"' in response.text + pending = await c.get(f"/api/connectors/oauth/pending/{state_id}", headers=auth) + assert pending.json()["status"] == "error" + assert "invalid_grant" in pending.json()["message"] diff --git a/tests/integration/test_knowledge_bases_api.py b/tests/integration/test_knowledge_bases_api.py index a5a325336b..0f928ae3c6 100644 --- a/tests/integration/test_knowledge_bases_api.py +++ b/tests/integration/test_knowledge_bases_api.py @@ -2,8 +2,11 @@ from __future__ import annotations +import asyncio from typing import Any +import pytest + from tests.support.auth import create_user @@ -48,3 +51,71 @@ async def test_embedding_options_require_knowledge_settings(env: Any) -> None: allowed = await client.get("/api/knowledge-bases/embedding-options", headers=settings_auth) assert allowed.status_code == 200, allowed.text assert "onnx" in allowed.json() + + +async def _wait_for_status(client: Any, auth: Any, kb_id: str, doc_id: str, want: str) -> None: + for _ in range(100): + rows = (await client.get(f"/api/knowledge-bases/{kb_id}/documents", headers=auth)).json() + if any(row["document_id"] == doc_id and row["status"] == want for row in rows): + return + await asyncio.sleep(0.05) + raise AssertionError(f"{doc_id} never reached {want}: {rows}") + + +async def test_move_document_into_another_knowledge_base( + env: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + from octop.api.routers import knowledge_bases as router + from octop.infra.knowledge import jobs, service + + for module in (router, service, jobs): + monkeypatch.setattr(module, "assert_knowledge_usable", lambda *_args: None) + monkeypatch.setattr( + jobs, "embed_knowledge_texts", lambda _services, texts: [[1.0, 0.0] for _ in texts] + ) + client, _server, admin_auth = env + source = ( + await client.post("/api/knowledge-bases", headers=admin_auth, json={"name": "产品资料库"}) + ).json() + target = ( + await client.post("/api/knowledge-bases", headers=admin_auth, json={"name": "客服话术"}) + ).json() + folder = await client.post( + f"/api/knowledge-bases/{target['id']}/folders", headers=admin_auth, json={"path": "售后"} + ) + assert folder.status_code in (200, 201), folder.text + created = await client.post( + f"/api/knowledge-bases/{source['id']}/documents/text", + headers=admin_auth, + json={"name": "退款流程", "format": "md", "content": "七天内可以无理由退款。"}, + ) + assert created.status_code == 201, created.text + doc_id = created.json()["document_id"] + await _wait_for_status(client, admin_auth, source["id"], doc_id, "ready") + + moved = await client.post( + f"/api/knowledge-bases/{source['id']}/documents/{doc_id}/move", + headers=admin_auth, + json={"target_folder": "售后", "target_kb_id": target["id"]}, + ) + + assert moved.status_code == 200, moved.text + assert (moved.json()["kb_id"], moved.json()["path"]) == (target["id"], "售后/退款流程.md") + assert moved.json()["status"] == "ready" + left = await client.get(f"/api/knowledge-bases/{source['id']}/documents", headers=admin_auth) + assert left.json() == [] + chunks = await client.get( + f"/api/knowledge-bases/{target['id']}/documents/{doc_id}/chunks", headers=admin_auth + ) + assert chunks.status_code == 200, chunks.text + assert "七天内可以无理由退款" in str(chunks.json()) + original = await client.get( + f"/api/knowledge-bases/{target['id']}/documents/{doc_id}/file", headers=admin_auth + ) + assert original.status_code == 200, original.text + assert "七天内可以无理由退款" in original.content.decode("utf-8") + bases = { + row["id"]: row["doc_count"] + for row in (await client.get("/api/knowledge-bases", headers=admin_auth)).json() + } + assert (bases[source["id"]], bases[target["id"]]) == (0, 1) diff --git a/tests/integration/test_pg_knowledge_graph.py b/tests/integration/test_pg_knowledge_graph.py new file mode 100644 index 0000000000..aa572548f3 --- /dev/null +++ b/tests/integration/test_pg_knowledge_graph.py @@ -0,0 +1,122 @@ +"""Knowledge-graph rows on a live PostgreSQL. + +Uses ``OCTOP_TEST_DATABASE_URL``. Drops only the ``public.knowledge_graph_*`` +tables. Does not reset the rest of ``public``. +""" + +from __future__ import annotations + +import os +from collections.abc import Iterator +from types import SimpleNamespace + +import pytest + +from octop.config import DatabaseConfig +from octop.infra.knowledge.graph import expand_query, graph_view, open_knowledge_graph +from octop.infra.knowledge.graph_store import ( + STATUS_PENDING, + STATUS_READY, + ChunkEntity, + ChunkGraph, + ChunkRelation, + PgKnowledgeGraphStore, +) +from tests.support.postgresql import requires_postgresql + +pytestmark = requires_postgresql + +_TABLES = ( + "public.knowledge_graph_entities", + "public.knowledge_graph_relations", + "public.knowledge_graph_documents", +) + + +def _drop_tables(database: DatabaseConfig) -> None: + import psycopg + + with psycopg.connect(database.postgresql_conninfo()) as conn: + for table in _TABLES: + conn.execute(f"DROP TABLE IF EXISTS {table}") + conn.commit() + + +@pytest.fixture +def database() -> Iterator[DatabaseConfig]: + from octop.config import _parse_database_url + + parsed = _parse_database_url(os.environ["OCTOP_TEST_DATABASE_URL"]) + config = DatabaseConfig( + driver="postgresql", + host=str(parsed["host"]), + port=int(parsed["port"]), + database=str(parsed["database"]), + user=str(parsed.get("user") or "postgres"), + password=parsed.get("password"), + url=str(parsed["url"]), + ) + _drop_tables(config) + yield config + _drop_tables(config) + + +def _chunk(ordinal: int, relations: list[tuple[str, str, str, str, float]]) -> ChunkGraph: + names = dict.fromkeys(end for source, target, *_ in relations for end in (source, target)) + return ChunkGraph( + ordinal=ordinal, + entities=tuple(ChunkEntity(name, "concept", f"About {name}.") for name in names), + relations=tuple(ChunkRelation(*relation) for relation in relations), + ) + + +def test_graph_round_trip_and_isolation_between_knowledge_bases( + database: DatabaseConfig, +) -> None: + services = SimpleNamespace(config=SimpleNamespace(database=database)) + store = open_knowledge_graph(services, "kb-1") + assert isinstance(store, PgKnowledgeGraphStore) + other = PgKnowledgeGraphStore("kb-2", database) + + assert store.exists() is False + store.replace_document( + "doc-1", + [ + _chunk(0, [("Apollo", "Alice", "is led by", "Alice leads Apollo.", 9)]), + _chunk(1, [("apollo", "alice", "is led by", "", 6)]), + _chunk(2, [("Alice", "Finance Board", "reports budgets to", "", 7)]), + ], + ) + store.replace_document("doc-2", [_chunk(0, [("100% Cotton", "a_c", "contains", "", 5)])]) + other.replace_document("doc-9", [_chunk(0, [("Zeus", "Bob", "is led by", "", 9)])]) + + assert store.exists() is True + assert store.counts() == (5, 3) + assert other.counts() == (2, 1) + assert store.entities(["apollo"])["apollo"].mentions == 2 + led = next(item for item in store.relations_of(["apollo"]) if item.label == "is led by") + assert led.weight == 15 + assert led.chunks == (("doc-1", 0), ("doc-1", 1)) + assert store.top_entity_keys(1) == ["alice"] + # % and _ in names and search fragments are literal characters on PostgreSQL too. + assert store.top_entity_keys(10, search="%") == ["100% cotton"] + assert store.match_keys("is 100% cotton soft?") == ["100% cotton"] + assert store.match_keys("abc") == [] + + expansion = expand_query(store, "Who approves the budget of Apollo?") + assert expansion.facts == ( + "Apollo -> is led by -> Alice: Alice leads Apollo.", + "Alice -> reports budgets to -> Finance Board", + ) + assert expansion.chunks == (("doc-1", 0), ("doc-1", 1)) + assert len(graph_view(store, limit=10)["edges"]) == 3 + + store.mark_document("doc-1", STATUS_PENDING) + assert store.counts() == (2, 1) + assert store.document_states()["doc-1"].status == STATUS_PENDING + assert store.document_states()["doc-2"].status == STATUS_READY + store.delete_document("doc-2") + assert store.counts() == (0, 0) + store.clear() + assert store.exists() is False + assert other.counts() == (2, 1) diff --git a/tests/integration/test_preferences_api.py b/tests/integration/test_preferences_api.py index c33a2e3ce4..3f054b00cf 100644 --- a/tests/integration/test_preferences_api.py +++ b/tests/integration/test_preferences_api.py @@ -4,6 +4,8 @@ import pytest +from tests.support.auth import create_user + @pytest.mark.asyncio async def test_preferences_bookmarks_roundtrip(env) -> None: @@ -116,3 +118,40 @@ async def test_preferences_sidebar_nav_roundtrip(env) -> None: r = await client.patch("/api/preferences", headers=auth, json={"sidebar_nav": None}) assert r.status_code == 200 assert r.json()["sidebar_nav"] is None + + +@pytest.mark.asyncio +async def test_preferences_onboarding_is_per_user(env) -> None: + client, _srv, admin = env + member = await create_user(client, admin, username="newcomer") + + for auth in (admin, member): + r = await client.get("/api/preferences", headers=auth) + assert r.status_code == 200 + assert r.json()["onboarding_completed"] is False + + r = await client.patch("/api/preferences", headers=member, json={"onboarding_completed": True}) + assert r.status_code == 200 + assert r.json()["onboarding_completed"] is True + + r = await client.get("/api/preferences", headers=member) + assert r.json()["onboarding_completed"] is True + r = await client.get("/api/preferences", headers=admin) + assert r.json()["onboarding_completed"] is False + + r = await client.patch("/api/preferences", headers=member, json={"onboarding_completed": False}) + assert r.status_code == 200 + assert r.json()["onboarding_completed"] is False + + +@pytest.mark.asyncio +async def test_preferences_onboarding_keeps_other_preferences(env) -> None: + client, _srv, auth = env + r = await client.patch("/api/preferences", headers=auth, json={"timezone": "Asia/Shanghai"}) + assert r.status_code == 200 + + r = await client.patch("/api/preferences", headers=auth, json={"onboarding_completed": True}) + assert r.status_code == 200 + body = r.json() + assert body["onboarding_completed"] is True + assert body["timezone"] == "Asia/Shanghai" diff --git a/tests/integration/test_providers_api.py b/tests/integration/test_providers_api.py index d8a4e3a989..89d31fbee7 100644 --- a/tests/integration/test_providers_api.py +++ b/tests/integration/test_providers_api.py @@ -182,3 +182,76 @@ async def test_admin_stream_usage_defaults_and_patch(env): ) assert patched.status_code == 200 assert patched.json()["stream_usage"] is False + + +async def test_magpie_discovery_is_typed_and_permissioned(env, monkeypatch): + from octop.infra.agents.providers.magpie import MagpieDiscovery + + async def found(): + return MagpieDiscovery(available=True, version="1.4.0") + + monkeypatch.setattr("octop.api.routers.providers.discover_local_magpie", found) + c, _, auth = env + ok = await c.get("/api/admin/providers/magpie/discovery", headers=auth) + assert ok.status_code == 200 + assert ok.json() == { + "available": True, + "base_url": "http://127.0.0.1:3425/v1", + "version": "1.4.0", + } + + async def missing(): + return MagpieDiscovery(available=False, version=None) + + monkeypatch.setattr("octop.api.routers.providers.discover_local_magpie", missing) + quiet = await c.get("/api/admin/providers/magpie/discovery", headers=auth) + assert quiet.status_code == 200 + assert quiet.json()["available"] is False + assert quiet.json()["version"] is None + assert "error" not in quiet.json() + + await c.post( + "/api/users", + headers=auth, + json={"username": "magpie-user", "password": "TestPass12", "role": "user"}, + ) + token = ( + await c.post( + "/api/auth/login", + json={"username": "magpie-user", "password": "TestPass12"}, + ) + ).json()["access_token"] + denied = await c.get( + "/api/admin/providers/magpie/discovery", + headers={"Authorization": f"Bearer {token}"}, + ) + assert denied.status_code == 403 + + +async def test_duplicate_magpie_create_does_not_overwrite(env): + c, _, auth = env + body = { + "name": "Magpie", + "kind": "openai", + "base_url": "http://127.0.0.1:3425/v1", + "api_key": "magpie-octop", + "extra_json": '{"integration": "magpie"}', + "models": [{"id": "vendor/org/model", "name": "Shown", "enabled": True}], + "stream_usage": True, + } + created = await c.post("/api/admin/providers", headers=auth, json=body) + assert created.status_code == 201 + saved = created.json() + assert saved["integration"] == "magpie" + assert saved["api_key"] == "magpie-octop" + assert saved["models"][0]["id"] == "vendor/org/model" + + again = await c.post("/api/admin/providers", headers=auth, json=body) + assert again.status_code == 409 + assert again.json()["error"]["code"] == "PROVIDER_NAME_TAKEN" + + listed = await c.get("/api/admin/providers", headers=auth) + rows = [row for row in listed.json() if row["name"] == "Magpie"] + assert len(rows) == 1 + assert rows[0]["api_key"] == "magpie-octop" + assert rows[0]["models"][0]["id"] == "vendor/org/model" diff --git a/tests/integration/test_user_roles_api.py b/tests/integration/test_user_roles_api.py index 77e341a6d9..2be15041f0 100644 --- a/tests/integration/test_user_roles_api.py +++ b/tests/integration/test_user_roles_api.py @@ -405,3 +405,129 @@ async def test_create_user_keeps_user_when_expert_template_missing(env): agents = await c.get("/api/agents", headers=auth, params={"scope": "all"}) owned = [row for row in agents.json() if row["user_id"] == created.json()["id"]] assert owned == [] + + +async def test_allowed_backend_types_limit_expert_create(env): + c, _srv, auth = env + created = await c.post( + "/api/users/roles", + headers=auth, + json={ + "user_role_name": "沙箱", + "permissions": ["browser"], + "policies": [ + {"name": "allowed_backend_types", "value": "docker,opensandbox,filesystem"}, + ], + }, + ) + assert created.status_code == 201, created.text + assert created.json()["allowed_backend_types"] == ["filesystem", "docker", "opensandbox"] + assert created.json()["policies"] == [ + {"name": "allowed_backend_types", "value": "filesystem,docker,opensandbox"}, + ] + + unknown = await c.post( + "/api/users/roles", + headers=auth, + json={ + "user_role_name": "坏类型", + "permissions": ["browser"], + "policies": [{"name": "allowed_backend_types", "value": "docker,nope"}], + }, + ) + assert unknown.status_code == 400, unknown.text + + user = await c.post( + "/api/users", + headers=auth, + json={ + "username": "boxed", + "password": TEST_PASSWORD, + "role": created.json()["user_role_id"], + "create_default_expert": False, + }, + ) + assert user.status_code == 201, user.text + assert user.json()["allowed_backend_types"] == ["filesystem", "docker", "opensandbox"] + + token = await login(c, username="boxed", password=TEST_PASSWORD) + me = await c.get("/api/auth/me", headers=bearer(token)) + assert me.status_code == 200, me.text + assert me.json()["allowed_backend_types"] == ["filesystem", "docker", "opensandbox"] + user_auth = bearer(token) + + denied = await c.post( + "/api/agents", + headers=user_auth, + json={ + "name": "host", + "config": {"backend": {"type": "local_shell", "root_dir": "/", "virtual_mode": True}}, + }, + ) + assert denied.status_code == 403, denied.text + assert denied.json()["error"]["code"] == "BACKEND_TYPE_NOT_ALLOWED" + + omitted = await c.post( + "/api/agents", + headers=user_auth, + json={"name": "omitted", "config": {}}, + ) + assert omitted.status_code == 403, omitted.text + assert omitted.json()["error"]["code"] == "BACKEND_TYPE_NOT_ALLOWED" + + created_agent = await c.post( + "/api/agents", + headers=user_auth, + json={ + "name": "files", + "config": {"backend": {"type": "filesystem", "root_dir": "/", "virtual_mode": True}}, + }, + ) + assert created_agent.status_code == 201, created_agent.text + agent_id = created_agent.json()["agent_id"] + + kept = await c.patch( + f"/api/agents/{agent_id}", + headers=user_auth, + json={ + "name": "files-renamed", + "config": {"backend": {"type": "filesystem", "root_dir": "/", "virtual_mode": True}}, + }, + ) + assert kept.status_code == 200, kept.text + + tightened = await c.patch( + f"/api/users/{user.json()['id']}", + headers=auth, + json={"allowed_backend_types": ["docker"]}, + ) + assert tightened.status_code == 200, tightened.text + assert tightened.json()["allowed_backend_types"] == ["docker"] + + still_kept = await c.patch( + f"/api/agents/{agent_id}", + headers=user_auth, + json={ + "name": "files-kept", + "config": {"backend": {"type": "filesystem", "root_dir": "/", "virtual_mode": True}}, + }, + ) + assert still_kept.status_code == 200, still_kept.text + + switched = await c.patch( + f"/api/agents/{agent_id}", + headers=user_auth, + json={ + "config": {"backend": {"type": "local_shell", "root_dir": "/", "virtual_mode": True}}, + }, + ) + assert switched.status_code == 403, switched.text + assert switched.json()["error"]["code"] == "BACKEND_TYPE_NOT_ALLOWED" + + cleared = await c.patch( + f"/api/users/{user.json()['id']}", + headers=auth, + json={"role": "user"}, + ) + assert cleared.status_code == 200, cleared.text + assert cleared.json()["allowed_backend_types"] is None diff --git a/tests/integration/test_users_api.py b/tests/integration/test_users_api.py index feeb997329..aff094fd0d 100644 --- a/tests/integration/test_users_api.py +++ b/tests/integration/test_users_api.py @@ -26,6 +26,7 @@ async def test_create_list_get_delete(env): assert r.json()["login_retry_after_seconds"] == 0 assert isinstance(r.json()["created_at"], int) assert r.json()["created_at"] > 0 + assert r.json()["last_login_at"] is None r = await c.delete(f"/api/users/{uid}", headers=auth) assert r.status_code == 204 diff --git a/tests/unit/agents/test_acp_host.py b/tests/unit/agents/test_acp_host.py new file mode 100644 index 0000000000..c1955379fa --- /dev/null +++ b/tests/unit/agents/test_acp_host.py @@ -0,0 +1,166 @@ +"""Host ACP probe / install status, without spawning a real CLI.""" + +from __future__ import annotations + +import pytest + +from octop.infra.agents.settings import acp_host + + +@pytest.mark.asyncio +async def test_ready_when_handshake_succeeds(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(acp_host, "_platform_key", lambda: "linux") + monkeypatch.setattr(acp_host, "_which", lambda name, env: f"/bin/{name}") + + async def ok(command: str, args: list[str], env: dict[str, str]) -> tuple[bool, str]: + assert command == "opencode" + assert args == ["acp"] + assert env + return True, "" + + monkeypatch.setattr(acp_host, "_handshake", ok) + result = await acp_host.acp_host_status(runner="opencode", locale="zh") + assert result["status"] == "ready" + assert result["acp_ok"] is True + assert result["can_auto_install"] is False + assert "无需再安装" in result["agent_prompt"] + assert "opencode.ai/v2/install" in result["scripts"]["linux"] + assert result["scripts"]["win32"].startswith("#") + + +@pytest.mark.asyncio +async def test_missing_binary_skips_handshake(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(acp_host, "_platform_key", lambda: "darwin") + monkeypatch.setattr(acp_host, "_which", lambda name, env: None) + monkeypatch.setattr(acp_host, "_shell_ok", lambda plat: True) + + async def boom(*_args: object, **_kwargs: object) -> tuple[bool, str]: + raise AssertionError("handshake should not run") + + monkeypatch.setattr(acp_host, "_handshake", boom) + result = await acp_host.acp_host_status(runner="kimi_code", locale="en") + assert result["status"] == "missing" + assert result["reason"] == "command_missing" + assert result["can_auto_install"] is True + assert "install.sh" in result["install_script"] + assert "Install Kimi Code" in result["agent_prompt"] + + +@pytest.mark.asyncio +async def test_adapter_requires_agent_binary(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(acp_host, "_platform_key", lambda: "linux") + + def which(name: str, env: dict[str, str]) -> str | None: + _ = env + return "/usr/bin/npx" if name == "npx" else None + + monkeypatch.setattr(acp_host, "_which", which) + + async def boom(*_args: object, **_kwargs: object) -> tuple[bool, str]: + raise AssertionError("handshake should not run") + + monkeypatch.setattr(acp_host, "_handshake", boom) + result = await acp_host.acp_host_status(runner="claude_code") + assert result["status"] == "missing" + assert result["reason"] == "agent_missing" + assert "claude.ai/install.sh" in result["scripts"]["darwin"] + + +@pytest.mark.asyncio +async def test_handshake_failure_is_degraded(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(acp_host, "_platform_key", lambda: "linux") + monkeypatch.setattr(acp_host, "_which", lambda name, env: f"/bin/{name}") + + async def fail(*_args: object, **_kwargs: object) -> tuple[bool, str]: + return False, "timed out waiting for ACP initialize" + + monkeypatch.setattr(acp_host, "_handshake", fail) + result = await acp_host.acp_host_status( + runner="custom", + command="my-agent", + args=["--acp"], + locale="en", + ) + assert result["status"] == "degraded" + assert result["reason"] == "handshake_timeout" + assert result["install_script"] == "" + assert result["can_auto_install"] is False + assert "my-agent" in result["agent_prompt"] + + +@pytest.mark.asyncio +async def test_install_then_probe(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(acp_host, "_platform_key", lambda: "win32") + monkeypatch.setattr(acp_host, "_shell_ok", lambda plat: True) + present = {"ok": False} + + def which(name: str, env: dict[str, str]) -> str | None: + _ = env + if present["ok"] and name == "agent": + return r"C:\agent.cmd" + return None + + def install(script: str, plat: str) -> tuple[bool, str]: + assert plat == "win32" + assert "cursor.com/install?win32=true" in script + present["ok"] = True + return True, "" + + async def ok(command: str, args: list[str], env: dict[str, str]) -> tuple[bool, str]: + assert command == "agent" + assert args == ["acp"] + assert env + return True, "" + + monkeypatch.setattr(acp_host, "_which", which) + monkeypatch.setattr(acp_host, "_run_install", install) + monkeypatch.setattr(acp_host, "_handshake", ok) + result = await acp_host.acp_host_status(runner="cursor_cli", attempt_install=True, locale="zh") + assert result["status"] == "installed" + assert "irm" in result["scripts"]["win32"] + + +@pytest.mark.asyncio +async def test_ask_returns_reply(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(acp_host, "_which", lambda name, env: f"/bin/{name}") + + async def answer(command: str, args: list[str], env: dict[str, str]) -> tuple[bool, str]: + assert command == "opencode" + assert args == ["acp"] + assert env + return True, "I am OpenCode." + + monkeypatch.setattr(acp_host, "_ask", answer) + result = await acp_host.acp_host_ask(runner="opencode") + assert result["ok"] is True + assert result["prompt"] == "who are you?" + assert result["text"] == "I am OpenCode." + + +@pytest.mark.asyncio +async def test_ask_skips_when_command_missing(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(acp_host, "_which", lambda name, env: None) + + async def boom(*_args: object, **_kwargs: object) -> tuple[bool, str]: + raise AssertionError("ask should not spawn") + + monkeypatch.setattr(acp_host, "_ask", boom) + result = await acp_host.acp_host_ask(runner="kimi_code") + assert result["ok"] is False + assert result["reason"] == "command_missing" + assert result["prompt"] == "who are you?" + + +@pytest.mark.asyncio +async def test_windows_opencode_skips_auto_install(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(acp_host, "_platform_key", lambda: "win32") + monkeypatch.setattr(acp_host, "_which", lambda name, env: None) + + def install(*_args: object, **_kwargs: object) -> tuple[bool, str]: + raise AssertionError("opencode has no Windows auto installer") + + monkeypatch.setattr(acp_host, "_run_install", install) + result = await acp_host.acp_host_status(runner="opencode", attempt_install=True) + assert result["status"] == "skipped" + assert result["can_auto_install"] is False + assert "WSL" in result["install_script"] diff --git a/tests/unit/agents/test_acp_permission_compat.py b/tests/unit/agents/test_acp_permission_compat.py new file mode 100644 index 0000000000..aa5ba2db61 --- /dev/null +++ b/tests/unit/agents/test_acp_permission_compat.py @@ -0,0 +1,70 @@ +"""Overlapping ACP permission requests must not drop the earlier waiter.""" + +from __future__ import annotations + +import asyncio + +import pytest +from octop_harness.acp.client import ACPHostedClient +from octop_harness.acp.models import ACPRunnerConfig + +from octop.infra.agents.acp_permission_compat import install_acp_permission_queue + +_OPTIONS = [ + {"optionId": "allow-once", "name": "Allow once", "kind": "allow_once"}, + {"optionId": "reject-once", "name": "Reject", "kind": "reject_once"}, +] + + +def _client() -> ACPHostedClient: + install_acp_permission_queue() + client = ACPHostedClient( + runner_name="cursor_cli", + runner_config=ACPRunnerConfig(command="agent", args=["acp"]), + cwd=".", + ) + + async def _on_message(_payload: dict[str, object], _is_last: bool) -> None: + return None + + client.start_prompt(_on_message) + return client + + +@pytest.mark.asyncio +async def test_second_permission_waits_until_the_first_is_answered() -> None: + client = _client() + first = asyncio.create_task( + client.request_permission( + _OPTIONS, + "session", + {"title": "Web search: one", "kind": "fetch"}, + ) + ) + await asyncio.sleep(0) + assert client.pending_permission is not None + assert client.pending_permission.summary == "Web search: one" + + second = asyncio.create_task( + client.request_permission( + _OPTIONS, + "session", + {"title": "Web search: two", "kind": "fetch"}, + ) + ) + await asyncio.sleep(0) + assert client.pending_permission.summary == "Web search: one" + assert not second.done() + + client.resolve_permission("allow-once") + first_result = await asyncio.wait_for(first, timeout=1) + assert getattr(first_result.outcome, "option_id", None) == "allow-once" + + await asyncio.sleep(0) + assert not second.done() + assert client.pending_permission is not None + assert client.pending_permission.summary == "Web search: two" + + client.resolve_permission("allow-once") + second_result = await asyncio.wait_for(second, timeout=1) + assert getattr(second_result.outcome, "option_id", None) == "allow-once" diff --git a/tests/unit/agents/test_acp_tool_name_compat.py b/tests/unit/agents/test_acp_tool_name_compat.py new file mode 100644 index 0000000000..bebd56fc33 --- /dev/null +++ b/tests/unit/agents/test_acp_tool_name_compat.py @@ -0,0 +1,57 @@ +"""A later ACP tool update must keep the title from the first event.""" + +from __future__ import annotations + +from types import SimpleNamespace + +from octop_harness.acp.client import ACPHostedClient +from octop_harness.acp.models import ACPRunnerConfig + +from octop.infra.agents.acp_tool_name_compat import install_acp_tool_titles + +_CALL = "call-0\nfc_0" + + +def _client() -> ACPHostedClient: + install_acp_tool_titles() + client = ACPHostedClient( + runner_name="cursor_cli", + runner_config=ACPRunnerConfig(command="agent", args=["acp"]), + cwd=".", + ) + client.start_prompt(lambda _payload, _is_last: None) + return client + + +def _update(title: str | None) -> SimpleNamespace: + return SimpleNamespace( + tool_call_id=_CALL, + title=title, + kind="other", + status="completed", + raw_input=None, + content=None, + raw_output=None, + ) + + +def test_titleless_update_keeps_the_first_tool_name() -> None: + client = _client() + opened = client._tool_event_from_state(_update("Web Search")) + assert opened is not None + assert opened["name"] == "Web Search" + + finished = client._tool_event_from_state(_update(None)) + assert finished is not None + assert finished["name"] == "Web Search" + assert finished["title"] == "Web Search" + + +def test_new_prompt_does_not_reuse_the_previous_title() -> None: + client = _client() + client._tool_event_from_state(_update("Web Search")) + client.start_prompt(lambda _payload, _is_last: None) + + finished = client._tool_event_from_state(_update(None)) + assert finished is not None + assert finished["name"] == "unknown" diff --git a/tests/unit/agents/test_agent_manager.py b/tests/unit/agents/test_agent_manager.py index 5a491404e6..7160e4ab80 100644 --- a/tests/unit/agents/test_agent_manager.py +++ b/tests/unit/agents/test_agent_manager.py @@ -494,10 +494,11 @@ async def test_install_team_host_enrich_uses_host_assignment(manager: AgentManag manager._install_team_host_dispatch() req = SimpleNamespace(thread_id="t~child", messages="我最近总失眠") out = await team._enrich_request("child", req) - assert isinstance(out.messages[0], SystemMessage) - assert isinstance(out.messages[1], HumanMessage) - assert out.messages[1].content == "请给出可执行的睡眠建议,聚焦作息而不是诊断" - assert "团队派工" in out.messages[0].content + assert not any(isinstance(m, SystemMessage) for m in out.messages) + assert len(out.messages) == 1 + assert isinstance(out.messages[0], HumanMessage) + assert out.messages[0].content == "请给出可执行的睡眠建议,聚焦作息而不是诊断" + assert "团队派工" in out.configurable["octop_team_dispatch_briefing"] @pytest.mark.asyncio @@ -1815,6 +1816,57 @@ def test_host_model_sdk_errors_keep_original_retry_budget( assert handler.call_count == cfg.model_retry_max_retries + 1 +def test_build_harness_config_applies_per_expert_retry_and_tool_cap(manager: AgentManager) -> None: + from langchain.agents.middleware import ( + ModelRetryMiddleware, + ToolCallLimitMiddleware, + ToolRetryMiddleware, + ) + + cfg = manager._build_harness_config( + _row( + config_json=json.dumps( + { + "tool_call_run_limit": 12, + "model_retry_max_retries": 4, + "model_retry_initial_delay": 0.25, + "tool_retry_max_retries": 1, + } + ) + ) + ) + limit = next(item for item in cfg.middleware or [] if isinstance(item, ToolCallLimitMiddleware)) + tool_retry = next( + item for item in cfg.middleware or [] if isinstance(item, ToolRetryMiddleware) + ) + model_retry = next( + item for item in cfg.middleware or [] if isinstance(item, ModelRetryMiddleware) + ) + assert limit.run_limit == 12 + assert limit.thread_limit is None + assert tool_retry.max_retries == 1 + assert model_retry.max_retries == 4 + assert model_retry.initial_delay == 0.25 + assert not cfg.model_retry_enabled + + disabled = manager._build_harness_config( + _row( + config_json=json.dumps( + { + "tool_call_limit_enabled": False, + "tool_retry_enabled": False, + "model_retry_enabled": False, + } + ) + ) + ) + assert not any( + isinstance(item, ToolCallLimitMiddleware | ToolRetryMiddleware | ModelRetryMiddleware) + for item in (disabled.middleware or []) + ) + assert not disabled.model_retry_enabled + + def test_runtime_uses_health_aware_model_failover(manager: AgentManager) -> None: from octop.infra.agents.middleware.model_failover import SafeModelFailoverMiddleware @@ -2945,6 +2997,37 @@ async def fake_load(_user_id: int, _server_name: str, _spec: dict[str, Any]) -> assert "notes" in missing +async def test_prepare_chat_mcp_stops_at_a_connector_that_lost_authorization( + manager: AgentManager, + monkeypatch: pytest.MonkeyPatch, +) -> None: + user_id = _seed_user(manager, "mcp-reauth") + svc = manager._connector_svc + svc.put_custom_servers( + user_id, + {"wiki": {"transport": "streamable_http", "url": "https://wiki.example.com/mcp"}}, + ) + servers = dict(svc.get_custom_servers(user_id)) + servers["wiki"] = {**servers["wiki"], "oauth": {"required": True, "expired": True}} + svc._save_custom_servers(user_id, servers) + agent_id = manager._repos.agent_repo.create(agent_id="REAUTH", user_id=user_id, name="a") + agent = MagicMock() + # The tools are still cached on the agent from before the token lapsed. + agent._mcp_tool_name_set = frozenset({"wiki_search"}) + agent._mcp_attached_user = {"wiki": user_id} + agent.config.mcp_server_configs = {} + load = AsyncMock() + monkeypatch.setattr(manager, "get_agent", lambda _agent_id: agent) + monkeypatch.setattr(manager, "_get_or_load_mcp_tools", load) + + missing = await manager.prepare_chat_mcp(agent_id, ["wiki"], connector_user_id=user_id) + + assert missing == ["wiki"] + load.assert_not_awaited() + blocked = await manager.connector_auth_blocked(user_id, missing) + assert [(b["server"], b["auth_status"]) for b in blocked] == [("wiki", "expired")] + + def test_prepare_stream_request_maps_max_iters_to_recursion_limit( manager: AgentManager, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/agents/test_compaction.py b/tests/unit/agents/test_compaction.py new file mode 100644 index 0000000000..3641a5d2c9 --- /dev/null +++ b/tests/unit/agents/test_compaction.py @@ -0,0 +1,79 @@ +"""Per-expert compaction settings.""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from octop.api.common.agent_runtime import AgentCompactionSettings +from octop.infra.agents.settings.compaction import ( + apply_compaction_update, + compaction_for_api, + harness_summarization_kwargs, + normalize_compaction, +) + +_SUPPORTED = frozenset( + { + "summarization_enabled", + "summarization_trigger", + "summarization_keep", + "summarization_trim_tokens", + } +) + + +def test_normalize_drops_defaults_and_keeps_overrides() -> None: + assert normalize_compaction(None) is None + assert normalize_compaction({"enabled": True}) is None + assert normalize_compaction({"enabled": False}) == {"enabled": False} + assert normalize_compaction( + { + "enabled": True, + "trigger_fraction": 0.7, + "keep_fraction": 0.2, + "trim_tokens": 4000, + } + ) == { + "trigger_fraction": 0.7, + "keep_fraction": 0.2, + "trim_tokens": 4000, + } + + +def test_normalize_drops_keep_that_is_not_below_the_trigger() -> None: + assert normalize_compaction({"trigger_fraction": 0.5, "keep_fraction": 0.5}) == { + "trigger_fraction": 0.5 + } + assert normalize_compaction({"keep_fraction": 0.9}) is None + + +def test_apply_compaction_update_clears_with_null() -> None: + cfg = {"compaction": {"trigger_fraction": 0.7}, "backend": {"type": "state"}} + assert apply_compaction_update(cfg, None) == {"backend": {"type": "state"}} + assert cfg["compaction"]["trigger_fraction"] == 0.7 + + +def test_harness_kwargs_follow_the_installed_fields() -> None: + cfg = {"compaction": {"enabled": False, "trigger_fraction": 0.6, "trim_tokens": 2000}} + assert harness_summarization_kwargs(cfg, supported_fields=frozenset()) == {} + assert harness_summarization_kwargs(cfg, supported_fields=_SUPPORTED) == { + "summarization_enabled": False + } + active = {"compaction": {"trigger_fraction": 0.6, "keep_fraction": 0.15, "trim_tokens": 2000}} + assert harness_summarization_kwargs(active, supported_fields=_SUPPORTED) == { + "summarization_trigger": ("fraction", 0.6), + "summarization_keep": ("fraction", 0.15), + "summarization_trim_tokens": 2000, + } + assert compaction_for_api({}) is None + + +def test_api_model_rejects_keep_at_or_above_the_trigger() -> None: + AgentCompactionSettings(trigger_fraction=0.7, keep_fraction=0.2, trim_tokens=4000) + with pytest.raises(ValidationError): + AgentCompactionSettings(keep_fraction=0.85) + with pytest.raises(ValidationError): + AgentCompactionSettings(trigger_fraction=0.4, keep_fraction=0.4) + with pytest.raises(ValidationError): + AgentCompactionSettings(trim_tokens=999) diff --git a/tests/unit/agents/test_execute_env.py b/tests/unit/agents/test_execute_env.py index 995de7a9eb..e19e5fabea 100644 --- a/tests/unit/agents/test_execute_env.py +++ b/tests/unit/agents/test_execute_env.py @@ -47,6 +47,7 @@ def test_inject_local_shell_uses_octop_auth_for_new_layout(tmp_path: Path, monke ) env = spec["env"] assert env["OCTOP_AGENT_ID"] == "agent-1" + assert env["OCTOP_TCCLI_PROFILE"] == "octop-1" assert (ws / ".octop" / "auth").is_dir() assert env["OCTOP_AUTH_DIR"] == str(ws / ".octop" / "auth") assert env["OCTOP_HOME"] == str(paths.root) @@ -162,4 +163,5 @@ def test_inject_docker_only_sets_platform_defaults(tmp_path: Path, monkeypatch) env = spec["env"] assert env["OCTOP_AGENT_ID"] == "agent-2" assert env["OCTOP_AUTH_DIR"] == str(ws / ".octop" / "auth") + assert spec["environment"]["OCTOP_TCCLI_PROFILE"] == "octop-1" assert "GLOBAL_KEY" not in env diff --git a/tests/unit/agents/test_managed.py b/tests/unit/agents/test_managed.py new file mode 100644 index 0000000000..3dcc1cb1e9 --- /dev/null +++ b/tests/unit/agents/test_managed.py @@ -0,0 +1,79 @@ +"""Managed agent kind and filesystem pin.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from octop.infra.agents.managed import ( + apply_managed_backend, + managed_default_workspace, + managed_project_root, + managed_skill_dir, + managed_subagent_dir, + managed_template_payload, + preserve_managed_config, + require_managed_fields, +) +from octop.infra.db.migrate import run_migrations +from octop.infra.db.pool import SqlitePool +from octop.infra.db.repos.agents import AgentRepo +from octop.infra.errors import OctopError + + +def test_repo_stores_managed_kind(tmp_path: Path) -> None: + pool = SqlitePool(tmp_path / "octop.db") + run_migrations(pool) + repo = AgentRepo(pool) + repo.create(agent_id="m1", user_id=None, name="coder", kind="managed") + row = repo.get("m1") + assert row is not None + assert row.kind == "managed" + + +def test_preserve_keeps_runner_and_filesystem_root() -> None: + config = { + "backend": {"type": "docker"}, + "workspace_dir": "/work", + "conversation_mode": "plan", + } + preserve_managed_config(config, {"managed_runner": "opencode"}) + assert config["backend"] == { + "type": "filesystem", + "root_dir": "/", + "virtual_mode": True, + } + assert config["managed_runner"] == "opencode" + assert "conversation_mode" not in config + assert config["memory"]["memory_enabled"] is False + assert config["memory"]["memory_capture_enabled"] is False + assert config["memory"]["memory_recall_inject_enabled"] is False + + +def test_managed_template_is_programming_seed() -> None: + payload = managed_template_payload() + prompts = payload["quick_prompts"] + assert len(prompts) == 6 + files = {item["name"]: item["content"] for item in payload["files"]} + assert "代码" in files["AGENTS.md"] + assert Path(payload["home"]) == Path.home() + assert managed_default_workspace("abc") == str(Path.home() / "abc") + + +def test_runner_project_dirs() -> None: + assert managed_project_root("codebuddy") == ".codebuddy" + assert managed_skill_dir("codebuddy") == ".codebuddy/skills" + assert managed_subagent_dir("codebuddy") == ".codebuddy/agents" + assert managed_project_root("unknown") == ".octop" + assert managed_skill_dir("claude_code") == ".claude/skills" + + +def test_require_managed_fields() -> None: + config: dict[str, Any] = {} + apply_managed_backend(config) + try: + require_managed_fields(config) + except OctopError as exc: + assert exc.status == 400 + else: + raise AssertionError("expected missing runner to fail") diff --git a/tests/unit/agents/test_model_failover_middleware.py b/tests/unit/agents/test_model_failover_middleware.py index eea88713fd..430bb688ae 100644 --- a/tests/unit/agents/test_model_failover_middleware.py +++ b/tests/unit/agents/test_model_failover_middleware.py @@ -298,3 +298,152 @@ async def handler(request: ModelRequest) -> ModelResponse: assert response.result[0].content == "ok" assert attempts == ["p/a", "q/c"] assert configurable["model"] == "q/c" + + +def _sdk_connection_error() -> Exception: + import httpx + from openai import APIConnectionError + + request = httpx.Request("POST", "http://127.0.0.1:3425/v1/chat/completions") + cause = httpx.ConnectError("All connection attempts failed") + try: + raise APIConnectionError(request=request) from cause + except APIConnectionError as exc: + return exc + + +@pytest.mark.asyncio +async def test_auto_failover_on_openai_connection_error(monkeypatch) -> None: + models = {"Magpie/a": _model("Magpie/a", 100_000), "openai/b": _model("openai/b", 100_000)} + configurable = {"model": "Magpie/a", "octop_auto_route": True} + middleware = SafeModelFailoverMiddleware( + health=ModelHealthRegistry(clock=lambda: 100.0), + candidates=lambda _current, _vision: [AutoRouteCandidate("openai/b", 100_000)], + model_for_ref=models.__getitem__, + max_retries=0, + initial_delay=0, + max_delay=0, + ) + monkeypatch.setattr(model_failover, "_configurable", lambda _request: configurable) + attempts: list[str] = [] + + async def handler(request: ModelRequest) -> ModelResponse: + attempts.append(request.model.model) + if request.model.model == "Magpie/a": + raise _sdk_connection_error() + return ModelResponse(result=[AIMessage(content="ok")]) + + response = await middleware.awrap_model_call( + ModelRequest(model=models["Magpie/a"], messages=[HumanMessage(content="hello")]), + handler, + ) + assert response.result[0].content == "ok" + assert attempts == ["Magpie/a", "openai/b"] + + +@pytest.mark.asyncio +async def test_pinned_model_does_not_switch_on_connection_error(monkeypatch) -> None: + model = _model("Magpie/a", 100_000) + middleware = SafeModelFailoverMiddleware( + health=ModelHealthRegistry(clock=lambda: 100.0), + candidates=lambda _current, _vision: [AutoRouteCandidate("openai/b", 100_000)], + model_for_ref=lambda ref: _model(ref, 100_000), + max_retries=0, + initial_delay=0, + max_delay=0, + ) + monkeypatch.setattr( + model_failover, + "_configurable", + lambda _request: {"model": "Magpie/a", "octop_auto_route": False}, + ) + handler = AsyncMock(side_effect=_sdk_connection_error()) + response = await middleware.awrap_model_call( + ModelRequest(model=model, messages=[HumanMessage(content="hello")]), + handler, + ) + assert "[model_call_failed]" in str(response.result[0].content) + assert handler.await_count == 1 + + +@pytest.mark.asyncio +async def test_failover_skips_models_omitted_by_permission_and_modality(monkeypatch) -> None: + models = { + "Magpie/a": _model("Magpie/a", 100_000), + "openai/text": _model("openai/text", 100_000), + "openai/vision": _model("openai/vision", 100_000), + } + middleware = SafeModelFailoverMiddleware( + health=ModelHealthRegistry(clock=lambda: 100.0), + candidates=lambda _current, needs_multimodal: ( + [AutoRouteCandidate("openai/vision", 100_000)] + if needs_multimodal + else [AutoRouteCandidate("openai/text", 8_000)] + ), + model_for_ref=models.__getitem__, + max_retries=0, + initial_delay=0, + max_delay=0, + ) + monkeypatch.setattr( + model_failover, + "_configurable", + lambda _request: {"model": "Magpie/a", "octop_auto_route": True}, + ) + monkeypatch.setattr(model_failover, "latest_human_needs_multimodal", lambda _messages: True) + attempts: list[str] = [] + + async def handler(request: ModelRequest) -> ModelResponse: + attempts.append(request.model.model) + if request.model.model == "Magpie/a": + raise _sdk_connection_error() + return ModelResponse(result=[AIMessage(content="ok")]) + + response = await middleware.awrap_model_call( + ModelRequest(model=models["Magpie/a"], messages=[HumanMessage(content="see")]), + handler, + ) + assert response.result[0].content == "ok" + assert attempts == ["Magpie/a", "openai/vision"] + + +@pytest.mark.asyncio +async def test_cooled_down_model_is_tried_again(monkeypatch) -> None: + clock = {"now": 100.0} + models = {"Magpie/a": _model("Magpie/a", 100_000)} + health = ModelHealthRegistry(clock=lambda: clock["now"]) + middleware = SafeModelFailoverMiddleware( + health=health, + candidates=lambda _current, _vision: [], + model_for_ref=models.__getitem__, + max_retries=0, + initial_delay=0, + max_delay=0, + ) + monkeypatch.setattr( + model_failover, + "_configurable", + lambda _request: {"model": "Magpie/a", "octop_auto_route": True}, + ) + calls = {"n": 0} + + async def handler(request: ModelRequest) -> ModelResponse: + del request + calls["n"] += 1 + if calls["n"] == 1: + raise _sdk_connection_error() + return ModelResponse(result=[AIMessage(content="back")]) + + first = await middleware.awrap_model_call( + ModelRequest(model=models["Magpie/a"], messages=[HumanMessage(content="hello")]), + handler, + ) + assert "[model_call_failed]" in str(first.result[0].content) + assert not health.is_routable("Magpie/a") + clock["now"] = 100.0 + 60.0 + second = await middleware.awrap_model_call( + ModelRequest(model=models["Magpie/a"], messages=[HumanMessage(content="hello")]), + handler, + ) + assert second.result[0].content == "back" + assert health.is_routable("Magpie/a") diff --git a/tests/unit/agents/test_provider_harness_factory.py b/tests/unit/agents/test_provider_harness_factory.py index 5c73b4c78a..0c63273e49 100644 --- a/tests/unit/agents/test_provider_harness_factory.py +++ b/tests/unit/agents/test_provider_harness_factory.py @@ -39,3 +39,20 @@ def test_sync_adds_and_removes_providers() -> None: harness_manager.remove_provider.assert_called_once_with("old") harness_manager.add_provider.assert_called_once_with(keep) + + +def test_sync_forwards_magpie_session_header_unchanged() -> None: + harness_manager = MagicMock() + magpie = ProviderConfig( + id="Magpie", + base_url="http://127.0.0.1:3425/v1", + api_key="magpie-octop", + name="Magpie", + headers={"User-Agent": "Octop/test"}, + session_header="X-Magpie-Session", + ) + sync_providers_to_harness(harness_manager, [magpie], shared_factory=None) + forwarded = harness_manager.add_provider.call_args.args[0] + assert forwarded.session_header == "X-Magpie-Session" + assert forwarded.headers == {"User-Agent": "Octop/test"} + assert "X-Magpie-Session" not in forwarded.headers diff --git a/tests/unit/agents/test_provider_store.py b/tests/unit/agents/test_provider_store.py index c280a8fdc4..76776d41a7 100644 --- a/tests/unit/agents/test_provider_store.py +++ b/tests/unit/agents/test_provider_store.py @@ -654,3 +654,82 @@ def test_list_auto_route_candidates_filters_and_preserves_capacity( assert [(item.ref, item.max_input_tokens) for item in vision_candidates] == [ ("p/vision", 32_000) ] + + +def test_magpie_runtime_headers_are_dynamic_and_do_not_pin_a_session(store: ProviderStore) -> None: + from octop import __version__ + + store._provider_repo.create( + name="Magpie", + kind="openai", + base_url="http://127.0.0.1:3425/v1", + api_key="magpie-octop", + extra_json=json.dumps({"integration": "magpie", "headers": {"User-Agent": "Custom/9"}}), + models_json=json.dumps([{"id": "group/a/b", "name": "Route", "enabled": True}]), + ) + store._provider_repo.create( + name="openai", + kind="openai", + base_url="https://api.openai.com/v1", + api_key="sk-test", + models_json=json.dumps([{"id": "gpt-4o", "name": "GPT", "enabled": True}]), + ) + configs = {item.name: item for item in store.build_harness_configs()} + magpie = configs["Magpie"] + assert magpie.session_header == "X-Magpie-Session" + assert magpie.headers.get("User-Agent") == "Custom/9" + assert "X-Magpie-Session" not in {key.lower() for key in magpie.headers} + assert configs["openai"].session_header is None + assert "Octop/" not in str(configs["openai"].headers) + + store._provider_repo.create( + name="Magpie Remote", + kind="openai", + base_url="https://gateway.example/v1", + api_key="real-key", + extra_json=json.dumps({"integration": "magpie"}), + models_json=json.dumps([{"id": "m", "enabled": True}]), + ) + remote = next(item for item in store.build_harness_configs() if item.name == "Magpie Remote") + assert remote.session_header == "X-Magpie-Session" + assert remote.headers["User-Agent"] == f"Octop/{__version__}" + assert "X-Magpie-Session" not in remote.headers + + +def test_magpie_model_removal_clears_stale_pins(tmp_path: Path) -> None: + from octop.infra.agents.providers.magpie import plan_magpie_sync + from octop.infra.agents.providers.store import clear_stale_pins_for_provider + + paths = PathLayout(tmp_path / ".octop") + paths.ensure_root() + db = SqlitePool(paths.db) + run_migrations(db) + services = build_shared_services(db=db, paths=paths, config=OctopConfig()) + existing = [ + { + "id": "keep", + "enabled": True, + "input": ["text", "image"], + "visibility": "groups", + "group_ids": ["design"], + }, + {"id": "gone", "enabled": True}, + ] + plan = plan_magpie_sync(existing, [{"id": "keep", "name": "Keep", "input": ["text"]}]) + assert plan.models[0]["group_ids"] == ["design"] + assert plan.models[0]["input"] == ["text", "image"] + services.repos.agent_repo.create( + agent_id="01GONE", + user_id=None, + name="pinned", + default_model="Magpie/gone", + ) + services.repos.settings_repo.set_active_model("Magpie", "gone") + cleared = clear_stale_pins_for_provider( + agent_repo=services.repos.agent_repo, + settings_repo=services.repos.settings_repo, + provider_name="Magpie", + models=plan.models, + ) + assert cleared == ["01GONE"] + assert services.repos.settings_repo.get_active_model() == ("", "") diff --git a/tests/unit/agents/test_runtime_limits.py b/tests/unit/agents/test_runtime_limits.py index 144854fae0..7c0adce465 100644 --- a/tests/unit/agents/test_runtime_limits.py +++ b/tests/unit/agents/test_runtime_limits.py @@ -2,14 +2,23 @@ from __future__ import annotations +from langchain.agents.middleware import ToolCallLimitMiddleware, ToolRetryMiddleware + from octop.infra.agents.settings.runtime_limits import ( CONFIGURABLE_MAX_INPUT_TOKENS, CONFIGURABLE_MODEL_SETTINGS, + DEFAULT_MODEL_RETRY_MAX_RETRIES, + DEFAULT_TOOL_CALL_RUN_LIMIT, + DEFAULT_TOOL_RETRY_MAX_RETRIES, agent_max_input_tokens, agent_model_settings, agent_recursion_limit, apply_agent_runtime_to_stream_request, + call_guard_middleware, resolve_context_max_tokens, + resolve_model_retry, + resolve_tool_call_run_limit, + resolve_tool_retry, ) @@ -69,3 +78,49 @@ def test_apply_agent_runtime_to_stream_request_merges_configurable() -> None: "max_tokens": 2048, } assert req["configurable"][CONFIGURABLE_MAX_INPUT_TOKENS] == 32000 + + +def test_call_guard_defaults_cap_tool_calls_and_retry_tool_exceptions() -> None: + assert resolve_tool_call_run_limit({}) == DEFAULT_TOOL_CALL_RUN_LIMIT + assert resolve_model_retry({}).enabled is True + assert resolve_model_retry({}).max_retries == DEFAULT_MODEL_RETRY_MAX_RETRIES + assert resolve_tool_retry({}).enabled is True + assert resolve_tool_retry({}).max_retries == DEFAULT_TOOL_RETRY_MAX_RETRIES + + middleware = call_guard_middleware({}) + limit = next(item for item in middleware if isinstance(item, ToolCallLimitMiddleware)) + tool_retry = next(item for item in middleware if isinstance(item, ToolRetryMiddleware)) + assert limit.run_limit == DEFAULT_TOOL_CALL_RUN_LIMIT + assert limit.thread_limit is None + assert limit.exit_behavior == "end" + assert tool_retry.max_retries == DEFAULT_TOOL_RETRY_MAX_RETRIES + assert tool_retry.initial_delay == 1.0 + assert tool_retry.max_delay == 60.0 + assert tool_retry.on_failure == "continue" + + +def test_call_guard_honors_overrides_and_can_be_disabled() -> None: + cfg = { + "tool_call_run_limit": 12, + "model_retry_enabled": False, + "model_retry_max_retries": 0, + "tool_retry_max_retries": 1, + "tool_retry_initial_delay": 0.5, + "tool_retry_max_delay": 2, + } + assert resolve_tool_call_run_limit(cfg) == 12 + assert resolve_model_retry(cfg).enabled is False + assert resolve_model_retry(cfg).max_retries == 0 + tool_retry = resolve_tool_retry(cfg) + assert tool_retry.max_retries == 1 + assert tool_retry.initial_delay == 0.5 + assert tool_retry.max_delay == 2.0 + + disabled = call_guard_middleware( + {"tool_call_limit_enabled": False, "tool_retry_enabled": False} + ) + assert disabled == [] + assert ( + resolve_tool_call_run_limit({"tool_call_limit_enabled": False, "tool_call_run_limit": 3}) + is None + ) diff --git a/tests/unit/agents/test_team_dispatch_briefing.py b/tests/unit/agents/test_team_dispatch_briefing.py new file mode 100644 index 0000000000..10179bdc8e --- /dev/null +++ b/tests/unit/agents/test_team_dispatch_briefing.py @@ -0,0 +1,156 @@ +"""Team dispatch briefing travels through configurable, not the conversation.""" + +from __future__ import annotations + +from typing import Any + +from langchain.agents import create_agent +from langchain.agents.middleware import AgentMiddleware, ModelRequest +from langchain_core.language_models.fake_chat_models import FakeListChatModel +from langchain_core.messages import HumanMessage, SystemMessage + +from octop.infra.agents.middleware.team_dispatch import ( + CONFIG_KEY, + TeamDispatchBriefingMiddleware, + _with_briefing, +) + +BRIEFING = "[团队派工,不是用户在直接问你]\n你是这个专家团队的成员。" +_MODEL = FakeListChatModel(responses=["ok"]) + + +def _patch_configurable(monkeypatch: Any, value: Any) -> None: + monkeypatch.setattr( + "octop.infra.agents.middleware.team_dispatch.get_config", + lambda: {"configurable": {} if value is None else {CONFIG_KEY: value}}, + ) + + +def _request(system: str | None, messages: list[Any] | None = None) -> ModelRequest[Any]: + return ModelRequest( + model=_MODEL, + messages=list(messages) if messages else [HumanMessage("任务正文")], + system_message=SystemMessage(content=system) if system is not None else None, + tools=[], + state={"messages": []}, + runtime=None, # type: ignore[arg-type] + ) + + +def test_with_briefing_no_configurable_key_leaves_request_untouched(monkeypatch: Any) -> None: + """场景:非团队派工轮次没有 briefing。预期:request 原样返回,system prompt 不变。""" + _patch_configurable(monkeypatch, None) + req = _request("agent-prompt") + + out = _with_briefing(req) + + assert out is req + assert out.system_message is not None + assert out.system_message.text == "agent-prompt" + + +def test_with_briefing_appends_to_existing_system_prompt(monkeypatch: Any) -> None: + """场景:成员已有 system prompt。预期:briefing 追加在其后,原文不丢。""" + _patch_configurable(monkeypatch, BRIEFING) + req = _request("agent-prompt") + + out = _with_briefing(req) + + text = out.system_message.text if out.system_message else "" + assert text.startswith("agent-prompt") + assert "团队派工" in text + assert out.messages == req.messages + + +def test_with_briefing_without_system_prompt_becomes_the_system_prompt(monkeypatch: Any) -> None: + """场景:成员没有 system prompt。预期:briefing 自己成为队首 system。""" + _patch_configurable(monkeypatch, BRIEFING) + req = _request(None) + + out = _with_briefing(req) + + assert out.system_message is not None + assert out.system_message.text.strip().startswith("[团队派工") + + +def test_with_briefing_blank_value_is_ignored(monkeypatch: Any) -> None: + """场景:briefing 是空白字符串。预期:不注入空 block,request 不变。""" + _patch_configurable(monkeypatch, " \n ") + req = _request("agent-prompt") + + out = _with_briefing(req) + + assert out is req + + +def test_with_briefing_is_stateless_across_model_calls(monkeypatch: Any) -> None: + """场景:一个 turn 内多次 model call 传入各自的 request。预期:每次只加一份,不累积。""" + _patch_configurable(monkeypatch, BRIEFING) + base = _request("agent-prompt") + + first = _with_briefing(base) + second = _with_briefing(base) + + assert (first.system_message.text if first.system_message else "").count("团队派工") == 1 + assert (second.system_message.text if second.system_message else "").count("团队派工") == 1 + + +class _BootstrapLike(AgentMiddleware[Any, Any]): + """Mirror harness bootstrap: drop conversation system messages, keep system prompt.""" + + def wrap_model_call( + self, + request: ModelRequest[Any], + handler: Any, + ) -> Any: + messages = [m for m in request.messages if not isinstance(m, SystemMessage)] + blocks: list[Any] = ( + list(request.system_message.content_blocks) if request.system_message else [] + ) + blocks.append({"type": "text", "text": "\n\n"}) + return handler( + request.override( + system_message=SystemMessage(content_blocks=blocks), + messages=messages, + ) + ) + + +class _Capture(AgentMiddleware[Any, Any]): + def __init__(self) -> None: + super().__init__() + self.system = "" + self.message_types: list[str] = [] + + def wrap_model_call( + self, + request: ModelRequest[Any], + handler: Any, + ) -> Any: + self.system = request.system_message.text if request.system_message else "" + self.message_types = [type(m).__name__ for m in request.messages] + return handler(request) + + +def test_briefing_survives_bootstrap_on_first_turn(monkeypatch: Any) -> None: + """场景:首轮 bootstrap 在外层先丢掉对话内 system message。 + + 预期:briefing 走 configurable,仍然进入队首 system prompt;对话里只剩任务。 + """ + _patch_configurable(monkeypatch, BRIEFING) + capture = _Capture() + agent = create_agent( + model=FakeListChatModel(responses=["ok"]), + tools=[], + middleware=[_BootstrapLike(), TeamDispatchBriefingMiddleware(), capture], + system_prompt="agent-prompt", + ) + + agent.invoke( + {"messages": [HumanMessage("任务正文")]}, + config={"configurable": {"thread_id": "t~member"}}, + ) + + assert "团队派工" in capture.system + assert "" in capture.system + assert "SystemMessage" not in capture.message_types diff --git a/tests/unit/api/test_acl_gate_coverage.py b/tests/unit/api/test_acl_gate_coverage.py index 41934b78d8..951e46b539 100644 --- a/tests/unit/api/test_acl_gate_coverage.py +++ b/tests/unit/api/test_acl_gate_coverage.py @@ -12,6 +12,7 @@ GATED_FILES = [ "routers/users.py", "routers/admin.py", + "routers/app_config.py", "routers/backup.py", "routers/security.py", "routers/providers.py", diff --git a/tests/unit/api/test_chat_serialize.py b/tests/unit/api/test_chat_serialize.py index 0216be8d69..8b4bb80a37 100644 --- a/tests/unit/api/test_chat_serialize.py +++ b/tests/unit/api/test_chat_serialize.py @@ -52,6 +52,16 @@ def test_serialize_history_message_keeps_client_message_id() -> None: assert entry["id"] == "1710000000000-ab12cd" +def test_serialize_history_message_keeps_empty_tool_result() -> None: + entry = _serialize_history_message( + ToolMessage(content="", tool_call_id="call-1", name="Read", status="success"), + ) + assert entry is not None + assert entry["content"] == [ + {"type": "tool_result", "id": "call-1", "name": "Read", "output": ""} + ] + + def test_serialize_history_message_includes_tool_artifact() -> None: msg = ToolMessage( content='{"octop_ui": {"renderer": "bilibili_player"}, "data_ref": "artifact"}', diff --git a/tests/unit/api/test_chat_ws_prepare_error.py b/tests/unit/api/test_chat_ws_prepare_error.py new file mode 100644 index 0000000000..d02e4e49b1 --- /dev/null +++ b/tests/unit/api/test_chat_ws_prepare_error.py @@ -0,0 +1,28 @@ +"""Error frames for dashboard turns rejected before they start.""" + +from __future__ import annotations + +from octop.api.routers.chat.ws import _prepare_error_frame +from octop.infra.errors import ErrorCode, OctopError + + +def test_frame_carries_the_error_code() -> None: + frame = _prepare_error_frame(OctopError(ErrorCode.CONNECTOR_MCP_LOAD_FAILED, "boom")) + + assert frame == { + "type": "error", + "message": "boom", + "error_code": "CONNECTOR_MCP_LOAD_FAILED", + } + + +def test_connector_auth_frame_says_which_connectors_to_authorize() -> None: + details = {"connectors": [{"server": "wiki"}], "servers": "Wiki"} + + frame = _prepare_error_frame( + OctopError(ErrorCode.CONNECTOR_AUTH_REQUIRED, "authorize Wiki", details=details) + ) + + assert frame["error_code"] == "CONNECTOR_AUTH_REQUIRED" + assert frame["error_details"] == details + assert frame["message"] == "authorize Wiki" diff --git a/tests/unit/api/test_knowledge_bases.py b/tests/unit/api/test_knowledge_bases.py index a8f8769047..011b30f64f 100644 --- a/tests/unit/api/test_knowledge_bases.py +++ b/tests/unit/api/test_knowledge_bases.py @@ -435,6 +435,154 @@ async def test_ocr_options_only_lists_image_capable_models() -> None: ] +def _rerank_provider(**overrides: object) -> SimpleNamespace: + values: dict[str, object] = { + "id": 4, + "name": "siliconflow", + "enabled": True, + "api_key": "sk-test", + "base_url": "https://example.test/v1", + "get_models": lambda: [ + {"id": "BAAI/bge-reranker-v2-m3", "name": "BGE Reranker"}, + {"id": "BAAI/bge-m3", "name": "BGE M3"}, + ], + } + values.update(overrides) + return SimpleNamespace(**values) + + +@pytest.mark.asyncio +async def test_rerank_options_list_enabled_providers_with_a_rerank_model() -> None: + from octop.api.routers import knowledge_bases + + providers = [ + _rerank_provider(), + _rerank_provider(id=5, name="disabled", enabled=False), + _rerank_provider(id=6, name="chat-only", get_models=lambda: [{"id": "chat-1"}]), + ] + server = SimpleNamespace( + services=SimpleNamespace(provider_repo=SimpleNamespace(list_all=lambda: providers)) + ) + + options = await knowledge_bases.rerank_options(server=server, _admin=object()) + + assert options == { + "providers": [ + { + "provider_id": "4", + "provider_name": "siliconflow", + "models": [{"id": "BAAI/bge-reranker-v2-m3", "name": "BGE Reranker"}], + } + ] + } + + +def _feature_server(monkeypatch: pytest.MonkeyPatch, provider: SimpleNamespace) -> SimpleNamespace: + """A server whose settings are a dict, with the embedding side of the feature stubbed.""" + from octop.api.routers import knowledge_bases + + values: dict[str, str] = {} + services = SimpleNamespace( + settings_repo=SimpleNamespace(get=values.get, set=values.__setitem__), + provider_repo=SimpleNamespace( + list_all=lambda: [provider], + get=lambda pid: provider if pid == provider.id else None, + ), + ) + monkeypatch.setattr(knowledge_bases, "set_feature_enabled", lambda *_a, **_k: None) + monkeypatch.setattr( + knowledge_bases, + "get_capability", + lambda *_a, **_k: { + "feature_enabled": False, + "selected_model": "", + "backend": "onnx", + "provider_id": "", + }, + ) + monkeypatch.setattr( + knowledge_bases, + "get_ocr_capability", + lambda *_a, **_k: {"enabled": False, "backend": "onnx", "model": "", "provider_id": ""}, + ) + return SimpleNamespace(services=services, values=values) + + +@pytest.mark.asyncio +async def test_put_feature_saves_rerank_and_reports_it(monkeypatch: pytest.MonkeyPatch) -> None: + from octop.api.routers import knowledge_bases + + server = _feature_server(monkeypatch, _rerank_provider()) + + capability = await knowledge_bases.put_feature( + knowledge_bases.FeatureBody( + enabled=False, + rerank_enabled=True, + rerank_provider_id="4", + rerank_model="BAAI/bge-reranker-v2-m3", + ), + request=_request(), + server=server, + _admin=object(), + ) + + assert capability["rerank"] == { + "enabled": True, + "provider_id": "4", + "model": "BAAI/bge-reranker-v2-m3", + "ready": True, + "usable": True, + } + + +@pytest.mark.asyncio +async def test_put_feature_refuses_a_model_that_is_not_a_reranker( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from octop.api.routers import knowledge_bases + + server = _feature_server(monkeypatch, _rerank_provider()) + + with pytest.raises(OctopError) as raised: + await knowledge_bases.put_feature( + knowledge_bases.FeatureBody( + enabled=False, + rerank_enabled=True, + rerank_provider_id="4", + rerank_model="BAAI/bge-m3", + ), + request=_request(), + server=server, + _admin=object(), + ) + + assert raised.value.code == ErrorCode.KNOWLEDGE_RERANK_NOT_READY + assert server.values == {} + + +@pytest.mark.asyncio +async def test_put_feature_leaves_rerank_alone_when_not_sent( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from octop.api.routers import knowledge_bases + + server = _feature_server(monkeypatch, _rerank_provider()) + server.values.update( + knowledge_rerank_enabled="true", + knowledge_rerank_provider_id="4", + knowledge_rerank_model="BAAI/bge-reranker-v2-m3", + ) + + capability = await knowledge_bases.put_feature( + knowledge_bases.FeatureBody(enabled=False), + request=_request(), + server=server, + _admin=object(), + ) + + assert capability["rerank"]["usable"] is True + + @pytest.mark.asyncio async def test_embedding_options_omits_non_recommended_onnx() -> None: from octop.api.routers import knowledge_bases @@ -629,6 +777,7 @@ def move_document(self, kb_id: str, doc_id: str, **kwargs: object) -> _Document: "actor_user_id": 7, "target_folder": "archive", "is_admin": False, + "target_kb_id": None, } ] assert result["document_id"] == "doc-1" @@ -636,10 +785,50 @@ def move_document(self, kb_id: str, doc_id: str, **kwargs: object) -> _Document: assert knowledge_bases.MoveDocumentBody().target_folder == "" +@pytest.mark.asyncio +async def test_move_document_to_another_base_queues_its_pending_files( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from octop.api.routers import knowledge_bases + + calls: list[dict[str, object]] = [] + moved = _Document(id="doc-1", path="act.md", filename="act.md", kb_id="kb-2") + queued: list[object] = [] + + class _FakeService: + def move_document(self, kb_id: str, doc_id: str, **kwargs: object) -> _Document: + calls.append(kwargs) + return moved + + def pending_file_ids(self, document: _Document) -> list[str]: + return ["doc-1", "doc-2"] if document is moved else [] + + monkeypatch.setattr(knowledge_bases, "_knowledge_service", lambda _server: _FakeService()) + monkeypatch.setattr( + knowledge_bases, + "enqueue_index_document", + lambda _services, kb_id, doc_id: queued.append((kb_id, doc_id)), + ) + + result = await knowledge_bases.move_document( + kb_id="kb-1", + doc_id="doc-1", + body=knowledge_bases.MoveDocumentBody(target_folder="", target_kb_id="kb-2"), + request=_request(), + server=SimpleNamespace(services=_services()), + user=SimpleNamespace(id=7, is_admin=False), + ) + + assert calls[0]["target_kb_id"] == "kb-2" + assert queued == [("kb-2", "doc-1"), ("kb-2", "doc-2")] + assert result["kb_id"] == "kb-2" + + @pytest.mark.asyncio @pytest.mark.parametrize( ("error", "code"), [ + (OctopError(ErrorCode.KNOWLEDGE_MOVE_BUSY, "x"), ErrorCode.KNOWLEDGE_MOVE_BUSY), (LookupError("knowledge folder not found"), ErrorCode.KNOWLEDGE_NOT_FOUND), (PermissionError("write access required"), ErrorCode.KNOWLEDGE_FORBIDDEN), (OctopError(ErrorCode.KNOWLEDGE_MOVE_INVALID, "x"), ErrorCode.KNOWLEDGE_MOVE_INVALID), @@ -833,3 +1022,271 @@ def test_other_prerequisite_failures_keep_the_generic_error(self) -> None: err = _map_knowledge_error(RuntimeError("embedding provider is not ready"), locale="zh") assert err.code is ErrorCode.KNOWLEDGE_PREREQUISITES_FAILED + + +def _chat_provider() -> SimpleNamespace: + return SimpleNamespace( + id=3, + name="Acme", + enabled=True, + api_key="sk-test", + base_url="https://example.test/v1", + get_models=lambda: [ + {"id": "chat-1", "name": "Chat One"}, + {"id": "embed-1", "name": "Embed", "embedding": True}, + ], + ) + + +@pytest.mark.asyncio +async def test_graph_options_lists_chat_models_per_provider() -> None: + from octop.api.routers import knowledge_bases + + disabled = _chat_provider() + disabled.enabled = False + server = SimpleNamespace( + services=SimpleNamespace( + provider_repo=SimpleNamespace(list_all=lambda: [_chat_provider(), disabled]) + ) + ) + + options = await knowledge_bases.graph_options(server=server, _admin=object()) + + assert options == { + "remote": [ + { + "provider_id": "3", + "provider_name": "Acme", + "models": [{"id": "chat-1", "name": "Chat One"}], + } + ] + } + + +def test_capability_payload_reports_graph_settings() -> None: + from octop.api.routers import knowledge_bases + + settings = { + "knowledge_graph_enabled": "true", + "knowledge_graph_provider_id": "3", + "knowledge_graph_model": "chat-1", + } + provider = _chat_provider() + server = SimpleNamespace( + services=SimpleNamespace( + settings_repo=SimpleNamespace(get=settings.get), + provider_repo=SimpleNamespace(get=lambda _id: provider), + config=None, + ) + ) + + assert knowledge_bases._capability_payload(server)["graph"] == { + "enabled": True, + "provider_id": "3", + "model": "chat-1", + "ready": True, + "usable": True, + } + + +@pytest.mark.asyncio +async def test_enabling_graph_without_a_chat_model_is_rejected_before_anything_is_saved( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from octop.api.routers import knowledge_bases + + saved: dict[str, str] = {} + provider = _chat_provider() + server = SimpleNamespace( + services=SimpleNamespace( + settings_repo=SimpleNamespace(get=saved.get, set=saved.__setitem__), + provider_repo=SimpleNamespace(get=lambda _id: provider, list_all=lambda: [provider]), + ) + ) + monkeypatch.setattr( + knowledge_bases, + "set_feature_enabled", + lambda *_a, **_k: pytest.fail("feature must not be saved"), + ) + + with pytest.raises(OctopError) as raised: + await knowledge_bases.put_feature( + knowledge_bases.FeatureBody( + enabled=False, graph_enabled=True, graph_provider_id="3", graph_model="embed-1" + ), + request=_request(), + server=server, + _admin=object(), + ) + + assert raised.value.code == ErrorCode.KNOWLEDGE_GRAPH_NOT_READY + assert raised.value.status == 409 + assert saved == {} + + +@pytest.mark.asyncio +async def test_put_feature_saves_graph_settings() -> None: + from octop.api.routers import knowledge_bases + + saved: dict[str, str] = {} + provider = _chat_provider() + server = SimpleNamespace( + services=SimpleNamespace( + settings_repo=SimpleNamespace(get=saved.get, set=saved.__setitem__), + provider_repo=SimpleNamespace(get=lambda _id: provider, list_all=lambda: [provider]), + config=None, + ) + ) + + capability = await knowledge_bases.put_feature( + knowledge_bases.FeatureBody( + enabled=False, graph_enabled=True, graph_provider_id="3", graph_model="chat-1" + ), + request=_request(), + server=server, + _admin=object(), + ) + + assert capability["graph"]["usable"] is True + assert saved["knowledge_graph_model"] == "chat-1" + + +@pytest.mark.asyncio +async def test_get_graph_passes_limit_and_search(monkeypatch: pytest.MonkeyPatch) -> None: + from octop.api.routers import knowledge_bases + + seen: dict[str, object] = {} + + def get_graph(kb_id: str, **kwargs: object) -> dict[str, object]: + seen.update(kb_id=kb_id, **kwargs) + return {"stats": {}, "nodes": [], "edges": []} + + monkeypatch.setattr( + knowledge_bases, "_knowledge_service", lambda _server: SimpleNamespace(get_graph=get_graph) + ) + + response = await knowledge_bases.get_graph( + "kb-1", + request=_request(), + limit=20, + q="apollo", + server=SimpleNamespace(services=_services()), + user=SimpleNamespace(id=5, is_admin=False), + ) + + assert response == {"stats": {}, "nodes": [], "edges": []} + assert seen == { + "kb_id": "kb-1", + "actor_user_id": 5, + "is_admin": False, + "limit": 20, + "search": "apollo", + } + + +@pytest.mark.asyncio +async def test_get_graph_entity_maps_missing_entity_to_not_found( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from octop.api.routers import knowledge_bases + + def missing(*_args: object, **_kwargs: object) -> None: + raise LookupError("knowledge graph entity not found") + + monkeypatch.setattr( + knowledge_bases, + "_knowledge_service", + lambda _server: SimpleNamespace(get_graph_entity=missing), + ) + + with pytest.raises(OctopError) as raised: + await knowledge_bases.get_graph_entity( + "kb-1", + request=_request(), + name="nobody", + server=SimpleNamespace(services=_services()), + user=SimpleNamespace(id=5, is_admin=False), + ) + + assert raised.value.code == ErrorCode.KNOWLEDGE_NOT_FOUND + + +@pytest.mark.asyncio +async def test_build_graph_enqueues_indexed_documents(monkeypatch: pytest.MonkeyPatch) -> None: + from octop.api.routers import knowledge_bases + + calls: list[tuple[object, str, list[str]]] = [] + server = SimpleNamespace(services=_services()) + monkeypatch.setattr( + knowledge_bases, + "_knowledge_service", + lambda _server: SimpleNamespace(graph_build_targets=lambda *_a, **_k: ["doc-1", "doc-2"]), + ) + monkeypatch.setattr(knowledge_bases, "_require_usable", lambda *_a, **_k: None) + + def enqueue(services: object, kb_id: str, doc_ids: list[str]) -> int: + calls.append((services, kb_id, doc_ids)) + return len(doc_ids) + + monkeypatch.setattr(knowledge_bases, "enqueue_base_graph", enqueue) + + response = await knowledge_bases.build_graph( + "kb-1", request=_request(), server=server, user=SimpleNamespace(id=1, is_admin=False) + ) + + assert response == {"enqueued": 2} + assert calls == [(server.services, "kb-1", ["doc-1", "doc-2"])] + + +@pytest.mark.asyncio +async def test_build_graph_without_an_extraction_model_maps_not_ready( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from octop.api.routers import knowledge_bases + + def not_ready(*_args: object, **_kwargs: object) -> None: + raise RuntimeError("knowledge graph extraction is not ready") + + monkeypatch.setattr( + knowledge_bases, + "_knowledge_service", + lambda _server: SimpleNamespace(graph_build_targets=not_ready), + ) + monkeypatch.setattr(knowledge_bases, "_require_usable", lambda *_a, **_k: None) + monkeypatch.setattr( + knowledge_bases, "enqueue_base_graph", lambda *_a: pytest.fail("must not be queued") + ) + + with pytest.raises(OctopError) as raised: + await knowledge_bases.build_graph( + "kb-1", + request=_request(), + server=SimpleNamespace(services=_services()), + user=SimpleNamespace(id=1, is_admin=False), + ) + + assert raised.value.code == ErrorCode.KNOWLEDGE_GRAPH_NOT_READY + + +@pytest.mark.asyncio +async def test_delete_graph_requires_write_access(monkeypatch: pytest.MonkeyPatch) -> None: + from octop.api.routers import knowledge_bases + + def forbidden(*_args: object, **_kwargs: object) -> None: + raise PermissionError("knowledge base write access is required") + + monkeypatch.setattr( + knowledge_bases, + "_knowledge_service", + lambda _server: SimpleNamespace(clear_graph=forbidden), + ) + + with pytest.raises(OctopError) as raised: + await knowledge_bases.delete_graph( + "kb-1", + request=_request(), + server=SimpleNamespace(services=_services()), + user=SimpleNamespace(id=2, is_admin=False), + ) + + assert raised.value.code == ErrorCode.KNOWLEDGE_FORBIDDEN diff --git a/tests/unit/backend/test_grep_safety.py b/tests/unit/backend/test_grep_safety.py index 09339706c1..4e60377630 100644 --- a/tests/unit/backend/test_grep_safety.py +++ b/tests/unit/backend/test_grep_safety.py @@ -109,7 +109,7 @@ def test_python_search_does_not_walk_proc() -> None: def test_dockerfiles_install_ripgrep() -> None: root = Path(__file__).resolve().parents[3] - for rel in ("docker/Dockerfile", "fnos/docker/Dockerfile"): + for rel in ("deploy/docker/Dockerfile", "fnos/docker/Dockerfile"): text = (root / rel).read_text(encoding="utf-8") assert "ripgrep" in text, f"{rel} must install ripgrep" diff --git a/tests/unit/cli/test_cron_create.py b/tests/unit/cli/test_cron_create.py new file mode 100644 index 0000000000..f665924711 --- /dev/null +++ b/tests/unit/cli/test_cron_create.py @@ -0,0 +1,61 @@ +"""Offline cron create: latest channel vs a pinned historical thread.""" + +from __future__ import annotations + +from pathlib import Path + +from octop.cli.support.db import open_cli_services +from octop.cli.support.offline_ops import create_cron_offline +from octop.infra.gateway.threads import ThreadRegistry +from octop.infra.utils.ulid import new_ulid + + +def test_create_cron_offline_pins_history_thread(tmp_path: Path) -> None: + aid = new_ulid() + with open_cli_services(tmp_path) as svc: + uid = svc.user_repo.create(username="u", password_hash="h", role="user") + svc.agent_repo.create(agent_id=aid, user_id=uid, name="bot") + session_key = ThreadRegistry.dashboard_key(agent_id=aid, user_id=uid) + svc.thread_repo.insert( + thread_id="thr_hist", + agent_id=aid, + user_id=uid, + channel_type=ThreadRegistry.CHANNEL_DASHBOARD, + session_key=session_key, + title="old chat", + last_active=10, + ) + svc.session_repo.upsert( + session_key=session_key, + agent_id=aid, + user_id=uid, + channel_type=ThreadRegistry.CHANNEL_DASHBOARD, + chat_type=ThreadRegistry.CHAT_TYPE_DM, + thread_id="thr_hist", + ) + + row = create_cron_offline( + agent_id=aid, + user_id=uid, + trigger="interval:60", + prompt="ping", + thread_id="thr_hist", + home=tmp_path, + ) + assert row["session_key"] == "thread:thr_hist" + + +def test_create_cron_offline_defaults_to_dashboard_channel(tmp_path: Path) -> None: + aid = new_ulid() + with open_cli_services(tmp_path) as svc: + uid = svc.user_repo.create(username="u", password_hash="h", role="user") + svc.agent_repo.create(agent_id=aid, user_id=uid, name="bot") + + row = create_cron_offline( + agent_id=aid, + user_id=uid, + trigger="interval:60", + prompt="ping", + home=tmp_path, + ) + assert row["session_key"] == ThreadRegistry.dashboard_key(agent_id=aid, user_id=uid) diff --git a/tests/unit/connectors/test_connector_reauth.py b/tests/unit/connectors/test_connector_reauth.py new file mode 100644 index 0000000000..8998e309c2 --- /dev/null +++ b/tests/unit/connectors/test_connector_reauth.py @@ -0,0 +1,252 @@ +"""Unit tests for connector authorization status and re-authorization.""" + +from __future__ import annotations + +import time +from pathlib import Path +from typing import Any +from unittest.mock import AsyncMock, patch + +import pytest + +from octop.config import OctopConfig +from octop.infra.connectors.auth_status import ( + custom_auth_status, +) +from octop.infra.connectors.builder import mcp_server_name +from octop.infra.connectors.custom_mcp import shared_mcp_server_name +from octop.infra.connectors.service import ConnectorService +from octop.infra.db.migrate import run_migrations +from octop.infra.db.pool import SqlitePool +from octop.infra.db.repos.connectors import ConnectorRepo +from octop.infra.db.repos.secrets import SecretRepo +from octop.infra.db.repos.settings import SettingsRepo +from octop.infra.utils.ulid import new_ulid + +NOW = 1_800_000_000 + + +@pytest.fixture +def db(tmp_path: Path) -> SqlitePool: + pool = SqlitePool(tmp_path / "octop.db") + run_migrations(pool) + return pool + + +@pytest.fixture +def svc(db: SqlitePool) -> ConnectorService: + return ConnectorService( + repo=ConnectorRepo(db), + secret_repo=SecretRepo(db), + settings_repo=SettingsRepo(db), + config=OctopConfig(), + ) + + +def _user(db: SqlitePool, name: str) -> int: + with db.transaction() as conn: + conn.execute( + "INSERT INTO users(username, password_hash, role, created_at) VALUES (?, 'x', 'user', 1)", + (name,), + ) + row = conn.execute("SELECT id FROM users WHERE username = ?", (name,)).fetchone() + assert row is not None + return int(row["id"]) + + +def _notion(svc: ConnectorService, user_id: int, creds: dict[str, Any]) -> str: + instance_id = new_ulid() + svc._repo.create( + instance_id=instance_id, + user_id=user_id, + kind="notion", + display_name="Notion", + mcp_server_name=mcp_server_name("notion", instance_id), + ) + svc.encrypt_and_store(instance_id=instance_id, payload=creds) + return instance_id + + +def _custom(svc: ConnectorService, user_id: int, servers: dict[str, dict[str, Any]]) -> None: + """Save custom servers, then write their stored OAuth state directly.""" + svc.put_custom_servers( + user_id, + {name: {k: v for k, v in spec.items() if k != "oauth"} for name, spec in servers.items()}, + ) + stored = dict(svc.get_custom_servers(user_id)) + for name, spec in servers.items(): + if "oauth" in spec: + stored[name] = {**stored[name], "oauth": spec["oauth"]} + svc._save_custom_servers(user_id, stored) + + +# -- status -------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("oauth", "expected"), + [ + (None, None), + ({"required": True}, "required"), + ({"required": True, "expired": True}, "expired"), + ({"access_token": "t", "expires_at": NOW + 60}, "ok"), + ({"access_token": "t", "expires_at": NOW - 60, "refresh_token": "r"}, "ok"), + ({"access_token": "t", "expires_at": NOW - 60}, "expired"), + ], +) +def test_custom_auth_status(oauth: dict[str, Any] | None, expected: str | None) -> None: + spec: dict[str, Any] = {"transport": "streamable_http", "url": "https://x.example/mcp"} + if oauth is not None: + spec["oauth"] = oauth + assert custom_auth_status(spec, now=NOW) == expected + + +# -- listing ------------------------------------------------------------------- + + +def test_list_reports_custom_server_status_for_owner_and_viewers( + svc: ConnectorService, db: SqlitePool +) -> None: + owner, viewer = _user(db, "u1"), _user(db, "u2") + _custom( + svc, + owner, + { + "needs": { + "transport": "streamable_http", + "url": "https://a.example/mcp", + "shared": True, + "oauth": {"required": True}, + }, + "plain": {"transport": "streamable_http", "url": "https://b.example/mcp"}, + }, + ) + + mine = {item["mcp_server_name"]: item for item in svc.list_instances_for_api(owner)} + assert mine["needs"]["auth_status"] == "required" + assert "auth_status" not in mine["plain"] + parent = svc._repo.get_by_user_kind(owner, "custom-mcp") + assert parent is not None + shared = {item["mcp_server_name"]: item for item in svc.list_instances_for_api(viewer)} + assert shared[shared_mcp_server_name(parent.instance_id, "needs")]["auth_status"] == "required" + + +def test_list_leaves_catalog_connectors_without_status( + svc: ConnectorService, db: SqlitePool +) -> None: + uid = _user(db, "u1") + _notion(svc, uid, {"access_token": "t", "expires_at": 1}) + + assert "auth_status" not in svc.list_instances_for_api(uid)[0] + + +# -- blocked servers ----------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_blocked_servers_lists_custom_servers_that_need_authorization( + svc: ConnectorService, db: SqlitePool +) -> None: + owner, viewer = _user(db, "u1"), _user(db, "u2") + expired = _notion(svc, owner, {"access_token": "t", "expires_at": 1}) + _custom( + svc, + owner, + { + "needs": { + "transport": "streamable_http", + "url": "https://a.example/mcp", + "shared": True, + "oauth": {"required": True}, + }, + "plain": {"transport": "streamable_http", "url": "https://b.example/mcp"}, + }, + ) + parent = svc._repo.get_by_user_kind(owner, "custom-mcp") + assert parent is not None + notion_name = mcp_server_name("notion", expired) + + with patch("octop.infra.connectors.service._token_rejected", new=AsyncMock(return_value=False)): + blocked = await svc.auth_blocked_servers(owner, [notion_name, "needs", "plain", "gone"]) + for_viewer = await svc.auth_blocked_servers( + viewer, [shared_mcp_server_name(parent.instance_id, "needs")] + ) + + # Catalog connectors keep their own token handling and are not reported. + assert [(b["server"], b["auth_status"], b["can_authorize"]) for b in blocked] == [ + ("needs", "required", True), + ] + assert blocked[0]["target"] == {"type": "custom_mcp", "server_name": "needs"} + # Someone else's shared server can only be fixed by its owner. + assert [(b["auth_status"], b["can_authorize"]) for b in for_viewer] == [("required", False)] + + +@pytest.mark.asyncio +async def test_a_revoked_token_is_detected_by_probe_and_flagged( + svc: ConnectorService, db: SqlitePool +) -> None: + uid = _user(db, "u1") + _custom( + svc, + uid, + { + "srv": { + "transport": "streamable_http", + "url": "https://a.example/mcp", + "oauth": {"access_token": "revoked", "expires_at": int(time.time()) + 3600}, + } + }, + ) + probe = AsyncMock(return_value=True) + + with patch("octop.infra.connectors.service._token_rejected", new=probe): + blocked = await svc.auth_blocked_servers(uid, ["srv"]) + + assert [b["auth_status"] for b in blocked] == ["expired"] + assert probe.await_args_list[0].args[1]["Authorization"] == "Bearer revoked" + assert svc.get_custom_servers(uid)["srv"]["oauth"] == {"required": True, "expired": True} + assert svc.list_instances_for_api(uid)[0]["auth_status"] == "expired" + + +@pytest.mark.asyncio +async def test_a_working_token_or_unreachable_server_is_not_an_auth_problem( + svc: ConnectorService, db: SqlitePool +) -> None: + uid = _user(db, "u1") + _custom( + svc, + uid, + { + "srv": { + "transport": "streamable_http", + "url": "https://a.example/mcp", + "oauth": {"access_token": "fine", "expires_at": int(time.time()) + 3600}, + } + }, + ) + + with patch( + "octop.infra.connectors.probe.probe_streamable_http_mcp", + new=AsyncMock(return_value={"ok": False, "error_type": "connection", "status_code": 502}), + ): + blocked = await svc.auth_blocked_servers(uid, ["srv"]) + + assert blocked == [] + assert svc.get_custom_servers(uid)["srv"]["oauth"]["access_token"] == "fine" + + +@pytest.mark.asyncio +async def test_token_rejected_reads_only_auth_errors() -> None: + from octop.infra.connectors import service + + with patch( + "octop.infra.connectors.probe.probe_streamable_http_mcp", + new=AsyncMock(return_value={"ok": False, "error_type": "auth", "status_code": 401}), + ): + assert await service._token_rejected("https://a.example/mcp", {}, kind="x") is True + with patch( + "octop.infra.connectors.probe.probe_streamable_http_mcp", + new=AsyncMock(side_effect=RuntimeError("boom")), + ): + assert await service._token_rejected("https://a.example/mcp", {}, kind="x") is False + assert await service._token_rejected("", {}, kind="x") is False diff --git a/tests/unit/connectors/test_custom_mcp.py b/tests/unit/connectors/test_custom_mcp.py index ad1568a3fc..71f53f9f87 100644 --- a/tests/unit/connectors/test_custom_mcp.py +++ b/tests/unit/connectors/test_custom_mcp.py @@ -603,7 +603,7 @@ async def test_ensure_fresh_custom_servers_marks_reauth_when_refresh_fails_expir preview = svc.get_custom_servers_for_api(uid) assert preview["srv"]["oauth"] == {"configured": False, "required": True} stored = svc.get_custom_servers(uid)["srv"]["oauth"] - assert stored == {"required": True} + assert stored == {"required": True, "expired": True} @pytest.mark.asyncio diff --git a/tests/unit/connectors/test_oauth_discovery.py b/tests/unit/connectors/test_oauth_discovery.py index 39b84211dd..310350a8b1 100644 --- a/tests/unit/connectors/test_oauth_discovery.py +++ b/tests/unit/connectors/test_oauth_discovery.py @@ -120,7 +120,7 @@ def test_mark_oauth_reauth_required_clears_tokens(): ), } marked = mark_oauth_reauth_required(spec) - assert marked["oauth"] == {"required": True} + assert marked["oauth"] == {"required": True, "expired": True} preview = redact_server_for_api(marked) assert preview["oauth"] == {"configured": False, "required": True} diff --git a/tests/unit/cron/test_cron_delivery.py b/tests/unit/cron/test_cron_delivery.py index c3843973ae..630df45934 100644 --- a/tests/unit/cron/test_cron_delivery.py +++ b/tests/unit/cron/test_cron_delivery.py @@ -288,6 +288,226 @@ async def _stream(_aid: str, _request: dict): assert gateway.push_session_text.await_args.args[1] == "最终学习内容" +@pytest.mark.asyncio +async def test_delivery_binds_channel_to_latest_conversation(tmp_path: Path) -> None: + db = SqlitePool(tmp_path / "octop.db") + run_migrations(db) + UserRepo(db).create(username="u", password_hash="h", role="user") + AgentRepo(db).create(agent_id="a1", user_id=1, name="bot") + session_key = ThreadRegistry.dashboard_key(agent_id="a1", user_id=1) + threads = ThreadRepo(db) + threads.insert( + thread_id="thr_open", + agent_id="a1", + user_id=1, + channel_type=ThreadRegistry.CHANNEL_DASHBOARD, + session_key=session_key, + title="正在看的旧对话", + last_active=10, + ) + threads.insert( + thread_id="thr_latest", + agent_id="a1", + user_id=1, + channel_type=ThreadRegistry.CHANNEL_DASHBOARD, + session_key=session_key, + title="最近一次会话", + last_active=20, + ) + threads.insert( + thread_id="thr_blank", + agent_id="a1", + user_id=1, + channel_type=ThreadRegistry.CHANNEL_DASHBOARD, + session_key=session_key, + title="空白新对话", + last_active=0, + ) + SessionRepo(db).upsert( + session_key=session_key, + agent_id="a1", + user_id=1, + channel_type=ThreadRegistry.CHANNEL_DASHBOARD, + chat_type=ThreadRegistry.CHAT_TYPE_DM, + thread_id="thr_open", + ) + registry = ThreadRegistry(session_repo=SessionRepo(db), thread_repo=threads) + + seen: list[str] = [] + + async def append(thread_id: str, _messages): + seen.append(thread_id) + return [ + HumanMessage(content="h", id="cron:x:human"), + AIMessage(content="记得喝水", id="cron:x:assistant"), + ] + + harness = MagicMock() + harness.aappend_messages = append + agent_manager = MagicMock() + agent_manager.get_agent.return_value = harness + + gateway = MagicMock() + gateway.thread_registry = registry + gateway.run_in_session = _run_locked + gateway.require_session = lambda _agent_id, key: registry.get_session(key) + gateway.push_session_text = AsyncMock() + gateway.notify_dashboard_push = AsyncMock() + + repos = MagicMock() + repos.user_repo.get.return_value = MagicMock(locale="zh") + repos.thread_message_repo.append_if_ready.return_value = 2 + + service = CronDeliveryService( + gateway=gateway, + agent_manager=agent_manager, + repos=repos, + ) + await service.deliver(_command(session_key=session_key)) + + assert seen == ["thr_latest"] + bound = registry.get_session(session_key) + assert bound is not None + assert bound.thread_id == "thr_latest" + + +@pytest.mark.asyncio +async def test_delivery_pins_historical_thread(tmp_path: Path) -> None: + db = SqlitePool(tmp_path / "octop.db") + run_migrations(db) + UserRepo(db).create(username="u", password_hash="h", role="user") + AgentRepo(db).create(agent_id="a1", user_id=1, name="bot") + session_key = ThreadRegistry.dashboard_key(agent_id="a1", user_id=1) + threads = ThreadRepo(db) + threads.insert( + thread_id="thr_old", + agent_id="a1", + user_id=1, + channel_type=ThreadRegistry.CHANNEL_DASHBOARD, + session_key=session_key, + title="历史会话", + last_active=10, + ) + threads.insert( + thread_id="thr_latest", + agent_id="a1", + user_id=1, + channel_type=ThreadRegistry.CHANNEL_DASHBOARD, + session_key=session_key, + title="最近一次会话", + last_active=20, + ) + SessionRepo(db).upsert( + session_key=session_key, + agent_id="a1", + user_id=1, + channel_type=ThreadRegistry.CHANNEL_DASHBOARD, + chat_type=ThreadRegistry.CHAT_TYPE_DM, + thread_id="thr_latest", + ) + registry = ThreadRegistry(session_repo=SessionRepo(db), thread_repo=threads) + seen: list[str] = [] + + async def append(thread_id: str, _messages): + seen.append(thread_id) + return [ + HumanMessage(content="h", id="cron:x:human"), + AIMessage(content="记得喝水", id="cron:x:assistant"), + ] + + harness = MagicMock() + harness.aappend_messages = append + agent_manager = MagicMock() + agent_manager.get_agent.return_value = harness + gateway = MagicMock() + gateway.thread_registry = registry + gateway.run_in_session = _run_locked + gateway.require_session = lambda _agent_id, key: registry.get_session(key) + gateway.push_session_text = AsyncMock() + gateway.notify_dashboard_push = AsyncMock() + repos = MagicMock() + repos.user_repo.get.return_value = MagicMock(locale="zh") + repos.thread_message_repo.append_if_ready.return_value = 2 + + service = CronDeliveryService( + gateway=gateway, + agent_manager=agent_manager, + repos=repos, + ) + await service.deliver(_command(session_key="thread:thr_old")) + + assert seen == ["thr_old"] + bound = registry.get_session(session_key) + assert bound is not None + assert bound.thread_id == "thr_old" + + +@pytest.mark.asyncio +async def test_fresh_thread_does_not_return_to_latest_conversation(tmp_path: Path) -> None: + db = SqlitePool(tmp_path / "octop.db") + run_migrations(db) + UserRepo(db).create(username="u", password_hash="h", role="user") + AgentRepo(db).create(agent_id="a1", user_id=1, name="bot") + session_key = ThreadRegistry.dashboard_key(agent_id="a1", user_id=1) + threads = ThreadRepo(db) + threads.insert( + thread_id="thr_latest", + agent_id="a1", + user_id=1, + channel_type=ThreadRegistry.CHANNEL_DASHBOARD, + session_key=session_key, + last_active=20, + ) + SessionRepo(db).upsert( + session_key=session_key, + agent_id="a1", + user_id=1, + channel_type=ThreadRegistry.CHANNEL_DASHBOARD, + chat_type=ThreadRegistry.CHAT_TYPE_DM, + thread_id="thr_latest", + channel_subject_id="1", + channel_chat_type=ThreadRegistry.CHAT_TYPE_DM, + ) + registry = ThreadRegistry(session_repo=SessionRepo(db), thread_repo=threads) + + seen: list[str] = [] + + async def append(thread_id: str, _messages): + seen.append(thread_id) + return [ + HumanMessage(content="h", id="cron:x:human"), + AIMessage(content="记得喝水", id="cron:x:assistant"), + ] + + harness = MagicMock() + harness.aappend_messages = append + agent_manager = MagicMock() + agent_manager.get_agent.return_value = harness + + gateway = MagicMock() + gateway.thread_registry = registry + gateway.run_in_session = _run_locked + gateway.require_session = lambda _agent_id, key: registry.get_session(key) + gateway.push_session_text = AsyncMock() + gateway.notify_dashboard_push = AsyncMock() + + repos = MagicMock() + repos.user_repo.get.return_value = MagicMock(locale="zh") + repos.thread_message_repo.append_if_ready.return_value = 2 + + service = CronDeliveryService( + gateway=gateway, + agent_manager=agent_manager, + repos=repos, + ) + await service.deliver(_command(session_key=session_key, fresh_thread=True)) + + assert seen and seen[0] != "thr_latest" + bound = registry.get_session(session_key) + assert bound is not None + assert bound.thread_id == seen[0] + + @pytest.mark.asyncio async def test_fresh_thread_resets_before_require_session() -> None: session = _session(channel_type="feishu") @@ -421,3 +641,58 @@ async def _stream(_aid: str, _request: dict): assert titled is not None assert titled.title == "喝水提醒" db.close() + + +@pytest.mark.asyncio +async def test_telegram_cron_reloads_topic_routing_from_persisted_session(tmp_path: Path) -> None: + from octop_gateway.models import ChannelSubject, InboundMessage + + from octop.infra.gateway.gateway import Gateway + from octop.infra.gateway.process.message_keys import ( + sanitize_im_metadata, + session_key_from_message, + ) + + db, _ = _seed_dashboard_thread(tmp_path, thread_id="dashboard-thread") + sessions = SessionRepo(db) + registry = ThreadRegistry(session_repo=sessions, thread_repo=ThreadRepo(db)) + inbound = InboundMessage( + channel_type="telegram", + channel_id="telegram-bot", + content=[], + channel_subject=ChannelSubject(subject_id="-100123", chat_type="group"), + metadata={ + "chat_id": "-100123", + "chat_type": "group", + "message_thread_id": 42, + "topic_isolation": True, + "message_id": 999, + }, + ) + key = session_key_from_message(inbound, agent_id="a1") + await registry.get_or_create_by_key( + session_key=key, + agent_id="a1", + user_id=1, + channel_type="telegram", + channel_channel_id="telegram-bot", + channel_metadata=sanitize_im_metadata(inbound), + ) + # Cron can create a fresh thread; routing must survive the reset and DB reload. + await registry.reset_by_session_key(key) + session = SessionRepo(db).get(key) + assert session is not None + gateway = MagicMock() + gateway.run_in_session = _run_locked + gateway.require_session.return_value = session + gateway.push_session_text = AsyncMock() + gateway.notify_dashboard_push = AsyncMock() + service = CronDeliveryService(gateway=gateway, agent_manager=MagicMock(), repos=MagicMock()) + await service.deliver(_command(session_key=key)) + delivered_session = gateway.push_session_text.call_args.args[0] + subject = Gateway._resolve_push_subject(gateway, delivered_session) + assert subject.metadata["chat_id"] == "-100123" + assert subject.metadata["message_thread_id"] == 42 + assert subject.metadata["channel_id"] == "telegram-bot" + assert "message_id" not in subject.metadata + db.close() diff --git a/tests/unit/cron/test_cron_manager.py b/tests/unit/cron/test_cron_manager.py index fccafb828e..381e64d1a6 100644 --- a/tests/unit/cron/test_cron_manager.py +++ b/tests/unit/cron/test_cron_manager.py @@ -392,6 +392,53 @@ async def test_create_stores_fresh_thread(tmp_path: Path) -> None: assert bool(row.fresh_thread) is True +@pytest.mark.asyncio +async def test_create_pins_history_thread(tmp_path: Path) -> None: + services = _make_services(tmp_path) + aid, uid = _make_agent(services) + channel_key = ThreadRegistry.dashboard_key(agent_id=aid, user_id=uid) + thread = MagicMock() + thread.agent_id = aid + thread.user_id = uid + thread.session_key = channel_key + gateway = _make_gateway() + gateway.thread_registry.get_thread = MagicMock(return_value=thread) + mgr = _make_manager(services, gateway=gateway) + + row = await mgr.create( + cron_id=_cron_id(), + agent_id=aid, + user_id=uid, + trigger="interval:120", + prompt="work", + session_key="thread:thr_old", + ) + + assert row.session_key == "thread:thr_old" + gateway.thread_registry.get_or_create_by_key.assert_awaited() + + +@pytest.mark.asyncio +async def test_create_rejects_missing_history_thread(tmp_path: Path) -> None: + from octop.infra.errors import OctopError # noqa: PLC0415 + + services = _make_services(tmp_path) + aid, uid = _make_agent(services) + gateway = _make_gateway() + gateway.thread_registry.get_thread = MagicMock(return_value=None) + mgr = _make_manager(services, gateway=gateway) + + with pytest.raises(OctopError): + await mgr.create( + cron_id=_cron_id(), + agent_id=aid, + user_id=uid, + trigger="interval:120", + prompt="work", + session_key="thread:missing", + ) + + # --------------------------------------------------------------------------- # get() / list_by_agent() / list_all() # --------------------------------------------------------------------------- diff --git a/tests/unit/cron/test_cron_tools.py b/tests/unit/cron/test_cron_tools.py index 8a3d57e804..3d68a01054 100644 --- a/tests/unit/cron/test_cron_tools.py +++ b/tests/unit/cron/test_cron_tools.py @@ -196,6 +196,59 @@ async def test_cronjob_create_uses_configurable_session_key(tmp_path: Path) -> N assert data["session_key"] == feishu_sk +@pytest.mark.asyncio +async def test_cronjob_create_pins_history_thread(tmp_path: Path) -> None: + services = _make_services(tmp_path) + agent_id = new_ulid() + user_id = services.repos.user_repo.create(username="hist", password_hash="x", role="user") + services.repos.agent_repo.create(agent_id=agent_id, user_id=user_id, name="a") + channel_key = ThreadRegistry.dashboard_key(agent_id=agent_id, user_id=user_id) + thread = MagicMock() + thread.agent_id = agent_id + thread.user_id = user_id + thread.session_key = channel_key + mgr = _make_manager(services) + mgr._gateway.thread_registry.get_thread = MagicMock(return_value=thread) + create = _tool_by_name(build_cronjob_tools(mgr), "cronjob_create") + + with _configurable(agent_id=agent_id, user=str(user_id), thread_id="thr_current"): + out = await create.ainvoke( + { + "trigger": "interval:30", + "prompt": "stay here", + "name": "stay", + "session_target": "history", + } + ) + data = json.loads(out) + assert data["session_key"] == "thread:thr_current" + + +@pytest.mark.asyncio +async def test_cronjob_create_fresh_thread_ignores_history_pin(tmp_path: Path) -> None: + services = _make_services(tmp_path) + agent_id = new_ulid() + user_id = services.repos.user_repo.create(username="fresh", password_hash="x", role="user") + services.repos.agent_repo.create(agent_id=agent_id, user_id=user_id, name="a") + mgr = _make_manager(services) + create = _tool_by_name(build_cronjob_tools(mgr), "cronjob_create") + channel_key = ThreadRegistry.dashboard_key(agent_id=agent_id, user_id=user_id) + + with _configurable(agent_id=agent_id, user=str(user_id), thread_id="thr_current"): + out = await create.ainvoke( + { + "trigger": "interval:30", + "prompt": "new chat", + "name": "new", + "session_target": "history", + "fresh_thread": True, + } + ) + data = json.loads(out) + assert data["session_key"] == channel_key + assert data["fresh_thread"] is True + + @pytest.mark.asyncio async def test_cronjob_create_persists_task_type(tmp_path: Path) -> None: services = _make_services(tmp_path) diff --git a/tests/unit/db/test_db_pool.py b/tests/unit/db/test_db_pool.py index adf1ebf605..911a6f9734 100644 --- a/tests/unit/db/test_db_pool.py +++ b/tests/unit/db/test_db_pool.py @@ -152,6 +152,7 @@ def test_run_migrations_idempotent(db: SqlitePool): assert {"mfa_version", "mfa_failed_count", "mfa_locked_until", "password_version"}.issubset( cols ) + assert "last_login_at" in cols assert { "user_totp", "user_webauthn_credentials", @@ -347,6 +348,7 @@ def test_migration_005_preserves_populated_users_and_constraints(tmp_path: Path) "mfa_failed_count": 0, "mfa_locked_until": 0, "password_version": 0, + "last_login_at": None, } assert dict(agent) == { "agent_id": "legacy-agent", diff --git a/tests/unit/db/test_repo_sessions_threads.py b/tests/unit/db/test_repo_sessions_threads.py index 949b91f869..10b84003c3 100644 --- a/tests/unit/db/test_repo_sessions_threads.py +++ b/tests/unit/db/test_repo_sessions_threads.py @@ -239,6 +239,50 @@ def test_unread_counts_for_threads(repos): assert counts == {"thr_a": 2} +def test_latest_activity_by_agent_is_per_viewer(repos): + _sessions, threads = repos + AgentRepo(threads._db).create(agent_id="a2", user_id=1, name="Agent 2") + peer_id = UserRepo(threads._db).create(username="other", password_hash="h", role="user") + own = ThreadRegistry.dashboard_key(agent_id="a1", user_id=1) + peer = ThreadRegistry.dashboard_key(agent_id="a1", user_id=peer_id) + empty = ThreadRegistry.dashboard_key(agent_id="a2", user_id=1) + threads.insert( + thread_id="thr_old", + agent_id="a1", + user_id=1, + channel_type="dashboard", + session_key=own, + last_active=10, + ) + threads.insert( + thread_id="thr_new", + agent_id="a1", + user_id=1, + channel_type="dashboard", + session_key=own, + last_active=40, + ) + threads.insert( + thread_id="thr_empty", + agent_id="a2", + user_id=1, + channel_type="dashboard", + session_key=empty, + last_active=0, + ) + threads.insert( + thread_id="thr_peer", + agent_id="a1", + user_id=peer_id, + channel_type="dashboard", + session_key=peer, + last_active=99, + ) + assert threads.latest_activity_by_agent(1, ["a1", "a2", "missing"]) == {"a1": 40} + assert threads.latest_activity_by_agent(peer_id, ["a1"]) == {"a1": 99} + assert threads.latest_activity_by_agent(1, []) == {} + + def test_threads_pinned_sort_first(repos): _sessions, threads = repos sk = ThreadRegistry.make_key(agent_id="a1", channel_type="dashboard", channel_subject_id="1") diff --git a/tests/unit/db/test_repo_users.py b/tests/unit/db/test_repo_users.py index 6555a7c843..b1c6bc3560 100644 --- a/tests/unit/db/test_repo_users.py +++ b/tests/unit/db/test_repo_users.py @@ -20,6 +20,27 @@ def repo(tmp_path: Path) -> UserRepo: return UserRepo(db) +def test_last_login_backfill_from_audit_does_not_clobber(repo: UserRepo): + uid = repo.create(username="alice", password_hash="h", role="user") + created = repo.get(uid) + assert created is not None + assert created.last_login_at is None + with repo._db.transaction() as conn: + conn.execute( + "INSERT INTO audit_log(ts, actor, action, target, payload) VALUES (?, ?, ?, ?, ?)", + (1_700_000_000, "alice", "auth.login", None, None), + ) + run_migrations(repo._db) + filled = repo.get(uid) + assert filled is not None + assert filled.last_login_at == 1_700_000_000 + repo.touch_last_login(uid, now=1_800_000_000) + run_migrations(repo._db) + kept = repo.get(uid) + assert kept is not None + assert kept.last_login_at == 1_800_000_000 + + def test_create_and_get(repo: UserRepo): uid = repo.create(username="alice", password_hash="h", role="admin", display_name="Alice") row = repo.get(uid) diff --git a/tests/unit/gateway/test_global_processor_team.py b/tests/unit/gateway/test_global_processor_team.py index d046fdb99a..998725b483 100644 --- a/tests/unit/gateway/test_global_processor_team.py +++ b/tests/unit/gateway/test_global_processor_team.py @@ -1932,6 +1932,172 @@ async def capture_push(channel_type: str, channel_id: str, subject: object, text assert pushed == [("feishu", "ch-feishu-1", "Researcher: TAM is 12B")] +@pytest.mark.asyncio +async def test_stream_peer_pushes_turn_media_to_im_sessions( + processor_env: dict, tmp_path: Path +) -> None: + import base64 + + from deepagents.backends.local_shell import LocalShellBackend + from langchain_core.messages import AIMessage + from octop_gateway.models import FileContent, ImageContent + from octop_harness.backends.workspace import BackendWorkspace + from octop_harness.request import ChatRequest + + from octop.infra.gateway.media.constants import OUTBOUND_DIR + + processor = processor_env["processor"] + gateway = processor_env["gateway"] + processor._agent_repo.create(agent_id="host", user_id=1, name="Host", kind="team") + _bind_im_session(processor_env) + registry = gateway.thread_registry + bad_key = registry.make_key( + agent_id="host", + channel_type="weixin", + channel_subject_id="wx_bad", + ) + registry._sessions.upsert( + session_key=bad_key, + agent_id="host", + user_id=1, + channel_type="weixin", + chat_type="dm", + thread_id="thr_parent", + channel_subject_id="wx_bad", + channel_id="ch-bad", + ) + + ws = tmp_path / "member-ws" + ws.mkdir() + root = str(ws) + workspace = BackendWorkspace(LocalShellBackend(root_dir=root, virtual_mode=False), root) + png_bytes = b"\x89PNG\r\nmember" + zip_bytes = b"PK\x03\x04member-zip" + old_bytes = b"PK\x03\x04history" + await workspace.aupload_bytes(f"{OUTBOUND_DIR}/shot.png", png_bytes) + await workspace.aupload_bytes(f"{OUTBOUND_DIR}/bundle.zip", zip_bytes) + await workspace.aupload_bytes(f"{OUTBOUND_DIR}/old.zip", old_bytes) + await workspace.aupload_bytes(f"{OUTBOUND_DIR}/notes.png", b"\x89PNG\r\nwrite") + + def block(kind: str, filename: str, mime: str) -> dict[str, object]: + rel = f"{OUTBOUND_DIR}/{filename}" + return { + "type": kind, + "source": { + "type": "url", + "url": f"file://{root}/{rel}", + "media_type": mime, + }, + "filename": filename, + "path": rel, + } + + frames: list[dict[str, object]] = [] + + async def capture_frame(frame: dict[str, object]) -> None: + frames.append(frame) + + gateway.ws_hub.register("conn-room", capture_frame) + gateway.ws_hub.subscribe("thr_parent", "conn-room") + + delivered: list[tuple[str, str, object]] = [] + attempts: list[str] = [] + + async def capture_content( + channel_type: str, + channel_id: str, + subject: object, + parts: list[object], + ) -> None: + attempts.append(channel_id) + subject_id = getattr(subject, "subject_id", "") + if channel_id == "ch-bad": + assert subject_id == "wx_bad" + raise RuntimeError("channel down") + assert channel_type == "feishu" + assert subject_id == "ou_1" + delivered.append((channel_id, subject_id, parts[0])) + + gateway.push_content = capture_content # type: ignore[method-assign] + + async def fake_stream(_agent_id: str, _request: dict[str, object]) -> object: + yield {"type": "tool_call_chunk", "id": "live-png", "name": "send_file_to_user", "args": ""} + yield { + "type": "tool_result", + "messages": [ + { + "tool_call_id": "old-zip", + "name": "send_file_to_user", + "content": block("file", "old.zip", "application/zip"), + }, + { + "tool_call_id": "live-png", + "name": "send_file_to_user", + "content": block("image", "shot.png", "image/png"), + }, + ], + } + yield {"type": "tool_call_chunk", "id": "live-zip", "name": "send_file_to_user", "args": ""} + yield { + "type": "tool_result", + "messages": [ + { + "tool_call_id": "old-zip", + "name": "send_file_to_user", + "content": block("file", "old.zip", "application/zip"), + }, + { + "tool_call_id": "live-png", + "name": "send_file_to_user", + "content": block("image", "shot.png", "image/png"), + }, + { + "tool_call_id": "live-zip", + "name": "send_file_to_user", + "content": block("file", "bundle.zip", "application/zip"), + }, + ], + } + yield {"type": "tool_call_chunk", "id": "live-write", "name": "write_file", "args": ""} + yield { + "type": "tool_result", + "messages": [ + { + "tool_call_id": "live-write", + "name": "write_file", + "content": block("image", "notes.png", "image/png"), + } + ], + } + yield {"type": "token", "content": "sent"} + + processor._agent_manager.stream = fake_stream + harness = MagicMock() + harness.workspace = workspace + harness.aget_history = AsyncMock(return_value=[AIMessage(content="sent")]) + processor._agent_manager.get_agent.return_value = harness + + result = await processor.teams.stream_peer_to_room( + ChatRequest(messages="task", thread_id="thr_parent~child", agent_id="child"), + room_thread_id="thr_parent", + speaker_id="child", + ) + + assert [type(row[2]) for row in delivered] == [ImageContent, FileContent] + image = delivered[0][2] + archive = delivered[1][2] + assert isinstance(image, ImageContent) + assert isinstance(archive, FileContent) + assert image.data is not None and base64.b64decode(image.data) == png_bytes + assert archive.filename == "bundle.zip" + assert archive.data is not None and base64.b64decode(archive.data) == zip_bytes + assert delivered[0][0] == "ch-feishu-1" + assert attempts.count("ch-bad") == 2 + assert attempts.count("ch-feishu-1") == 2 + assert result["stream_text"] == "sent" + assert [frame.get("type") for frame in frames].count("tool_result") == 3 + + @pytest.mark.asyncio async def test_fan_in_skips_dashboard_when_pushing_channels(processor_env: dict) -> None: from langchain_core.messages import AIMessage diff --git a/tests/unit/gateway/test_group_chat_regressions.py b/tests/unit/gateway/test_group_chat_regressions.py new file mode 100644 index 0000000000..44c120de9a --- /dev/null +++ b/tests/unit/gateway/test_group_chat_regressions.py @@ -0,0 +1,161 @@ +"""Host-owned group choice liveness, quote prompts and rejection notices.""" + +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest +from octop_gateway.models import TextContent +from tests.unit.gateway.test_telegram_bridge import setup_bridge + +from octop.infra.gateway.channels.telegram_bridge import TelegramBridge, _ButtonReply, _ButtonState + + +def _bridge(thread_binding): + hitl = SimpleNamespace( + store=SimpleNamespace( + get_pending=lambda pid, **kw: SimpleNamespace( + ask_question_index=0, user_id=1, session_key="s", thread_id="new-thread" + ) + ) + ) + gw = SimpleNamespace( + processor=SimpleNamespace(hitl_coordinator=hitl), + thread_registry=SimpleNamespace(get_bound_thread_id=lambda _: thread_binding), + _repos=SimpleNamespace( + agent_repo=SimpleNamespace(get=lambda _: SimpleNamespace(user_id=1)) + ), + ) + return TelegramBridge(gw, lambda m: None, "tg") + + +def _reply(state): + from octop_gateway.models import ChannelSubject, InboundMessage, TextContent + + subject = ChannelSubject(subject_id="-100", chat_type="group", metadata={"chat_id": "-100"}) + base = InboundMessage( + channel_id="tg", + channel_type="telegram", + tenant_id="sites", + channel_subject=subject, + content=[TextContent(text="1")], + metadata={"chat_id": "-100", "user_id": "7", "is_group": True}, + ) + return _ButtonReply(**base.model_dump(), button_state=state) + + +@pytest.mark.parametrize("binding", ["old-thread", "new-thread"]) +def test_choice_button_survives_group_thread_rebind(binding): + """群里每轮都会重绑 thread;只要 pending 记录仍属于该会话/用户,按钮就该生效。""" + bridge = _bridge(binding) + state = _ButtonState( + owner_id=1, + thread_id="old-thread", + pending_id="p1", + question_index=0, + model_page=None, + text="1", + ) + assert bridge._button_reject_reason(_reply(state), state, "sites", "s") is None + + +def test_stale_choice_button_is_still_rejected(): + bridge = _bridge("new-thread") + state = _ButtonState( + owner_id=1, + thread_id="old-thread", + pending_id="p1", + question_index=5, + model_page=None, + text="1", + ) + assert ( + bridge._button_reject_reason(_reply(state), state, "sites", "s") == "question_index_changed" + ) + + +@pytest.mark.asyncio +async def test_rejected_button_sends_notice_without_typing(): + """按钮失效必须给用户提示,而不是静默丢弃;不先发 typing 白占限流额度。""" + bridge, msg, channel, record = setup_bridge() + sent = [] + typing = [] + + async def fake_send(subject, text): + sent.append(text) + + async def fake_typing(subject): + typing.append(subject) + + channel.push_text = fake_send + channel._send_typing_indicator = fake_typing + channel.enqueue = MagicMock() + bridge.gateway.thread_registry.get_bound_thread_id = lambda _: "someone-else" + # 组装一条按钮回复 + reply = msg.model_copy( + update={"content": [TextContent(text="1")], "metadata": {"user_id": "telegram-user"}} + ) + from octop.infra.gateway.channels.telegram_bridge import _ButtonReply, _ButtonState + + state = _ButtonState( + owner_id=0, thread_id="t1", pending_id="p1", question_index=0, model_page=None, text="1" + ) + button_reply = _ButtonReply(**reply.model_dump(), button_state=state) + _ = [event async for event in bridge(button_reply)] + assert typing == [], "提示前不发 typing" + assert sent == ["This card does not belong to the current session."] + channel.enqueue.assert_not_called() + + +def _nav_state(): + return _ButtonState( + owner_id=1, thread_id="old-thread", pending_id=None, question_index=0, model_page=1, text="" + ) + + +def test_model_menu_buttons_survive_thread_rebind_in_groups(): + """群聊里别人的新消息会重绑 thread,翻页等导航按钮不能因此失效。""" + bridge = _bridge("new-thread") + reply = _reply(_nav_state()) + assert bridge._button_reject_reason(reply, reply.button_state, "sites", "s") is None + + +def test_model_menu_buttons_stay_strict_in_private_chats(): + bridge = _bridge("new-thread") + reply = _reply(_nav_state()) + reply = reply.model_copy(update={"metadata": {**reply.metadata, "is_group": False}}) + assert bridge._button_reject_reason(reply, reply.button_state, "sites", "s") == "thread_rebound" + + +@pytest.mark.asyncio +async def test_reject_notice_follows_user_locale(): + bridge, msg, channel, record = setup_bridge() + bridge.gateway._repos.user_repo = SimpleNamespace(get=lambda _id: SimpleNamespace(locale="zh")) + sent = [] + + async def fake_send(subject, text): + sent.append(text) + + channel.push_text = fake_send + state = _ButtonState( + owner_id=0, thread_id="t1", pending_id="p1", question_index=0, model_page=None, text="1" + ) + reply = msg.model_copy(update={"content": [TextContent(text="1")]}) + _ = [e async for e in bridge(_ButtonReply(**reply.model_dump(), button_state=state))] + assert sent == ["该卡片不属于当前会话"] + + +@pytest.mark.asyncio +async def test_choice_card_falls_back_to_text_when_buttons_fail(): + """send_buttons 抛错时被扣住的文本卡片必须补发,不能丢。""" + bridge, msg, channel, record = setup_bridge() + channel.send_buttons.side_effect = RuntimeError("telegram down") + events = [event async for event in bridge(msg)] + assert [e.type.value for e in events].count("message") == 1 + assert events[-1].type.value == "completed" + + +@pytest.mark.asyncio +async def test_buttons_are_registered_with_hitl_aligned_ttl(): + bridge, msg, channel, record = setup_bridge() + _ = [event async for event in bridge(msg)] + assert channel.send_buttons.call_args.kwargs["ttl"] == 1800.0 diff --git a/tests/unit/gateway/test_harness_request.py b/tests/unit/gateway/test_harness_request.py index 38e7b7ecc0..24357e2ab3 100644 --- a/tests/unit/gateway/test_harness_request.py +++ b/tests/unit/gateway/test_harness_request.py @@ -13,6 +13,33 @@ from octop.infra.gateway.process.harness_request import build_content_from_message +@pytest.mark.asyncio +@pytest.mark.parametrize("channel_type", ["telegram", "qq"]) +@pytest.mark.parametrize("locale,label", [("en", "[Quoted message]"), ("zh", "[引用消息]")]) +@pytest.mark.parametrize("own_reply,explicit", [(False, False), (True, False), (True, True)]) +async def test_quoted_message_is_added_only_to_model_content( + channel_type, locale, label, own_reply, explicit +): + msg = InboundMessage( + channel_id="tg", + channel_type=channel_type, + content=[TextContent(text="explain this")], + metadata={ + "reply_to_text": "whole message", + "reply_to_is_own_bot": own_reply, + "reply_quote_text": "selection" if explicit else "", + }, + ) + result = await build_content_from_message(msg, locale=locale) + assert msg.text == "explain this" + if own_reply and not explicit: + assert result == "explain this" + else: + assert result.startswith(label) + assert ("selection" if explicit else "whole message") in result + assert result.endswith("explain this") + + @pytest.mark.asyncio async def test_group_context_is_rendered_as_background_inside_current_turn() -> None: msg = InboundMessage( diff --git a/tests/unit/gateway/test_message_keys.py b/tests/unit/gateway/test_message_keys.py index 3885cee7ae..8b1842be2d 100644 --- a/tests/unit/gateway/test_message_keys.py +++ b/tests/unit/gateway/test_message_keys.py @@ -104,6 +104,77 @@ def test_sanitize_im_metadata_persists_weixin_routing() -> None: assert "message_id" not in meta +def _telegram_message(**metadata: object) -> InboundMessage: + msg = _msg(subject_id="-100123", channel_type="telegram") + msg.metadata = {"chat_type": "group", "chat_id": -100123, **metadata} + return msg + + +def test_telegram_topic_isolation_is_opt_in() -> None: + legacy = session_key_from_message(_telegram_message(), agent_id="a1") + assert legacy == "a1:telegram:-100123:group" + for flag in (None, False, "true"): + msg = _telegram_message(topic_isolation=flag, message_thread_id=42) + assert session_key_from_message(msg, agent_id="a1") == legacy + + +def test_telegram_isolation_separates_chat_topic_instance_and_agent() -> None: + msg = _telegram_message(topic_isolation=True, message_thread_id=42) + first = session_key_from_message(msg, agent_id="a1") + msg.metadata["message_thread_id"] = 43 + second = session_key_from_message(msg, agent_id="a1") + msg.channel_id = "ch2" + third = session_key_from_message(msg, agent_id="a1") + msg.metadata["chat_id"] = -100456 + fourth = session_key_from_message(msg, agent_id="a1") + fifth = session_key_from_message(msg, agent_id="a2") + assert len({first, second, third, fourth, fifth}) == 5 + assert first == "a1:telegram:ch1/-100123/42:group" + + +def test_telegram_missing_topic_preserves_legacy_sessions() -> None: + for chat_type in ("dm", "group"): + for topic in ({}, {"message_thread_id": None}): + msg = _telegram_message(topic_isolation=True, chat_type=chat_type, **topic) + assert session_key_from_message(msg, agent_id="a1") == ( + f"a1:telegram:-100123:{chat_type}" + ) + + +def test_telegram_zero_topic_is_present_and_explicit_session_wins() -> None: + for topic_id in (None, 0, 42): + msg = _telegram_message(topic_isolation=True, message_thread_id=topic_id) + msg.metadata["session_key"] = " existing-session " + assert session_key_from_message(msg, agent_id="a1") == "existing-session" + for empty in ("", " ", None): + msg.metadata["session_key"] = empty + subject = "-100123" if topic_id is None else f"ch1/-100123/{topic_id}" + assert session_key_from_message(msg, agent_id="a1") == f"a1:telegram:{subject}:group" + + +def test_topic_isolation_metadata_does_not_change_other_channels() -> None: + msg = _telegram_message(topic_isolation=True, message_thread_id=42) + msg.channel_type = "qq" + assert session_key_from_message(msg, agent_id="a1") == "a1:qq:-100123:group" + + +def test_telegram_durable_topic_metadata_excludes_source_reply_ids() -> None: + msg = _telegram_message( + topic_isolation=True, + message_thread_id=42, + message_id=123, + source_message_id=123, + reply_to_message_id=122, + ) + assert sanitize_im_metadata(msg) == { + "channel_type": "telegram", + "chat_type": "group", + "chat_id": -100123, + "topic_isolation": True, + "message_thread_id": 42, + } + + def test_session_metadata_keeps_weixin_token_under_a_durable_key() -> None: from octop.infra.gateway.process.message_keys import session_metadata_from_message diff --git a/tests/unit/gateway/test_processor_default_open.py b/tests/unit/gateway/test_processor_default_open.py index 7bda1b8fa1..eb7e583f58 100644 --- a/tests/unit/gateway/test_processor_default_open.py +++ b/tests/unit/gateway/test_processor_default_open.py @@ -312,6 +312,7 @@ async def test_resolve_turn_mcp_servers_raises_when_prepare_fails() -> None: agent_manager.merge_turn_mcp_servers = MagicMock(return_value=["bad__1"]) agent_manager.prepare_chat_mcp = AsyncMock(return_value=["bad__1"]) agent_manager.default_mcp_servers = MagicMock(return_value=[]) + agent_manager.connector_auth_blocked = AsyncMock(return_value=[]) processor = GlobalProcessor( agent_manager=agent_manager, @@ -332,3 +333,42 @@ async def test_resolve_turn_mcp_servers_raises_when_prepare_fails() -> None: explicit=None, ) assert ei.value.code == ErrorCode.CONNECTOR_MCP_LOAD_FAILED + + +@pytest.mark.asyncio +async def test_resolve_turn_mcp_servers_names_connectors_that_need_authorization() -> None: + blocked = [ + { + "server": "notion__1", + "display_name": "Notion", + "auth_status": "expired", + "can_authorize": True, + "target": {"type": "catalog", "kind": "notion", "instance_id": "1"}, + } + ] + agent_manager = MagicMock() + agent_manager.merge_turn_mcp_servers = MagicMock(return_value=["notion__1", "ok"]) + agent_manager.prepare_chat_mcp = AsyncMock(return_value=["notion__1"]) + agent_manager.default_mcp_servers = MagicMock(return_value=[]) + agent_manager.connector_auth_blocked = AsyncMock(return_value=blocked) + + processor = GlobalProcessor( + agent_manager=agent_manager, + thread_registry=MagicMock(), + audit_repo=MagicMock(), + agent_repo=MagicMock(), + user_repo=MagicMock(), + connector_repo=MagicMock(), + dispatcher=SlashDispatcher(), + usage_repo=None, + gateway=None, + ) + + with pytest.raises(OctopError) as ei: + await processor._resolve_turn_mcp_servers(agent_id="a1", user_id=1, explicit=None) + + assert ei.value.code == ErrorCode.CONNECTOR_AUTH_REQUIRED + assert ei.value.status == 409 + assert ei.value.details["connectors"] == blocked + agent_manager.connector_auth_blocked.assert_awaited_once_with(1, ["notion__1"]) + assert ei.value.localized_message("zh") == "以下连接器需要授权后才能使用:Notion" diff --git a/tests/unit/gateway/test_stream_close_chain.py b/tests/unit/gateway/test_stream_close_chain.py new file mode 100644 index 0000000000..42cf5aabcd --- /dev/null +++ b/tests/unit/gateway/test_stream_close_chain.py @@ -0,0 +1,96 @@ +import asyncio +import logging +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest +from octop_gateway.models import ChannelSubject, InboundMessage, MessageEvent, TextContent + +from octop.infra.gateway.channels.telegram_bridge import TelegramBridge, _ButtonReply, _ButtonState + + +def _bridge_with(inner): + msg = InboundMessage( + channel_id="tg", + channel_type="telegram", + tenant_id="agent", + channel_subject=ChannelSubject(subject_id="chat"), + content=[TextContent(text="hi")], + metadata={"user_id": "u", "chat_id": "chat"}, + ) + hitl = SimpleNamespace(resolve_ask_pending=lambda *a, **k: None) + gateway = SimpleNamespace( + channel_manager=SimpleNamespace(get_channel=lambda _: None), + processor=SimpleNamespace(hitl_coordinator=hitl), + thread_registry=SimpleNamespace(get_bound_thread_id=lambda _: None), + _repos=SimpleNamespace( + agent_repo=SimpleNamespace(get=lambda _: SimpleNamespace(user_id=1)) + ), + _agent_manager=SimpleNamespace(cancel_stream=AsyncMock()), + ) + return TelegramBridge(gateway, inner, "tg"), msg + + +@pytest.mark.asyncio +async def test_bridge_closes_inner_stream_when_consumer_closes_early(): + closed = asyncio.Event() + + async def inner(msg): + try: + yield MessageEvent.text("a") + yield MessageEvent.text("b") + await asyncio.sleep(3600) + finally: + closed.set() + + bridge, msg = _bridge_with(inner) + it = bridge(msg) + first = await it.__anext__() + assert first.content[0].text == "a" + assert not closed.is_set() + await it.aclose() + assert closed.is_set(), "inner stream must finalize with the outer close" + + +@pytest.mark.asyncio +async def test_bridge_inner_cancelled_stream_runs_cleanup(): + closed = asyncio.Event() + + async def inner(msg): + try: + yield MessageEvent.text("a") + await asyncio.sleep(3600) + finally: + closed.set() + + bridge, msg = _bridge_with(inner) + + async def consume(): + async for _ in bridge(msg): + await asyncio.sleep(3600) + + task = asyncio.create_task(consume()) + await asyncio.sleep(0.05) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + # 取消传播会打断 finally 中的 await,内层 aclose 由事件循环终结器 + # 异步补完——修复保证它必然被调度(而非留给 GC 悬置)。 + await asyncio.wait_for(closed.wait(), timeout=1) + assert closed.is_set() + + +def test_button_validation_failure_is_logged(caplog): + bridge, msg = _bridge_with(lambda m: None) + state = _ButtonState( + owner_id=None, + thread_id=None, + pending_id=None, + question_index=0, + model_page=None, + text="", + ) + reply = _ButtonReply(**msg.model_dump(), button_state=state) + with caplog.at_level(logging.WARNING, logger="octop.infra.gateway.channels.telegram_bridge"): + assert bridge._button_reject_reason_call(reply) is not None + assert any("button reply dropped" in rec.message for rec in caplog.records) diff --git a/tests/unit/gateway/test_telegram_bridge.py b/tests/unit/gateway/test_telegram_bridge.py new file mode 100644 index 0000000000..086e63bd82 --- /dev/null +++ b/tests/unit/gateway/test_telegram_bridge.py @@ -0,0 +1,498 @@ +"""Host bridge tests run with both released and new Gateway transports.""" + +import asyncio +from dataclasses import dataclass +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest +from octop_gateway.models import ( + ChannelSubject, + InboundMessage, + MessageEvent, + MessageEventType, + TextContent, +) + +from octop.infra.errors import OctopError +from octop.infra.gateway.channels.telegram_bridge import TelegramBridge, validate_telegram_config +from octop.infra.gateway.hitl.coordinator import HitlChannelCoordinator, HitlStreamContext +from octop.infra.gateway.process.message_keys import session_key_from_message + + +def setup_bridge(): + msg = InboundMessage( + channel_id="tg", + channel_type="telegram", + tenant_id="agent", + channel_subject=ChannelSubject(subject_id="chat"), + content=[TextContent(text="hello")], + metadata={"user_id": "telegram-user", "chat_id": "chat", "message_thread_id": 7}, + ) + hitl = HitlChannelCoordinator() + record = hitl.register_from_request( + { + "action_requests": [ + { + "name": "ask_user_question", + "args": { + "questions": [ + { + "question": "Pick", + "options": [{"label": "Alpha"}, {"label": "Beta"}], + }, + {"question": "Next", "options": [{"label": "Gamma"}]}, + ] + }, + } + ] + }, + ctx=HitlStreamContext( + thread_id="thread", + agent_id="agent", + user_id=1, + session_key=session_key_from_message(msg, agent_id="agent"), + channel_type="telegram", + ), + ) + channel = SimpleNamespace( + send_buttons=AsyncMock(), + _process_inbound=AsyncMock(), + enqueue=MagicMock(), + get_debounce_key=lambda msg: "chat", + ) + + async def run_in_session(channel_id, key, operation): + assert (channel_id, key) == ("tg", "chat") + return await operation() + + gateway = SimpleNamespace( + channel_manager=SimpleNamespace( + get_channel=lambda _: channel, run_in_session=run_in_session + ), + processor=SimpleNamespace(hitl_coordinator=hitl), + thread_registry=SimpleNamespace(get_bound_thread_id=lambda _: "thread"), + _repos=SimpleNamespace( + agent_repo=SimpleNamespace(get=lambda _: SimpleNamespace(user_id=1)) + ), + _agent_manager=SimpleNamespace(cancel_stream=MagicMock()), + ) + + async def processor(msg): + yield MessageEvent.text("question", metadata={"octop_hitl_pending_id": record.pending_id}) + yield MessageEvent.completed() + + return TelegramBridge(gateway, processor, "tg"), msg, channel, record + + +async def drain_button(bridge, channel): + reply = channel.enqueue.call_args.args[0] + processor = bridge.processor + + async def deliver(msg): + await channel._process_inbound(msg, msg.channel_subject) + if False: + yield + + bridge.processor = deliver + try: + _ = [event async for event in bridge(reply)] + finally: + bridge.processor = processor + + +@pytest.mark.asyncio +async def test_callback_returns_before_session_lock_and_agent_turn(): + bridge, msg, channel, record = setup_bridge() + channel.enqueue = MagicMock() + entered = asyncio.Event() + release = asyncio.Event() + + async def busy_session(channel_id, key, operation): + entered.set() + await release.wait() + await operation() + + bridge.gateway.channel_manager.run_in_session = busy_session + action = bridge._action(msg, msg.channel_subject, "1", record.pending_id) + task = asyncio.create_task(action()) + try: + await asyncio.sleep(0) + assert task.done(), "Telegram callback is waiting for session execution" + await task + channel.enqueue.assert_called_once() + channel._process_inbound.assert_not_awaited() + finally: + release.set() + await task + + +@pytest.mark.asyncio +async def test_choice_uses_original_scope_and_rejects_stale_siblings(): + bridge, msg, channel, record = setup_bridge() + # 卡片文本被按钮取代:只发按钮 + 终止事件,不再重复发一条选项文本。 + events = [event async for event in bridge(msg)] + assert [event.type for event in events] == [MessageEventType.COMPLETED] + subject, text, actions = channel.send_buttons.call_args.args + assert subject.metadata["user_id"] == "telegram-user" + assert subject.metadata["message_thread_id"] == 7 + assert [label for label, _ in actions] == ["Alpha", "Beta"] + await actions[0][1]() + await drain_button(bridge, channel) + reply = channel._process_inbound.call_args.args[0] + assert reply.text == "1" + assert reply.tenant_id == "agent" + record.ask_question_index = 1 + await actions[1][1]() + await drain_button(bridge, channel) + channel._process_inbound.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_choice_advances_existing_question_collector(): + bridge, msg, channel, record = setup_bridge() + hitl = bridge.gateway.processor.hitl_coordinator + + async def process_reply(reply, subject): + message, next_card = hitl.collect_ask_reply(record, reply.text, locale="en") + assert message is None + assert "Next" in next_card + + channel._process_inbound.side_effect = process_reply + _ = [event async for event in bridge(msg)] + await channel.send_buttons.call_args.args[2][1][1]() + await drain_button(bridge, channel) + assert record.ask_question_index == 1 + assert record.ask_answers == ["Beta"] + + +@pytest.mark.asyncio +async def test_multiselect_and_security_pause_do_not_offer_choices(): + bridge, msg, channel, record = setup_bridge() + record.action_requests[0]["args"]["questions"][0]["multi_select"] = True + _ = [event async for event in bridge(msg)] + channel.send_buttons.assert_not_awaited() + record.action_requests = [{"name": "execute", "args": {"command": "test"}}] + _ = [event async for event in bridge(msg)] + channel.send_buttons.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_expired_or_rebound_pause_does_not_resume(): + bridge, msg, channel, record = setup_bridge() + _ = [event async for event in bridge(msg)] + action = channel.send_buttons.call_args.args[2][0][1] + bridge.gateway.thread_registry.get_bound_thread_id = lambda _: "new-thread" + await action() + await drain_button(bridge, channel) + channel._process_inbound.assert_not_awaited() + bridge.gateway.thread_registry.get_bound_thread_id = lambda _: "thread" + record.status = "expired" + await action() + await drain_button(bridge, channel) + channel._process_inbound.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_native_cancel_signals_task_runtime(): + import asyncio + + bridge, msg, channel, record = setup_bridge() + + async def cancelled(msg): + raise asyncio.CancelledError + yield # pragma: no cover + + bridge.processor = cancelled + with pytest.raises(asyncio.CancelledError): + _ = [event async for event in bridge(msg)] + bridge.gateway._agent_manager.cancel_stream.assert_called_once_with("agent", "thread") + + +@pytest.mark.asyncio +async def test_older_gateway_falls_back_to_text(): + bridge, msg, channel, record = setup_bridge() + channel.send_buttons = None + assert len([event async for event in bridge(msg)]) == 2 + channel._process_inbound.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_invoke_mode_preserves_question_and_buttons_precede_completed(): + from octop_gateway.models import MessageEventType + + from octop.infra.gateway.process.response_mode import processor_for_response_mode + + bridge, msg, channel, record = setup_bridge() + bridge.processor = processor_for_response_mode(bridge.processor, "invoke") + async for event in bridge(msg): + if event.type == MessageEventType.COMPLETED: + channel.send_buttons.assert_awaited_once() + break # BaseChannel stops consuming here. + + +@pytest.mark.asyncio +async def test_model_choices_dispatch_existing_slash_path(): + bridge, msg, channel, record = setup_bridge() + msg = msg.model_copy( + update={ + "content": [TextContent(text="/model")], + "metadata": {**msg.metadata, "reply_quote_text": "quoted source"}, + } + ) + bridge.gateway.processor.hitl_coordinator.store.mark_resolved(record.pending_id, "expired") + provider = SimpleNamespace( + name="provider", + enabled=1, + api_key="key", + get_models=lambda: [{"id": "model", "enabled": True}], + ) + bridge.gateway._repos.provider_repo = SimpleNamespace(list_all=lambda: [provider]) + _ = [event async for event in bridge(msg)] + actions = channel.send_buttons.call_args.args[2] + assert actions[0][0] == "provider/model" + await actions[0][1]() + await drain_button(bridge, channel) + assert channel._process_inbound.call_args.args[0].text == "/model provider/model" + + +@pytest.mark.asyncio +async def test_quoted_turn_does_not_change_native_answer_control_text(): + bridge, msg, channel, record = setup_bridge() + msg = msg.model_copy(update={"metadata": {**msg.metadata, "reply_quote_text": "quoted source"}}) + _ = [event async for event in bridge(msg)] + await channel.send_buttons.call_args.args[2][0][1]() + await drain_button(bridge, channel) + assert channel._process_inbound.call_args.args[0].text == "1" + + +@pytest.mark.asyncio +async def test_create_and_update_reject_before_persisting(monkeypatch): + import octop_gateway.channels.telegram as telegram + + from octop.infra.gateway.gateway import ChannelCreateSpec, Gateway + + @dataclass + class OldConfig: + bot_token: str = "" + + monkeypatch.setattr(telegram, "TelegramConfig", OldConfig) + repos = MagicMock() + repos.channel_repo.get.return_value = SimpleNamespace(kind="telegram", config_json="{}") + gateway = Gateway(agent_manager=MagicMock(), repos=repos) + with pytest.raises(OctopError) as error: + await gateway.create_channel( + ChannelCreateSpec( + channel_id="tg", + agent_id="a", + user_id=1, + kind="telegram", + name="tg", + config={"streaming_mode": "edit"}, + ) + ) + assert "Gateway" in error.value.localized_message("en") + repos.channel_repo.create.assert_not_called() + with pytest.raises(OctopError): + await gateway.update_channel("tg", config_json='{"topic_isolation": true}') + repos.channel_repo.update.assert_not_called() + + +def test_old_gateway_options_fail_explicitly(monkeypatch): + import octop_gateway.channels.telegram as telegram + + @dataclass + class OldConfig: + bot_token: str = "" + + monkeypatch.setattr(telegram, "TelegramConfig", OldConfig) + validate_telegram_config("telegram", {"bot_token": "token"}) + with pytest.raises(OctopError, match="streaming_mode"): + validate_telegram_config("telegram", {"bot_token": "token", "streaming_mode": "edit"}) + validate_telegram_config("other", {"streaming_mode": "edit"}) + + +@pytest.mark.asyncio +async def test_model_pagination_and_revoked_model(): + bridge, msg, channel, record = setup_bridge() + msg = msg.model_copy(update={"content": [TextContent(text="/models")]}) + models = [{"id": f"m{i:03}", "enabled": True} for i in range(105)] + models.extend([{"id": "disabled", "enabled": False}, {"id": "embedding", "task": "embedding"}]) + provider = SimpleNamespace(name="p", enabled=1, api_key="key", get_models=lambda: models) + bridge.gateway._repos.provider_repo = SimpleNamespace(list_all=lambda: [provider]) + bridge.gateway.processor.hitl_coordinator.store.mark_resolved(record.pending_id, "expired") + seen = [] + + async def processor(message): + seen.append(message.text) + yield MessageEvent.completed() + + bridge.processor = processor + _ = [event async for event in bridge(msg)] + assert seen == ["/models"] + all_labels = set() + for page in range(3): + actions = channel.send_buttons.call_args.args[2] + assert len(actions) <= 50 + all_labels.update(label for label, _ in actions if label.startswith("p/")) + if page < 2: + await next(action for label, action in actions if label == "→")() + await drain_button(bridge, channel) + assert all_labels == {f"p/m{i:03}" for i in range(105)} + last_action = next(action for label, action in actions if label == "p/m104") + models[104]["enabled"] = False + await last_action() + await drain_button(bridge, channel) + channel._process_inbound.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_callback_rejects_owner_change(): + bridge, msg, channel, record = setup_bridge() + _ = [event async for event in bridge(msg)] + action = channel.send_buttons.call_args.args[2][0][1] + bridge.gateway._repos.agent_repo.get = lambda _: SimpleNamespace(user_id=2) + await action() + await drain_button(bridge, channel) + channel._process_inbound.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_unrelated_turn_does_not_reveal_pending_question(): + bridge, msg, channel, record = setup_bridge() + + async def processor(message): + yield MessageEvent.completed() + + bridge.processor = processor + _ = [event async for event in bridge(msg)] + channel.send_buttons.assert_not_awaited() + + +def test_legacy_enabled_is_host_owned(): + validate_telegram_config("telegram", {"bot_token": "token", "enabled": "true"}) + + +def test_invalid_gateway_option_values_rejected(): + with pytest.raises(OctopError): + validate_telegram_config("telegram", {"group_activation": "unsupported"}) + + +@pytest.mark.asyncio +async def test_other_group_cannot_receive_pending_buttons(): + bridge, msg, channel, record = setup_bridge() + msg = msg.model_copy(update={"channel_subject": ChannelSubject(subject_id="another-chat")}) + _ = [event async for event in bridge(msg)] + channel.send_buttons.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_failed_model_command_does_not_offer_menu(): + bridge, msg, channel, record = setup_bridge() + msg = msg.model_copy(update={"content": [TextContent(text="/models")]}) + + async def processor(message): + yield MessageEvent.error_event("denied") + yield MessageEvent.completed() + + bridge.processor = processor + _ = [event async for event in bridge(msg)] + channel.send_buttons.assert_not_awaited() + + +@pytest.mark.parametrize("name", ["topic_isolation", "allow_unaddressed_commands"]) +def test_string_boolean_is_not_silently_accepted(name): + with pytest.raises(OctopError): + validate_telegram_config("telegram", {name: "false"}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("change", ["none", "owner", "thread", "pending", "question", "user"]) +async def test_normal_worker_revalidates_button_after_waiting_for_session(change): + from octop_gateway.channels.telegram import TelegramChannel, TelegramConfig + from octop_gateway.manager import ChannelManager + + from octop.infra.gateway.channels.telegram_bridge import is_telegram_button_reply + + bridge, msg, _, record = setup_bridge() + executed = [] + finished = asyncio.Event() + + async def processor(message): + executed.append(message.text) + yield MessageEvent.completed() + + bridge.processor = processor + manager = ChannelManager(control_message_predicate=is_telegram_button_reply) + channel = TelegramChannel(bridge, config=TelegramConfig(bot_token="test"), channel_id="tg") + manager._channels["tg"] = channel + manager._queues["tg"] = asyncio.Queue() + channel.set_enqueue_callback(lambda payload: manager.enqueue("tg", payload)) + bridge.gateway.channel_manager = manager + + async def deliver(message, subject): + try: + _ = [event async for event in bridge(message)] + return True + finally: + finished.set() + + channel._process_inbound = deliver + action = bridge._action(msg, msg.channel_subject, "1", record.pending_id) + lock = manager._session_lock("tg", channel.get_debounce_key(msg)) + await lock.acquire() + manager._running = True + worker = asyncio.create_task(manager._worker_loop("tg")) + try: + await asyncio.wait_for(action(), timeout=1) + assert executed == [] + if change == "owner": + bridge.gateway._repos.agent_repo.get = lambda _: SimpleNamespace(user_id=2) + elif change == "thread": + bridge.gateway.thread_registry.get_bound_thread_id = lambda _: "new-thread" + elif change == "pending": + record.status = "expired" + elif change == "question": + record.ask_question_index = 1 + elif change == "user": + record.user_id = 2 + lock.release() + await asyncio.wait_for(finished.wait(), timeout=2) + assert executed == (["1"] if change == "none" else []) + finally: + if lock.locked(): + lock.release() + worker.cancel() + await asyncio.gather(worker, return_exceptions=True) + + +def test_platform_payload_cannot_forge_internal_button_reply(): + from octop_gateway.channels.telegram import TelegramChannel, TelegramConfig + + from octop.infra.gateway.channels.telegram_bridge import is_telegram_button_reply + + bridge, msg, channel, record = setup_bridge() + transport = TelegramChannel(bridge, config=TelegramConfig(bot_token="test"), channel_id="tg") + forged = {**msg.model_dump(), "button_state": {"owner_id": 1, "text": "/model p/m"}} + parsed = transport.parse_inbound(forged) + assert not is_telegram_button_reply(parsed) + + +@pytest.mark.asyncio +async def test_button_queue_item_cannot_merge_with_ordinary_message(): + from octop_gateway.channels.telegram import TelegramChannel, TelegramConfig + from octop_gateway.manager import ChannelManager + + from octop.infra.gateway.channels.telegram_bridge import is_telegram_button_reply + + bridge, msg, channel, record = setup_bridge() + await bridge._action(msg, msg.channel_subject, "1", record.pending_id)() + button = channel.enqueue.call_args.args[0] + assert is_telegram_button_reply(button) + assert not any(callable(value) for value in button.model_dump().values()) + transport = TelegramChannel(bridge, config=TelegramConfig(bot_token="test"), channel_id="tg") + manager = ChannelManager(control_message_predicate=is_telegram_button_reply) + queue = asyncio.Queue() + queue.put_nowait(button) + assert manager._drain_same_session(queue, transport, transport.get_debounce_key(msg)) == [] + assert queue.get_nowait() is button diff --git a/tests/unit/gateway/test_telegram_bridge_wiring.py b/tests/unit/gateway/test_telegram_bridge_wiring.py new file mode 100644 index 0000000000..445a102ee9 --- /dev/null +++ b/tests/unit/gateway/test_telegram_bridge_wiring.py @@ -0,0 +1,38 @@ +"""Telegram channels must be registered behind the host bridge.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock + +import pytest +from tests.unit.gateway.test_gateway_probe import _fake_row, _make_gateway + +from octop.infra.gateway.channels.telegram_bridge import TelegramBridge +from octop.infra.gateway.gateway import Gateway + + +async def _registered_processor(gw: Gateway, kind: str, config_json: str): + gw._processor = MagicMock() + gw._channel_manager = MagicMock() + gw._channel_manager.add_channel = AsyncMock() + gw._channel_manager.get_channel = MagicMock(return_value=None) + row = _fake_row() + row.kind = kind + row.config_json = config_json + await gw._register_channel(row) + return gw._channel_manager.add_channel.call_args.kwargs["processor"] + + +@pytest.mark.asyncio +async def test_telegram_channel_is_wrapped_by_bridge(tmp_path) -> None: + gw = _make_gateway(tmp_path) + processor = await _registered_processor(gw, "telegram", '{"bot_token":"123:abc"}') + assert isinstance(processor, TelegramBridge) + assert not isinstance(processor.processor, TelegramBridge) + + +@pytest.mark.asyncio +async def test_other_channels_are_not_wrapped(tmp_path) -> None: + gw = _make_gateway(tmp_path) + processor = await _registered_processor(gw, "feishu", '{"app_id":"x","app_secret":"y"}') + assert not isinstance(processor, TelegramBridge) diff --git a/tests/unit/i18n/test_channel.py b/tests/unit/i18n/test_channel.py index ad36a1c034..192ee4082f 100644 --- a/tests/unit/i18n/test_channel.py +++ b/tests/unit/i18n/test_channel.py @@ -3,6 +3,7 @@ from __future__ import annotations from octop.i18n import channel_tool_hint_end, channel_tool_hint_start +from octop.i18n.domains.channel import telegram_quote_prompt, telegram_reject_notice def test_channel_tool_hint_start_zh(): @@ -14,3 +15,17 @@ def test_channel_tool_hint_start_zh(): def test_channel_tool_hint_end_en(): assert channel_tool_hint_end("Read file", "en") == "✅ Read file done" + + +def test_telegram_reject_notice_known_and_unknown_reason(): + assert telegram_reject_notice("user_mismatch", "zh") == "该卡片不属于当前会话" + assert telegram_reject_notice("pending_expired_or_resolved", "en").startswith("This card") + assert telegram_reject_notice("brand_new_reason", "en") == ( + "The action did not take effect. Please try again." + ) + assert telegram_reject_notice("brand_new_reason", "zh") == "操作未生效,请重试" + + +def test_telegram_quote_prompt_locales(): + assert telegram_quote_prompt("hi", "en") == "[Quoted message]\nhi\n\n[Current message]\n" + assert telegram_quote_prompt("hi", "zh") == "[引用消息]\nhi\n\n[当前消息]\n" diff --git a/tests/unit/i18n/test_tools.py b/tests/unit/i18n/test_tools.py index f5e4e463f1..33166d2c2a 100644 --- a/tests/unit/i18n/test_tools.py +++ b/tests/unit/i18n/test_tools.py @@ -18,6 +18,12 @@ def test_tool_display_name_known_zh(): assert tool_display_name("write_env_file", "en") == "Write env file" +def test_external_agent_approval_has_a_display_name(): + assert tool_display_name("external-agent", "zh") == "托管智能体" + assert tool_display_name("external-agent", "en") == "Managed agent" + assert "external-agent" not in {entry.name for entry in hitl_tool_catalog()} + + def test_tool_display_name_unknown_passthrough(): assert tool_display_name("custom_plugin_tool", "en") == "custom_plugin_tool" diff --git a/tests/unit/infra/cloud/test_tencent_mirror.py b/tests/unit/infra/cloud/test_tencent_mirror.py index e4ce59683a..0591572ff0 100644 --- a/tests/unit/infra/cloud/test_tencent_mirror.py +++ b/tests/unit/infra/cloud/test_tencent_mirror.py @@ -43,14 +43,13 @@ def home(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: "octop.infra.cloud.tencent.Path.home", lambda: tmp_path, ) - monkeypatch.delenv("OCTOP_TCCLI_PROFILE_ISOLATION", raising=False) return tmp_path def test_mirror_writes_credential_and_pointer(home: Path) -> None: assert mirror_to_tccli(_credential(), uid=1) == "ok" - path = home / ".tccli" / "default.credential" + path = home / ".tccli" / "octop-1.credential" payload = json.loads(path.read_text()) assert payload["type"] == "oauth" assert payload["secretId"] == "AKIDtest000000000000" @@ -62,11 +61,7 @@ def test_mirror_writes_credential_and_pointer(home: Path) -> None: # read-only flag, so the Unix mode is meaningful on POSIX. if os.name == "posix": assert stat.S_IMODE(path.stat().st_mode) == 0o600 - # 指针文件告诉技能脚本该读哪个 profile - pointer = home / ".tccli" / ".octop-profile" - assert pointer.read_text().strip() == "default" - if os.name == "posix": - assert stat.S_IMODE(pointer.stat().st_mode) == 0o600 + assert not (home / ".tccli" / ".octop-profile").exists() def test_mirror_skips_without_tccli_dir(home: Path) -> None: @@ -84,14 +79,14 @@ def boom(*_args: object, **_kwargs: object) -> None: monkeypatch.setattr("octop.infra.cloud.tencent._write_private", boom) assert mirror_to_tccli(_credential(), uid=1) == "failed" - assert not (home / ".tccli" / "default.credential").exists() + assert not (home / ".tccli" / "octop-1.credential").exists() def test_credential_from_tccli_roundtrip(home: Path) -> None: original = _credential() assert mirror_to_tccli(original, uid=1) == "ok" - parsed = credential_from_tccli(home / ".tccli" / "default.credential") + parsed = credential_from_tccli(home / ".tccli" / "octop-1.credential") assert parsed == original @@ -103,21 +98,14 @@ def test_credential_from_tccli_rejects_static_key_file(home: Path) -> None: credential_from_tccli(path) -def test_profile_isolation_is_opt_in(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.delenv("OCTOP_TCCLI_PROFILE_ISOLATION", raising=False) - assert tccli_profile_for_user(7) == "default" - - monkeypatch.setenv("OCTOP_TCCLI_PROFILE_ISOLATION", "1") +def test_profile_is_per_user() -> None: assert tccli_profile_for_user(7) == "octop-7" - - # 无用户上下文(CLI 直接调用)时不能用隔离名 assert tccli_profile_for_user(None) == "default" -def test_isolated_profile_writes_separate_file(home: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("OCTOP_TCCLI_PROFILE_ISOLATION", "1") +def test_isolated_profile_writes_separate_file(home: Path) -> None: assert mirror_to_tccli(_credential(), uid=3) == "ok" assert (home / ".tccli" / "octop-3.credential").is_file() assert not (home / ".tccli" / "default.credential").exists() - assert (home / ".tccli" / ".octop-profile").read_text().strip() == "octop-3" + assert not (home / ".tccli" / ".octop-profile").exists() diff --git a/tests/unit/infra/cloud/test_tencent_store.py b/tests/unit/infra/cloud/test_tencent_store.py index bc091d0686..9cc28f1772 100644 --- a/tests/unit/infra/cloud/test_tencent_store.py +++ b/tests/unit/infra/cloud/test_tencent_store.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json from pathlib import Path from octop.infra.cloud.tencent import ( @@ -69,6 +70,26 @@ def test_shared_store_adopts_local_file_without_overwriting(tmp_path: Path) -> N assert shared.load(7) is None +def test_shared_store_seals_credential_and_reads_legacy_plaintext(tmp_path: Path) -> None: + pool = SqlitePool(tmp_path / "x.db") + run_migrations(pool) + secrets = SecretRepo(pool) + root = tmp_path / "home" + shared = CloudCredentialStore(root, secrets) + shared.save(4, _credential("AKID-sealed")) + blob = secrets.get("cloud.cred.4") + assert blob is not None + assert b"secret-key-value" not in blob + loaded = shared.load(4) + assert loaded is not None + assert loaded.secret_id == "AKID-sealed" + + secrets.put("cloud.cred.5", json.dumps(_credential("AKID-legacy").to_dict()).encode()) + legacy = shared.load(5) + assert legacy is not None + assert legacy.secret_id == "AKID-legacy" + + def test_open_uses_files_unless_postgresql(tmp_path: Path) -> None: pool = SqlitePool(tmp_path / "x.db") run_migrations(pool) diff --git a/tests/unit/knowledge/test_graph.py b/tests/unit/knowledge/test_graph.py new file mode 100644 index 0000000000..86b4f7644b --- /dev/null +++ b/tests/unit/knowledge/test_graph.py @@ -0,0 +1,389 @@ +"""Unit tests for building a knowledge graph and using it at retrieval time.""" + +from __future__ import annotations + +import asyncio +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from octop.infra.db.migrate import run_migrations +from octop.infra.db.pool import SqlitePool +from octop.infra.db.repos.knowledge import KnowledgeRepo +from octop.infra.db.repos.settings import SettingsRepo +from octop.infra.db.repos.users import UserRepo +from octop.infra.knowledge import graph +from octop.infra.knowledge.graph_store import ( + STATUS_FAILED, + STATUS_PENDING, + STATUS_READY, + ChunkEntity, + ChunkGraph, + ChunkRelation, + SqliteKnowledgeGraphStore, +) +from octop.infra.knowledge.index import KnowledgeIndex + + +def _graph(ordinal: int, relations: list[tuple[str, str, str, str, float]]) -> ChunkGraph: + names = dict.fromkeys(end for source, target, *_ in relations for end in (source, target)) + return ChunkGraph( + ordinal=ordinal, + entities=tuple(ChunkEntity(name, "concept", f"About {name}.") for name in names), + relations=tuple(ChunkRelation(*relation) for relation in relations), + ) + + +@pytest.fixture +def env(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> SimpleNamespace: + monkeypatch.setenv("OCTOP_HOME", str(tmp_path / "home")) + pool = SqlitePool(tmp_path / "octop.db") + run_migrations(pool) + repo = KnowledgeRepo(pool) + owner = UserRepo(pool).create(username="owner", password_hash="h", role="user") + base = repo.create_base(owner_user_id=owner, name="Docs") + services = SimpleNamespace( + knowledge_repo=repo, + settings_repo=SettingsRepo(pool), + provider_repo=SimpleNamespace(get=lambda _id: None), + ) + return SimpleNamespace(services=services, repo=repo, kb_id=base.id, owner=owner) + + +def _document(env: SimpleNamespace, name: str, chunks: list[str], *, status: str = "ready") -> str: + document = env.repo.create_document( + kb_id=env.kb_id, filename=name, content_type="text/markdown", byte_size=1, status=status + ) + KnowledgeIndex(env.kb_id).replace_doc_chunks(document.id, chunks, [[1.0, 0.0] for _ in chunks]) + return str(document.id) + + +def test_build_extracts_every_chunk_and_marks_the_document_ready(env: SimpleNamespace) -> None: + doc_id = _document(env, "plan.md", ["Alice leads Apollo.", "Apollo ships in May."]) + calls: list[tuple[str, int, str]] = [] + + def extractor(*, filename: str, ordinal: int, text: str) -> ChunkGraph: + calls.append((filename, ordinal, text)) + return _graph(ordinal, [("Alice", "Apollo", "leads", text, 8)]) + + graph.build_document_graph(env.services, env.kb_id, doc_id, extractor=extractor) + + store = SqliteKnowledgeGraphStore(env.kb_id) + assert sorted(calls) == [ + ("plan.md", 0, "Alice leads Apollo."), + ("plan.md", 1, "Apollo ships in May."), + ] + assert store.counts() == (2, 1) + assert store.document_states()[doc_id].status == STATUS_READY + assert store.relations_of(["alice"])[0].weight == 16 + + +def test_build_keeps_what_it_could_extract_and_notes_the_rest(env: SimpleNamespace) -> None: + doc_id = _document(env, "plan.md", ["good", "bad"]) + + def extractor(*, filename: str, ordinal: int, text: str) -> ChunkGraph: + if text == "bad": + raise ValueError("knowledge graph extraction returned invalid JSON") + return _graph(ordinal, [("Alice", "Apollo", "leads", "", 8)]) + + graph.build_document_graph(env.services, env.kb_id, doc_id, extractor=extractor) + + state = SqliteKnowledgeGraphStore(env.kb_id).document_states()[doc_id] + assert state.status == STATUS_READY + assert state.error == "1 of 2 passages could not be extracted" + + +def test_build_fails_the_document_when_no_passage_could_be_extracted( + env: SimpleNamespace, +) -> None: + doc_id = _document(env, "plan.md", ["one", "two"]) + + def extractor(**_kwargs: object) -> ChunkGraph: + raise RuntimeError("provider is down") + + with pytest.raises(RuntimeError, match="failed for every passage"): + graph.build_document_graph(env.services, env.kb_id, doc_id, extractor=extractor) + + state = SqliteKnowledgeGraphStore(env.kb_id).document_states()[doc_id] + assert state.status == STATUS_FAILED + assert "every passage" in state.error + + +def test_build_without_an_extraction_model_fails_the_document(env: SimpleNamespace) -> None: + doc_id = _document(env, "plan.md", ["one"]) + + with pytest.raises(RuntimeError, match="knowledge graph extraction is not ready"): + graph.build_document_graph(env.services, env.kb_id, doc_id) + + assert SqliteKnowledgeGraphStore(env.kb_id).document_states()[doc_id].status == STATUS_FAILED + + +def test_build_forgets_a_document_that_no_longer_exists(env: SimpleNamespace) -> None: + store = SqliteKnowledgeGraphStore(env.kb_id) + store.replace_document("gone", [_graph(0, [("Alice", "Apollo", "leads", "", 8)])]) + + graph.build_document_graph(env.services, env.kb_id, "gone", extractor=lambda **_: None) + + assert store.document_states() == {} + assert store.counts() == (0, 0) + + +def test_reindexing_leaves_knowledge_bases_without_a_graph_alone(env: SimpleNamespace) -> None: + doc_id = _document(env, "plan.md", ["one"]) + + assert graph.mark_document_reindexed(env.services, env.kb_id, doc_id) is False + assert not SqliteKnowledgeGraphStore(env.kb_id).path.exists() + + +def test_reindexing_drops_stale_rows_and_waits_when_extraction_is_unavailable( + env: SimpleNamespace, +) -> None: + doc_id = _document(env, "plan.md", ["one"]) + store = SqliteKnowledgeGraphStore(env.kb_id) + store.replace_document(doc_id, [_graph(0, [("Alice", "Apollo", "leads", "", 8)])]) + + assert graph.mark_document_reindexed(env.services, env.kb_id, doc_id) is False + + assert store.counts() == (0, 0) + assert store.document_states()[doc_id].status == STATUS_PENDING + + +def test_reindexing_asks_for_extraction_when_a_model_is_ready( + env: SimpleNamespace, monkeypatch: pytest.MonkeyPatch +) -> None: + doc_id = _document(env, "plan.md", ["one"]) + SqliteKnowledgeGraphStore(env.kb_id).mark_document(doc_id, STATUS_READY) + monkeypatch.setattr(graph, "get_graph_capability", lambda *_args: {"usable": True}) + + assert graph.mark_document_reindexed(env.services, env.kb_id, doc_id) is True + + +def test_enqueue_base_graph_queues_documents_and_forgets_deleted_ones( + env: SimpleNamespace, monkeypatch: pytest.MonkeyPatch +) -> None: + graph.reset_graph_semaphore_for_tests() + kept = _document(env, "plan.md", ["one"]) + store = SqliteKnowledgeGraphStore(env.kb_id) + store.replace_document("gone", [_graph(0, [("Alice", "Apollo", "leads", "", 8)])]) + built: list[str] = [] + monkeypatch.setattr( + graph, "build_document_graph", lambda _services, _kb, doc_id: built.append(doc_id) + ) + + async def run() -> int: + count = graph.enqueue_base_graph(env.services, env.kb_id, [kept]) + await asyncio.sleep(0) + await asyncio.gather( + *(task for task in asyncio.all_tasks() if task is not asyncio.current_task()) + ) + return count + + assert asyncio.run(run()) == 1 + assert built == [kept] + assert set(store.document_states()) == {kept} + assert store.document_states()[kept].status == STATUS_PENDING + + +def test_a_failed_background_build_does_not_raise( + env: SimpleNamespace, monkeypatch: pytest.MonkeyPatch +) -> None: + graph.reset_graph_semaphore_for_tests() + + def boom(*_args: object) -> None: + raise RuntimeError("provider is down") + + monkeypatch.setattr(graph, "build_document_graph", boom) + + async def run() -> None: + await graph.enqueue_document_graph(env.services, env.kb_id, "doc-1") + + asyncio.run(run()) + + +def test_startup_requeues_documents_left_pending_or_processing( + env: SimpleNamespace, monkeypatch: pytest.MonkeyPatch +) -> None: + graph.reset_graph_semaphore_for_tests() + store = SqliteKnowledgeGraphStore(env.kb_id) + store.replace_document("done", [_graph(0, [("Alice", "Apollo", "leads", "", 8)])]) + store.mark_document("queued", STATUS_PENDING) + store.mark_document("interrupted", "processing") + store.mark_document("failed", STATUS_FAILED, error="boom") + built: list[str] = [] + monkeypatch.setattr( + graph, "build_document_graph", lambda _services, _kb, doc_id: built.append(doc_id) + ) + usable = {"usable": False} + monkeypatch.setattr(graph, "get_graph_capability", lambda *_args: usable) + + async def run() -> None: + graph.resume_pending_graph_jobs(env.services) + await asyncio.sleep(0) + await asyncio.gather( + *(task for task in asyncio.all_tasks() if task is not asyncio.current_task()) + ) + + asyncio.run(run()) + assert built == [] + + usable["usable"] = True + asyncio.run(run()) + assert sorted(built) == ["interrupted", "queued"] + + +def test_startup_survives_unreadable_graph_storage( + env: SimpleNamespace, monkeypatch: pytest.MonkeyPatch +) -> None: + def boom(*_args: object) -> None: + raise RuntimeError("graph storage is unavailable") + + monkeypatch.setattr(graph, "get_graph_capability", lambda *_args: {"usable": True}) + monkeypatch.setattr(graph, "open_knowledge_graph", boom) + + graph.resume_pending_graph_jobs(env.services) + + +def _seed(env: SimpleNamespace) -> SqliteKnowledgeGraphStore: + store = SqliteKnowledgeGraphStore(env.kb_id) + store.replace_document( + "doc-1", + [ + _graph(0, [("Apollo", "Alice", "is led by", "Alice leads Apollo.", 9)]), + _graph(1, [("Alice", "Finance Board", "reports budgets to", "", 7)]), + _graph(2, [("Zeus", "Bob", "is led by", "", 9)]), + ], + ) + return store + + +def test_expand_query_returns_direct_then_indirect_relations(env: SimpleNamespace) -> None: + expansion = graph.expand_query(_seed(env), "Who approves the budget of Apollo?") + + assert expansion.facts == ( + "Apollo -> is led by -> Alice: Alice leads Apollo.", + "Alice -> reports budgets to -> Finance Board", + ) + # The passage a second hop away adds its relation, but is not pulled in itself. + assert expansion.chunks == (("doc-1", 0),) + + +def test_expand_query_ignores_names_inside_longer_ascii_words(env: SimpleNamespace) -> None: + store = SqliteKnowledgeGraphStore(env.kb_id) + store.replace_document("doc-1", [_graph(0, [("AI", "Zeus", "powers", "", 9)])]) + + assert graph.expand_query(store, "How do we maintain it?") == graph.GraphExpansion() + assert graph.expand_query(store, "What does AI power?").facts == ("AI -> powers -> Zeus",) + assert graph.expand_query(store, "AI是什么").facts == ("AI -> powers -> Zeus",) + + +def test_expand_query_without_a_named_entity_adds_nothing(env: SimpleNamespace) -> None: + assert graph.expand_query(_seed(env), "What is the weather?") == graph.GraphExpansion() + assert ( + graph.expand_query(SqliteKnowledgeGraphStore("empty"), "Apollo") == graph.GraphExpansion() + ) + + +def test_expand_query_includes_passages_that_only_mention_the_entity( + env: SimpleNamespace, +) -> None: + store = SqliteKnowledgeGraphStore(env.kb_id) + store.replace_document( + "doc-1", [ChunkGraph(ordinal=4, entities=(ChunkEntity("Apollo", "product"),))] + ) + + expansion = graph.expand_query(store, "Tell me about Apollo") + + assert expansion.facts == () + assert expansion.chunks == (("doc-1", 4),) + + +def test_graph_view_merges_relations_between_a_pair_into_one_edge(env: SimpleNamespace) -> None: + store = SqliteKnowledgeGraphStore(env.kb_id) + store.replace_document( + "doc-1", + [ + _graph( + 0, + [ + ("Alice", "Apollo", "leads", "", 8), + ("Apollo", "Alice", "is owned by", "", 4), + ("Bob", "Apollo", "tests", "", 2), + ], + ) + ], + ) + + view = graph.graph_view(store, limit=10) + + assert [(node["id"], node["degree"]) for node in view["nodes"]] == [ + ("alice", 1), + ("apollo", 2), + ("bob", 1), + ] + assert view["edges"] == [ + {"source": "alice", "target": "apollo", "labels": ["leads", "is owned by"], "weight": 12.0}, + {"source": "bob", "target": "apollo", "labels": ["tests"], "weight": 2.0}, + ] + assert [node["id"] for node in graph.graph_view(store, limit=10, search="bo")["nodes"]] == [ + "bob" + ] + assert graph.graph_view(store, limit=1)["edges"] == [] + + +def test_graph_stats_count_document_states(env: SimpleNamespace) -> None: + store = SqliteKnowledgeGraphStore(env.kb_id) + assert graph.graph_stats(store, ["a", "b"])["built"] is False + store.replace_document("a", [_graph(0, [("Alice", "Apollo", "leads", "", 8)])]) + store.replace_document("b", [], error="1 of 2 passages could not be extracted") + store.mark_document("c", STATUS_PENDING) + store.mark_document("d", STATUS_FAILED, error="boom") + store.mark_document("deleted", STATUS_READY) + + assert graph.graph_stats(store, ["a", "b", "c", "d", "new"]) == { + "built": True, + "entities": 2, + "relations": 1, + "documents_total": 5, + "documents_ready": 2, + "documents_partial": 1, + "documents_pending": 1, + "documents_failed": 1, + } + + +def test_entity_view_lists_relations_and_source_passages(env: SimpleNamespace) -> None: + doc_id = _document(env, "plan.md", ["Alice leads Apollo.", "Unrelated."]) + hidden = _document(env, "draft.md", ["Alice drafts."], status="pending") + store = SqliteKnowledgeGraphStore(env.kb_id) + store.replace_document(doc_id, [_graph(0, [("Alice", "Apollo", "leads", "Why.", 8)])]) + store.replace_document(hidden, [_graph(0, [("Alice", "Draft", "writes", "", 3)])]) + + view = graph.entity_view(env.services, env.kb_id, " ALICE ") + + assert view["entity"] == { + "id": "alice", + "name": "Alice", + "type": "concept", + "description": "About Alice.", + "mentions": 2, + } + assert view["relations"][0] == { + "source": {"id": "alice", "name": "Alice"}, + "target": {"id": "apollo", "name": "Apollo"}, + "label": "leads", + "description": "Why.", + "weight": 8.0, + } + # Passages of documents that are not searchable are left out. + assert view["passages"] == [ + { + "doc_id": doc_id, + "filename": "plan.md", + "path": "plan.md", + "ordinal": 0, + "text": "Alice leads Apollo.", + } + ] + with pytest.raises(LookupError, match="knowledge graph entity not found"): + graph.entity_view(env.services, env.kb_id, "nobody") diff --git a/tests/unit/knowledge/test_graph_extract.py b/tests/unit/knowledge/test_graph_extract.py new file mode 100644 index 0000000000..b73d607577 --- /dev/null +++ b/tests/unit/knowledge/test_graph_extract.py @@ -0,0 +1,212 @@ +"""Unit tests for knowledge-graph settings and reply parsing.""" + +from __future__ import annotations + +import json +from types import SimpleNamespace + +import pytest + +from octop.infra.knowledge import graph_extract +from octop.infra.knowledge.graph_extract import ( + MAX_ENTITIES_PER_CHUNK, + GraphExtractor, + get_graph_capability, + graph_models, + parse_extraction, + set_graph_settings, +) + + +def _provider(*, enabled: bool = True, api_key: str = "key") -> SimpleNamespace: + return SimpleNamespace( + id=7, + name="acme", + enabled=enabled, + api_key=api_key, + base_url="https://llm.example/v1", + get_models=lambda: [ + {"id": "chat-1", "name": "Chat One"}, + {"id": "embed-1", "name": "Embed", "type": "embedding"}, + ], + ) + + +def _repo(provider: SimpleNamespace | None) -> SimpleNamespace: + return SimpleNamespace(get=lambda provider_id: provider if provider_id == 7 else None) + + +def test_graph_models_lists_only_chat_models_of_a_usable_provider() -> None: + assert graph_models(_provider()) == [{"id": "chat-1", "name": "Chat One"}] + assert graph_models(_provider(enabled=False)) == [] + assert graph_models(_provider(api_key="")) == [] + assert graph_models(None) == [] + + +def test_capability_is_usable_only_when_enabled_with_a_ready_model() -> None: + settings = { + "knowledge_graph_enabled": "true", + "knowledge_graph_provider_id": "7", + "knowledge_graph_model": "chat-1", + } + + assert get_graph_capability(settings.get, _repo(_provider())) == { + "enabled": True, + "provider_id": "7", + "model": "chat-1", + "ready": True, + "usable": True, + } + assert get_graph_capability(settings.get, _repo(None))["usable"] is False + settings["knowledge_graph_model"] = "embed-1" + assert get_graph_capability(settings.get, _repo(_provider()))["ready"] is False + assert get_graph_capability({}.get)["enabled"] is False + + +def test_enabling_requires_a_chat_model_and_disabling_does_not() -> None: + settings: dict[str, str] = {} + + with pytest.raises(ValueError, match="knowledge graph model is not available"): + set_graph_settings( + settings.__setitem__, + enabled=True, + provider_id="7", + model="embed-1", + provider_repo=_repo(_provider()), + ) + assert settings == {} + + set_graph_settings( + settings.__setitem__, + enabled=True, + provider_id=" 7 ", + model="chat-1", + provider_repo=_repo(_provider()), + ) + assert settings == { + "knowledge_graph_enabled": "true", + "knowledge_graph_provider_id": "7", + "knowledge_graph_model": "chat-1", + } + + set_graph_settings(settings.__setitem__, enabled=False, provider_id=None, model=None) + assert settings["knowledge_graph_enabled"] == "false" + + +def test_parse_reads_an_object_wrapped_in_a_fence_and_prose() -> None: + reply = ( + "Here is the graph:\n```json\n" + + json.dumps( + { + "entities": [ + {"name": "Alice Chen", "type": "Person", "description": "Leads Apollo."}, + {"name": "Apollo", "type": "spaceship", "description": ""}, + ], + "relations": [ + { + "source": "alice chen", + "target": "Apollo", + "relation": "leads", + "description": "Alice leads Apollo.", + "strength": 42, + } + ], + } + ) + + "\n```" + ) + + graph = parse_extraction(reply, ordinal=3) + + assert graph.ordinal == 3 + assert [(item.name, item.type) for item in graph.entities] == [ + ("Alice Chen", "person"), + ("Apollo", "other"), + ] + relation = graph.relations[0] + assert (relation.source, relation.target, relation.label) == ("Alice Chen", "Apollo", "leads") + assert relation.strength == 10.0 + + +def test_parse_drops_relations_the_passage_cannot_back() -> None: + reply = json.dumps( + { + "entities": [{"name": "Alice"}, {"name": "Bob"}, {"name": "x"}, "junk"], + "relations": [ + {"source": "Alice", "target": "Carol", "relation": "knows"}, + {"source": "Alice", "target": "Alice", "relation": "is"}, + {"source": "Alice", "target": "Bob", "relation": ""}, + {"source": "Alice", "target": "Bob", "relation": "manages", "strength": "high"}, + {"source": "Alice", "target": "Bob", "relation": "Manages"}, + None, + ], + } + ) + + graph = parse_extraction(reply, ordinal=0) + + assert [item.name for item in graph.entities] == ["Alice", "Bob"] + assert [(item.label, item.strength) for item in graph.relations] == [("manages", 5.0)] + + +def test_parse_caps_entities_per_chunk() -> None: + reply = json.dumps({"entities": [{"name": f"Entity {n}"} for n in range(40)]}) + + assert len(parse_extraction(reply, ordinal=0).entities) == MAX_ENTITIES_PER_CHUNK + + +def test_parse_accepts_an_empty_graph() -> None: + graph = parse_extraction('{"entities":[],"relations":[]}', ordinal=0) + + assert graph.entities == () and graph.relations == () + + +@pytest.mark.parametrize("reply", ["", "no json here", "{not json}", "[1, 2]"]) +def test_parse_rejects_replies_without_a_json_object(reply: str) -> None: + with pytest.raises(ValueError, match="knowledge graph extraction returned"): + parse_extraction(reply, ordinal=0) + + +def test_extractor_refuses_to_start_without_a_ready_model() -> None: + services = SimpleNamespace( + settings_repo=SimpleNamespace(get=lambda _key: None), provider_repo=_repo(None) + ) + + with pytest.raises(RuntimeError, match="knowledge graph extraction is not ready"): + GraphExtractor(services) + + +def test_extractor_sends_the_passage_and_parses_the_reply( + monkeypatch: pytest.MonkeyPatch, +) -> None: + settings = { + "knowledge_graph_enabled": "true", + "knowledge_graph_provider_id": "7", + "knowledge_graph_model": "chat-1", + } + prompts: list[str] = [] + + class _Model: + def invoke(self, messages: list[object]) -> SimpleNamespace: + prompts.append(str(messages[0].content)) # type: ignore[attr-defined] + return SimpleNamespace(content='{"entities":[{"name":"Apollo","type":"product"}]}') + + built: list[tuple[object, str]] = [] + + def build(provider: object, *, model_id: str) -> _Model: + built.append((provider, model_id)) + return _Model() + + monkeypatch.setattr(graph_extract, "build_probe_chat_model", build) + provider = _provider() + services = SimpleNamespace( + settings_repo=SimpleNamespace(get=settings.get), provider_repo=_repo(provider) + ) + + graph = GraphExtractor(services)(filename="plan.md", ordinal=2, text="Apollo ships in May.") + + assert built == [(provider, "chat-1")] + assert "Document: plan.md" in prompts[0] + assert "Apollo ships in May." in prompts[0] + assert graph.ordinal == 2 + assert [item.name for item in graph.entities] == ["Apollo"] diff --git a/tests/unit/knowledge/test_graph_store.py b/tests/unit/knowledge/test_graph_store.py new file mode 100644 index 0000000000..0840f4fa67 --- /dev/null +++ b/tests/unit/knowledge/test_graph_store.py @@ -0,0 +1,208 @@ +"""Unit tests for the per-knowledge-base entity graph storage.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from octop.infra.knowledge.graph_store import ( + STATUS_FAILED, + STATUS_PENDING, + STATUS_READY, + ChunkEntity, + ChunkGraph, + ChunkRelation, + SqliteKnowledgeGraphStore, + entity_key, +) + + +@pytest.fixture +def store(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> SqliteKnowledgeGraphStore: + monkeypatch.setenv("OCTOP_HOME", str(tmp_path / "home")) + return SqliteKnowledgeGraphStore("kb-1") + + +def _chunk( + ordinal: int, entities: list[tuple[str, str, str]], relations: list[tuple] +) -> ChunkGraph: + return ChunkGraph( + ordinal=ordinal, + entities=tuple( + ChunkEntity(name, kind, description) for name, kind, description in entities + ), + relations=tuple(ChunkRelation(*relation) for relation in relations), + ) + + +def test_entity_key_ignores_case_and_spacing() -> None: + assert entity_key(" Tencent Cloud ") == entity_key("tencent cloud") == "tencent cloud" + + +def test_reads_on_a_knowledge_base_without_a_graph_create_nothing( + store: SqliteKnowledgeGraphStore, +) -> None: + assert store.exists() is False + assert store.counts() == (0, 0) + assert store.match_keys("anything") == [] + store.delete_document("doc-1") + store.clear() + assert not store.path.exists() + + +def test_entities_are_merged_by_name_across_chunks_and_documents( + store: SqliteKnowledgeGraphStore, +) -> None: + store.replace_document( + "doc-1", + [ + _chunk(0, [("Tencent Cloud", "organization", "A cloud provider.")], []), + _chunk(1, [("tencent cloud", "organization", "")], []), + ], + ) + store.replace_document( + "doc-2", + [_chunk(0, [("Tencent Cloud", "product", "A cloud provider based in Shenzhen.")], [])], + ) + + merged = store.entities(["tencent cloud"])["tencent cloud"] + + assert merged.name == "Tencent Cloud" + assert merged.type == "organization" + assert merged.description == "A cloud provider based in Shenzhen." + assert merged.mentions == 3 + assert store.counts() == (1, 0) + assert store.entity_chunks("tencent cloud") == [("doc-1", 0), ("doc-1", 1), ("doc-2", 0)] + + +def test_relations_are_merged_with_summed_weight_and_source_chunks( + store: SqliteKnowledgeGraphStore, +) -> None: + entities = [("Alice", "person", ""), ("Apollo", "product", ""), ("Bob", "person", "")] + store.replace_document( + "doc-1", + [ + _chunk(0, entities, [("Alice", "Apollo", "leads", "Alice leads Apollo.", 8)]), + _chunk( + 1, + entities, + [("alice", "apollo", "leads", "", 6), ("Bob", "Apollo", "tests", "", 3)], + ), + ], + ) + + relations = store.relations_of(["alice"]) + + assert len(relations) == 1 + assert (relations[0].source, relations[0].target, relations[0].label) == ( + "alice", + "apollo", + "leads", + ) + assert relations[0].weight == 14 + assert relations[0].description == "Alice leads Apollo." + assert relations[0].chunks == (("doc-1", 0), ("doc-1", 1)) + assert [item.label for item in store.relations_of(["apollo"])] == ["leads", "tests"] + assert [item.label for item in store.relations_among(["alice", "apollo"])] == ["leads"] + assert store.counts() == (3, 2) + + +def test_self_relations_and_blank_names_are_not_stored(store: SqliteKnowledgeGraphStore) -> None: + store.replace_document( + "doc-1", + [ + _chunk( + 0, + [("Alice", "person", ""), (" ", "person", "")], + [("Alice", "alice", "is", "", 5), ("Alice", "", "knows", "", 5)], + ) + ], + ) + + assert store.counts() == (1, 0) + + +def test_top_entity_keys_rank_by_mentions_and_filter_by_fragment( + store: SqliteKnowledgeGraphStore, +) -> None: + store.replace_document( + "doc-1", + [ + _chunk(0, [("Apollo", "product", ""), ("Zeus", "product", "")], []), + _chunk(1, [("Apollo", "product", ""), ("100% Cotton", "concept", "")], []), + ], + ) + + assert store.top_entity_keys(2) == ["apollo", "100% cotton"] + assert store.top_entity_keys(10, search="ZEU") == ["zeus"] + # % is a literal character in a search fragment, not a wildcard. + assert store.top_entity_keys(10, search="%") == ["100% cotton"] + assert store.top_entity_keys(0) == [] + + +def test_match_keys_finds_names_inside_a_query(store: SqliteKnowledgeGraphStore) -> None: + store.replace_document( + "doc-1", + [ + _chunk( + 0, + [("阿波罗项目", "product", ""), ("Apollo", "product", ""), ("a_c", "other", "")], + [], + ) + ], + ) + + assert store.match_keys("谁负责阿波罗项目的预算?") == ["阿波罗项目"] + assert store.match_keys("Who leads APOLLO?") == ["apollo"] + # _ is a literal character in a name, not a single-character wildcard. + assert store.match_keys("abc") == [] + assert store.match_keys("") == [] + + +def test_replacing_a_document_drops_what_it_contributed_before( + store: SqliteKnowledgeGraphStore, +) -> None: + store.replace_document("doc-1", [_chunk(0, [("Old", "concept", "")], [])]) + store.replace_document("doc-1", [_chunk(0, [("New", "concept", "")], [])], error="note") + + assert store.top_entity_keys(10) == ["new"] + state = store.document_states()["doc-1"] + assert (state.status, state.error) == (STATUS_READY, "note") + + +def test_marking_a_document_not_ready_drops_its_rows_but_keeps_the_graph( + store: SqliteKnowledgeGraphStore, +) -> None: + store.replace_document("doc-1", [_chunk(0, [("Apollo", "product", "")], [])]) + + store.mark_document("doc-1", STATUS_PENDING) + + assert store.counts() == (0, 0) + assert store.exists() is True + store.mark_document("doc-1", STATUS_FAILED, error="boom") + assert store.document_states()["doc-1"].error == "boom" + + +def test_delete_document_and_clear(store: SqliteKnowledgeGraphStore) -> None: + store.replace_document("doc-1", [_chunk(0, [("Apollo", "product", "")], [])]) + store.replace_document("doc-2", [_chunk(0, [("Zeus", "product", "")], [])]) + + store.delete_document("doc-1") + + assert store.top_entity_keys(10) == ["zeus"] + assert set(store.document_states()) == {"doc-2"} + store.clear() + assert store.exists() is False + assert store.counts() == (0, 0) + + +def test_graphs_of_two_knowledge_bases_are_separate( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("OCTOP_HOME", str(tmp_path / "home")) + first, second = SqliteKnowledgeGraphStore("kb-1"), SqliteKnowledgeGraphStore("kb-2") + first.replace_document("doc-1", [_chunk(0, [("Apollo", "product", "")], [])]) + + assert second.exists() is False + assert first.path != second.path diff --git a/tests/unit/knowledge/test_jobs.py b/tests/unit/knowledge/test_jobs.py index 9a51febced..a4798c02ce 100644 --- a/tests/unit/knowledge/test_jobs.py +++ b/tests/unit/knowledge/test_jobs.py @@ -116,6 +116,30 @@ def test_process_document_indexes_chunks_and_marks_ready( assert KnowledgeIndex(kb.id).search([1.0, 0.0], 1)[0].doc_id == doc.id +def test_process_document_skips_a_document_moved_to_another_base( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("OCTOP_HOME", str(tmp_path / "home")) + pool = SqlitePool(tmp_path / "octop.db") + run_migrations(pool) + repo = KnowledgeRepo(pool) + owner_id = UserRepo(pool).create(username="owner", password_hash="h", role="user") + old = repo.create_base(owner_user_id=owner_id, name="Old") + new = repo.create_base(owner_user_id=owner_id, name="New") + doc = repo.create_document( + kb_id=old.id, filename="notes.md", content_type="text/markdown", byte_size=5 + ) + repo.move_to_base(doc, target_kb_id=new.id, target_folder="", max_documents=0) + services = SimpleNamespace(knowledge_repo=repo, settings_repo=SettingsRepo(pool)) + monkeypatch.setattr(jobs, "assert_knowledge_usable", lambda *_args: None) + + jobs.process_document(services, old.id, doc.id) + + moved = repo.get_document(doc.id) + assert moved is not None + assert (moved.kb_id, moved.status) == (new.id, "pending") + + def test_process_document_indexes_ocr_text(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("OCTOP_HOME", str(tmp_path / "home")) pool = SqlitePool(tmp_path / "octop.db") @@ -192,3 +216,47 @@ def test_resume_pending_index_jobs_resets_processing_and_enqueues( assert repo.get_document(doc.id).status == "pending" assert enqueued == [(kb.id, doc.id)] + + +def test_indexing_refreshes_an_existing_graph_after_the_index_slot( + monkeypatch: pytest.MonkeyPatch, +) -> None: + jobs.reset_index_semaphore_for_tests() + events: list[str] = [] + monkeypatch.setattr(jobs, "process_document", lambda *_args: events.append("indexed")) + monkeypatch.setattr( + jobs, "mark_document_reindexed", lambda _services, _kb, doc_id: doc_id == "with-graph" + ) + monkeypatch.setattr( + jobs, + "enqueue_document_graph", + lambda _services, _kb, doc_id: events.append(f"graph:{doc_id}"), + ) + + async def run() -> None: + await jobs.enqueue_index_document(object(), "kb", "with-graph") + await jobs.enqueue_index_document(object(), "kb", "without-graph") + + asyncio.run(run()) + + assert events == ["indexed", "graph:with-graph", "indexed"] + + +def test_indexing_succeeds_when_the_graph_cannot_be_checked( + monkeypatch: pytest.MonkeyPatch, +) -> None: + jobs.reset_index_semaphore_for_tests() + + def boom(*_args: object) -> bool: + raise RuntimeError("graph storage is unavailable") + + monkeypatch.setattr(jobs, "process_document", lambda *_args: None) + monkeypatch.setattr(jobs, "mark_document_reindexed", boom) + monkeypatch.setattr( + jobs, "enqueue_document_graph", lambda *_args: pytest.fail("must not be queued") + ) + + async def run() -> None: + await jobs.enqueue_index_document(object(), "kb", "doc") + + asyncio.run(run()) diff --git a/tests/unit/knowledge/test_rerank.py b/tests/unit/knowledge/test_rerank.py new file mode 100644 index 0000000000..6d01005862 --- /dev/null +++ b/tests/unit/knowledge/test_rerank.py @@ -0,0 +1,322 @@ +"""Reranking retrieved knowledge chunks with a provider's /rerank endpoint.""" + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import httpx +import pytest + +from octop.infra.db.migrate import run_migrations +from octop.infra.db.pool import SqlitePool +from octop.infra.db.repos.knowledge import KnowledgeRepo +from octop.infra.db.repos.settings import SettingsRepo +from octop.infra.db.repos.users import UserRepo +from octop.infra.knowledge import rerank as rerank_module +from octop.infra.knowledge import retrieve as retrieve_module +from octop.infra.knowledge.index import KnowledgeIndex +from octop.infra.knowledge.rerank import ( + candidate_count, + get_rerank_capability, + rerank_models, + rerank_scores, + set_rerank_settings, +) + + +def _provider(**overrides: Any) -> SimpleNamespace: + values: dict[str, Any] = { + "id": 3, + "name": "siliconflow", + "enabled": True, + "api_key": "sk-test", + "base_url": "https://api.example.test/v1/", + "extra_json": None, + "get_models": lambda: [ + {"id": "BAAI/bge-reranker-v2-m3", "name": "BGE Reranker"}, + {"id": "custom-ranker", "task": "rerank"}, + {"id": "BAAI/bge-m3", "name": "BGE M3"}, + {"id": "chat-1"}, + ], + } + values.update(overrides) + return SimpleNamespace(**values) + + +class _Settings: + def __init__(self, **values: str) -> None: + self.values = dict(values) + + def get(self, key: str) -> str | None: + return self.values.get(key) + + def set(self, key: str, value: str) -> None: + self.values[key] = value + + +def _repo(provider: Any | None) -> SimpleNamespace: + return SimpleNamespace(get=lambda pid: provider if provider and pid == provider.id else None) + + +def _enabled_settings() -> _Settings: + return _Settings( + knowledge_rerank_enabled="true", + knowledge_rerank_provider_id="3", + knowledge_rerank_model="BAAI/bge-reranker-v2-m3", + ) + + +def test_only_rerank_models_are_offered() -> None: + assert rerank_models(_provider()) == [ + {"id": "BAAI/bge-reranker-v2-m3", "name": "BGE Reranker"}, + {"id": "custom-ranker", "name": "custom-ranker"}, + ] + + +@pytest.mark.parametrize(("k", "expected"), [(1, 20), (8, 32), (20, 50), (60, 60)]) +def test_candidate_count_widens_the_fetch(k: int, expected: int) -> None: + assert candidate_count(k) == expected + + +def test_capability_is_off_by_default() -> None: + assert get_rerank_capability(_Settings().get, _repo(_provider())) == { + "enabled": False, + "provider_id": "", + "model": "", + "ready": False, + "usable": False, + } + + +def test_enabling_saves_the_choice() -> None: + settings = _Settings() + repo = _repo(_provider()) + + set_rerank_settings( + settings.set, enabled=True, provider_id="3", model="custom-ranker", provider_repo=repo + ) + + assert get_rerank_capability(settings.get, repo) == { + "enabled": True, + "provider_id": "3", + "model": "custom-ranker", + "ready": True, + "usable": True, + } + + +@pytest.mark.parametrize( + ("provider", "provider_id", "model"), + [ + (_provider(), "3", "BAAI/bge-m3"), # an embedding model, not a reranker + (_provider(), "3", ""), + (_provider(), "9", "custom-ranker"), + (_provider(enabled=False), "3", "custom-ranker"), + (_provider(api_key=""), "3", "custom-ranker"), + ], +) +def test_enabling_needs_a_provider_that_serves_the_model( + provider: Any, provider_id: str, model: str +) -> None: + settings = _Settings() + with pytest.raises(ValueError, match="knowledge rerank"): + set_rerank_settings( + settings.set, + enabled=True, + provider_id=provider_id, + model=model, + provider_repo=_repo(provider), + ) + assert settings.values == {} + + +def test_disabling_keeps_the_last_choice_when_none_is_sent() -> None: + settings = _enabled_settings() + set_rerank_settings( + settings.set, enabled=False, provider_id=None, model=None, provider_repo=_repo(None) + ) + assert settings.values["knowledge_rerank_enabled"] == "false" + assert settings.values["knowledge_rerank_model"] == "" + + +def test_a_stored_choice_stops_being_usable_when_the_provider_goes_away() -> None: + capability = get_rerank_capability(_enabled_settings().get, _repo(None)) + assert (capability["enabled"], capability["ready"], capability["usable"]) == ( + True, + False, + False, + ) + + +def _stub_http(monkeypatch: pytest.MonkeyPatch, payload: Any, status: int = 200) -> list[Any]: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(status, json=payload) + + real_client = httpx.Client + monkeypatch.setattr( + rerank_module.httpx, + "Client", + lambda **kwargs: real_client(transport=httpx.MockTransport(handler), **kwargs), + ) + return requests + + +def _services(settings: _Settings, provider: Any | None = None) -> SimpleNamespace: + return SimpleNamespace(settings_repo=settings, provider_repo=_repo(provider or _provider())) + + +def test_scores_come_back_aligned_with_the_texts(monkeypatch: pytest.MonkeyPatch) -> None: + requests = _stub_http( + monkeypatch, + {"results": [{"index": 2, "relevance_score": 0.9}, {"index": 0, "relevance_score": 0.2}]}, + ) + + scores = rerank_scores(_services(_enabled_settings()), "question", ["a", "b", "c"]) + + assert scores == [0.2, float("-inf"), 0.9] + [request] = requests + assert str(request.url) == "https://api.example.test/v1/rerank" + assert request.headers["Authorization"] == "Bearer sk-test" + assert json.loads(request.content) == { + "model": "BAAI/bge-reranker-v2-m3", + "query": "question", + "documents": ["a", "b", "c"], + "top_n": 3, + } + + +@pytest.mark.parametrize( + "payload", + [ + {"output": {"results": [{"index": 0, "relevance_score": 0.5}]}}, # DashScope + [{"index": 0, "score": 0.5}], # text-embeddings-inference + ], +) +def test_other_response_shapes_are_read(monkeypatch: pytest.MonkeyPatch, payload: Any) -> None: + _stub_http(monkeypatch, payload) + assert rerank_scores(_services(_enabled_settings()), "q", ["a"]) == [0.5] + + +@pytest.mark.parametrize( + "payload", + [ + {"data": []}, + {"results": [{"index": 5, "relevance_score": 0.1}]}, + {"results": [{"index": 0, "relevance_score": "high"}]}, + {"results": ["nope"]}, + ], +) +def test_a_malformed_response_raises(monkeypatch: pytest.MonkeyPatch, payload: Any) -> None: + _stub_http(monkeypatch, payload) + with pytest.raises(RuntimeError, match="knowledge rerank response"): + rerank_scores(_services(_enabled_settings()), "q", ["a"]) + + +def test_an_http_error_raises(monkeypatch: pytest.MonkeyPatch) -> None: + _stub_http(monkeypatch, {"error": "bad key"}, status=401) + with pytest.raises(httpx.HTTPStatusError): + rerank_scores(_services(_enabled_settings()), "q", ["a"]) + + +def test_no_request_is_made_when_rerank_is_off(monkeypatch: pytest.MonkeyPatch) -> None: + requests = _stub_http(monkeypatch, {"results": []}) + assert rerank_scores(_services(_Settings()), "q", ["a"]) is None + assert rerank_scores(_services(_enabled_settings()), "q", []) is None + assert requests == [] + + +def _indexed_services(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> tuple[Any, int, str]: + """A base whose vector order is ``far`` < ``near`` for the query vector [1, 0].""" + monkeypatch.setenv("OCTOP_HOME", str(tmp_path / "home")) + pool = SqlitePool(tmp_path / "octop.db") + run_migrations(pool) + services = SimpleNamespace( + knowledge_repo=KnowledgeRepo(pool), + settings_repo=SettingsRepo(pool), + user_repo=UserRepo(pool), + provider_repo=_repo(_provider()), + ) + user = services.user_repo.create(username="reader", password_hash="h", role="user") + base = services.knowledge_repo.create_base(owner_user_id=user, name="Docs") + document = services.knowledge_repo.create_document( + kb_id=base.id, + filename="facts.md", + content_type="text/markdown", + byte_size=1, + status="ready", + ) + KnowledgeIndex(base.id).replace_doc_chunks( + document.id, + ["looks similar", "answers the question", "unrelated"], + [[1.0, 0.0], [0.8, 0.6], [0.0, 1.0]], + ) + monkeypatch.setattr(retrieve_module, "assert_knowledge_usable", lambda *_args: None) + monkeypatch.setattr( + retrieve_module, "embed_knowledge_texts", lambda _services, _texts: [[1.0, 0.0]] + ) + return services, user, base.id + + +def _retrieve(services: Any, user: int, base_id: str, k: int = 1) -> str: + return asyncio.run( + retrieve_module.retrieve_context( + services, + user_id=user, + is_admin=False, + query="what answers the question?", + knowledge_base_ids=[base_id], + k=k, + ) + ) + + +def _enable(services: Any) -> None: + for key, value in _enabled_settings().values.items(): + services.settings_repo.set(key, value) + + +def test_retrieval_keeps_the_rerankers_pick( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + services, user, base_id = _indexed_services(tmp_path, monkeypatch) + assert "looks similar" in _retrieve(services, user, base_id) + + _enable(services) + seen: list[list[str]] = [] + + def scores(_services: Any, _query: str, texts: list[str]) -> list[float]: + seen.append(texts) + return [0.1 if text != "answers the question" else 0.9 for text in texts] + + monkeypatch.setattr(retrieve_module, "rerank_scores", scores) + + context = _retrieve(services, user, base_id) + + # k=1, yet the reranker saw every candidate and its choice replaced the nearest vector. + assert seen == [["looks similar", "answers the question", "unrelated"]] + assert "answers the question" in context + assert "looks similar" not in context + + +def test_retrieval_falls_back_to_vector_order_when_rerank_fails( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + services, user, base_id = _indexed_services(tmp_path, monkeypatch) + _enable(services) + + def boom(*_args: Any) -> list[float]: + raise httpx.ConnectError("provider is down") + + monkeypatch.setattr(retrieve_module, "rerank_scores", boom) + + context = _retrieve(services, user, base_id) + + assert "looks similar" in context + assert "knowledge rerank failed" in caplog.text diff --git a/tests/unit/knowledge/test_retrieve.py b/tests/unit/knowledge/test_retrieve.py index d2f7299381..a351a39aa4 100644 --- a/tests/unit/knowledge/test_retrieve.py +++ b/tests/unit/knowledge/test_retrieve.py @@ -12,6 +12,12 @@ from octop.infra.db.repos.users import UserRepo from octop.infra.knowledge import retrieve as retrieve_module from octop.infra.knowledge.citations import CITATIONS_MARKER_PREFIX +from octop.infra.knowledge.graph_store import ( + ChunkEntity, + ChunkGraph, + ChunkRelation, + SqliteKnowledgeGraphStore, +) from octop.infra.knowledge.index import KnowledgeIndex @@ -97,3 +103,110 @@ def test_format_context_cites_only_documents_inside_the_budget() -> None: marker = text.split(CITATIONS_MARKER_PREFIX, 1)[1] assert "a.md" in marker assert "b.md" not in marker + + +def _graph_services(tmp_path, monkeypatch): # type: ignore[no-untyped-def] + monkeypatch.setenv("OCTOP_HOME", str(tmp_path / "home")) + pool = SqlitePool(tmp_path / "octop.db") + run_migrations(pool) + services = SimpleNamespace( + knowledge_repo=KnowledgeRepo(pool), + settings_repo=SettingsRepo(pool), + user_repo=UserRepo(pool), + ) + owner = services.user_repo.create(username="owner", password_hash="h", role="user") + base = services.knowledge_repo.create_base(owner_user_id=owner, name="Docs") + document = services.knowledge_repo.create_document( + kb_id=base.id, + filename="plan.md", + content_type="text/markdown", + byte_size=1, + status="ready", + ) + # Four passages rank by similarity; the fifth is unrelated by wording. + texts = ["Apollo overview", "Apollo timeline", "Apollo risks", "Apollo team", "Budget rules"] + vectors = [[1.0, 0.0], [0.9, 0.1], [0.8, 0.2], [0.7, 0.3], [0.0, 1.0]] + KnowledgeIndex(base.id).replace_doc_chunks(document.id, texts, vectors) + SqliteKnowledgeGraphStore(base.id).replace_document( + document.id, + [ + ChunkGraph( + ordinal=4, + entities=(ChunkEntity("Apollo"), ChunkEntity("Finance Board")), + relations=( + ChunkRelation( + "Apollo", "Finance Board", "is funded by", "The board signs off.", 9 + ), + ), + ) + ], + ) + monkeypatch.setattr(retrieve_module, "assert_knowledge_usable", lambda *_args: None) + monkeypatch.setattr( + retrieve_module, "embed_knowledge_texts", lambda _services, _texts: [[1.0, 0.0]] + ) + return services, owner, base + + +def _retrieve(services, owner, base, **kwargs): # type: ignore[no-untyped-def] + return asyncio.run( + retrieve_module.retrieve_context( + services, + user_id=owner, + is_admin=False, + query="Who funds Apollo?", + knowledge_base_ids=[base.id], + **kwargs, + ) + ) + + +def test_retrieve_context_adds_graph_passage_and_relations(tmp_path, monkeypatch) -> None: + services, owner, base = _graph_services(tmp_path, monkeypatch) + services.settings_repo.set("knowledge_graph_enabled", "true") + + context = _retrieve(services, owner, base, k=4) + + # The linked passage takes the last of four slots; the weakest vector hit gives way. + assert "Budget rules" in context + assert "Apollo team" not in context + assert context.index("Apollo overview") < context.index("Budget rules") + assert "- Apollo -> is funded by -> Finance Board: The board signs off." in context + assert context.index("Budget rules") < context.index("- Apollo -> is funded by") + assert context.rstrip().endswith("-->") + + +def test_retrieve_context_ignores_the_graph_when_the_feature_is_off(tmp_path, monkeypatch) -> None: + services, owner, base = _graph_services(tmp_path, monkeypatch) + + context = _retrieve(services, owner, base, k=4) + + assert "Apollo team" in context + assert "Budget rules" not in context + assert "is funded by" not in context + + +def test_retrieve_context_small_k_keeps_vector_passages(tmp_path, monkeypatch) -> None: + services, owner, base = _graph_services(tmp_path, monkeypatch) + services.settings_repo.set("knowledge_graph_enabled", "true") + + context = _retrieve(services, owner, base, k=2) + + assert "Apollo overview" in context and "Apollo timeline" in context + assert "Budget rules" not in context + assert "is funded by" in context + + +def test_retrieve_context_survives_a_broken_graph(tmp_path, monkeypatch) -> None: + services, owner, base = _graph_services(tmp_path, monkeypatch) + services.settings_repo.set("knowledge_graph_enabled", "true") + + def boom(*_args: object, **_kwargs: object) -> None: + raise RuntimeError("graph storage is unavailable") + + monkeypatch.setattr(retrieve_module, "expand_query", boom) + + context = _retrieve(services, owner, base, k=4) + + assert "Apollo team" in context + assert "is funded by" not in context diff --git a/tests/unit/knowledge/test_service_acl.py b/tests/unit/knowledge/test_service_acl.py index a5286527b3..0adbff2fbf 100644 --- a/tests/unit/knowledge/test_service_acl.py +++ b/tests/unit/knowledge/test_service_acl.py @@ -14,6 +14,7 @@ from octop.infra.db.repos.users import UserRepo from octop.infra.errors import ErrorCode, OctopError from octop.infra.knowledge import service as service_module +from octop.infra.knowledge.index import KnowledgeIndex from octop.infra.knowledge.service import MAX_DOCS_PER_KB, KnowledgeService from octop.infra.utils.paths import PathLayout @@ -406,6 +407,176 @@ def test_move_does_not_cross_knowledge_bases(service: KnowledgeService) -> None: service.move_document(other.id, doc.id, actor_user_id=owner, target_folder="") +def _two_bases(service: KnowledgeService) -> tuple[int, str, str]: + """owner, a source base holding ``notes/law/act.md`` and ``notes/readme.md``, and an + empty target base with an ``inbox`` folder. Every file is indexed with one chunk.""" + owner, kb_id = _base_with_tree(service) + repo = service._services.knowledge_repo + for row in repo.list_documents(kb_id): + if not row.is_dir: + KnowledgeIndex(kb_id).replace_doc_chunks(row.id, [f"text of {row.path}"], [[1.0, 0.0]]) + repo.update_document(row.id, status="ready", chunk_count=1) + repo.update_base(kb_id, embedding_dim=2) + target = service.create_base(owner_user_id=owner, name="Target") + service.create_folder(target.id, actor_user_id=owner, path="inbox") + return owner, kb_id, target.id + + +def test_move_file_to_another_base_takes_its_original_and_chunks( + service: KnowledgeService, +) -> None: + owner, kb_id, target_id = _two_bases(service) + repo = service._services.knowledge_repo + doc = repo.get_document_by_path(kb_id, "notes/readme.md") + + moved = service.move_document( + kb_id, doc.id, actor_user_id=owner, target_folder="inbox", target_kb_id=target_id + ) + + assert (moved.id, moved.kb_id, moved.path) == (doc.id, target_id, "inbox/readme.md") + assert (moved.status, moved.chunk_count) == ("ready", 1) + assert repo.get_document_by_path(kb_id, "notes/readme.md") is None + assert (repo.get_base(kb_id).doc_count, repo.get_base(target_id).doc_count) == (1, 1) + assert repo.get_base(target_id).embedding_dim == 2 + blobs = service._blobs() + assert blobs.read_bytes(target_id, doc.id, "readme.md") == b"x" + assert not blobs.exists(kb_id, doc.id, "readme.md") + assert KnowledgeIndex(target_id).read_doc_chunks(doc.id) == [ + ("text of notes/readme.md", [1.0, 0.0], {}) + ] + assert KnowledgeIndex(kb_id).list_doc_chunks(doc.id) == [] + + +def test_move_folder_to_another_base_takes_its_subtree(service: KnowledgeService) -> None: + owner, kb_id, target_id = _two_bases(service) + repo = service._services.knowledge_repo + folder = repo.get_document_by_path(kb_id, "notes") + + service.move_document( + kb_id, folder.id, actor_user_id=owner, target_folder="", target_kb_id=target_id + ) + + assert repo.list_documents(kb_id) == [repo.get_document_by_path(kb_id, "archive")] + assert sorted(row.path for row in repo.list_documents(target_id)) == [ + "inbox", + "notes", + "notes/law", + "notes/law/act.md", + "notes/readme.md", + ] + act = repo.get_document_by_path(target_id, "notes/law/act.md") + assert KnowledgeIndex(target_id).list_doc_chunks(act.id) == [(0, "text of notes/law/act.md")] + assert (repo.get_base(kb_id).doc_count, repo.get_base(target_id).doc_count) == (0, 2) + + +def test_move_to_a_base_with_another_model_indexes_again(service: KnowledgeService) -> None: + owner, kb_id, target_id = _two_bases(service) + repo = service._services.knowledge_repo + repo.update_base(target_id, embedding_model="other-model") + doc = repo.get_document_by_path(kb_id, "notes/readme.md") + + moved = service.move_document( + kb_id, doc.id, actor_user_id=owner, target_folder="", target_kb_id=target_id + ) + + assert (moved.status, moved.chunk_count) == ("pending", 0) + assert service.pending_file_ids(moved) == [doc.id] + assert KnowledgeIndex(target_id).list_doc_chunks(doc.id) == [] + assert KnowledgeIndex(kb_id).list_doc_chunks(doc.id) == [] + assert service._blobs().exists(target_id, doc.id, "readme.md") + + +def test_move_to_another_base_keeps_a_failed_file_failed(service: KnowledgeService) -> None: + owner, kb_id, target_id = _two_bases(service) + repo = service._services.knowledge_repo + doc = repo.get_document_by_path(kb_id, "notes/readme.md") + repo.update_document(doc.id, status="failed", error_message="bad file", chunk_count=0) + + moved = service.move_document( + kb_id, doc.id, actor_user_id=owner, target_folder="", target_kb_id=target_id + ) + + assert (moved.status, moved.error_message) == ("failed", "bad file") + + +def test_move_to_another_base_waits_for_indexing(service: KnowledgeService) -> None: + owner, kb_id, target_id = _two_bases(service) + repo = service._services.knowledge_repo + repo.update_document( + repo.get_document_by_path(kb_id, "notes/law/act.md").id, status="processing" + ) + + with pytest.raises(OctopError) as raised: + service.move_document( + kb_id, + repo.get_document_by_path(kb_id, "notes").id, + actor_user_id=owner, + target_folder="", + target_kb_id=target_id, + ) + + assert raised.value.code is ErrorCode.KNOWLEDGE_MOVE_BUSY + assert repo.get_document_by_path(kb_id, "notes/law/act.md") is not None + + +def test_move_to_a_full_base_changes_nothing(service: KnowledgeService) -> None: + owner, kb_id, target_id = _two_bases(service) + repo = service._services.knowledge_repo + service.update_base(target_id, actor_user_id=owner, max_documents=1) + folder = repo.get_document_by_path(kb_id, "notes") + + with pytest.raises(OctopError) as raised: + service.move_document( + kb_id, folder.id, actor_user_id=owner, target_folder="", target_kb_id=target_id + ) + + assert raised.value.code is ErrorCode.KNOWLEDGE_DOC_LIMIT + assert repo.get_document(folder.id).kb_id == kb_id + assert repo.get_base(target_id).doc_count == 0 + assert repo.get_base(kb_id).doc_count == 2 + + +def test_move_to_another_base_rejects_a_name_collision(service: KnowledgeService) -> None: + owner, kb_id, target_id = _two_bases(service) + repo = service._services.knowledge_repo + service.create_folder(target_id, actor_user_id=owner, path="notes") + + with pytest.raises(OctopError) as raised: + service.move_document( + kb_id, + repo.get_document_by_path(kb_id, "notes").id, + actor_user_id=owner, + target_folder="", + target_kb_id=target_id, + ) + + assert raised.value.code is ErrorCode.KNOWLEDGE_MOVE_NAME_TAKEN + + +def test_move_to_another_base_needs_its_folder(service: KnowledgeService) -> None: + owner, kb_id, target_id = _two_bases(service) + doc = service._services.knowledge_repo.get_document_by_path(kb_id, "notes/readme.md") + + with pytest.raises(LookupError, match="folder not found"): + service.move_document( + kb_id, doc.id, actor_user_id=owner, target_folder="archive", target_kb_id=target_id + ) + + +def test_move_into_a_base_the_actor_cannot_write_is_rejected(service: KnowledgeService) -> None: + owner, kb_id, _target_id = _two_bases(service) + stranger = service._services.user_repo.create(username="s", password_hash="h", role="user") + theirs = service._services.knowledge_repo.create_base( + owner_user_id=stranger, name="Theirs", shared=True + ) + doc = service._services.knowledge_repo.get_document_by_path(kb_id, "notes/readme.md") + + with pytest.raises(PermissionError, match="write"): + service.move_document( + kb_id, doc.id, actor_user_id=owner, target_folder="", target_kb_id=theirs.id + ) + + def test_shared_reader_cannot_move(service: KnowledgeService) -> None: users = service._services.user_repo owner = users.create(username="owner", password_hash="h", role="user") @@ -417,3 +588,87 @@ def test_shared_reader_cannot_move(service: KnowledgeService) -> None: with pytest.raises(PermissionError, match="write"): service.move_document(kb.id, folder.id, actor_user_id=viewer, target_folder="b") + + +def _graph_fixture(service: KnowledgeService) -> tuple[str, int, int, str]: + from octop.infra.knowledge.graph_store import ( + ChunkEntity, + ChunkGraph, + ChunkRelation, + SqliteKnowledgeGraphStore, + ) + from octop.infra.knowledge.index import KnowledgeIndex + + users = service._services.user_repo + owner = users.create(username="owner", password_hash="h", role="user") + reader = users.create(username="reader", password_hash="h", role="user") + kb = service.create_base(owner_user_id=owner, name="Docs", shared=True) + document = service._repo.create_document( + kb_id=kb.id, filename="plan.md", content_type="text/markdown", byte_size=1, status="ready" + ) + service._repo.create_document( + kb_id=kb.id, filename="draft.md", content_type="text/markdown", byte_size=1 + ) + KnowledgeIndex(kb.id).replace_doc_chunks(document.id, ["Alice leads Apollo."], [[1.0, 0.0]]) + SqliteKnowledgeGraphStore(kb.id).replace_document( + document.id, + [ + ChunkGraph( + ordinal=0, + entities=(ChunkEntity("Alice", "person"), ChunkEntity("Apollo", "product")), + relations=(ChunkRelation("Alice", "Apollo", "leads"),), + ) + ], + ) + return kb.id, owner, reader, document.id + + +def test_readers_can_view_a_graph_but_not_build_or_delete_it(service: KnowledgeService) -> None: + kb_id, owner, reader, doc_id = _graph_fixture(service) + outsider = service._services.user_repo.create(username="out", password_hash="h", role="user") + service.update_base(kb_id, actor_user_id=owner, shared=False) + with pytest.raises(PermissionError): + service.get_graph(kb_id, actor_user_id=outsider) + service.update_base(kb_id, actor_user_id=owner, shared=True) + + view = service.get_graph(kb_id, actor_user_id=reader) + + assert view["stats"]["built"] is True + assert view["stats"]["documents_total"] == 1 + assert [node["name"] for node in view["nodes"]] == ["Alice", "Apollo"] + assert view["edges"][0]["labels"] == ["leads"] + entity = service.get_graph_entity(kb_id, "alice", actor_user_id=reader) + assert entity["passages"][0]["doc_id"] == doc_id + with pytest.raises(PermissionError): + service.graph_build_targets(kb_id, actor_user_id=reader) + with pytest.raises(PermissionError): + service.clear_graph(kb_id, actor_user_id=reader) + + +def test_graph_build_needs_an_extraction_model_and_targets_indexed_documents( + service: KnowledgeService, monkeypatch: pytest.MonkeyPatch +) -> None: + kb_id, owner, _reader, doc_id = _graph_fixture(service) + + with pytest.raises(RuntimeError, match="knowledge graph extraction is not ready"): + service.graph_build_targets(kb_id, actor_user_id=owner) + + monkeypatch.setattr(service_module, "get_graph_capability", lambda *_args: {"usable": True}) + assert service.graph_build_targets(kb_id, actor_user_id=owner) == [doc_id] + + +def test_clearing_a_graph_and_deleting_a_document_remove_graph_rows( + service: KnowledgeService, +) -> None: + from octop.infra.knowledge.graph_store import SqliteKnowledgeGraphStore + + kb_id, owner, _reader, doc_id = _graph_fixture(service) + store = SqliteKnowledgeGraphStore(kb_id) + + service.delete_document(kb_id, doc_id, actor_user_id=owner) + assert store.counts() == (0, 0) + assert store.document_states() == {} + + store.mark_document("other", "ready") + service.clear_graph(kb_id, actor_user_id=owner) + assert service.get_graph(kb_id, actor_user_id=owner)["stats"]["built"] is False diff --git a/tests/unit/proactive/test_scheduler.py b/tests/unit/proactive/test_scheduler.py index 36709e830d..81916d73c6 100644 --- a/tests/unit/proactive/test_scheduler.py +++ b/tests/unit/proactive/test_scheduler.py @@ -557,6 +557,26 @@ async def test_scheduler_start_all_includes_default_on_agent( await asyncio.sleep(0.01) +@pytest.mark.asyncio +async def test_scheduler_skips_managed_agents( + db: SqlitePool, + config_repo: ProactiveCareConfigRepo, +) -> None: + from octop.infra.db.repos.sessions import SessionRepo + + uid = UserRepo(db).create(username="bob", password_hash="h", role="admin") + managed_id = new_ulid() + AgentRepo(db).create(agent_id=managed_id, user_id=uid, name="coder", kind="managed") + scheduler = ProactiveCareScheduler( + care_service=AsyncMock(), + config_repo=config_repo, + session_repo=SessionRepo(db), + agent_repo=AgentRepo(db), + ) + scheduler.ensure_scheduled(managed_id) + assert managed_id not in scheduler._tasks + + @pytest.mark.asyncio async def test_shutdown_cancels_long_sleep_quickly( agent_id: str, diff --git a/tests/unit/providers/test_magpie.py b/tests/unit/providers/test_magpie.py new file mode 100644 index 0000000000..b8aeb3bbdc --- /dev/null +++ b/tests/unit/providers/test_magpie.py @@ -0,0 +1,253 @@ +"""Magpie identity, discovery, and catalog sync.""" + +from __future__ import annotations + +from typing import Any + +import httpx +import pytest + +from octop.infra.agents.providers.magpie import ( + MAGPIE_DEFAULT_BASE_URL, + MAGPIE_HELLO_URL, + MAGPIE_LOCAL_API_KEY, + MAGPIE_SESSION_HEADER, + discover_local_magpie, + ensure_magpie_headers, + initial_magpie_models, + is_magpie_default_base_url, + is_magpie_provider, + magpie_catalog_model, + magpie_preset, + plan_magpie_sync, +) + + +def test_preset_is_empty_openai_catalog() -> None: + preset = magpie_preset() + assert preset["id"] == "magpie" + assert preset["name"] == "Magpie" + assert preset["protocol"] == "openai" + assert preset["base_url"] == MAGPIE_DEFAULT_BASE_URL + assert preset["models"] == [] + assert preset["default_api_key"] == MAGPIE_LOCAL_API_KEY + assert preset["integration"] == "magpie" + + +@pytest.mark.parametrize( + ("base_url", "expected"), + [ + ("http://127.0.0.1:3425/v1", True), + ("http://127.0.0.1:3425/v1/", True), + ("https://127.0.0.1:3425/v1", False), + ("http://localhost:3425/v1", False), + ("http://127.0.0.1:3426/v1", False), + ("http://127.0.0.1:3425/v1/extra", False), + ("http://127.0.0.1/v1", False), + ("http://magpie.example/v1", False), + ("http://127.0.0.1:invalid/v1", False), + ("http://127.0.0.1:65536/v1", False), + ], +) +def test_default_base_url_matches_scheme_host_port_and_path(base_url: str, expected: bool) -> None: + assert is_magpie_default_base_url(base_url) is expected + + +def test_identity_prefers_integration_then_name_then_exact_url() -> None: + assert is_magpie_provider( + provider_name="custom", + base_url="https://gateway.example/v1", + extra={"integration": "magpie"}, + ) + assert is_magpie_provider(provider_name="Magpie", base_url="https://other.example/v1") + assert is_magpie_provider(provider_name="custom", base_url="http://127.0.0.1:3425/v1") + assert not is_magpie_provider( + provider_name="my-magpie-gateway", base_url="https://api.example/v1" + ) + assert not is_magpie_provider( + provider_name="OpenCode Go", base_url="https://opencode.ai/zen/go/v1" + ) + + +def test_headers_add_user_agent_and_keep_overrides() -> None: + from octop import __version__ + + headers = ensure_magpie_headers(None) + assert headers["User-Agent"] == f"Octop/{__version__}" + assert MAGPIE_SESSION_HEADER not in headers + + custom = ensure_magpie_headers({"user-agent": "Mine/1", "X-Magpie-Session": "keep"}) + assert custom["user-agent"] == "Mine/1" + assert custom["X-Magpie-Session"] == "keep" + + probe = ensure_magpie_headers({}, probe=True) + assert probe[MAGPIE_SESSION_HEADER] + assert probe[MAGPIE_SESSION_HEADER] != "thread" + + +def test_catalog_keeps_nested_ids_and_skips_non_chat() -> None: + kept = magpie_catalog_model( + { + "id": "vendor/org/model", + "display_name": "Shown", + "context_window": 128000, + "max_output_tokens": 4096, + "modalities": ["text", "image", "embedding"], + "reasoning_config": {"supported": True, "adapter": "thinking"}, + } + ) + assert kept == { + "id": "vendor/org/model", + "name": "Shown", + "context_window": 128000, + "max_tokens": 4096, + "input": ["text", "image"], + "reasoning": True, + "reasoning_config": {"supported": True, "adapter": "thinking"}, + } + assert magpie_catalog_model({"id": "group/route", "modalities": ["text"]}) == { + "id": "group/route", + "name": "group/route", + "input": ["text"], + } + assert magpie_catalog_model( + { + "id": "codex/gpt-6.1-sol", + "display_name": "GPT-6.1-Sol", + "type": "model", + "context_window": 500000, + "max_output_tokens": 128000, + "modalities": {"input": ["text", "image"]}, + "reasoning": True, + } + ) == { + "id": "codex/gpt-6.1-sol", + "name": "GPT-6.1-Sol", + "context_window": 500000, + "max_tokens": 128000, + "input": ["text", "image"], + "reasoning": True, + } + assert magpie_catalog_model({"id": "text-embedding-3-large", "type": "embedding"}) is None + assert magpie_catalog_model({"id": "video-model", "type": "video"}) is None + assert magpie_catalog_model( + {"id": "mystery", "reasoning_config": {"adapter": "not-a-real-adapter"}} + ) == {"id": "mystery", "name": "mystery"} + + +def test_duplicate_ids_and_empty_catalog() -> None: + assert initial_magpie_models([]) is None + created = initial_magpie_models( + [ + {"id": "a/b/c", "name": "A"}, + {"id": "a/b/c", "name": "dup"}, + {"id": "group/x", "name": "G"}, + ] + ) + assert created is not None + assert [model["id"] for model in created] == ["a/b/c", "group/x"] + assert all(model["enabled"] is True for model in created) + + +def test_sync_preserves_overrides_and_lists_removals() -> None: + existing = [ + { + "id": "keep/me", + "name": "Old", + "enabled": False, + "input": ["text", "image"], + "context_window": 1000, + "visibility": "groups", + "group_ids": ["design"], + }, + {"id": "gone", "name": "Gone", "enabled": True, "visibility": "all"}, + ] + fetched = [ + {"id": "keep/me", "name": "New name", "context_window": 999999, "input": ["text"]}, + {"id": "keep/me", "name": "dup"}, + {"id": "added/one", "name": "Added"}, + ] + plan = plan_magpie_sync(existing, fetched) + assert plan.added == ["added/one"] + assert plan.removed == ["gone"] + kept = plan.models[0] + assert kept["enabled"] is False + assert kept["input"] == ["text", "image"] + assert kept["context_window"] == 1000 + assert kept["group_ids"] == ["design"] + assert kept["name"] == "Old" + assert plan.models[1]["id"] == "added/one" + assert plan.models[1]["enabled"] is False + + +def test_empty_refresh_removes_every_model() -> None: + plan = plan_magpie_sync([{"id": "a", "enabled": True}, {"id": "b", "enabled": True}], []) + assert plan.models == [] + assert plan.removed == ["a", "b"] + assert plan.added == [] + + +class _HelloClient: + def __init__(self, **kwargs: Any) -> None: + self.kwargs = kwargs + + async def __aenter__(self) -> _HelloClient: + return self + + async def __aexit__(self, *_exc: object) -> bool: + return False + + async def get(self, url: str) -> httpx.Response: + assert url == MAGPIE_HELLO_URL + responder = self.kwargs["responder"] + result = responder() + if isinstance(result, Exception): + raise result + return result + + +def _response( + status: int, payload: Any | None = None, *, text: str | None = None +) -> httpx.Response: + request = httpx.Request("GET", MAGPIE_HELLO_URL) + if text is not None: + return httpx.Response(status, text=text, request=request) + return httpx.Response(status, json=payload, request=request) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("responder", "available", "version"), + [ + (lambda: _response(200, {"name": "magpie", "version": "1.2.3"}), True, "1.2.3"), + (lambda: _response(200, {"name": "other", "version": "1"}), False, None), + (lambda: _response(200, text="not-json"), False, None), + (lambda: _response(302, {"name": "magpie"}), False, None), + (lambda: _response(500, {"name": "magpie"}), False, None), + (lambda: httpx.ConnectError("connection refused"), False, None), + (lambda: httpx.TimeoutException("timed out"), False, None), + ], +) +async def test_discover_local_magpie( + monkeypatch: pytest.MonkeyPatch, + responder: Any, + available: bool, + version: str | None, +) -> None: + monkeypatch.setenv("HTTP_PROXY", "http://proxy.example:8080") + monkeypatch.setenv("HTTPS_PROXY", "http://proxy.example:8080") + created: list[_HelloClient] = [] + + def factory(**kwargs: Any) -> _HelloClient: + client = _HelloClient(**kwargs, responder=responder) + created.append(client) + return client + + monkeypatch.setattr("httpx.AsyncClient", factory) + found = await discover_local_magpie() + assert found.available is available + assert found.base_url == MAGPIE_DEFAULT_BASE_URL + assert found.version == version + assert created[0].kwargs["follow_redirects"] is False + assert created[0].kwargs["trust_env"] is False + assert created[0].kwargs["timeout"] == 1.0 diff --git a/tests/unit/providers/test_magpie_session.py b/tests/unit/providers/test_magpie_session.py new file mode 100644 index 0000000000..3b1b306062 --- /dev/null +++ b/tests/unit/providers/test_magpie_session.py @@ -0,0 +1,28 @@ +"""Session header isolation for the harness version Octop actually imports.""" + +from __future__ import annotations + +import asyncio + +import httpx +from octop_harness.llm.session_header import install_request_hook, session_header_scope + + +async def test_magpie_session_header_follows_the_active_thread() -> None: + seen: dict[str, str] = {} + + def handler(request: httpx.Request) -> httpx.Response: + seen[request.headers["X-Magpie-Session"]] = request.headers["user-agent"] + return httpx.Response(200, json={"ok": True}) + + transport = httpx.MockTransport(handler) + client = httpx.AsyncClient(transport=transport, headers={"User-Agent": "Octop/test"}) + install_request_hook(client, "X-Magpie-Session") + + async def one(thread_id: str) -> None: + with session_header_scope(thread_id): + await client.post("http://127.0.0.1:3425/v1/chat/completions") + + await asyncio.gather(one("thread-a"), one("thread-b")) + await client.aclose() + assert seen == {"thread-a": "Octop/test", "thread-b": "Octop/test"} diff --git a/tests/unit/providers/test_turn_access.py b/tests/unit/providers/test_turn_access.py new file mode 100644 index 0000000000..5336eae4f2 --- /dev/null +++ b/tests/unit/providers/test_turn_access.py @@ -0,0 +1,53 @@ +"""AUTO failover candidates stay inside the turn user's model grants.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from octop.infra.agents.providers.store import AutoRouteCandidate +from octop.infra.agents.providers.turn_access import ( + filter_auto_route_candidates, + turn_model_access, +) +from octop.infra.db.migrate import run_migrations +from octop.infra.db.pool import SqlitePool +from octop.infra.db.services import RepoBundle +from octop.infra.users.groups import RESOURCE_MODEL, GroupAccess + + +@pytest.fixture +def repos(tmp_path: Path) -> RepoBundle: + pool = SqlitePool(tmp_path / "octop.db") + run_migrations(pool) + return RepoBundle.from_pool(pool) + + +def test_unbound_turn_keeps_every_candidate(repos: RepoBundle) -> None: + candidates = [AutoRouteCandidate("openai/gpt", 1000), AutoRouteCandidate("Magpie/a", 1000)] + assert filter_auto_route_candidates(candidates, repos=repos) == candidates + + +def test_turn_user_cannot_fail_over_to_a_hidden_model(repos: RepoBundle) -> None: + user_id = repos.user_repo.create(username="member", role="user") + outsider = repos.user_repo.create(username="outsider", role="user") + group = repos.user_group_repo.create(name="design", description=None) + repos.user_group_member_repo.add(group.user_group_id, user_id) + GroupAccess(repos).set_visibility( + RESOURCE_MODEL, + "openai/private", + "groups", + [group.user_group_id], + ) + candidates = [ + AutoRouteCandidate("Magpie/public", 8000), + AutoRouteCandidate("openai/private", 8000), + ] + with turn_model_access(outsider, is_admin=False): + allowed = filter_auto_route_candidates(candidates, repos=repos) + assert [item.ref for item in allowed] == ["Magpie/public"] + + with turn_model_access(user_id, is_admin=False): + allowed = filter_auto_route_candidates(candidates, repos=repos) + assert [item.ref for item in allowed] == ["Magpie/public", "openai/private"] diff --git a/tests/unit/test_app_config_file.py b/tests/unit/test_app_config_file.py new file mode 100644 index 0000000000..5e49a70186 --- /dev/null +++ b/tests/unit/test_app_config_file.py @@ -0,0 +1,200 @@ +"""config.json admin editor: merge, redact, and reject invalid documents.""" + +from __future__ import annotations + +import json + +import pytest + +from octop.infra.app_config import read_app_config, save_raw_config, save_structured_config +from octop.infra.errors import ErrorCode, OctopError + + +def _body(**overrides: object) -> dict: + database = { + "driver": "sqlite", + "sqlite_path": "octop.db", + "host": "127.0.0.1", + "port": 5432, + "database": "octop", + "user": "octop", + "password": None, + "url": None, + } + tls = { + "enabled": False, + "mode": "", + "domains": [], + "cert_file": "", + "key_file": "", + "issued_at": "", + "expires_at": "", + "acme_staging": False, + "http_port": 80, + } + body = { + "bind_host": "127.0.0.1", + "port": 8088, + "log_level": "info", + "access_token_ttl_seconds": 86400, + "login_max_attempts": 5, + "login_lockout_seconds": 900, + "cors_origins": [], + "default_timezone": "Asia/Shanghai", + "enable_dashboard": True, + "enable_api_docs": False, + "require_setup_password": True, + "max_upload_mb": 100, + "database": database, + "tls": tls, + } + body.update(overrides) + return body + + +def test_structured_save_keeps_unknown_keys_and_password(tmp_path) -> None: + path = tmp_path / "config.json" + secret = "super-secret-db-password" + path.write_text( + json.dumps( + { + "port": 8088, + "plugins": {"demo": {"enabled": False}}, + "bundled_plugins_seeded": ["demo"], + "database": { + "driver": "postgresql", + "host": "db.internal", + "port": 5432, + "database": "octop", + "user": "octop", + "password": secret, + }, + } + ), + encoding="utf-8", + ) + body = _body() + body["database"] = { + **body["database"], + "driver": "postgresql", + "host": "db.internal", + "password": None, + } + body["port"] = 9090 + save_structured_config(path, body) + + saved = json.loads(path.read_text(encoding="utf-8")) + assert saved["plugins"] == {"demo": {"enabled": False}} + assert saved["bundled_plugins_seeded"] == ["demo"] + assert saved["port"] == 9090 + assert saved["database"]["password"] == secret + + view = read_app_config(path) + assert view.valid is True + assert view.config is not None + assert view.config["database"]["password"] is None + assert view.has_database_password is True + assert secret in view.raw + assert secret not in json.dumps(view.config) + + +def test_clear_and_replace_database_password(tmp_path) -> None: + path = tmp_path / "config.json" + path.write_text( + json.dumps( + { + "database": { + "driver": "postgresql", + "host": "127.0.0.1", + "port": 5432, + "database": "octop", + "user": "octop", + "password": "old", + } + } + ), + encoding="utf-8", + ) + body = _body() + body["database"] = {**body["database"], "driver": "postgresql", "password": ""} + save_structured_config(path, body) + assert "password" not in json.loads(path.read_text(encoding="utf-8"))["database"] + + body["database"] = {**body["database"], "password": "next"} + save_structured_config(path, body) + assert json.loads(path.read_text(encoding="utf-8"))["database"]["password"] == "next" + + +def test_database_url_replaces_discrete_fields(tmp_path) -> None: + path = tmp_path / "config.json" + path.write_text("{}", encoding="utf-8") + body = _body() + body["database"] = { + **body["database"], + "driver": "postgresql", + "url": "postgresql://app:s3cret@db.example:5433/octop?sslmode=require", + } + save_structured_config(path, body) + saved = json.loads(path.read_text(encoding="utf-8"))["database"] + assert saved["host"] == "db.example" + assert saved["port"] == 5433 + assert saved["user"] == "app" + assert saved["password"] == "s3cret" + assert saved["url"].endswith("sslmode=require") + + +def test_invalid_save_does_not_change_the_file(tmp_path) -> None: + path = tmp_path / "config.json" + original = json.dumps({"port": 8088, "plugins": {"demo": {"enabled": True}}}) + path.write_text(original, encoding="utf-8") + body = _body(port=70000) + with pytest.raises(OctopError) as exc: + save_structured_config(path, body) + assert exc.value.code is ErrorCode.APP_CONFIG_INVALID + assert "70000" not in str(exc.value) or "port" in str(exc.value) + assert path.read_text(encoding="utf-8") == original + + +def test_raw_save_rejects_invalid_json_and_keeps_extra_keys(tmp_path) -> None: + path = tmp_path / "config.json" + path.write_text("{}\n", encoding="utf-8") + with pytest.raises(OctopError) as exc: + save_raw_config(path, "{") + assert exc.value.code is ErrorCode.APP_CONFIG_INVALID + assert path.read_text(encoding="utf-8") == "{}\n" + + save_raw_config( + path, + json.dumps({"port": 8088, "plugins": {"demo": {"enabled": True}}, "log_level": "debug"}), + ) + saved = json.loads(path.read_text(encoding="utf-8")) + assert saved["plugins"]["demo"]["enabled"] is True + assert saved["log_level"] == "debug" + + +def test_corrupt_file_is_not_replaced_by_a_form_save(tmp_path) -> None: + path = tmp_path / "config.json" + path.write_text("{ not json", encoding="utf-8") + view = read_app_config(path) + assert view.valid is False + assert view.config is None + with pytest.raises(OctopError) as exc: + save_structured_config(path, _body()) + assert exc.value.code is ErrorCode.CONFIG_FILE_CORRUPT + assert path.read_text(encoding="utf-8") == "{ not json" + + +def test_env_override_and_restart_flag(tmp_path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("OCTOP_PORT", "9091") + monkeypatch.delenv("OCTOP_SERVICE_MODE", raising=False) + monkeypatch.delenv("OCTOP_DESKTOP", raising=False) + monkeypatch.delenv("OCTOP_DESKTOP_SHELL", raising=False) + monkeypatch.delenv("OCTOP_GREEN_PACKAGES", raising=False) + view = read_app_config(tmp_path / "missing.json") + assert "OCTOP_PORT" in view.overrides + assert view.restart_available is False + + monkeypatch.setenv("OCTOP_SERVICE_MODE", "systemd") + view = read_app_config(tmp_path / "missing.json") + assert view.service_mode == "systemd" + assert view.restart_available is True diff --git a/tests/unit/test_docker_compose_database_env.py b/tests/unit/test_docker_compose_database_env.py index 23fef9f7f6..16c85d4cf0 100644 --- a/tests/unit/test_docker_compose_database_env.py +++ b/tests/unit/test_docker_compose_database_env.py @@ -21,10 +21,10 @@ def _compose_environment_entries(compose_text: str) -> str: def test_docker_compose_forwards_database_env_keys() -> None: - """docker/.env alone does not inject vars — they must be listed under environment.""" + """deploy/docker/.env alone does not inject vars — they must be listed under environment.""" repo = Path(__file__).resolve().parents[2] for name in ("docker-compose.yml", "docker-compose.cn.yml"): - text = (repo / "docker" / name).read_text(encoding="utf-8") + text = (repo / "deploy" / "docker" / name).read_text(encoding="utf-8") env_text = _compose_environment_entries(text) missing = [k for k in _DATABASE_ENV_KEYS if f"{k}=" not in env_text] assert not missing, f"{name} environment missing: {missing}" diff --git a/tests/unit/test_docker_entrypoint.py b/tests/unit/test_docker_entrypoint.py index 9528170de7..fb99990dc5 100644 --- a/tests/unit/test_docker_entrypoint.py +++ b/tests/unit/test_docker_entrypoint.py @@ -21,7 +21,7 @@ bash_only = pytest.mark.skipif(bash_exe is None, reason="requires a bash shell") REPO = Path(__file__).resolve().parents[2] -ENTRYPOINT = REPO / "docker" / "docker-entrypoint.sh" +ENTRYPOINT = REPO / "deploy" / "docker" / "docker-entrypoint.sh" def _bash_path(path: Path) -> str: diff --git a/tests/unit/test_docker_knowledge_dependencies.py b/tests/unit/test_docker_knowledge_dependencies.py index 888232e0c0..b6491b640a 100644 --- a/tests/unit/test_docker_knowledge_dependencies.py +++ b/tests/unit/test_docker_knowledge_dependencies.py @@ -9,7 +9,7 @@ def test_docker_syncs_retain_knowledge_extras() -> None: """A later exact uv sync must not prune extras installed in an earlier layer.""" repo = Path(__file__).resolve().parents[2] - dockerfile = (repo / "docker" / "Dockerfile").read_text(encoding="utf-8") + dockerfile = (repo / "deploy" / "docker" / "Dockerfile").read_text(encoding="utf-8") commands = dockerfile.replace("\\\n", " ").splitlines() syncs = [shlex.split(line.strip()) for line in commands if "uv sync " in line] assert len(syncs) == 2 @@ -42,7 +42,7 @@ def _desktop_smoke_import(dockerfile: str) -> bool: def test_fnos_source_dockerfile_installs_knowledge_extras() -> None: """The manual FnOS source image must bake the same knowledge runtimes. - Published FnOS Docker packages pull the image built from docker/Dockerfile + Published FnOS Docker packages pull the image built from deploy/docker/Dockerfile and do not build this file. """ repo = Path(__file__).resolve().parents[2] @@ -56,7 +56,7 @@ def test_variant_is_validated_and_compiler_is_purged_in_the_install_layer() -> N """A typo must not silently build the full image, and gcc must not stay in it.""" repo = Path(__file__).resolve().parents[2] for rel, install_marker in ( - ("docker/Dockerfile", "uv sync "), + ("deploy/docker/Dockerfile", "uv sync "), ("fnos/docker/Dockerfile", "uv pip install "), ): text = (repo / rel).read_text(encoding="utf-8") @@ -78,7 +78,7 @@ def test_project_sync_purges_compiler_only_when_it_is_installed() -> None: with ``Unable to locate package build-essential``. """ repo = Path(__file__).resolve().parents[2] - text = (repo / "docker" / "Dockerfile").read_text(encoding="utf-8") + text = (repo / "deploy" / "docker" / "Dockerfile").read_text(encoding="utf-8") layers = [layer for layer in text.split("\nRUN ") if "uv sync " in layer] assert len(layers) == 2 project = layers[1] @@ -89,7 +89,7 @@ def test_project_sync_purges_compiler_only_when_it_is_installed() -> None: def test_images_bake_desktop_without_starting_or_downloading_laya() -> None: """Both images install the desktop stack and leave Xvnc stopped.""" repo = Path(__file__).resolve().parents[2] - bake = (repo / "docker" / "bake-desktop.sh").read_text(encoding="utf-8") + bake = (repo / "deploy" / "docker" / "bake-desktop.sh").read_text(encoding="utf-8") install = (repo / "src/octop/infra/control/desktop/scripts/linux/v1.0/install.sh").read_text( encoding="utf-8" ) @@ -100,7 +100,7 @@ def test_images_bake_desktop_without_starting_or_downloading_laya() -> None: assert "firefox-esr" in bake assert "chromium-browser" in install assert 'if [ "$NO_START" = 1 ]' in install - for rel in ("docker/Dockerfile", "fnos/docker/Dockerfile"): + for rel in ("deploy/docker/Dockerfile", "fnos/docker/Dockerfile"): text = (repo / rel).read_text(encoding="utf-8") assert "bake-desktop.sh" in text assert "playwright install" not in text diff --git a/tests/unit/test_docker_nodejs.py b/tests/unit/test_docker_nodejs.py index 9a612e947c..ab2f5e700c 100644 --- a/tests/unit/test_docker_nodejs.py +++ b/tests/unit/test_docker_nodejs.py @@ -8,12 +8,12 @@ def test_images_ship_nodejs_runtime() -> None: """Both Dockerfiles copy Node 20 from the frontend stage into the runtime. - Published FnOS Docker packages pull the image built from docker/Dockerfile. + Published FnOS Docker packages pull the image built from deploy/docker/Dockerfile. The FnOS source Dockerfile must keep the same runtime so a manual build does not drop Node. """ repo = Path(__file__).resolve().parents[2] - for rel in ("docker/Dockerfile", "fnos/docker/Dockerfile"): + for rel in ("deploy/docker/Dockerfile", "fnos/docker/Dockerfile"): text = (repo / rel).read_text(encoding="utf-8") assert "COPY --from=frontend-builder /usr/local/bin/node /usr/local/bin/node" in text assert ( diff --git a/tests/unit/test_docker_nofile.py b/tests/unit/test_docker_nofile.py index 43f33c860a..1592995ab0 100644 --- a/tests/unit/test_docker_nofile.py +++ b/tests/unit/test_docker_nofile.py @@ -11,9 +11,9 @@ @pytest.mark.parametrize( "relative_path", [ - "docker/docker-compose.yml", - "docker/docker-compose.cn.yml", - "docker/docker-compose.mobile.yml", + "deploy/docker/docker-compose.yml", + "deploy/docker/docker-compose.cn.yml", + "deploy/docker/docker-compose.mobile.yml", "fnos/docker/app/docker/docker-compose.yaml", ], ) diff --git a/tests/unit/test_langfuse_env_names.py b/tests/unit/test_langfuse_env_names.py index c4fd369abd..519a150cf0 100644 --- a/tests/unit/test_langfuse_env_names.py +++ b/tests/unit/test_langfuse_env_names.py @@ -1,6 +1,6 @@ """Every Langfuse variable shipped to the container must have a reader. -`.env.example` and `docker/docker-compose.yml` advertise the observability +`.env.example` and `deploy/docker/docker-compose.yml` advertise the observability knobs to self-hosters. Octop itself keeps the enable switch in the settings table (`observability_langfuse_enabled`), while the credentials fall back to the Langfuse SDK's own environment variables — so a name neither side reads @@ -16,7 +16,7 @@ REPO_ROOT = Path(__file__).resolve().parents[2] ENV_EXAMPLE = REPO_ROOT / ".env.example" -COMPOSE = REPO_ROOT / "docker" / "docker-compose.yml" +COMPOSE = REPO_ROOT / "deploy" / "docker" / "docker-compose.yml" def _shipped_langfuse_names() -> set[str]: diff --git a/tests/unit/test_provider_fetch_models.py b/tests/unit/test_provider_fetch_models.py index c5a8d90ef4..1442eb12dd 100644 --- a/tests/unit/test_provider_fetch_models.py +++ b/tests/unit/test_provider_fetch_models.py @@ -237,4 +237,52 @@ async def test_fetch_models_server_error_friendly() -> None: assert result["ok"] is False assert "temporarily unavailable" in result["error"].lower() - assert "503" not in result["error"] + + +@pytest.mark.asyncio +async def test_fetch_magpie_catalog_keeps_nested_ids_and_user_agent() -> None: + from octop import __version__ + + response = _mock_response( + 200, + { + "data": [ + { + "id": "vendor/org/model", + "display_name": "Shown", + "context_window": 32000, + "max_output_tokens": 1024, + "modalities": ["text", "image"], + }, + {"id": "vendor/org/model"}, + {"id": "text-embedding-3-large", "type": "embedding"}, + {"id": "group/route"}, + ] + }, + ) + mock_client = AsyncMock() + mock_client.get = AsyncMock(return_value=response) + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=None) + + with patch("octop.infra.agents.providers.probe.httpx.AsyncClient", return_value=mock_client): + result = await fetch_openai_compatible_models( + base_url="https://gateway.example/v1", + api_key="real-key", + provider_name="Desk", + extra={"integration": "magpie"}, + ) + + assert result["ok"] is True + assert [model["id"] for model in result["models"]] == [ + "group/route", + "vendor/org/model", + ] + shown = next(model for model in result["models"] if model["id"] == "vendor/org/model") + assert shown["name"] == "Shown" + assert shown["context_window"] == 32000 + assert shown["max_tokens"] == 1024 + assert shown["input"] == ["text", "image"] + headers = mock_client.get.await_args.kwargs["headers"] + assert headers["User-Agent"] == f"Octop/{__version__}" + assert "X-Magpie-Session" not in headers diff --git a/tests/unit/test_provider_preset_expansion.py b/tests/unit/test_provider_preset_expansion.py index 430f08c110..603c4c5be8 100644 --- a/tests/unit/test_provider_preset_expansion.py +++ b/tests/unit/test_provider_preset_expansion.py @@ -114,6 +114,14 @@ def test_load_provider_presets_integration() -> None: go_oai = next(p for p in presets if p["id"] == "opencode-go-openai") assert go_oai["base_url"] == "https://opencode.ai/zen/go/v1" + magpie = next(p for p in presets if p["id"] == "magpie") + assert magpie["name"] == "Magpie" + assert magpie["base_url"] == "http://127.0.0.1:3425/v1" + assert magpie["protocol"] == "openai" + assert magpie["models"] == [] + assert magpie["integration"] == "magpie" + assert magpie["default_api_key"] == "magpie-octop" + go_ant = next(p for p in presets if p["id"] == "opencode-go-anthropic") assert go_ant["base_url"] == "https://opencode.ai/zen/go" diff --git a/tests/unit/test_storage_spec.py b/tests/unit/test_storage_spec.py index 9f5cfcc9a8..dca4c8ce2d 100644 --- a/tests/unit/test_storage_spec.py +++ b/tests/unit/test_storage_spec.py @@ -273,6 +273,65 @@ def test_enrich_docker_backend_spec_defaults() -> None: assert "agent_id" not in fixed or fixed.get("agent_id") == "ZE6GR2" +def test_omit_docker_volumes_drops_host_mounts() -> None: + from octop.infra.backend.docker_spec import ( + enrich_docker_backend_spec, + omit_docker_volumes, + ) + + mounted = { + "type": "docker", + "image": "python:3.12-slim", + "volumes": {"/var/lib/octop": {"bind": "/host", "mode": "rw"}}, + } + stripped = omit_docker_volumes(mounted) + assert "volumes" not in stripped + assert stripped["image"] == "python:3.12-slim" + assert "volumes" in mounted + + composite = omit_docker_volumes( + { + "type": "composite", + "default": {"type": "local_shell", "root_dir": "workspace-root"}, + "routes": { + "/box": { + "type": "docker", + "image": "python:3.12-slim", + "volumes": ["/etc:/host:ro"], + } + }, + } + ) + assert "volumes" not in composite["routes"]["/box"] + assert composite["default"]["root_dir"] == "workspace-root" + + enriched = enrich_docker_backend_spec(mounted, agent_id="ZE6GR2") + assert "volumes" not in enriched + assert enriched["agent_id"] == "ZE6GR2" + + +def test_row_to_backend_spec_docker_ignores_volumes() -> None: + row = BackendRow( + id=1, + name="sandbox", + kind="docker", + endpoint=None, + access_key=None, + secret_key=None, + bucket="python:3.12-slim", + region=None, + config_json='{"memory": "512m", "volumes": {"/": {"bind": "/host", "mode": "rw"}}}', + note=None, + enabled=1, + created_at=0, + updated_at=0, + ) + spec = row_to_backend_spec(row) + assert spec is not None + assert spec["memory"] == "512m" + assert "volumes" not in spec + + def test_inject_docker_global_environment_sets_file() -> None: from octop.infra.backend.docker_spec import inject_docker_global_environment diff --git a/tests/unit/test_user_manager.py b/tests/unit/test_user_manager.py index 3ed95f95c4..a3610dda58 100644 --- a/tests/unit/test_user_manager.py +++ b/tests/unit/test_user_manager.py @@ -59,6 +59,22 @@ async def test_authenticate_success(manager: UserManager): assert user.username == "a" +async def test_authenticate_records_last_login(manager: UserManager): + await manager.create(username="a", password="TestPass12", role=Role.USER) + before = manager._services.user_repo.get_by_username("a") + assert before is not None + assert before.last_login_at is None + assert await manager.authenticate("a", "wrong-password") is None + untouched = manager._services.user_repo.get_by_username("a") + assert untouched is not None + assert untouched.last_login_at is None + assert await manager.authenticate("a", "TestPass12") is not None + after = manager._services.user_repo.get_by_username("a") + assert after is not None + assert after.last_login_at is not None + assert after.last_login_at > 0 + + async def test_authenticate_by_email(manager: UserManager): await manager.create( username="alice", diff --git a/tests/unit/users/test_permissions.py b/tests/unit/users/test_permissions.py index c31710b9a7..9cb8d752fa 100644 --- a/tests/unit/users/test_permissions.py +++ b/tests/unit/users/test_permissions.py @@ -87,6 +87,7 @@ def test_categories_match_nav_groups() -> None: assert {k for k, p in PERMISSIONS.items() if p.category == "control"} == { "terminal", "browser", + "cloud", "desktop", "mobile", } diff --git a/tests/unit/users/test_preferences.py b/tests/unit/users/test_preferences.py index 794361bf04..9891542d68 100644 --- a/tests/unit/users/test_preferences.py +++ b/tests/unit/users/test_preferences.py @@ -8,11 +8,13 @@ from octop.infra.users.preferences import ( ModelReasoningPreference, get_model_reasoning_from_json, + get_onboarding_completed_from_json, get_preferred_model_from_json, get_remote_browser_bookmarks_from_json, get_sidebar_nav_from_json, get_timezone_from_preferences_json, merge_model_preferences_json, + merge_onboarding_completed_json, merge_preferences_json, merge_sidebar_nav_json, validate_remote_browser_bookmarks, @@ -154,3 +156,26 @@ def test_clear_preferred_model_keeps_reasoning_defaults() -> None: cleared = merge_model_preferences_json(raw, preferred_model=None) assert get_preferred_model_from_json(cleared) is None assert "token/glm-5" in get_model_reasoning_from_json(cleared) + + +def test_onboarding_completed_defaults_to_false() -> None: + assert get_onboarding_completed_from_json(None) is False + assert get_onboarding_completed_from_json("") is False + assert get_onboarding_completed_from_json("not json") is False + # Only a real boolean counts; stray truthy values do not hide the guide. + assert get_onboarding_completed_from_json('{"onboarding_completed": "yes"}') is False + + +def test_onboarding_completed_roundtrip_preserves_other_keys() -> None: + raw = merge_preferences_json( + None, + validate_remote_browser_bookmarks([{"url": "https://a.example", "title": "A"}]), + ) + done = merge_onboarding_completed_json(raw, True) + assert get_onboarding_completed_from_json(done) is True + assert get_remote_browser_bookmarks_from_json(done)[0].url == "https://a.example" + + reset = merge_onboarding_completed_json(done, False) + assert get_onboarding_completed_from_json(reset) is False + assert "onboarding_completed" not in reset + assert get_remote_browser_bookmarks_from_json(reset)[0].url == "https://a.example" diff --git a/tests/unit/users/test_resource_policy.py b/tests/unit/users/test_resource_policy.py index 3700684d72..2c66654543 100644 --- a/tests/unit/users/test_resource_policy.py +++ b/tests/unit/users/test_resource_policy.py @@ -14,6 +14,7 @@ from octop.infra.db.repos.users import UserRepo from octop.infra.errors import ErrorCode, OctopError from octop.infra.users.resource_policy import ( + POLICY_ALLOWED_BACKEND_TYPES, POLICY_MAX_AGENTS, POLICY_TOKEN_QUOTA, POLICY_WORKSPACE_ROOT_DIR, @@ -22,9 +23,11 @@ assert_backend_within_user_root, assert_token_quota_available, clamp_backend_to_workspace_root, + normalize_allowed_backend_types, normalize_workspace_root_dir, persisted_workspace_within_root, public_policy_fields, + raise_if_backend_types_disallowed, ) @@ -326,6 +329,7 @@ def test_user_policy_repo_named_rows(db: SqlitePool, tmp_path: Path) -> None: "workspace_root_dir": str(jail), "token_quota": 42, "max_agents": None, + "allowed_backend_types": None, } root = policies.get(uid, POLICY_WORKSPACE_ROOT_DIR) assert root is not None @@ -465,3 +469,91 @@ def test_migration_rebuilds_legacy_kv_table(tmp_path: Path) -> None: cols = {r["name"] for r in conn.execute("PRAGMA table_info(user_policies)")} assert "key" not in cols assert "policy_id" in cols + + +def test_normalize_allowed_backend_types_orders_and_rejects_unknown() -> None: + assert normalize_allowed_backend_types(None) is None + assert normalize_allowed_backend_types("") is None + assert normalize_allowed_backend_types([]) is None + assert ( + normalize_allowed_backend_types(["docker", "opensandbox", "filesystem", "docker"]) + == "filesystem,docker,opensandbox" + ) + assert normalize_allowed_backend_types(["shell", "docker"]) == "local_shell,docker" + assert normalize_allowed_backend_types(["local_shell", "shell"]) == "local_shell" + with pytest.raises(OctopError) as exc: + normalize_allowed_backend_types(["docker", "nope"]) + assert exc.value.code is ErrorCode.FORBIDDEN + + +def test_raise_if_backend_types_disallowed_walks_named_and_composite(db: SqlitePool) -> None: + users = UserRepo(db) + policies = UserPolicyRepo(db) + uid = users.create(username="alice", password_hash="h", role="user") + kinds = {"box": "docker", "disk": "filesystem", "host": "shell"} + + def named_kind(name: str) -> str | None: + return kinds.get(name) + + raise_if_backend_types_disallowed( + policies, + uid, + {"type": "local_shell"}, + named_kind=named_kind, + ) + policies.set(uid, POLICY_ALLOWED_BACKEND_TYPES, "docker,opensandbox") + spec = { + "type": "composite", + "default": {"type": "named", "name": "box"}, + "routes": {"/data": {"type": "named", "name": "disk"}}, + } + with pytest.raises(OctopError) as exc: + raise_if_backend_types_disallowed(policies, uid, spec, named_kind=named_kind) + assert exc.value.code is ErrorCode.BACKEND_TYPE_NOT_ALLOWED + assert exc.value.details["types"] == "filesystem" + raise_if_backend_types_disallowed( + policies, + uid, + {"type": "named", "name": "box"}, + named_kind=named_kind, + require_present=True, + ) + with pytest.raises(OctopError) as missing: + raise_if_backend_types_disallowed( + policies, + uid, + None, + named_kind=named_kind, + require_present=True, + ) + assert missing.value.details["types"] == "local_shell" + # A stored filesystem route can stay; a new shell route cannot. + raise_if_backend_types_disallowed( + policies, + uid, + spec, + named_kind=named_kind, + previous=spec, + ) + with pytest.raises(OctopError) as shell_blocked: + raise_if_backend_types_disallowed( + policies, + uid, + {"type": "named", "name": "host"}, + named_kind=named_kind, + previous=spec, + ) + assert shell_blocked.value.details["types"] == "local_shell" + policies.set(uid, POLICY_ALLOWED_BACKEND_TYPES, "local_shell") + raise_if_backend_types_disallowed( + policies, + uid, + {"type": "named", "name": "host"}, + named_kind=named_kind, + ) + raise_if_backend_types_disallowed( + policies, + uid, + {"type": "local_shell"}, + named_kind=named_kind, + ) diff --git a/uv.lock b/uv.lock index 587386649e..cdc8d7a021 100644 --- a/uv.lock +++ b/uv.lock @@ -2589,7 +2589,7 @@ wheels = [ [[package]] name = "octop" -version = "1.0.3b5" +version = "1.0.3b6" source = { editable = "." } dependencies = [ { name = "acme" }, @@ -2707,7 +2707,7 @@ requires-dist = [ { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.10" }, { name = "octop-browser", specifier = ">=1.0.0" }, { name = "octop-gateway", specifier = ">=1.0.3" }, - { name = "octop-harness", extras = ["all"], specifier = ">=1.0.9" }, + { name = "octop-harness", extras = ["all"], specifier = ">=1.0.10" }, { name = "octop-memory", specifier = ">=1.0.4" }, { name = "onnxruntime", marker = "extra == 'knowledge-ocr'", specifier = ">=1.17" }, { name = "openpyxl", specifier = ">=3.1" }, @@ -2803,7 +2803,7 @@ wheels = [ [[package]] name = "octop-harness" -version = "1.0.9" +version = "1.0.10" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "deepagents" }, @@ -2821,9 +2821,9 @@ dependencies = [ { name = "octop-memory" }, { name = "pyyaml" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/02/d9/259fa4d2662fcb909a7fab8662e7cdf84f4e514f8375685ccfd366c608ba/octop_harness-1.0.9.tar.gz", hash = "sha256:146f21964210acfe7203957f746102699a67e283d4e6a04e8ccc8a2683b67e72", size = 1021267, upload-time = "2026-11-10T12:21:15.589Z" } +sdist = { url = "https://files.pythonhosted.org/packages/22/d7/cc7ada944e0c5dec954fd5cad5c18acec6067dc113610ea68dde081f8962/octop_harness-1.0.10.tar.gz", hash = "sha256:1dbd95a0ce5c0dd2b24828df7763db86a6acbb1b6cfd719acf6bc31f264fd12b", size = 1031842, upload-time = "2026-11-15T11:49:28.761Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/76/99/7bb8a7a5b3b542c928a5b23aeedd05d96b4211f6940c7d174c4de298b07d/octop_harness-1.0.9-py3-none-any.whl", hash = "sha256:b01e2f6fd691ede8d7df70c0ab9a84b5b4ace0394fb6533eb444573b3c9aa1a6", size = 1220134, upload-time = "2026-11-10T12:21:14.076Z" }, + { url = "https://files.pythonhosted.org/packages/7b/17/6945030f10986dae0d1638c337b7ab23095590502be61c2a9c3cd8bab5ae/octop_harness-1.0.10-py3-none-any.whl", hash = "sha256:6c01f0604a2ceb9a8607dbb517b1e1d07f9f2751dcef6c0517042d5e3e8d28c4", size = 1230546, upload-time = "2026-11-15T11:49:27.298Z" }, ] [package.optional-dependencies]