From 6ee50a4692f6cece614823a456e39c024dd02085 Mon Sep 17 00:00:00 2001 From: Henry Zhou <739287804@qq.com> Date: Tue, 3 Nov 2026 16:42:56 +0800 Subject: [PATCH 1/4] feat: add image inspector plugin # Conflicts: # CHANGELOG.md # Conflicts: # CHANGELOG.md --- CHANGELOG.md | 4 + plugins/image-inspector/README.md | 40 ++++ plugins/image-inspector/main.py | 158 ++++++++++++++++ plugins/image-inspector/plugin.yaml | 7 + tests/unit/test_image_inspector_plugin.py | 217 ++++++++++++++++++++++ 5 files changed, 426 insertions(+) create mode 100644 plugins/image-inspector/README.md create mode 100644 plugins/image-inspector/main.py create mode 100644 plugins/image-inspector/plugin.yaml create mode 100644 tests/unit/test_image_inspector_plugin.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c2828353a4..a35bf5917d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,12 +11,16 @@ - 安装向导连接已有用户的 PostgreSQL 时,确认后保留这些账户并去登录。SQLite 仍要空库 - 知识库预览可以在原文和已入库切片之间切换。文件名悬停后可重命名,扩展名保持不变 - Docker 与飞牛增加精简版。默认 `:` / `:latest` 仍是含桌面、OCR 和本地向量的完整镜像;精简版标签是 `:-lite` 和 `:lite` +- 新增独立 `image-inspector` 工具插件:生成图片后可按需调用 `inspect_image`,将工作区原图送回当前多模态模型检查排版、遮挡、裁切和间距(#2283) ### 修复 - PostgreSQL 控制面下,知识库检索读 pgvector,不再查到空的本地 `index.sqlite` - Docker 镜像在无头构建时不再导入 pynput。飞牛 Docker 包缺失时,已打好的本地包仍会挂到 Release - 精简版只有 amd64 和 arm64 都发布成功才打通用标签。完整飞牛包不再等待缺失的精简镜像,Release 下载区只保留实际挂上的飞牛包 - Docker 构建在安装依赖的同一层删除编译器,避免编译工具留在最终镜像里 +- `image-inspector` 的视觉检查模型调用与父级会话流隔离,内部质检文本不会再被推送到 IM 或保存成额外的 AI 消息(#2286) +- `image-inspector` 支持自定义 `OCTOP_HOME` / `OCTOP_DATA_DIR` 和 Docker sandbox 的 `.harness-materialize` 文件路径,避免可见图片被误报为不存在 +- `image-inspector` 遇到宿主机不可访问的沙箱路径时,继续尝试已允许的数据目录与 materialized 路径,避免在找到可读图片前提前报错(#2286) - 插件市场和专家市场的对象源站可用 `OCTOP_MARKET_REMOTE_URL` 配置,默认仍是 `https://download.octop.cloud`。门户目录地址继续用 `OCTOP_PORTAL_URL`,默认 `https://octop.cloud`。内网按同一接口自建市场的说明见 `docs/private-market.md` - 插件市场和专家市场的对象源站可用 `OCTOP_MARKET_REMOTE_ORIGIN` 配置,默认仍是 `https://download.octop.cloud`。门户目录地址继续用 `OCTOP_PORTAL_ORIGIN`,默认 `https://octop.cloud`。内网按同一接口自建市场的说明见 `docs/private-market.md` - 技能市场源站可用 `SKILL_MARKET_ORIGIN` 配置,默认仍是 `https://api.skillhub.cn`。设了该变量后,搜索、榜单和安装只请求这个源站。自建接口见 `docs/skill-market.md` diff --git a/plugins/image-inspector/README.md b/plugins/image-inspector/README.md new file mode 100644 index 0000000000..f83211949d --- /dev/null +++ b/plugins/image-inspector/README.md @@ -0,0 +1,40 @@ +# Image Inspector + +An optional Octop tool plugin that invokes the current multimodal model to +review a workspace image and returns a textual visual-quality assessment. + +Use it for layout defects that OCR cannot detect: + +- clipped or overlapping text; +- misaligned or uneven cards; +- insufficient spacing; +- weak contrast or hierarchy; +- incorrect visual density. + +## Install independently + +```bash +octop plugin install ./plugins/image-inspector --force +``` + +Enable the plugin in Admin -> Plugins, then enable `inspect_image` for the +agent in Tool management. + +## Agent workflow + +1. Generate an image with `generate_image`. +2. Read the returned `localPath`. +3. Call `inspect_image` with that exact path when visual review is needed. +4. Use the returned text assessment before delivering or revising the image. + +The tool is manual. It does not run automatically after generation and does not +modify the image. Each call sends the image as a data URL to a dedicated model +request, so it consumes model tokens in addition to the outer agent turn. + +## Security + +- Images must be under `$OCTOP_HOME/workspaces/`. +- Supported formats: PNG, JPEG, WebP. +- Maximum size: 20 MiB. +- HTTP(S) URL fetching is not supported. +- The plugin does not modify Octop package files. diff --git a/plugins/image-inspector/main.py b/plugins/image-inspector/main.py new file mode 100644 index 0000000000..72576091cb --- /dev/null +++ b/plugins/image-inspector/main.py @@ -0,0 +1,158 @@ +"""Return local workspace images to the current multimodal model.""" + +from __future__ import annotations + +import asyncio +import base64 +import io +import os +from dataclasses import dataclass +from pathlib import Path + +from langchain_core.messages import HumanMessage, SystemMessage +from octop_harness.plugins import PluginContext +from PIL import Image, UnidentifiedImageError + +_MAX_IMAGE_BYTES = 20 * 1024 * 1024 +_MEDIA_TYPES = { + "PNG": "image/png", + "JPEG": "image/jpeg", + "WEBP": "image/webp", +} +_SYSTEM_PROMPT = """You are a precise visual quality reviewer. +Inspect the supplied image itself. Do not rely on OCR or guess unseen content. +Report visible defects involving clipping, overlap, alignment, spacing, +contrast, hierarchy, incorrect relative-date labels, or other visual mistakes. +State uncertain observations explicitly and do not invent details.""" + + +@dataclass(frozen=True) +class _LoadedImage: + path: Path + media_type: str + data: bytes + width: int + height: int + + +def _error(message: str) -> str: + return f"Error: {message}" + + +def _workspace_root() -> Path: + home = Path(os.environ.get("OCTOP_HOME") or (Path.home() / ".octop")).expanduser() + return (home / "workspaces").resolve() + + +def _load_image(path: str) -> _LoadedImage | str: + raw = path.strip() + if not raw: + return _error("path is required") + + candidate = Path(raw).expanduser() + if not candidate.is_absolute(): + candidate = Path.cwd() / candidate + + try: + resolved = candidate.resolve(strict=True) + except FileNotFoundError: + return _error(f"image file not found: {raw}") + except OSError as exc: + return _error(f"cannot resolve image path: {exc}") + + allowed_root = _workspace_root() + try: + resolved.relative_to(allowed_root) + except ValueError: + return _error(f"path is outside the Octop workspaces directory: {resolved}") + + if not resolved.is_file(): + return _error(f"not a regular image file: {resolved}") + + try: + size_bytes = resolved.stat().st_size + except OSError as exc: + return _error(f"cannot stat image file: {exc}") + if size_bytes > _MAX_IMAGE_BYTES: + return _error(f"image is too large: {size_bytes} bytes; maximum is {_MAX_IMAGE_BYTES}") + + try: + data = resolved.read_bytes() + with Image.open(io.BytesIO(data)) as image: + image_format = str(image.format or "").upper() + width, height = image.size + image.verify() + except Image.DecompressionBombError: + return _error("image dimensions exceed Pillow safety limits") + except (UnidentifiedImageError, OSError, ValueError) as exc: + return _error(f"unsupported image format: {exc}") + + media_type = _MEDIA_TYPES.get(image_format) + if media_type is None: + return _error( + f"unsupported image format {image_format or 'unknown'}; use PNG, JPEG, or WebP" + ) + + return _LoadedImage( + path=resolved, + media_type=media_type, + data=data, + width=width, + height=height, + ) + + +def setup(ctx: PluginContext) -> None: + async def inspect_image(path: str, focus: str = "") -> str: + """Inspect a workspace image with the Agent's multimodal model.""" + loaded = await asyncio.to_thread(_load_image, path) + if isinstance(loaded, str): + return loaded + + encoded = base64.b64encode(loaded.data).decode("ascii") + focus_text = focus.strip() or ( + "Review the overall visual quality and identify concrete defects." + ) + messages = [ + SystemMessage(content=_SYSTEM_PROMPT), + HumanMessage( + content=[ + { + "type": "text", + "text": ( + f"Visually inspect {loaded.path.name} " + f"({loaded.width}x{loaded.height}).\nFocus: {focus_text}" + ), + }, + { + "type": "image_url", + "image_url": { + "url": f"data:{loaded.media_type};base64,{encoded}", + }, + }, + ], + ), + ] + try: + access = ctx.get_model_access("auto", messages=messages) + result = await access.ainvoke(messages) + except Exception as exc: # noqa: BLE001 — returned to the agent as a tool error + detail = str(exc).strip()[:240] or exc.__class__.__name__ + return _error(f"visual review model call failed: {detail}") + + text = str(result).strip() + if not text: + return _error("visual review model returned an empty result") + return text + + ctx.tool( + "inspect_image", + inspect_image, + description=( + "Inspect a local PNG, JPEG, or WebP image from the Octop workspace " + "with the current multimodal model and return a textual visual-quality review. " + "Call manually after generate_image when the generated image will be delivered " + "or judged for visual quality. Pass the exact localPath returned by " + "generate_image. Do not use read_file for images. This tool does not fetch URLs." + ), + ) diff --git a/plugins/image-inspector/plugin.yaml b/plugins/image-inspector/plugin.yaml new file mode 100644 index 0000000000..45c87bfb8b --- /dev/null +++ b/plugins/image-inspector/plugin.yaml @@ -0,0 +1,7 @@ +id: image-inspector +version: 0.1.0 +name: Image Inspector +description: Optional visual review for workspace images generated by Octop. +icon: "🔍" +kind: tool +entry: main.py diff --git a/tests/unit/test_image_inspector_plugin.py b/tests/unit/test_image_inspector_plugin.py new file mode 100644 index 0000000000..f1e59499a6 --- /dev/null +++ b/tests/unit/test_image_inspector_plugin.py @@ -0,0 +1,217 @@ +"""Tests for the standalone image-inspector tool plugin.""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Any + +import pytest +from langchain_core.tools import StructuredTool +from octop_harness.plugins import PluginRegistry, build_plugin_tools, load_plugin_dir +from PIL import Image + +_PLUGIN_DIR = Path(__file__).resolve().parents[2] / "plugins" / "image-inspector" + + +@pytest.fixture(autouse=True) +def _reset_registry() -> None: + PluginRegistry.reset() + yield + PluginRegistry.reset() + + +def test_plugin_loads_and_registers_inspect_image() -> None: + loaded = load_plugin_dir(_PLUGIN_DIR, install_deps=False) + + assert loaded.manifest.id == "image-inspector" + assert loaded.manifest.kind == "tool" + assert [tool.name for tool in loaded.tools] == ["inspect_image"] + + tools = build_plugin_tools(global_plugins={"image-inspector": True}) + assert len(tools) == 1 + assert isinstance(tools[0], StructuredTool) + assert "generate_image" in tools[0].description + + +class _FakeModelAccess: + def __init__(self, response: str) -> None: + self.response = response + self.calls: list[tuple[list[Any], dict[str, Any]]] = [] + + async def ainvoke(self, messages: list[Any], **kwargs: Any) -> str: + self.calls.append((messages, kwargs)) + return self.response + + +class _FakeModelFactory: + def __init__(self, access: _FakeModelAccess) -> None: + self.access = access + self.specs: list[tuple[Any, dict[str, Any]]] = [] + + def get_for( + self, + spec: Any, + *, + get_protocol: Any = None, + **kwargs: Any, + ) -> _FakeModelAccess: + self.specs.append((spec, kwargs)) + return self.access + + +def _image_tool( + response: str = "Visual review result", +) -> tuple[StructuredTool, _FakeModelAccess, _FakeModelFactory]: + loaded = load_plugin_dir(_PLUGIN_DIR, install_deps=False) + assert [tool.name for tool in loaded.tools] == ["inspect_image"] + access = _FakeModelAccess(response) + factory = _FakeModelFactory(access) + assert loaded.context is not None + loaded.context.bind_model_factory(factory) + tools = build_plugin_tools(global_plugins={"image-inspector": True}) + assert len(tools) == 1 + return tools[0], access, factory + + +def _workspace_image( + tmp_path: Path, + *, + fmt: str = "PNG", + suffix: str = ".png", +) -> Path: + path = tmp_path / "workspaces" / "main" / "generated" / f"image{suffix}" + path.parent.mkdir(parents=True) + Image.new("RGB", (2, 3), (255, 0, 0)).save(path, format=fmt) + return path + + +async def test_inspect_image_calls_multimodal_model_and_returns_text( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("OCTOP_HOME", str(tmp_path)) + image_path = _workspace_image(tmp_path) + tool, access, factory = _image_tool("Layout is clean; no clipping detected.") + + result = await tool.ainvoke({"path": str(image_path), "focus": "check clipping and alignment"}) + + assert result == "Layout is clean; no clipping detected." + assert factory.specs[0][0] == "auto" + assert "messages" in factory.specs[0][1] + assert len(access.calls) == 1 + messages = access.calls[0][0] + human_content = messages[1].content + assert isinstance(human_content, list) + image_blocks = [block for block in human_content if block["type"] == "image_url"] + assert len(image_blocks) == 1 + assert image_blocks[0]["image_url"]["url"].startswith("data:image/png;base64,") + text_blocks = [block for block in human_content if block["type"] == "text"] + assert "check clipping and alignment" in text_blocks[0]["text"] + + +@pytest.mark.parametrize( + ("fmt", "suffix", "expected_media_type"), + [ + ("JPEG", ".jpg", "image/jpeg"), + ("WEBP", ".webp", "image/webp"), + ], +) +async def test_inspect_image_supports_common_raster_formats( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + fmt: str, + suffix: str, + expected_media_type: str, +) -> None: + monkeypatch.setenv("OCTOP_HOME", str(tmp_path)) + image_path = _workspace_image(tmp_path, fmt=fmt, suffix=suffix) + + tool, access, _ = _image_tool() + result = await tool.ainvoke({"path": str(image_path)}) + + assert result == "Visual review result" + human_content = access.calls[0][0][1].content + image_block = next(block for block in human_content if block["type"] == "image_url") + assert image_block["image_url"]["url"].startswith(f"data:{expected_media_type};base64,") + + +async def test_inspect_image_rejects_outside_workspace( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("OCTOP_HOME", str(tmp_path)) + outside = tmp_path / "outside.png" + Image.new("RGB", (1, 1)).save(outside, format="PNG") + + tool, access, _ = _image_tool() + result = await tool.ainvoke({"path": str(outside)}) + + assert isinstance(result, str) + assert result.startswith("Error: path is outside") + assert access.calls == [] + + +async def test_inspect_image_rejects_missing_file( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("OCTOP_HOME", str(tmp_path)) + path = tmp_path / "workspaces" / "main" / "generated" / "missing.png" + + tool, access, _ = _image_tool() + result = await tool.ainvoke({"path": str(path)}) + + assert isinstance(result, str) + assert "image file not found" in result + assert access.calls == [] + + +async def test_inspect_image_rejects_directory( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("OCTOP_HOME", str(tmp_path)) + path = tmp_path / "workspaces" / "main" / "generated" / "folder.png" + path.mkdir(parents=True) + + tool, access, _ = _image_tool() + result = await tool.ainvoke({"path": str(path)}) + + assert isinstance(result, str) + assert "not a regular image file" in result + assert access.calls == [] + + +async def test_inspect_image_rejects_unsupported_content( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("OCTOP_HOME", str(tmp_path)) + path = tmp_path / "workspaces" / "main" / "generated" / "not-image.png" + path.parent.mkdir(parents=True) + path.write_text("plain text", encoding="utf-8") + + tool, access, _ = _image_tool() + result = await tool.ainvoke({"path": str(path)}) + + assert isinstance(result, str) + assert "unsupported image format" in result + assert access.calls == [] + + +async def test_inspect_image_rejects_oversized_file( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("OCTOP_HOME", str(tmp_path)) + image_path = _workspace_image(tmp_path) + tool, access, _ = _image_tool() + module = sys.modules["harness_plugin_image_inspector"] + monkeypatch.setattr(module, "_MAX_IMAGE_BYTES", 1) + + result = await tool.ainvoke({"path": str(image_path)}) + + assert isinstance(result, str) + assert "too large" in result + assert access.calls == [] From 1659286c3487ce06dd5b7bdb426958e5958491b6 Mon Sep 17 00:00:00 2001 From: Henry Zhou <739287804@qq.com> Date: Mon, 2 Nov 2026 17:29:54 +0800 Subject: [PATCH 2/4] fix: isolate image inspector model stream # Conflicts: # CHANGELOG.md --- plugins/image-inspector/README.md | 4 +- plugins/image-inspector/main.py | 13 ++++- tests/unit/test_image_inspector_plugin.py | 64 ++++++++++++++++++++--- 3 files changed, 70 insertions(+), 11 deletions(-) diff --git a/plugins/image-inspector/README.md b/plugins/image-inspector/README.md index f83211949d..30dd5e25c5 100644 --- a/plugins/image-inspector/README.md +++ b/plugins/image-inspector/README.md @@ -29,7 +29,9 @@ agent in Tool management. The tool is manual. It does not run automatically after generation and does not modify the image. Each call sends the image as a data URL to a dedicated model -request, so it consumes model tokens in addition to the outer agent turn. +request, so it consumes model tokens in addition to the outer agent turn. That +request is isolated from the parent conversation stream; its review is returned +only as the tool result and is not sent to the user as a separate message. ## Security diff --git a/plugins/image-inspector/main.py b/plugins/image-inspector/main.py index 72576091cb..15447d8e9f 100644 --- a/plugins/image-inspector/main.py +++ b/plugins/image-inspector/main.py @@ -10,10 +10,12 @@ from pathlib import Path from langchain_core.messages import HumanMessage, SystemMessage +from octop_harness.llm.invoke_helpers import stringify_content from octop_harness.plugins import PluginContext from PIL import Image, UnidentifiedImageError _MAX_IMAGE_BYTES = 20 * 1024 * 1024 +_INTERNAL_LC_SOURCE = "image_inspector" _MEDIA_TYPES = { "PNG": "image/png", "JPEG": "image/jpeg", @@ -135,12 +137,19 @@ async def inspect_image(path: str, focus: str = "") -> str: ] try: access = ctx.get_model_access("auto", messages=messages) - result = await access.ainvoke(messages) + # The parent LangGraph stream must not expose this nested review call. + result = await access.chat_model.ainvoke( + messages, + config={ + "callbacks": [], + "metadata": {"lc_source": _INTERNAL_LC_SOURCE}, + }, + ) except Exception as exc: # noqa: BLE001 — returned to the agent as a tool error detail = str(exc).strip()[:240] or exc.__class__.__name__ return _error(f"visual review model call failed: {detail}") - text = str(result).strip() + text = stringify_content(getattr(result, "content", result)) if not text: return _error("visual review model returned an empty result") return text diff --git a/tests/unit/test_image_inspector_plugin.py b/tests/unit/test_image_inspector_plugin.py index f1e59499a6..f7d58a0678 100644 --- a/tests/unit/test_image_inspector_plugin.py +++ b/tests/unit/test_image_inspector_plugin.py @@ -7,6 +7,10 @@ from typing import Any import pytest +from langchain_core.callbacks import BaseCallbackHandler +from langchain_core.language_models.fake_chat_models import FakeMessagesListChatModel +from langchain_core.messages import AIMessage +from langchain_core.runnables.config import var_child_runnable_config from langchain_core.tools import StructuredTool from octop_harness.plugins import PluginRegistry, build_plugin_tools, load_plugin_dir from PIL import Image @@ -35,6 +39,12 @@ def test_plugin_loads_and_registers_inspect_image() -> None: class _FakeModelAccess: + def __init__(self, response: str) -> None: + self.response = response + self.chat_model = _FakeChatModel(response) + + +class _FakeChatModel: def __init__(self, response: str) -> None: self.response = response self.calls: list[tuple[list[Any], dict[str, Any]]] = [] @@ -60,6 +70,19 @@ def get_for( return self.access +class _ChatModelStartCounter(BaseCallbackHandler): + def __init__(self) -> None: + self.starts = 0 + + def on_chat_model_start( + self, + serialized: dict[str, Any], + messages: list[list[Any]], + **kwargs: Any, + ) -> None: + self.starts += 1 + + def _image_tool( response: str = "Visual review result", ) -> tuple[StructuredTool, _FakeModelAccess, _FakeModelFactory]: @@ -99,8 +122,8 @@ async def test_inspect_image_calls_multimodal_model_and_returns_text( assert result == "Layout is clean; no clipping detected." assert factory.specs[0][0] == "auto" assert "messages" in factory.specs[0][1] - assert len(access.calls) == 1 - messages = access.calls[0][0] + assert len(access.chat_model.calls) == 1 + messages = access.chat_model.calls[0][0] human_content = messages[1].content assert isinstance(human_content, list) image_blocks = [block for block in human_content if block["type"] == "image_url"] @@ -110,6 +133,31 @@ async def test_inspect_image_calls_multimodal_model_and_returns_text( assert "check clipping and alignment" in text_blocks[0]["text"] +async def test_inspect_image_isolates_visual_review_from_parent_stream( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("OCTOP_HOME", str(tmp_path)) + image_path = _workspace_image(tmp_path) + loaded = load_plugin_dir(_PLUGIN_DIR, install_deps=False) + access = _FakeModelAccess("Visual review result") + access.chat_model = FakeMessagesListChatModel( + responses=[AIMessage(content="Visual review result")] + ) + assert loaded.context is not None + loaded.context.bind_model_factory(_FakeModelFactory(access)) + tool = build_plugin_tools(global_plugins={"image-inspector": True})[0] + parent_handler = _ChatModelStartCounter() + token = var_child_runnable_config.set({"callbacks": [parent_handler]}) + try: + result = await tool.ainvoke({"path": str(image_path)}) + finally: + var_child_runnable_config.reset(token) + + assert result == "Visual review result" + assert parent_handler.starts == 0 + + @pytest.mark.parametrize( ("fmt", "suffix", "expected_media_type"), [ @@ -131,7 +179,7 @@ async def test_inspect_image_supports_common_raster_formats( result = await tool.ainvoke({"path": str(image_path)}) assert result == "Visual review result" - human_content = access.calls[0][0][1].content + human_content = access.chat_model.calls[0][0][1].content image_block = next(block for block in human_content if block["type"] == "image_url") assert image_block["image_url"]["url"].startswith(f"data:{expected_media_type};base64,") @@ -149,7 +197,7 @@ async def test_inspect_image_rejects_outside_workspace( assert isinstance(result, str) assert result.startswith("Error: path is outside") - assert access.calls == [] + assert access.chat_model.calls == [] async def test_inspect_image_rejects_missing_file( @@ -164,7 +212,7 @@ async def test_inspect_image_rejects_missing_file( assert isinstance(result, str) assert "image file not found" in result - assert access.calls == [] + assert access.chat_model.calls == [] async def test_inspect_image_rejects_directory( @@ -180,7 +228,7 @@ async def test_inspect_image_rejects_directory( assert isinstance(result, str) assert "not a regular image file" in result - assert access.calls == [] + assert access.chat_model.calls == [] async def test_inspect_image_rejects_unsupported_content( @@ -197,7 +245,7 @@ async def test_inspect_image_rejects_unsupported_content( assert isinstance(result, str) assert "unsupported image format" in result - assert access.calls == [] + assert access.chat_model.calls == [] async def test_inspect_image_rejects_oversized_file( @@ -214,4 +262,4 @@ async def test_inspect_image_rejects_oversized_file( assert isinstance(result, str) assert "too large" in result - assert access.calls == [] + assert access.chat_model.calls == [] From d80e930c0d3f65a4f49b0eaaff3a467bedff2ca2 Mon Sep 17 00:00:00 2001 From: Henry Zhou <739287804@qq.com> Date: Mon, 2 Nov 2026 17:30:19 +0800 Subject: [PATCH 3/4] fix(image-inspector): support data and sandbox paths # Conflicts: # CHANGELOG.md --- plugins/image-inspector/README.md | 15 +- plugins/image-inspector/main.py | 163 +++++++++++++++++++--- tests/unit/test_image_inspector_plugin.py | 58 +++++++- 3 files changed, 210 insertions(+), 26 deletions(-) diff --git a/plugins/image-inspector/README.md b/plugins/image-inspector/README.md index 30dd5e25c5..4cce394f70 100644 --- a/plugins/image-inspector/README.md +++ b/plugins/image-inspector/README.md @@ -27,15 +27,18 @@ agent in Tool management. 3. Call `inspect_image` with that exact path when visual review is needed. 4. Use the returned text assessment before delivering or revising the image. -The tool is manual. It does not run automatically after generation and does not -modify the image. Each call sends the image as a data URL to a dedicated model -request, so it consumes model tokens in addition to the outer agent turn. That -request is isolated from the parent conversation stream; its review is returned -only as the tool result and is not sent to the user as a separate message. +The tool is manual. It resolves images from `OCTOP_HOME`, `OCTOP_DATA_DIR`, and +the default `~/.octop` layout, including Docker-sandbox files materialized under +`.harness-materialize/`. It does not run automatically after generation and does +not modify the image. Each call sends the image as a data URL to a dedicated +model request, so it consumes model tokens in addition to the outer agent turn. +That request is isolated from the parent conversation stream; its review is +returned only as the tool result and is not sent to the user as a separate +message. ## Security -- Images must be under `$OCTOP_HOME/workspaces/`. +- Images must be under an Octop `agents/` or `workspaces/` directory. - Supported formats: PNG, JPEG, WebP. - Maximum size: 20 MiB. - HTTP(S) URL fetching is not supported. diff --git a/plugins/image-inspector/main.py b/plugins/image-inspector/main.py index 15447d8e9f..60803aa716 100644 --- a/plugins/image-inspector/main.py +++ b/plugins/image-inspector/main.py @@ -41,9 +41,147 @@ def _error(message: str) -> str: return f"Error: {message}" -def _workspace_root() -> Path: - home = Path(os.environ.get("OCTOP_HOME") or (Path.home() / ".octop")).expanduser() - return (home / "workspaces").resolve() +def _resolved(path: Path) -> Path: + try: + return path.expanduser().resolve() + except OSError: + return path.expanduser() + + +def _data_roots() -> list[Path]: + candidates: list[Path] = [] + home = os.environ.get("OCTOP_HOME", "").strip() + data_dir = os.environ.get("OCTOP_DATA_DIR", "").strip() + if home: + candidates.append(Path(home)) + if data_dir: + candidates.append(Path(data_dir)) + if Path(data_dir).name != ".octop": + candidates.append(Path(data_dir) / ".octop") + candidates.append(Path.home() / ".octop") + + roots: list[Path] = [] + seen: set[Path] = set() + for candidate in candidates: + resolved = _resolved(candidate) + if resolved in seen: + continue + seen.add(resolved) + roots.append(resolved) + return roots + + +def _allowed_roots() -> list[Path]: + roots: list[Path] = [] + for data_root in _data_roots(): + for name in ("agents", "workspaces"): + roots.append((data_root / name).resolve()) + return roots + + +def _path_hint(raw: str) -> tuple[str, str, Path] | None: + parts = [part for part in raw.replace("\\", "/").split("/") if part] + for index, part in enumerate(parts): + if part in {"agents", "workspaces"} and index + 2 < len(parts): + return part, parts[index + 1], Path(*parts[index + 2 :]) + return None + + +def _path_is_allowed(path: Path) -> bool: + for root in _allowed_roots(): + try: + path.relative_to(root) + return True + except ValueError: + continue + return False + + +def _candidate_paths(raw: str) -> list[Path]: + candidate = Path(raw).expanduser() + candidates = [candidate] + normalized = raw.replace("\\", "/") + hint = _path_hint(raw) + + for root in _data_roots(): + if normalized.startswith("/.octop/"): + candidates.append(root / normalized.lstrip("/")) + elif not candidate.is_absolute(): + candidates.append(root / candidate) + + if hint is None: + continue + namespace, agent_id, relative = hint + for resolved_namespace in dict.fromkeys((namespace, "agents", "workspaces")): + workspace = root / resolved_namespace / agent_id + candidates.append(workspace / relative) + candidates.append(workspace / ".harness-materialize" / relative) + return candidates + + +def _unique_workspace_match(relative: Path) -> list[Path]: + matches: list[Path] = [] + seen: set[Path] = set() + for root in _allowed_roots(): + if not root.is_dir(): + continue + try: + workspaces = root.iterdir() + except OSError: + continue + for workspace in workspaces: + if not workspace.is_dir(): + continue + for base in (workspace, workspace / ".harness-materialize"): + candidate = base / relative + try: + resolved = candidate.resolve(strict=True) + except (FileNotFoundError, NotADirectoryError, OSError): + continue + if resolved.is_file() and resolved not in seen: + seen.add(resolved) + matches.append(resolved) + return matches + + +def _resolve_image_path(raw: str) -> Path | str: + direct = Path(raw).expanduser() + seen: set[Path] = set() + for candidate in _candidate_paths(raw): + try: + resolved = candidate.resolve(strict=True) + except (FileNotFoundError, NotADirectoryError): + continue + except OSError as exc: + return _error(f"cannot resolve image path: {exc}") + if resolved in seen: + continue + seen.add(resolved) + if _path_is_allowed(resolved): + return resolved + + if not direct.is_absolute() and _path_hint(raw) is None: + matches = _unique_workspace_match(direct) + if len(matches) == 1: + return matches[0] + if len(matches) > 1: + return _error(f"relative image path is ambiguous across agent workspaces: {raw}") + + if direct.is_absolute(): + direct_resolved: Path | None + try: + direct_resolved = direct.resolve(strict=True) + except (FileNotFoundError, NotADirectoryError): + direct_resolved = None + except OSError as exc: + return _error(f"cannot resolve image path: {exc}") + if direct_resolved is not None: + return _error("path is outside the Octop agents/workspaces directories") + + return _error( + f"image file not found: {raw}. If it was created in a sandbox backend, " + "materialize it first with send_file_to_user or pass the .harness-materialize path." + ) def _load_image(path: str) -> _LoadedImage | str: @@ -51,22 +189,9 @@ def _load_image(path: str) -> _LoadedImage | str: if not raw: return _error("path is required") - candidate = Path(raw).expanduser() - if not candidate.is_absolute(): - candidate = Path.cwd() / candidate - - try: - resolved = candidate.resolve(strict=True) - except FileNotFoundError: - return _error(f"image file not found: {raw}") - except OSError as exc: - return _error(f"cannot resolve image path: {exc}") - - allowed_root = _workspace_root() - try: - resolved.relative_to(allowed_root) - except ValueError: - return _error(f"path is outside the Octop workspaces directory: {resolved}") + resolved = _resolve_image_path(raw) + if isinstance(resolved, str): + return resolved if not resolved.is_file(): return _error(f"not a regular image file: {resolved}") diff --git a/tests/unit/test_image_inspector_plugin.py b/tests/unit/test_image_inspector_plugin.py index f7d58a0678..ebb2758c7b 100644 --- a/tests/unit/test_image_inspector_plugin.py +++ b/tests/unit/test_image_inspector_plugin.py @@ -102,8 +102,9 @@ def _workspace_image( *, fmt: str = "PNG", suffix: str = ".png", + relative: Path = Path("workspaces/main/generated/image.png"), ) -> Path: - path = tmp_path / "workspaces" / "main" / "generated" / f"image{suffix}" + path = tmp_path / relative.with_suffix(suffix) path.parent.mkdir(parents=True) Image.new("RGB", (2, 3), (255, 0, 0)).save(path, format=fmt) return path @@ -263,3 +264,58 @@ async def test_inspect_image_rejects_oversized_file( assert isinstance(result, str) assert "too large" in result assert access.chat_model.calls == [] + + +async def test_inspect_image_uses_octop_data_dir( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("OCTOP_HOME", raising=False) + monkeypatch.setenv("OCTOP_DATA_DIR", str(tmp_path / "data")) + image_path = _workspace_image( + tmp_path / "data" / ".octop", + relative=Path("workspaces/main/generated/image.png"), + ) + + tool, access, _ = _image_tool() + result = await tool.ainvoke({"path": "/.octop/workspaces/main/generated/image.png"}) + + assert result == "Visual review result" + assert len(access.chat_model.calls) == 1 + assert image_path.is_file() + + +async def test_inspect_image_resolves_docker_materialize_path( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("OCTOP_HOME", str(tmp_path)) + materialized = _workspace_image( + tmp_path, + relative=Path("agents/main/.harness-materialize/analysis/page.png"), + ) + + tool, access, _ = _image_tool() + result = await tool.ainvoke({"path": "/root/.octop/workspaces/main/analysis/page.png"}) + + assert result == "Visual review result" + assert len(access.chat_model.calls) == 1 + assert materialized.is_file() + + +async def test_inspect_image_resolves_unique_relative_agent_path( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("OCTOP_HOME", str(tmp_path)) + image_path = _workspace_image( + tmp_path, + relative=Path("agents/main/analysis/page.png"), + ) + + tool, access, _ = _image_tool() + result = await tool.ainvoke({"path": "analysis/page.png"}) + + assert result == "Visual review result" + assert len(access.chat_model.calls) == 1 + assert image_path.is_file() From 00f1d5be0f780d755bbbee710b6fd25e1442ee90 Mon Sep 17 00:00:00 2001 From: Henry Zhou <739287804@qq.com> Date: Mon, 2 Nov 2026 17:30:43 +0800 Subject: [PATCH 4/4] fix(image-inspector): remap paths after host access errors # Conflicts: # CHANGELOG.md --- plugins/image-inspector/main.py | 6 ++++-- tests/unit/test_image_inspector_plugin.py | 26 +++++++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/plugins/image-inspector/main.py b/plugins/image-inspector/main.py index 60803aa716..07c500ec72 100644 --- a/plugins/image-inspector/main.py +++ b/plugins/image-inspector/main.py @@ -152,8 +152,10 @@ def _resolve_image_path(raw: str) -> Path | str: resolved = candidate.resolve(strict=True) except (FileNotFoundError, NotADirectoryError): continue - except OSError as exc: - return _error(f"cannot resolve image path: {exc}") + except OSError: + # A sandbox-facing path may be inaccessible on the host. Try the + # allowed data-root/materialized candidates before reporting failure. + continue if resolved in seen: continue seen.add(resolved) diff --git a/tests/unit/test_image_inspector_plugin.py b/tests/unit/test_image_inspector_plugin.py index ebb2758c7b..63aebfa2eb 100644 --- a/tests/unit/test_image_inspector_plugin.py +++ b/tests/unit/test_image_inspector_plugin.py @@ -319,3 +319,29 @@ async def test_inspect_image_resolves_unique_relative_agent_path( assert result == "Visual review result" assert len(access.chat_model.calls) == 1 assert image_path.is_file() + + +async def test_inspect_image_remaps_inaccessible_virtual_path( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("OCTOP_HOME", str(tmp_path)) + materialized = _workspace_image( + tmp_path, + relative=Path("agents/main/.harness-materialize/analysis/page.png"), + ) + virtual = tmp_path / "inaccessible" / "workspaces" / "main" / "analysis" / "page.png" + original_resolve = Path.resolve + + def resolve(path: Path, *args: Any, **kwargs: Any) -> Path: + if path == virtual: + raise PermissionError("virtual path is inaccessible on the host") + return original_resolve(path, *args, **kwargs) + + monkeypatch.setattr(Path, "resolve", resolve) + tool, access, _ = _image_tool() + result = await tool.ainvoke({"path": str(virtual)}) + + assert result == "Visual review result" + assert len(access.chat_model.calls) == 1 + assert materialized.is_file()