diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a0dd92cba..90412f61aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ ## [Unreleased] +### 新增 +- 新增独立 `image-inspector` 工具插件:生成图片后可按需调用 `inspect_image`,将工作区原图送回当前多模态模型检查排版、遮挡、裁切和间距(#2283) + +### 修复 +- `image-inspector` 的视觉检查模型调用与父级会话流隔离,内部质检文本不会再被推送到 IM 或保存成额外的 AI 消息(#2286) +- `image-inspector` 支持自定义 `OCTOP_HOME` / `OCTOP_DATA_DIR` 和 Docker sandbox 的 `.harness-materialize` 文件路径,避免可见图片被误报为不存在 +- `image-inspector` 遇到宿主机不可访问的沙箱路径时,继续尝试已允许的数据目录与 materialized 路径,避免在找到可读图片前提前报错(#2286) + ## [1.0.3b7] - 2026-11-17 ### 新增 diff --git a/plugins/image-inspector/README.md b/plugins/image-inspector/README.md new file mode 100644 index 0000000000..4cce394f70 --- /dev/null +++ b/plugins/image-inspector/README.md @@ -0,0 +1,45 @@ +# Image Inspector + +An optional Octop tool plugin that invokes the current multimodal model to +review a workspace image and returns a textual visual-quality assessment. + +Use it for layout defects that OCR cannot detect: + +- clipped or overlapping text; +- misaligned or uneven cards; +- insufficient spacing; +- weak contrast or hierarchy; +- incorrect visual density. + +## Install independently + +```bash +octop plugin install ./plugins/image-inspector --force +``` + +Enable the plugin in Admin -> Plugins, then enable `inspect_image` for the +agent in Tool management. + +## Agent workflow + +1. Generate an image with `generate_image`. +2. Read the returned `localPath`. +3. Call `inspect_image` with that exact path when visual review is needed. +4. Use the returned text assessment before delivering or revising the image. + +The tool is manual. It resolves images from `OCTOP_HOME`, `OCTOP_DATA_DIR`, and +the default `~/.octop` layout, including Docker-sandbox files materialized under +`.harness-materialize/`. It does not run automatically after generation and does +not modify the image. Each call sends the image as a data URL to a dedicated +model request, so it consumes model tokens in addition to the outer agent turn. +That request is isolated from the parent conversation stream; its review is +returned only as the tool result and is not sent to the user as a separate +message. + +## Security + +- Images must be under an Octop `agents/` or `workspaces/` directory. +- Supported formats: PNG, JPEG, WebP. +- Maximum size: 20 MiB. +- HTTP(S) URL fetching is not supported. +- The plugin does not modify Octop package files. diff --git a/plugins/image-inspector/main.py b/plugins/image-inspector/main.py new file mode 100644 index 0000000000..07c500ec72 --- /dev/null +++ b/plugins/image-inspector/main.py @@ -0,0 +1,294 @@ +"""Return local workspace images to the current multimodal model.""" + +from __future__ import annotations + +import asyncio +import base64 +import io +import os +from dataclasses import dataclass +from pathlib import Path + +from langchain_core.messages import HumanMessage, SystemMessage +from octop_harness.llm.invoke_helpers import stringify_content +from octop_harness.plugins import PluginContext +from PIL import Image, UnidentifiedImageError + +_MAX_IMAGE_BYTES = 20 * 1024 * 1024 +_INTERNAL_LC_SOURCE = "image_inspector" +_MEDIA_TYPES = { + "PNG": "image/png", + "JPEG": "image/jpeg", + "WEBP": "image/webp", +} +_SYSTEM_PROMPT = """You are a precise visual quality reviewer. +Inspect the supplied image itself. Do not rely on OCR or guess unseen content. +Report visible defects involving clipping, overlap, alignment, spacing, +contrast, hierarchy, incorrect relative-date labels, or other visual mistakes. +State uncertain observations explicitly and do not invent details.""" + + +@dataclass(frozen=True) +class _LoadedImage: + path: Path + media_type: str + data: bytes + width: int + height: int + + +def _error(message: str) -> str: + return f"Error: {message}" + + +def _resolved(path: Path) -> Path: + try: + return path.expanduser().resolve() + except OSError: + return path.expanduser() + + +def _data_roots() -> list[Path]: + candidates: list[Path] = [] + home = os.environ.get("OCTOP_HOME", "").strip() + data_dir = os.environ.get("OCTOP_DATA_DIR", "").strip() + if home: + candidates.append(Path(home)) + if data_dir: + candidates.append(Path(data_dir)) + if Path(data_dir).name != ".octop": + candidates.append(Path(data_dir) / ".octop") + candidates.append(Path.home() / ".octop") + + roots: list[Path] = [] + seen: set[Path] = set() + for candidate in candidates: + resolved = _resolved(candidate) + if resolved in seen: + continue + seen.add(resolved) + roots.append(resolved) + return roots + + +def _allowed_roots() -> list[Path]: + roots: list[Path] = [] + for data_root in _data_roots(): + for name in ("agents", "workspaces"): + roots.append((data_root / name).resolve()) + return roots + + +def _path_hint(raw: str) -> tuple[str, str, Path] | None: + parts = [part for part in raw.replace("\\", "/").split("/") if part] + for index, part in enumerate(parts): + if part in {"agents", "workspaces"} and index + 2 < len(parts): + return part, parts[index + 1], Path(*parts[index + 2 :]) + return None + + +def _path_is_allowed(path: Path) -> bool: + for root in _allowed_roots(): + try: + path.relative_to(root) + return True + except ValueError: + continue + return False + + +def _candidate_paths(raw: str) -> list[Path]: + candidate = Path(raw).expanduser() + candidates = [candidate] + normalized = raw.replace("\\", "/") + hint = _path_hint(raw) + + for root in _data_roots(): + if normalized.startswith("/.octop/"): + candidates.append(root / normalized.lstrip("/")) + elif not candidate.is_absolute(): + candidates.append(root / candidate) + + if hint is None: + continue + namespace, agent_id, relative = hint + for resolved_namespace in dict.fromkeys((namespace, "agents", "workspaces")): + workspace = root / resolved_namespace / agent_id + candidates.append(workspace / relative) + candidates.append(workspace / ".harness-materialize" / relative) + return candidates + + +def _unique_workspace_match(relative: Path) -> list[Path]: + matches: list[Path] = [] + seen: set[Path] = set() + for root in _allowed_roots(): + if not root.is_dir(): + continue + try: + workspaces = root.iterdir() + except OSError: + continue + for workspace in workspaces: + if not workspace.is_dir(): + continue + for base in (workspace, workspace / ".harness-materialize"): + candidate = base / relative + try: + resolved = candidate.resolve(strict=True) + except (FileNotFoundError, NotADirectoryError, OSError): + continue + if resolved.is_file() and resolved not in seen: + seen.add(resolved) + matches.append(resolved) + return matches + + +def _resolve_image_path(raw: str) -> Path | str: + direct = Path(raw).expanduser() + seen: set[Path] = set() + for candidate in _candidate_paths(raw): + try: + resolved = candidate.resolve(strict=True) + except (FileNotFoundError, NotADirectoryError): + continue + except OSError: + # A sandbox-facing path may be inaccessible on the host. Try the + # allowed data-root/materialized candidates before reporting failure. + continue + if resolved in seen: + continue + seen.add(resolved) + if _path_is_allowed(resolved): + return resolved + + if not direct.is_absolute() and _path_hint(raw) is None: + matches = _unique_workspace_match(direct) + if len(matches) == 1: + return matches[0] + if len(matches) > 1: + return _error(f"relative image path is ambiguous across agent workspaces: {raw}") + + if direct.is_absolute(): + direct_resolved: Path | None + try: + direct_resolved = direct.resolve(strict=True) + except (FileNotFoundError, NotADirectoryError): + direct_resolved = None + except OSError as exc: + return _error(f"cannot resolve image path: {exc}") + if direct_resolved is not None: + return _error("path is outside the Octop agents/workspaces directories") + + return _error( + f"image file not found: {raw}. If it was created in a sandbox backend, " + "materialize it first with send_file_to_user or pass the .harness-materialize path." + ) + + +def _load_image(path: str) -> _LoadedImage | str: + raw = path.strip() + if not raw: + return _error("path is required") + + resolved = _resolve_image_path(raw) + if isinstance(resolved, str): + return resolved + + if not resolved.is_file(): + return _error(f"not a regular image file: {resolved}") + + try: + size_bytes = resolved.stat().st_size + except OSError as exc: + return _error(f"cannot stat image file: {exc}") + if size_bytes > _MAX_IMAGE_BYTES: + return _error(f"image is too large: {size_bytes} bytes; maximum is {_MAX_IMAGE_BYTES}") + + try: + data = resolved.read_bytes() + with Image.open(io.BytesIO(data)) as image: + image_format = str(image.format or "").upper() + width, height = image.size + image.verify() + except Image.DecompressionBombError: + return _error("image dimensions exceed Pillow safety limits") + except (UnidentifiedImageError, OSError, ValueError) as exc: + return _error(f"unsupported image format: {exc}") + + media_type = _MEDIA_TYPES.get(image_format) + if media_type is None: + return _error( + f"unsupported image format {image_format or 'unknown'}; use PNG, JPEG, or WebP" + ) + + return _LoadedImage( + path=resolved, + media_type=media_type, + data=data, + width=width, + height=height, + ) + + +def setup(ctx: PluginContext) -> None: + async def inspect_image(path: str, focus: str = "") -> str: + """Inspect a workspace image with the Agent's multimodal model.""" + loaded = await asyncio.to_thread(_load_image, path) + if isinstance(loaded, str): + return loaded + + encoded = base64.b64encode(loaded.data).decode("ascii") + focus_text = focus.strip() or ( + "Review the overall visual quality and identify concrete defects." + ) + messages = [ + SystemMessage(content=_SYSTEM_PROMPT), + HumanMessage( + content=[ + { + "type": "text", + "text": ( + f"Visually inspect {loaded.path.name} " + f"({loaded.width}x{loaded.height}).\nFocus: {focus_text}" + ), + }, + { + "type": "image_url", + "image_url": { + "url": f"data:{loaded.media_type};base64,{encoded}", + }, + }, + ], + ), + ] + try: + access = ctx.get_model_access("auto", messages=messages) + # The parent LangGraph stream must not expose this nested review call. + result = await access.chat_model.ainvoke( + messages, + config={ + "callbacks": [], + "metadata": {"lc_source": _INTERNAL_LC_SOURCE}, + }, + ) + except Exception as exc: # noqa: BLE001 — returned to the agent as a tool error + detail = str(exc).strip()[:240] or exc.__class__.__name__ + return _error(f"visual review model call failed: {detail}") + + text = stringify_content(getattr(result, "content", result)) + if not text: + return _error("visual review model returned an empty result") + return text + + ctx.tool( + "inspect_image", + inspect_image, + description=( + "Inspect a local PNG, JPEG, or WebP image from the Octop workspace " + "with the current multimodal model and return a textual visual-quality review. " + "Call manually after generate_image when the generated image will be delivered " + "or judged for visual quality. Pass the exact localPath returned by " + "generate_image. Do not use read_file for images. This tool does not fetch URLs." + ), + ) diff --git a/plugins/image-inspector/plugin.yaml b/plugins/image-inspector/plugin.yaml new file mode 100644 index 0000000000..45c87bfb8b --- /dev/null +++ b/plugins/image-inspector/plugin.yaml @@ -0,0 +1,7 @@ +id: image-inspector +version: 0.1.0 +name: Image Inspector +description: Optional visual review for workspace images generated by Octop. +icon: "🔍" +kind: tool +entry: main.py diff --git a/tests/unit/test_image_inspector_plugin.py b/tests/unit/test_image_inspector_plugin.py new file mode 100644 index 0000000000..63aebfa2eb --- /dev/null +++ b/tests/unit/test_image_inspector_plugin.py @@ -0,0 +1,347 @@ +"""Tests for the standalone image-inspector tool plugin.""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Any + +import pytest +from langchain_core.callbacks import BaseCallbackHandler +from langchain_core.language_models.fake_chat_models import FakeMessagesListChatModel +from langchain_core.messages import AIMessage +from langchain_core.runnables.config import var_child_runnable_config +from langchain_core.tools import StructuredTool +from octop_harness.plugins import PluginRegistry, build_plugin_tools, load_plugin_dir +from PIL import Image + +_PLUGIN_DIR = Path(__file__).resolve().parents[2] / "plugins" / "image-inspector" + + +@pytest.fixture(autouse=True) +def _reset_registry() -> None: + PluginRegistry.reset() + yield + PluginRegistry.reset() + + +def test_plugin_loads_and_registers_inspect_image() -> None: + loaded = load_plugin_dir(_PLUGIN_DIR, install_deps=False) + + assert loaded.manifest.id == "image-inspector" + assert loaded.manifest.kind == "tool" + assert [tool.name for tool in loaded.tools] == ["inspect_image"] + + tools = build_plugin_tools(global_plugins={"image-inspector": True}) + assert len(tools) == 1 + assert isinstance(tools[0], StructuredTool) + assert "generate_image" in tools[0].description + + +class _FakeModelAccess: + def __init__(self, response: str) -> None: + self.response = response + self.chat_model = _FakeChatModel(response) + + +class _FakeChatModel: + def __init__(self, response: str) -> None: + self.response = response + self.calls: list[tuple[list[Any], dict[str, Any]]] = [] + + async def ainvoke(self, messages: list[Any], **kwargs: Any) -> str: + self.calls.append((messages, kwargs)) + return self.response + + +class _FakeModelFactory: + def __init__(self, access: _FakeModelAccess) -> None: + self.access = access + self.specs: list[tuple[Any, dict[str, Any]]] = [] + + def get_for( + self, + spec: Any, + *, + get_protocol: Any = None, + **kwargs: Any, + ) -> _FakeModelAccess: + self.specs.append((spec, kwargs)) + return self.access + + +class _ChatModelStartCounter(BaseCallbackHandler): + def __init__(self) -> None: + self.starts = 0 + + def on_chat_model_start( + self, + serialized: dict[str, Any], + messages: list[list[Any]], + **kwargs: Any, + ) -> None: + self.starts += 1 + + +def _image_tool( + response: str = "Visual review result", +) -> tuple[StructuredTool, _FakeModelAccess, _FakeModelFactory]: + loaded = load_plugin_dir(_PLUGIN_DIR, install_deps=False) + assert [tool.name for tool in loaded.tools] == ["inspect_image"] + access = _FakeModelAccess(response) + factory = _FakeModelFactory(access) + assert loaded.context is not None + loaded.context.bind_model_factory(factory) + tools = build_plugin_tools(global_plugins={"image-inspector": True}) + assert len(tools) == 1 + return tools[0], access, factory + + +def _workspace_image( + tmp_path: Path, + *, + fmt: str = "PNG", + suffix: str = ".png", + relative: Path = Path("workspaces/main/generated/image.png"), +) -> Path: + path = tmp_path / relative.with_suffix(suffix) + path.parent.mkdir(parents=True) + Image.new("RGB", (2, 3), (255, 0, 0)).save(path, format=fmt) + return path + + +async def test_inspect_image_calls_multimodal_model_and_returns_text( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("OCTOP_HOME", str(tmp_path)) + image_path = _workspace_image(tmp_path) + tool, access, factory = _image_tool("Layout is clean; no clipping detected.") + + result = await tool.ainvoke({"path": str(image_path), "focus": "check clipping and alignment"}) + + assert result == "Layout is clean; no clipping detected." + assert factory.specs[0][0] == "auto" + assert "messages" in factory.specs[0][1] + assert len(access.chat_model.calls) == 1 + messages = access.chat_model.calls[0][0] + human_content = messages[1].content + assert isinstance(human_content, list) + image_blocks = [block for block in human_content if block["type"] == "image_url"] + assert len(image_blocks) == 1 + assert image_blocks[0]["image_url"]["url"].startswith("data:image/png;base64,") + text_blocks = [block for block in human_content if block["type"] == "text"] + assert "check clipping and alignment" in text_blocks[0]["text"] + + +async def test_inspect_image_isolates_visual_review_from_parent_stream( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("OCTOP_HOME", str(tmp_path)) + image_path = _workspace_image(tmp_path) + loaded = load_plugin_dir(_PLUGIN_DIR, install_deps=False) + access = _FakeModelAccess("Visual review result") + access.chat_model = FakeMessagesListChatModel( + responses=[AIMessage(content="Visual review result")] + ) + assert loaded.context is not None + loaded.context.bind_model_factory(_FakeModelFactory(access)) + tool = build_plugin_tools(global_plugins={"image-inspector": True})[0] + parent_handler = _ChatModelStartCounter() + token = var_child_runnable_config.set({"callbacks": [parent_handler]}) + try: + result = await tool.ainvoke({"path": str(image_path)}) + finally: + var_child_runnable_config.reset(token) + + assert result == "Visual review result" + assert parent_handler.starts == 0 + + +@pytest.mark.parametrize( + ("fmt", "suffix", "expected_media_type"), + [ + ("JPEG", ".jpg", "image/jpeg"), + ("WEBP", ".webp", "image/webp"), + ], +) +async def test_inspect_image_supports_common_raster_formats( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + fmt: str, + suffix: str, + expected_media_type: str, +) -> None: + monkeypatch.setenv("OCTOP_HOME", str(tmp_path)) + image_path = _workspace_image(tmp_path, fmt=fmt, suffix=suffix) + + tool, access, _ = _image_tool() + result = await tool.ainvoke({"path": str(image_path)}) + + assert result == "Visual review result" + human_content = access.chat_model.calls[0][0][1].content + image_block = next(block for block in human_content if block["type"] == "image_url") + assert image_block["image_url"]["url"].startswith(f"data:{expected_media_type};base64,") + + +async def test_inspect_image_rejects_outside_workspace( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("OCTOP_HOME", str(tmp_path)) + outside = tmp_path / "outside.png" + Image.new("RGB", (1, 1)).save(outside, format="PNG") + + tool, access, _ = _image_tool() + result = await tool.ainvoke({"path": str(outside)}) + + assert isinstance(result, str) + assert result.startswith("Error: path is outside") + assert access.chat_model.calls == [] + + +async def test_inspect_image_rejects_missing_file( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("OCTOP_HOME", str(tmp_path)) + path = tmp_path / "workspaces" / "main" / "generated" / "missing.png" + + tool, access, _ = _image_tool() + result = await tool.ainvoke({"path": str(path)}) + + assert isinstance(result, str) + assert "image file not found" in result + assert access.chat_model.calls == [] + + +async def test_inspect_image_rejects_directory( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("OCTOP_HOME", str(tmp_path)) + path = tmp_path / "workspaces" / "main" / "generated" / "folder.png" + path.mkdir(parents=True) + + tool, access, _ = _image_tool() + result = await tool.ainvoke({"path": str(path)}) + + assert isinstance(result, str) + assert "not a regular image file" in result + assert access.chat_model.calls == [] + + +async def test_inspect_image_rejects_unsupported_content( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("OCTOP_HOME", str(tmp_path)) + path = tmp_path / "workspaces" / "main" / "generated" / "not-image.png" + path.parent.mkdir(parents=True) + path.write_text("plain text", encoding="utf-8") + + tool, access, _ = _image_tool() + result = await tool.ainvoke({"path": str(path)}) + + assert isinstance(result, str) + assert "unsupported image format" in result + assert access.chat_model.calls == [] + + +async def test_inspect_image_rejects_oversized_file( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("OCTOP_HOME", str(tmp_path)) + image_path = _workspace_image(tmp_path) + tool, access, _ = _image_tool() + module = sys.modules["harness_plugin_image_inspector"] + monkeypatch.setattr(module, "_MAX_IMAGE_BYTES", 1) + + result = await tool.ainvoke({"path": str(image_path)}) + + assert isinstance(result, str) + assert "too large" in result + assert access.chat_model.calls == [] + + +async def test_inspect_image_uses_octop_data_dir( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("OCTOP_HOME", raising=False) + monkeypatch.setenv("OCTOP_DATA_DIR", str(tmp_path / "data")) + image_path = _workspace_image( + tmp_path / "data" / ".octop", + relative=Path("workspaces/main/generated/image.png"), + ) + + tool, access, _ = _image_tool() + result = await tool.ainvoke({"path": "/.octop/workspaces/main/generated/image.png"}) + + assert result == "Visual review result" + assert len(access.chat_model.calls) == 1 + assert image_path.is_file() + + +async def test_inspect_image_resolves_docker_materialize_path( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("OCTOP_HOME", str(tmp_path)) + materialized = _workspace_image( + tmp_path, + relative=Path("agents/main/.harness-materialize/analysis/page.png"), + ) + + tool, access, _ = _image_tool() + result = await tool.ainvoke({"path": "/root/.octop/workspaces/main/analysis/page.png"}) + + assert result == "Visual review result" + assert len(access.chat_model.calls) == 1 + assert materialized.is_file() + + +async def test_inspect_image_resolves_unique_relative_agent_path( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("OCTOP_HOME", str(tmp_path)) + image_path = _workspace_image( + tmp_path, + relative=Path("agents/main/analysis/page.png"), + ) + + tool, access, _ = _image_tool() + result = await tool.ainvoke({"path": "analysis/page.png"}) + + assert result == "Visual review result" + assert len(access.chat_model.calls) == 1 + assert image_path.is_file() + + +async def test_inspect_image_remaps_inaccessible_virtual_path( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("OCTOP_HOME", str(tmp_path)) + materialized = _workspace_image( + tmp_path, + relative=Path("agents/main/.harness-materialize/analysis/page.png"), + ) + virtual = tmp_path / "inaccessible" / "workspaces" / "main" / "analysis" / "page.png" + original_resolve = Path.resolve + + def resolve(path: Path, *args: Any, **kwargs: Any) -> Path: + if path == virtual: + raise PermissionError("virtual path is inaccessible on the host") + return original_resolve(path, *args, **kwargs) + + monkeypatch.setattr(Path, "resolve", resolve) + tool, access, _ = _image_tool() + result = await tool.ainvoke({"path": str(virtual)}) + + assert result == "Visual review result" + assert len(access.chat_model.calls) == 1 + assert materialized.is_file()