From 4ebc8053f63b2fc55c10d9375100b517bfffe0da Mon Sep 17 00:00:00 2001 From: Lebedev Nikita Date: Tue, 6 Oct 2026 14:39:26 +0400 Subject: [PATCH 1/3] fix: route research discovery to the Compose search service --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 8f4f5ab..10c9772 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -53,6 +53,7 @@ services: ports: - "127.0.0.1:11235:11235" environment: + SEARXNG_URL: "http://local-web-searxng:8080" LOCAL_WEB_RESEARCH_DB: "/app/work/research.sqlite3" LOCAL_WEB_BROWSER_CDP_URL: "${LOCAL_WEB_BROWSER_CDP_URL:-}" CRAWL4AI_DEFAULT_PROXY: "socks5://byedpi:1080" From 669cc37d619880b04d35dd02aa6839b4b6cd04c2 Mon Sep 17 00:00:00 2001 From: Lebedev Nikita Date: Tue, 6 Oct 2026 14:43:20 +0400 Subject: [PATCH 2/3] fix: validate service routes and expose research search gaps --- .github/workflows/ci.yml | 1 + docs/QUALITY.md | 2 +- scripts/quality/check.py | 1 + scripts/quality/compose_routes.py | 70 +++++++++++++++++++++++++++++++ service/research/router.py | 7 ++++ service/static/index.html | 2 +- service/static/view.js | 7 ++++ tests/test_quality_routes.py | 54 ++++++++++++++++++++++++ 8 files changed, 142 insertions(+), 2 deletions(-) create mode 100644 scripts/quality/compose_routes.py create mode 100644 tests/test_quality_routes.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4db9187..1927984 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,7 @@ name: Quality on: push: + branches: [main] pull_request: workflow_dispatch: permissions: diff --git a/docs/QUALITY.md b/docs/QUALITY.md index af30aad..6d94826 100644 --- a/docs/QUALITY.md +++ b/docs/QUALITY.md @@ -5,7 +5,7 @@ Install uv 0.11.13 and Python 3.12 or 3.13. Run `make setup`, `make hooks`, then `uv run --frozen pre-commit install --hook-type pre-commit --hook-type pre-push`, and `uv run --frozen python scripts/quality/check.py`. -The same command runs before commit, before push, and in CI. It checks lock freshness +The same command runs before commit, before push, and in CI. Docker Compose CLI is required; configuration validation runs offline without a daemon or service startup. The route guard rejects host loopback/published-port addresses for cross-service traffic. It checks lock freshness with `uv lock --check --offline`; frozen sync alone does not verify freshness. Ruff format and correctness lint cover service, MCP, tests and quality scripts. Mypy checks application/MCP and quality scripts, including untyped function bodies; diff --git a/scripts/quality/check.py b/scripts/quality/check.py index e6813d3..9f2a522 100644 --- a/scripts/quality/check.py +++ b/scripts/quality/check.py @@ -20,6 +20,7 @@ def main() -> None: os.chdir(ROOT) run("uv", "lock", "--check", "--offline") run(sys.executable, "scripts/quality/policy.py") + run(sys.executable, "scripts/quality/compose_routes.py") run( sys.executable, "-m", diff --git a/scripts/quality/compose_routes.py b/scripts/quality/compose_routes.py new file mode 100644 index 0000000..e797673 --- /dev/null +++ b/scripts/quality/compose_routes.py @@ -0,0 +1,70 @@ +"""Validate container-to-container endpoints in normalized Compose configuration.""" + +from __future__ import annotations + +import json +import subprocess +from typing import Any +from urllib.parse import urlsplit + +ROUTES = ( + ("crawl4ai", "SEARXNG_URL", "searxng"), + ("mcp", "SEARXNG_URL", "searxng"), + ("mcp", "CRAWL4AI_URL", "crawl4ai"), +) + + +def validate_routes(config: dict[str, Any]) -> None: + services = config.get("services", {}) + errors = [] + for caller, variable, target in ROUTES: + if caller not in services or target not in services: + errors.append(f"{caller}.{variable}: required caller/target service is missing") + continue + endpoint = services[caller].get("environment", {}).get(variable) + if not isinstance(endpoint, str) or not endpoint.strip(): + errors.append(f"{caller}.{variable}: explicit cross-service endpoint is required") + continue + destination = services[target] + hosts = {target, destination.get("container_name", target)} + internal_ports = { + int(port["target"]) + for port in destination.get("ports", []) + if isinstance(port, dict) and port.get("target") is not None + } + internal_ports.update(int(port) for port in destination.get("expose", [])) + try: + url = urlsplit(endpoint) + port = url.port or {"http": 80, "https": 443}.get(url.scheme) + valid = ( + url.scheme in {"http", "https"} + and url.hostname in hosts + and not url.username + and not url.password + and port in internal_ports + ) + except ValueError: + valid = False + if not valid: + errors.append( + f"{caller}.{variable}: must address declared {target} service/container " + "using its internal target port, not loopback or a published host port" + ) + if errors: + raise ValueError("\n".join(errors)) + + +def main() -> None: + result = subprocess.run( + ["docker", "compose", "config", "--format", "json"], + check=True, + capture_output=True, + text=True, + timeout=30, + ) + validate_routes(json.loads(result.stdout)) + print("Compose cross-service routes passed (normalized config; no daemon required)") + + +if __name__ == "__main__": + main() diff --git a/service/research/router.py b/service/research/router.py index 73853f3..f1405b9 100644 --- a/service/research/router.py +++ b/service/research/router.py @@ -85,6 +85,13 @@ def export(run_id: str, format: str = Query(default="markdown", pattern="^(markd json.dumps(data["coverage"]["counts"]), "", ] + gaps = [ + search + for search in data["coverage"]["searches"] + if search["data"].get("error") or search["data"].get("engine_errors") + ] + if gaps: + lines.extend(["## Search gaps", "", json.dumps(gaps, ensure_ascii=False), ""]) for item in data["sources"]: lines.extend( [ diff --git a/service/static/index.html b/service/static/index.html index db135e0..6ef9a07 100644 --- a/service/static/index.html +++ b/service/static/index.html @@ -7,7 +7,7 @@
RESEARCH / WORKSPACELOCAL FIRST
START WITH A QUESTION

Turn questions into evidence.

Discover sources, collect readable content, and keep the trail behind every finding.

Refine discovery Optional
- +
Local Web Research · Evidence before conclusions
SOURCE DETAIL

Import a browser result

If automated collection needs a browser, use Chrome DevTools, Playwright MCP, or agent-browser with the appropriate session. This form records content you have actually collected; it does not launch a browser.

Saved content

Collection history

diff --git a/service/static/view.js b/service/static/view.js index ec9a1d7..dd3afa5 100644 --- a/service/static/view.js +++ b/service/static/view.js @@ -51,6 +51,13 @@ export function renderCoverage(coverage, sources) { return card; })); if (coverage) { + $('collection-budget').textContent = `Requests ${coverage.requests_used || 0}/${coverage.max_requests || '—'} · Saved characters ${coverage.content_chars || 0}/${coverage.max_content_chars || '—'} · Exact content duplicates ${coverage.content_duplicates || 0}`; + const failures = (coverage.searches || []).filter(item => item.data?.error) + .map(item => `${item.query} / page ${item.page}: ${item.data.error}`); + const engines = (coverage.engine_errors || []).map(item => Array.isArray(item) ? item.join(': ') : String(item)); + const gaps = [...failures, ...engines]; + $('search-gaps').hidden = !gaps.length; + $('search-gaps').textContent = gaps.length ? `Search gaps (${gaps.length}): ${gaps.slice(0, 5).join(' · ')}. Retry discovery or use alternative queries; coverage remains incomplete.` : ''; $('coverage').setAttribute('title', `Requests: ${coverage.requests_used || 0}/${coverage.max_requests || '—'} · Content characters: ${coverage.content_chars || 0}/${coverage.max_content_chars || '—'}`); } } diff --git a/tests/test_quality_routes.py b/tests/test_quality_routes.py new file mode 100644 index 0000000..02a7b47 --- /dev/null +++ b/tests/test_quality_routes.py @@ -0,0 +1,54 @@ +"""Regression cases for the installed crawler's missing internal search route.""" + +from copy import deepcopy + +import pytest + +from scripts.quality.compose_routes import ROUTES, validate_routes + + +@pytest.fixture +def graph(): + return { + "services": { + "searxng": { + "container_name": "search-instance", + "ports": [{"target": 8080, "published": "8088"}], + }, + "crawl4ai": { + "container_name": "reader-instance", + "ports": [{"target": 11235, "published": "11236"}], + "environment": {"SEARXNG_URL": "http://search-instance:8080"}, + }, + "mcp": { + "environment": { + "SEARXNG_URL": "http://searxng:8080", + "CRAWL4AI_URL": "http://reader-instance:11235", + }, + }, + } + } + + +def test_valid_service_and_container_name_routes(graph): + validate_routes(graph) + alternate = deepcopy(graph) + alternate["services"]["crawl4ai"]["environment"]["SEARXNG_URL"] = "http://searxng:8080" + alternate["services"]["mcp"]["environment"]["CRAWL4AI_URL"] = "http://crawl4ai:11235" + validate_routes(alternate) + + +@pytest.mark.parametrize("caller,variable,target", ROUTES) +@pytest.mark.parametrize("failure", ["missing", "localhost", "published-port"]) +def test_reject_broken_cross_service_routes(graph, caller, variable, target, failure): + environment = graph["services"][caller]["environment"] + if failure == "missing": + environment.pop(variable) + elif failure == "localhost": + internal = graph["services"][target]["ports"][0]["target"] + environment[variable] = f"http://127.0.0.1:{internal}" + else: + published = graph["services"][target]["ports"][0]["published"] + environment[variable] = f"http://{target}:{published}" + with pytest.raises(ValueError, match=f"{caller}.{variable}"): + validate_routes(graph) From cc170c44877b6316869b9ed3d4827ac40864ac02 Mon Sep 17 00:00:00 2001 From: Lebedev Nikita Date: Tue, 6 Oct 2026 14:44:35 +0400 Subject: [PATCH 3/3] fix: preserve generated API documentation assets --- service/app_runtime.py | 4 +++- tests/test_service_api.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/service/app_runtime.py b/service/app_runtime.py index f7ebae8..7bac027 100644 --- a/service/app_runtime.py +++ b/service/app_runtime.py @@ -38,7 +38,9 @@ async def local_ui_security(request: Request, call_next): response = await call_next(request) response.headers["X-Content-Type-Options"] = "nosniff" response.headers["Referrer-Policy"] = "no-referrer" - if response.headers.get("content-type", "").startswith("text/html"): + if request.url.path in {"/", "/index.html"} and response.headers.get( + "content-type", "" + ).startswith("text/html"): response.headers["Content-Security-Policy"] = ( "default-src 'self'; script-src 'self'; style-src 'self'; " "img-src 'self' data:; connect-src 'self'; frame-ancestors 'none'; base-uri 'self'; form-action 'self'" diff --git a/tests/test_service_api.py b/tests/test_service_api.py index 0b0c584..ebb00d4 100644 --- a/tests/test_service_api.py +++ b/tests/test_service_api.py @@ -61,6 +61,7 @@ def test_dashboard_assets_and_csrf_boundary(): ) assert denied.status_code == 403 assert client.get("/health").json()["status"] == "healthy" + assert "content-security-policy" not in client.get("/docs").headers def test_config_helpers_and_proxy(tmp_path, monkeypatch):