Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
name: Quality
on:
push:
branches: [main]
pull_request:
workflow_dispatch:
permissions:
Expand Down
1 change: 1 addition & 0 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ services:
ports:
- "127.0.0.1:11235:11235"
environment:
SEARXNG_URL: "http://local-web-searxng:8080"
LOCAL_WEB_RESEARCH_DB: "/app/work/research.sqlite3"
LOCAL_WEB_BROWSER_CDP_URL: "${LOCAL_WEB_BROWSER_CDP_URL:-}"
CRAWL4AI_DEFAULT_PROXY: "socks5://byedpi:1080"
Expand Down
2 changes: 1 addition & 1 deletion docs/QUALITY.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ Install uv 0.11.13 and Python 3.12 or 3.13. Run `make setup`, `make hooks`, then
`uv run --frozen pre-commit install --hook-type pre-commit --hook-type pre-push`,
and `uv run --frozen python scripts/quality/check.py`.

The same command runs before commit, before push, and in CI. It checks lock freshness
The same command runs before commit, before push, and in CI. Docker Compose CLI is required; configuration validation runs offline without a daemon or service startup. The route guard rejects host loopback/published-port addresses for cross-service traffic. It checks lock freshness
with `uv lock --check --offline`; frozen sync alone does not verify freshness.
Ruff format and correctness lint cover service, MCP, tests and quality scripts.
Mypy checks application/MCP and quality scripts, including untyped function bodies;
Expand Down
1 change: 1 addition & 0 deletions scripts/quality/check.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ def main() -> None:
os.chdir(ROOT)
run("uv", "lock", "--check", "--offline")
run(sys.executable, "scripts/quality/policy.py")
run(sys.executable, "scripts/quality/compose_routes.py")
run(
sys.executable,
"-m",
Expand Down
70 changes: 70 additions & 0 deletions scripts/quality/compose_routes.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
"""Validate container-to-container endpoints in normalized Compose configuration."""

from __future__ import annotations

import json
import subprocess
from typing import Any
from urllib.parse import urlsplit

ROUTES = (
("crawl4ai", "SEARXNG_URL", "searxng"),
("mcp", "SEARXNG_URL", "searxng"),
("mcp", "CRAWL4AI_URL", "crawl4ai"),
)


def validate_routes(config: dict[str, Any]) -> None:
services = config.get("services", {})
errors = []
for caller, variable, target in ROUTES:
if caller not in services or target not in services:
errors.append(f"{caller}.{variable}: required caller/target service is missing")
continue
endpoint = services[caller].get("environment", {}).get(variable)
if not isinstance(endpoint, str) or not endpoint.strip():
errors.append(f"{caller}.{variable}: explicit cross-service endpoint is required")
continue
destination = services[target]
hosts = {target, destination.get("container_name", target)}
internal_ports = {
int(port["target"])
for port in destination.get("ports", [])
if isinstance(port, dict) and port.get("target") is not None
}
internal_ports.update(int(port) for port in destination.get("expose", []))
try:
url = urlsplit(endpoint)
port = url.port or {"http": 80, "https": 443}.get(url.scheme)
valid = (
url.scheme in {"http", "https"}
and url.hostname in hosts
and not url.username
and not url.password
and port in internal_ports
)
except ValueError:
valid = False
if not valid:
errors.append(
f"{caller}.{variable}: must address declared {target} service/container "
"using its internal target port, not loopback or a published host port"
)
if errors:
raise ValueError("\n".join(errors))


def main() -> None:
result = subprocess.run(
["docker", "compose", "config", "--format", "json"],
check=True,
capture_output=True,
text=True,
timeout=30,
)
validate_routes(json.loads(result.stdout))
print("Compose cross-service routes passed (normalized config; no daemon required)")


if __name__ == "__main__":
main()
4 changes: 3 additions & 1 deletion service/app_runtime.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,9 @@ async def local_ui_security(request: Request, call_next):
response = await call_next(request)
response.headers["X-Content-Type-Options"] = "nosniff"
response.headers["Referrer-Policy"] = "no-referrer"
if response.headers.get("content-type", "").startswith("text/html"):
if request.url.path in {"/", "/index.html"} and response.headers.get(
"content-type", ""
).startswith("text/html"):
response.headers["Content-Security-Policy"] = (
"default-src 'self'; script-src 'self'; style-src 'self'; "
"img-src 'self' data:; connect-src 'self'; frame-ancestors 'none'; base-uri 'self'; form-action 'self'"
Expand Down
7 changes: 7 additions & 0 deletions service/research/router.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,13 @@ def export(run_id: str, format: str = Query(default="markdown", pattern="^(markd
json.dumps(data["coverage"]["counts"]),
"",
]
gaps = [
search
for search in data["coverage"]["searches"]
if search["data"].get("error") or search["data"].get("engine_errors")
]
if gaps:
lines.extend(["## Search gaps", "", json.dumps(gaps, ensure_ascii=False), ""])
for item in data["sources"]:
lines.extend(
[
Expand Down
2 changes: 1 addition & 1 deletion service/static/index.html
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
<aside class="sidebar" aria-label="Research navigation"><a class="brand" href="/" aria-label="Local Web home"><span class="brand-mark">LW</span><span>Local Web<small>Research workspace</small></span></a><button id="new-run" class="primary">+ New research</button><div class="sidebar-heading">YOUR RESEARCH <button id="refresh-runs" class="icon-button" aria-label="Refresh research list">↻</button></div><nav id="run-list" aria-label="Saved research runs"></nav><div class="local-note"><span class="online-dot"></span> Local workspace<p>Saved evidence. Traceable sources.<br>No claim of complete web coverage.</p></div></aside>
<main id="main" tabindex="-1"><header class="topbar"><span class="eyebrow">RESEARCH / WORKSPACE</span><span class="local-badge">LOCAL FIRST</span></header><div id="notice" role="status" aria-live="polite" hidden></div>
<section id="create-panel" class="panel create-panel"><div class="section-heading"><span class="eyebrow">START WITH A QUESTION</span><h1>Turn questions into evidence.</h1><p>Discover sources, collect readable content, and keep the trail behind every finding.</p></div><form id="run-form" data-testid="run-form"><div class="field"><label for="title">Research title</label><input id="title" name="title" data-testid="title" required maxlength="200" placeholder="e.g. The future of local AI tools"></div><div class="field"><label for="query">Research question</label><textarea id="query" name="query" data-testid="query" required maxlength="1000" rows="3" placeholder="What would you like to investigate?"></textarea></div><details class="advanced"><summary>Refine discovery <span>Optional</span></summary><div class="form-grid"><div class="field"><label for="variants">Query variants <small>One per line</small></label><textarea id="variants" rows="3" placeholder="Alternative searches"></textarea></div><div class="field"><label for="subtopics">Subtopics <small>One per line</small></label><textarea id="subtopics" rows="3" placeholder="Questions to cover"></textarea></div><div class="field"><label for="domains">Preferred domains <small>Comma separated</small></label><input id="domains" placeholder="example.org, docs.example.com"></div><div class="field"><label for="language">Search language</label><select id="language"><option value="en">English</option><option value="ru">Russian</option><option value="all">Any language</option></select></div><div class="field"><label for="budget">Source budget</label><input id="budget" type="number" min="1" max="100" value="20"></div></div></details><div class="form-footer"><span>You control discovery and collection.</span><button type="submit" class="primary">Create research <span aria-hidden="true">→</span></button></div></form></section>
<section id="run-panel" hidden><div class="run-heading"><div><span class="eyebrow">EVIDENCE COLLECTION</span><h1 id="run-title" tabindex="-1"></h1><p id="run-query"></p></div><span id="run-status" class="badge"></span></div><div class="actions"><button id="discover" data-testid="discover" class="primary">Discover sources</button><button id="collect" data-testid="collect">Collect next 5</button><button id="refresh">↻ Refresh</button><div class="export-actions"><a id="export-markdown" class="button" download>Export Markdown</a><a id="export-json" class="button" download>JSON</a></div></div><div id="coverage" class="coverage" aria-label="Collection coverage"></div><div id="browser-status" class="browser-status" role="status">Checking browser collection availability…</div><div class="coverage-note">Coverage describes the sources discovered in this run. The total relevant web universe is unknown.</div><section class="panel source-panel"><div class="source-heading"><div><h2>Source library <span id="source-count" data-testid="source-count">0</span></h2><p>Inspect content, collection history, and next steps.</p></div><div class="filters"><label class="sr-only" for="source-search">Search sources</label><input id="source-search" type="search" placeholder="Search title, URL or reason"><label class="sr-only" for="source-filter">Filter source status</label><select id="source-filter" data-testid="source-filter"><option value="">All statuses</option></select></div></div><div class="table-scroll"><table><thead><tr><th scope="col">Source</th><th scope="col">Status</th><th scope="col">Method / attempts</th><th scope="col">Discovery trail</th><th scope="col">Action</th></tr></thead><tbody id="sources"></tbody></table></div><div id="source-empty" class="empty" hidden><span aria-hidden="true">◎</span><h3>No sources yet</h3><p>Discover sources to begin building your evidence library.</p></div></section></section>
<section id="run-panel" hidden><div class="run-heading"><div><span class="eyebrow">EVIDENCE COLLECTION</span><h1 id="run-title" tabindex="-1"></h1><p id="run-query"></p></div><span id="run-status" class="badge"></span></div><div class="actions"><button id="discover" data-testid="discover" class="primary">Discover sources</button><button id="collect" data-testid="collect">Collect next 5</button><button id="refresh">↻ Refresh</button><div class="export-actions"><a id="export-markdown" class="button" download>Export Markdown</a><a id="export-json" class="button" download>JSON</a></div></div><div id="coverage" class="coverage" aria-label="Collection coverage"></div><div id="collection-budget" class="coverage-note" aria-live="polite"></div><div id="search-gaps" class="browser-status" role="status" hidden></div><div id="browser-status" class="browser-status" role="status">Checking browser collection availability…</div><div class="coverage-note">Coverage describes the sources discovered in this run. The total relevant web universe is unknown.</div><section class="panel source-panel"><div class="source-heading"><div><h2>Source library <span id="source-count" data-testid="source-count">0</span></h2><p>Inspect content, collection history, and next steps.</p></div><div class="filters"><label class="sr-only" for="source-search">Search sources</label><input id="source-search" type="search" placeholder="Search title, URL or reason"><label class="sr-only" for="source-filter">Filter source status</label><select id="source-filter" data-testid="source-filter"><option value="">All statuses</option></select></div></div><div class="table-scroll"><table><thead><tr><th scope="col">Source</th><th scope="col">Status</th><th scope="col">Method / attempts</th><th scope="col">Discovery trail</th><th scope="col">Action</th></tr></thead><tbody id="sources"></tbody></table></div><div id="source-empty" class="empty" hidden><span aria-hidden="true">◎</span><h3>No sources yet</h3><p>Discover sources to begin building your evidence library.</p></div></section></section>
<footer>Local Web Research <span>·</span> Evidence before conclusions</footer></main></div>
<dialog id="source-dialog" aria-labelledby="detail-title"><div id="detail-notice" role="status" aria-live="polite" hidden></div><div class="drawer-header"><span class="eyebrow">SOURCE DETAIL</span><button id="close-detail" class="icon-button" aria-label="Close source details">✕</button></div><h2 id="detail-title"></h2><a id="detail-url" target="_blank" rel="noopener noreferrer"></a><div id="detail-meta" class="detail-meta"></div><div id="detail-reason" class="reason"></div><div class="actions"><button id="retry-source">Retry collection</button><button id="skip-source">Skip source</button></div><details id="browser-import"><summary>Import a browser result</summary><p class="muted">If automated collection needs a browser, use Chrome DevTools, Playwright MCP, or agent-browser with the appropriate session. This form records content you have actually collected; it does not launch a browser.</p><form id="browser-form"><div class="field"><label for="browser-url">Final URL</label><input id="browser-url" type="url" required></div><div class="field"><label for="browser-title">Browser page title</label><input id="browser-title" required maxlength="300" placeholder="Title from the collected browser page"></div><div class="field"><label for="browser-method">Collection method</label><select id="browser-method"><option value="playwright-mcp">Playwright MCP</option><option value="chrome-devtools">Chrome DevTools</option><option value="agent-browser">agent-browser</option></select></div><div class="field"><label for="browser-markdown">Collected Markdown</label><textarea id="browser-markdown" rows="6" required placeholder="Paste content collected in the browser"></textarea></div><button class="primary" type="submit">Save browser result</button></form></details><h3>Saved content</h3><pre id="detail-content" tabindex="0"></pre><h3>Collection history</h3><pre id="detail-history" tabindex="0"></pre></dialog>
</body></html>
7 changes: 7 additions & 0 deletions service/static/view.js
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,13 @@ export function renderCoverage(coverage, sources) {
return card;
}));
if (coverage) {
$('collection-budget').textContent = `Requests ${coverage.requests_used || 0}/${coverage.max_requests || '—'} · Saved characters ${coverage.content_chars || 0}/${coverage.max_content_chars || '—'} · Exact content duplicates ${coverage.content_duplicates || 0}`;
const failures = (coverage.searches || []).filter(item => item.data?.error)
.map(item => `${item.query} / page ${item.page}: ${item.data.error}`);
const engines = (coverage.engine_errors || []).map(item => Array.isArray(item) ? item.join(': ') : String(item));
const gaps = [...failures, ...engines];
$('search-gaps').hidden = !gaps.length;
$('search-gaps').textContent = gaps.length ? `Search gaps (${gaps.length}): ${gaps.slice(0, 5).join(' · ')}. Retry discovery or use alternative queries; coverage remains incomplete.` : '';
$('coverage').setAttribute('title', `Requests: ${coverage.requests_used || 0}/${coverage.max_requests || '—'} · Content characters: ${coverage.content_chars || 0}/${coverage.max_content_chars || '—'}`);
}
}
Expand Down
54 changes: 54 additions & 0 deletions tests/test_quality_routes.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
"""Regression cases for the installed crawler's missing internal search route."""

from copy import deepcopy

import pytest

from scripts.quality.compose_routes import ROUTES, validate_routes


@pytest.fixture
def graph():
return {
"services": {
"searxng": {
"container_name": "search-instance",
"ports": [{"target": 8080, "published": "8088"}],
},
"crawl4ai": {
"container_name": "reader-instance",
"ports": [{"target": 11235, "published": "11236"}],
"environment": {"SEARXNG_URL": "http://search-instance:8080"},
},
"mcp": {
"environment": {
"SEARXNG_URL": "http://searxng:8080",
"CRAWL4AI_URL": "http://reader-instance:11235",
},
},
}
}


def test_valid_service_and_container_name_routes(graph):
validate_routes(graph)
alternate = deepcopy(graph)
alternate["services"]["crawl4ai"]["environment"]["SEARXNG_URL"] = "http://searxng:8080"
alternate["services"]["mcp"]["environment"]["CRAWL4AI_URL"] = "http://crawl4ai:11235"
validate_routes(alternate)


@pytest.mark.parametrize("caller,variable,target", ROUTES)
@pytest.mark.parametrize("failure", ["missing", "localhost", "published-port"])
def test_reject_broken_cross_service_routes(graph, caller, variable, target, failure):
environment = graph["services"][caller]["environment"]
if failure == "missing":
environment.pop(variable)
elif failure == "localhost":
internal = graph["services"][target]["ports"][0]["target"]
environment[variable] = f"http://127.0.0.1:{internal}"
else:
published = graph["services"][target]["ports"][0]["published"]
environment[variable] = f"http://{target}:{published}"
with pytest.raises(ValueError, match=f"{caller}.{variable}"):
validate_routes(graph)
1 change: 1 addition & 0 deletions tests/test_service_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ def test_dashboard_assets_and_csrf_boundary():
)
assert denied.status_code == 403
assert client.get("/health").json()["status"] == "healthy"
assert "content-security-policy" not in client.get("/docs").headers


def test_config_helpers_and_proxy(tmp_path, monkeypatch):
Expand Down