From a9314281b190c2ee95eb27f40275e5e5c9071546 Mon Sep 17 00:00:00 2001 From: William Loo Date: Tue, 3 Nov 2026 13:34:08 -0800 Subject: [PATCH 1/6] feat(website): the browser client, env website new and env website dev (FD-4683) This PR's change, rebased as one commit onto #445. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../examples/website/notes/README.md | 11 +- .../examples/website/notes/web/src/main.js | 33 +- .../website/notes/web/src/site-client.js | 84 +++ .../src/agentenv_protocol/website/__init__.py | 4 +- .../website/client/site-client.d.ts | 36 ++ .../website/client/site-client.js | 84 +++ .../tests/test_website_notes_example.py | 6 + src/agent_env/cli/env/website.py | 143 ++++- src/agent_env/cli/env/website_dev.py | 336 +++++++++++ src/agent_env/cli/env/website_scaffold.py | 534 ++++++++++++++++++ .../Dockerfile.dockerignore.tmpl | 6 + .../cli/env/website_template/Dockerfile.tmpl | 22 + .../cli/env/website_template/README.md.tmpl | 40 ++ .../env/website_template/data/seed.json.tmpl | 11 + .../cli/env/website_template/env.toml.tmpl | 3 + .../cli/env/website_template/gitignore.tmpl | 3 + .../env/website_template/site-tools.py.tmpl | 36 ++ .../cli/env/website_template/site.py.tmpl | 78 +++ .../env/website_template/web/index.html.tmpl | 24 + .../website_template/web/package.json.tmpl | 14 + .../web/src/main-tools.js.tmpl | 68 +++ .../env/website_template/web/src/main.js.tmpl | 85 +++ .../website_template/web/src/style.css.tmpl | 46 ++ .../website_template/web/vite.config.js.tmpl | 11 + tst/unit/cli/website_dev_cli_test.py | 310 ++++++++++ tst/unit/cli/website_new_cli_test.py | 252 +++++++++ tst/unit/cli/website_server_image_cli_test.py | 15 +- tst/unit/config/test_explain_and_sources.py | 3 +- 28 files changed, 2258 insertions(+), 40 deletions(-) create mode 100644 packages/agentenv-protocol/examples/website/notes/web/src/site-client.js create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/website/client/site-client.d.ts create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/website/client/site-client.js create mode 100644 src/agent_env/cli/env/website_dev.py create mode 100644 src/agent_env/cli/env/website_scaffold.py create mode 100644 src/agent_env/cli/env/website_template/Dockerfile.dockerignore.tmpl create mode 100644 src/agent_env/cli/env/website_template/Dockerfile.tmpl create mode 100644 src/agent_env/cli/env/website_template/README.md.tmpl create mode 100644 src/agent_env/cli/env/website_template/data/seed.json.tmpl create mode 100644 src/agent_env/cli/env/website_template/env.toml.tmpl create mode 100644 src/agent_env/cli/env/website_template/gitignore.tmpl create mode 100644 src/agent_env/cli/env/website_template/site-tools.py.tmpl create mode 100644 src/agent_env/cli/env/website_template/site.py.tmpl create mode 100644 src/agent_env/cli/env/website_template/web/index.html.tmpl create mode 100644 src/agent_env/cli/env/website_template/web/package.json.tmpl create mode 100644 src/agent_env/cli/env/website_template/web/src/main-tools.js.tmpl create mode 100644 src/agent_env/cli/env/website_template/web/src/main.js.tmpl create mode 100644 src/agent_env/cli/env/website_template/web/src/style.css.tmpl create mode 100644 src/agent_env/cli/env/website_template/web/vite.config.js.tmpl create mode 100644 tst/unit/cli/website_dev_cli_test.py create mode 100644 tst/unit/cli/website_new_cli_test.py diff --git a/packages/agentenv-protocol/examples/website/notes/README.md b/packages/agentenv-protocol/examples/website/notes/README.md index 6bd558790..861d6d643 100644 --- a/packages/agentenv-protocol/examples/website/notes/README.md +++ b/packages/agentenv-protocol/examples/website/notes/README.md @@ -9,7 +9,7 @@ them. The page shows the universe's time. | `website.toml` | The site's spec: its name, domains, health path, source, seed, collections, personas, roles, routes, state read and clock (see `docs/website-spec.md`). | | `site.py` | The server: the `Notes` WebsiteEnvironment, declared from `website.toml`, its `@api` routes, and `serve()` on `PORT` (8000). | | `data/seed.json` | The seed data: two users (`ann`, `ben`), their notes, and `current_user`, the acting user. | -| `web/` | A minimal Vite app (`index.html`, `src/main.js`, `src/style.css`). Its assets (`base: "./"`) and API calls are relative to the page, so it works at `/` and under the gateway's `/website/notes/` alike; opened at a path without its final slash, such as `/website/notes`, the page first reloads itself with it. | +| `web/` | A minimal Vite app (`index.html`, `src/main.js`, `src/style.css`, and `src/site-client.js`, a copy of the SDK's browser client). Its assets (`base: "./"`) and API calls are relative to the page, so it works at `/` and under the gateway's `/website/notes/` alike; opened at a path without its final slash, such as `/website/notes`, the page first reloads itself with it. | | `Dockerfile` | The server image: Node builds `web/`, then Python installs this repo's `packages/agentenv-protocol` with uvicorn. It installs `curl`, which the gateway's compose health check runs. | | `env.toml` | The website env as a bundle entry, pointing at `website.toml`. | @@ -38,7 +38,8 @@ them. The page shows the universe's time. - **Time.** Notes are stamped with `self.now()`, which follows the gateway's clock once `clock/v1`'s `sync_time` names it. The frontend reads `X-Universe-Now` from every API response into `universeNow()` and shows it in the header. It uses this helper rather than `clock_patch=True`, so the page works the same against a server written without the - SDK. + SDK. The helper and the page's `api()`, `session()` and `signIn()` come from `src/site-client.js`, a copy of + `agentenv_protocol/website/client/site-client.js`, which a test keeps it the same as. - **Readiness.** `@ready_check seeded` holds `/healthz` at 503 until the store has users, for example after a `data/reset` and before the next `data/add`. @@ -54,6 +55,10 @@ PYTHONPATH=/packages/agentenv-protocol/src python site.py Open http://localhost:8000. To work on the frontend, run `npm run dev` in `web/` while `site.py` runs; Vite proxies `/api` to port 8000. +Or run `agent-env env website dev` from this folder: it builds `web/` when it is out of date and rebuilds it as it +changes, serves `site.py` with reload, syncs it to a local stand-in for the gateway's clock, loads `data/seed.json` +through the data plane after every reload, and runs the conformance check once. + ## Test it ```bash @@ -88,7 +93,7 @@ the container before using it. ```bash agent-env env website put --id notes-site \ --server-dockerfile packages/agentenv-protocol/examples/website/notes/Dockerfile \ - --server-docker-context . --health-path /healthz + --server-docker-context . ``` Run it from the repo root. `environment_name` comes from the card, which the spec names `notes`. The stored env diff --git a/packages/agentenv-protocol/examples/website/notes/web/src/main.js b/packages/agentenv-protocol/examples/website/notes/web/src/main.js index 61ead9d31..3996ac11e 100644 --- a/packages/agentenv-protocol/examples/website/notes/web/src/main.js +++ b/packages/agentenv-protocol/examples/website/notes/web/src/main.js @@ -1,6 +1,6 @@ import "./style.css"; +import { api, followUniverseClock, session, signIn, universeNow } from "./site-client.js"; -let offsetMs = 0; let filter = "all"; let renders = 0; const saving = new Map(); @@ -9,27 +9,6 @@ const landed = new Map(); const drafts = new Map(); const CLOCK_REFRESH_MS = 5000; -export function universeNow() { - return new Date(Date.now() + offsetMs); -} - -function followUniverse(response) { - const at = Date.parse(response.headers.get("X-Universe-Now") || ""); - if (!Number.isNaN(at)) offsetMs = at - Date.now(); -} - -async function api(path, { method = "GET", body } = {}) { - const response = await fetch(path, { - method, - headers: body === undefined ? {} : { "Content-Type": "application/json" }, - body: body === undefined ? undefined : JSON.stringify(body), - }); - followUniverse(response); - const data = await response.json().catch(() => null); - if (!response.ok) throw new Error(data?.error || `${method} ${path}: ${response.status}`); - return data; -} - const $ = (id) => document.getElementById(id); function setDraft(id, kind, value, saved) { @@ -77,7 +56,7 @@ function queueSave(id, fields) { } function showError(error) { - $("error").textContent = error ? error.message : ""; + $("error").textContent = error ? error.error || error.message : ""; $("error").hidden = !error; } @@ -193,9 +172,9 @@ async function run(action) { } async function start() { - const [session, users] = await Promise.all([api("api/session"), api("api/users")]); + const [signedIn, users] = await Promise.all([session(), api("api/users")]); $("user").replaceChildren(...users.map((user) => new Option(user.name || user.id, user.id))); - let signedInAs = session.user ?? ""; + let signedInAs = signedIn.user ?? ""; $("user").value = signedInAs; $("user").addEventListener("change", (event) => { const user = event.target.value; @@ -208,7 +187,7 @@ async function start() { const failed = pending.find((state) => state.error); if (failed) throw failed.error; } - await api("api/sign-in", { method: "POST", body: { user } }); + await signIn(user); signedInAs = user; drafts.clear(); } finally { @@ -245,7 +224,7 @@ async function start() { tick(); setInterval(tick, 1000); - setInterval(() => fetch("healthz").then(followUniverse).catch(() => {}), CLOCK_REFRESH_MS); + followUniverseClock({ path: "healthz", everyMs: CLOCK_REFRESH_MS }); await render(); } diff --git a/packages/agentenv-protocol/examples/website/notes/web/src/site-client.js b/packages/agentenv-protocol/examples/website/notes/web/src/site-client.js new file mode 100644 index 000000000..2d5b65a39 --- /dev/null +++ b/packages/agentenv-protocol/examples/website/notes/web/src/site-client.js @@ -0,0 +1,84 @@ +// The browser client of a site on agentenv_protocol.website: JSON calls to the site's API, the universe's time from +// X-Universe-Now, and the signed-in user. It has no dependencies. Paths are relative to the page, so a site works at +// "/" and under the gateway's "/website//" alike: a leading "/" is dropped. + +let offsetMs = 0; + +// An answer other than success: `status` (0 when the server couldn't be reached), and the site's `code` and `error`. +export class ApiError extends Error { + constructor(status, code, error, body) { + super(error); + this.name = "ApiError"; + this.status = status; + this.code = code; + this.error = error; + this.body = body; + } +} + +// The universe's time: the X-Universe-Now of the latest answer, extrapolated at real speed since it came. An idle +// page keeps it right with followUniverseClock(). +export function universeNow() { + return new Date(Date.now() + offsetMs); +} + +// Read X-Universe-Now from `path`, the site's health path, every `everyMs`, so universeNow() follows a clock that +// stops or runs at another speed while the page makes no calls. Returns a function that stops it. +export function followUniverseClock({ path = "api/health", everyMs = 5000 } = {}) { + const id = setInterval(() => fetch(relative(path), { credentials: "same-origin" }).then(follow).catch(() => {}), everyMs); + return () => clearInterval(id); +} + +function follow(response) { + const at = Date.parse(response.headers.get("X-Universe-Now") || ""); + if (!Number.isNaN(at)) offsetMs = at - Date.now(); +} + +function relative(path) { + return String(path).replace(/^\/+/, ""); +} + +// Call the site's API: `body` is sent as JSON; the answer's JSON is returned, or an ApiError thrown. +export async function api(path, { method = "GET", body, headers = {} } = {}) { + const url = relative(path); + let response; + try { + response = await fetch(url, { + method, + credentials: "same-origin", + headers: body === undefined ? headers : { "Content-Type": "application/json", ...headers }, + body: body === undefined ? undefined : JSON.stringify(body), + }); + } catch (cause) { + throw new ApiError(0, "network_error", `${method} ${url}: ${cause?.message || cause}`, null); + } + follow(response); + const text = await response.text(); + let data = null; + try { + data = text ? JSON.parse(text) : null; + } catch { + data = text; + } + if (!response.ok) { + const error = data && typeof data === "object" && data.error ? data.error : `${method} ${url}: ${response.status}`; + const code = data && typeof data === "object" && data.code ? data.code : "http_error"; + throw new ApiError(response.status, code, error, data); + } + return data; +} + +// Who the browser is signed in as: {user, role, signed_in}. A browser with no session is signed in as the site's +// acting user. +export function session() { + return api("api/session"); +} + +// Sign in as `user` through the site's own sign-in route, `api/sign-in` unless `path` names another. +export function signIn(user, { path = "api/sign-in" } = {}) { + return api(path, { method: "POST", body: { user } }); +} + +export function signOut() { + return api("api/session", { method: "DELETE" }); +} diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/website/__init__.py index 455f9010c..f28e270be 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/__init__.py @@ -4,7 +4,9 @@ package, and uvicorn to serve. The class is optional: a server in any language that meets the contract works, which ``python -m agentenv_protocol.website.conformance `` checks. Whether a site is ready is its ``WebsiteReadiness`` answer to ``ready``, which ``website_readiness`` reads of one site and -``env_readiness`` of every site of an env. What a task left a site with is its data plane's ``data/get``.""" +``env_readiness`` of every site of an env. What a task left a site with is its data plane's ``data/get``. A site's pages +can call its API with ``client/site-client.js``, a dependency-free ES module (with its ``.d.ts``) to copy into them: +``api()``, ``universeNow()`` from ``X-Universe-Now``, ``session()``, ``signIn()`` and ``signOut()``.""" from __future__ import annotations diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/client/site-client.d.ts b/packages/agentenv-protocol/src/agentenv_protocol/website/client/site-client.d.ts new file mode 100644 index 000000000..4d37b8a22 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/client/site-client.d.ts @@ -0,0 +1,36 @@ +/** An answer other than success: `status` (0 when the server couldn't be reached), and the site's `code` and `error`. */ +export declare class ApiError extends Error { + readonly status: number; + readonly code: string; + readonly error: string; + readonly body: unknown; + constructor(status: number, code: string, error: string, body: unknown); +} + +export interface ApiOptions { + method?: "GET" | "POST" | "PUT" | "PATCH" | "DELETE" | string; + body?: unknown; + headers?: Record; +} + +export interface Session { + user: string | null; + role: string; + signed_in: boolean; +} + +/** The universe's time: the X-Universe-Now of the latest answer, extrapolated at real speed since it came. */ +export declare function universeNow(): Date; +/** Read X-Universe-Now from `path`, the site's health path, every `everyMs` (5000), so universeNow() follows a clock that stops or runs at another speed; returns a function that stops it. */ +export declare function followUniverseClock(options?: { path?: string; everyMs?: number }): () => void; + +/** Call the site's API at `path`, relative to the page; returns the answer's JSON or throws an ApiError. */ +export declare function api(path: string, options?: ApiOptions): Promise; + +/** Who the browser is signed in as; a browser with no session is signed in as the site's acting user. */ +export declare function session(): Promise; + +/** Sign in as `user` through the site's own sign-in route, `api/sign-in` unless `path` names another. */ +export declare function signIn(user: string, options?: { path?: string }): Promise; + +export declare function signOut(): Promise; diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/client/site-client.js b/packages/agentenv-protocol/src/agentenv_protocol/website/client/site-client.js new file mode 100644 index 000000000..2d5b65a39 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/client/site-client.js @@ -0,0 +1,84 @@ +// The browser client of a site on agentenv_protocol.website: JSON calls to the site's API, the universe's time from +// X-Universe-Now, and the signed-in user. It has no dependencies. Paths are relative to the page, so a site works at +// "/" and under the gateway's "/website//" alike: a leading "/" is dropped. + +let offsetMs = 0; + +// An answer other than success: `status` (0 when the server couldn't be reached), and the site's `code` and `error`. +export class ApiError extends Error { + constructor(status, code, error, body) { + super(error); + this.name = "ApiError"; + this.status = status; + this.code = code; + this.error = error; + this.body = body; + } +} + +// The universe's time: the X-Universe-Now of the latest answer, extrapolated at real speed since it came. An idle +// page keeps it right with followUniverseClock(). +export function universeNow() { + return new Date(Date.now() + offsetMs); +} + +// Read X-Universe-Now from `path`, the site's health path, every `everyMs`, so universeNow() follows a clock that +// stops or runs at another speed while the page makes no calls. Returns a function that stops it. +export function followUniverseClock({ path = "api/health", everyMs = 5000 } = {}) { + const id = setInterval(() => fetch(relative(path), { credentials: "same-origin" }).then(follow).catch(() => {}), everyMs); + return () => clearInterval(id); +} + +function follow(response) { + const at = Date.parse(response.headers.get("X-Universe-Now") || ""); + if (!Number.isNaN(at)) offsetMs = at - Date.now(); +} + +function relative(path) { + return String(path).replace(/^\/+/, ""); +} + +// Call the site's API: `body` is sent as JSON; the answer's JSON is returned, or an ApiError thrown. +export async function api(path, { method = "GET", body, headers = {} } = {}) { + const url = relative(path); + let response; + try { + response = await fetch(url, { + method, + credentials: "same-origin", + headers: body === undefined ? headers : { "Content-Type": "application/json", ...headers }, + body: body === undefined ? undefined : JSON.stringify(body), + }); + } catch (cause) { + throw new ApiError(0, "network_error", `${method} ${url}: ${cause?.message || cause}`, null); + } + follow(response); + const text = await response.text(); + let data = null; + try { + data = text ? JSON.parse(text) : null; + } catch { + data = text; + } + if (!response.ok) { + const error = data && typeof data === "object" && data.error ? data.error : `${method} ${url}: ${response.status}`; + const code = data && typeof data === "object" && data.code ? data.code : "http_error"; + throw new ApiError(response.status, code, error, data); + } + return data; +} + +// Who the browser is signed in as: {user, role, signed_in}. A browser with no session is signed in as the site's +// acting user. +export function session() { + return api("api/session"); +} + +// Sign in as `user` through the site's own sign-in route, `api/sign-in` unless `path` names another. +export function signIn(user, { path = "api/sign-in" } = {}) { + return api(path, { method: "POST", body: { user } }); +} + +export function signOut() { + return api("api/session", { method: "DELETE" }); +} diff --git a/packages/agentenv-protocol/tests/test_website_notes_example.py b/packages/agentenv-protocol/tests/test_website_notes_example.py index 2b4308a65..2b61707d2 100644 --- a/packages/agentenv-protocol/tests/test_website_notes_example.py +++ b/packages/agentenv-protocol/tests/test_website_notes_example.py @@ -237,3 +237,9 @@ def test_the_built_page_loads_its_assets_and_api_under_the_gateways_prefix() -> assert not [path for path in called if path.startswith("/")], called assert browser.get(urljoin(page_url, "api/session")).json()["user"] == "ann" assert [u["id"] for u in browser.get(urljoin(page_url, "api/users")).json()] == ["ann", "ben"] + + +def test_the_page_uses_the_sdks_browser_client_as_the_package_ships_it() -> None: + shipped = Path(website.__code__.co_filename).parent / "client" / "site-client.js" + assert (NOTES_DIR / "web" / "src" / "site-client.js").read_text(encoding="utf-8") == shipped.read_text(encoding="utf-8") + assert 'from "./site-client.js"' in (NOTES_DIR / "web" / "src" / "main.js").read_text(encoding="utf-8") diff --git a/src/agent_env/cli/env/website.py b/src/agent_env/cli/env/website.py index 7332e3df9..de7d75aa0 100644 --- a/src/agent_env/cli/env/website.py +++ b/src/agent_env/cli/env/website.py @@ -1,11 +1,14 @@ import asyncio import os +import subprocess import sys from pathlib import Path import click from agent_env.artifact import DockerImageArtifact, EnvironmentArtifact +from agent_env.cli.env import website_dev +from agent_env.cli.env.website_scaffold import ScaffoldError, scaffold, tools_from_json, tools_from_mcp from agent_env.cli.utils import ( deployed_env_from_instance, build_platform_option, @@ -29,6 +32,132 @@ def website(): pass +@website.command() +@click.argument("name") +@click.option("--dir", "directory", default=None, type=click.Path(file_okay=False, path_type=Path), + help="Where to write the site (defaults to ./NAME)") +@click.option("--from-mcp", "from_mcp", default=None, type=click.Path(exists=True, path_type=Path), + help="An MCP server's source, a folder or .py file, whose @tool methods become stub @api routes; read " + "without running it") +@click.option("--from-tools-json", "from_tools_json", default=None, type=click.Path(exists=True, dir_okay=False, + path_type=Path), help="A JSON file of the tools to make stub @api routes of: tools/list's answer, a " + "list of {name, description, inputSchema}, or an environment card") +@click.option("--domain", default=None, help="The domain the site is declared at (defaults to NAME.test)") +@click.option("--force", is_flag=True, default=False, help="Write into a directory that isn't empty") +def new(name: str, directory: Path | None, from_mcp: Path | None, from_tools_json: Path | None, domain: str | None, + force: bool): + """Make a new website env NAME like the Notes example: site.py, data/seed.json, a Vite app in web/ with the SDK's + browser client, a Dockerfile, env.toml and a README. With --from-mcp or --from-tools-json, each of a server's tools + is a stub @api route of the same name, and the seed has a collection for each.""" + if from_mcp and from_tools_json: + click.echo("Error: --from-mcp and --from-tools-json are mutually exclusive", err=True) + sys.exit(1) + target = directory or Path(name) + try: + tools, source = None, None + if from_mcp: + tools, source = tools_from_mcp(from_mcp), str(from_mcp) + elif from_tools_json: + tools, source = tools_from_json(from_tools_json), str(from_tools_json) + written = scaffold(target, name, domain=domain, tools=tools, source=source, force=force) + except ScaffoldError as e: + click.echo(f"Error: {e}", err=True) + sys.exit(1) + click.echo(f"Created website env {name!r} in {target} ({len(written)} files" + + (f", {len(tools)} stub route(s) from {source}" if tools is not None else "") + ").") + click.echo(f"\nNext:\n cd {target}\n agent-env env website dev\n" + f" agent-env env website put --id {name} --server-dockerfile Dockerfile") + + +@website.command() +@click.option("--dir", "directory", default=".", show_default=True, type=click.Path(exists=True, file_okay=False, + path_type=Path), help="The site's folder, with its site.py") +@click.option("--site", "site_file", default="site.py", show_default=True, help="The site's server, in --dir") +@click.option("--host", default="127.0.0.1", show_default=True, help="The address the site listens on") +@click.option("--port", default=8000, show_default=True, type=int, help="The port the site listens on") +@click.option("--clock-port", default=0, type=int, help="The local clock's port (default: any free one)") +@click.option("--clock-time", default=None, help="The universe's time the local clock starts at, RFC3339 (default: now)") +@click.option("--seed", default="data/seed.json", show_default=True, + help="The seed loaded through the data plane, in --dir; skipped when the file isn't there") +@click.option("--web/--no-web", "build_web", default=True, show_default=True, + help="Build web/ when it is out of date, if it has a package.json") +@click.option("--watch/--no-watch", default=True, show_default=True, help="Rebuild web/ as its sources change") +@click.option("--check/--no-check", default=True, show_default=True, + help="Run the website/v1 conformance check once the site is up") +def dev(directory: Path, site_file: str, host: str, port: int, clock_port: int, clock_time: str | None, seed: str, + build_web: bool, watch: bool, check: bool): + """Run a site locally for authoring, without Docker: build web/ when it is out of date, serve site.py with + uvicorn's reload, start a local clock standing in for the gateway's, sync the site to it with clock/v1's + sync_time and load the seed through the data plane (again after every reload), print the URLs, and run the + conformance check once. Ctrl-C stops it all.""" + import uvicorn + + from agentenv_protocol.website.clock import parse_time + + site_dir = directory.resolve() + site_path = site_dir / site_file + if not site_path.is_file(): + click.echo(f"Error: no {site_file} in {site_dir}", err=True) + sys.exit(1) + try: + start = parse_time(clock_time) if clock_time else None + except ValueError: + click.echo(f"Error: --clock-time {clock_time!r} is not an RFC3339 time", err=True) + sys.exit(1) + try: + website_dev.site_class(site_path) + except Exception as e: + click.echo(f"Error: can't load {site_path}: {e}", err=True) + sys.exit(1) + + web = site_dir / "web" + watcher = None + if build_web and (web / "package.json").is_file(): + if not website_dev.npm_available(): + click.echo("Warning: web/ has a package.json but npm isn't on PATH; serving web/dist as it is", err=True) + else: + commands, watch_command = website_dev.web_plan(web, watch=watch) + for command in commands: + click.echo(f"[dev] web/: {' '.join(command)}") + try: + website_dev.run_npm(command, web) + except subprocess.CalledProcessError as e: + click.echo(f"Error: {' '.join(command)} failed with exit code {e.returncode}", err=True) + sys.exit(1) + if watch_command: + click.echo(f"[dev] web/: {' '.join(watch_command)} (in the background)") + watcher = website_dev.run_npm(watch_command, web, background=True) + + seed_path = site_dir / seed + clock = website_dev.DevClock(start, port=clock_port).start() + base_url = f"http://{'127.0.0.1' if host in ('0.0.0.0', '::') else host}:{port}" + supervisor = website_dev.Supervisor(base_url, clock.url, seed_path if seed_path.is_file() else None, check=check, + echo=click.echo).start() + clock_base = clock.url.rsplit("/clock/", 1)[0] + click.echo(f""" +[dev] Site: {base_url}/ +[dev] Card: {base_url}/.well-known/agent-env.json +[dev] Clock: {clock.url} +[dev] curl -X PUT {clock_base}/clock/advance -d '{{"seconds": 3600}}' +[dev] curl -X PUT {clock_base}/clock/set-time -d '{{"virtual_time": "2030-01-01T09:00:00Z"}}' +[dev] Check: python -m agentenv_protocol.website.conformance {base_url} +""") + excludes = [str(web)] if web.is_dir() else [] + os.environ[website_dev.SITE_ENV] = str(site_path) + try: + uvicorn.run("agent_env.cli.env.website_dev:create_app", factory=True, host=host, port=port, reload=True, + reload_dirs=[str(site_dir)], reload_includes=["*.py", "*.json"], reload_excludes=excludes or None) + finally: + supervisor.stop() + clock.close() + if watcher is not None: + watcher.terminate() + try: + watcher.wait(timeout=5) + except subprocess.TimeoutExpired: + watcher.kill() + + @website.command() @click.option("--id", "env_id", required=True, help="Env id") @click.option("--server-dockerfile", "--backend-dockerfile", "backend_dockerfile", default=None, type=click.Path(exists=True), @@ -47,8 +176,9 @@ def website(): @click.option("--frontend-docker-context", default=None, type=click.Path(exists=True), help="Frontend Docker build context (defaults to Dockerfile's directory)") @click.option("--frontend-dockerfile-github-url", default=None, help="GitHub URL to a separate frontend Dockerfile (deprecated)") @click.option("--frontend-docker-context-github-url", default=None, help="GitHub URL to frontend build context directory") -@click.option("--health-path", default=AGENT_ENV_WEBSITE_HEALTH_PATH, show_default=True, - help="The path that answers 200 once the site is usable") +@click.option("--health-path", default=None, + help="Deprecated: a site says when it's usable by its card's website/v1 ready. The path that answers " + f"200 once a site without it is usable (defaults to {AGENT_ENV_WEBSITE_HEALTH_PATH})") @environment_name_options @env_provider_type_option("What deploys the env: 'gateway' (a gateway in front of the website), or the type of an installed " "agent_env.env_providers plugin", "website") @@ -66,7 +196,7 @@ def put( frontend_docker_context: str | None, frontend_dockerfile_github_url: str | None, frontend_docker_context_github_url: str | None, - health_path: str, + health_path: str | None, environment_name: str | None, env_provider_type: str, metadata_pairs: tuple[str, ...], @@ -93,9 +223,12 @@ def put( click.echo("Error: --server-image-ref needs --environment-name, which a registry image's card can't give here", err=True) sys.exit(1) - if problem := health_path_problem(health_path): + if health_path is not None and (problem := health_path_problem(health_path)): click.echo(f"Error: {problem}", err=True) sys.exit(1) + if health_path is not None: + click.echo("Warning: --health-path is deprecated; a site says when it's usable by its card's website/v1 ready", + err=True) if frontend_dockerfile or frontend_dockerfile_github_url: click.echo("Warning: a separate frontend image is deprecated; build the site's pages into its server image", err=True) @@ -107,7 +240,7 @@ def put( sys.exit(1) key, value = pair.split("=", 1) user_metadata[key] = value - site = {"health_path": health_path} + site = {"health_path": health_path} if health_path is not None else {} if use_github: if environment_name is None: diff --git a/src/agent_env/cli/env/website_dev.py b/src/agent_env/cli/env/website_dev.py new file mode 100644 index 000000000..e5af25a4e --- /dev/null +++ b/src/agent_env/cli/env/website_dev.py @@ -0,0 +1,336 @@ +"""The pieces of ``agent-env env website dev``, which runs a site for authoring without Docker: the site's ``web/`` +built when it's out of date, ``site.py`` served by uvicorn with reload, a local stand-in for the gateway's clock, and a +supervisor that syncs each freshly started server to that clock (``clock/v1``'s ``sync_time``) and loads its seed +through the data plane (``data/reset`` then ``data/add``), as a deploy would. A server started by a reload is told +from the one before by ``BOOT_HEADER``, which ``create_app`` stamps on every answer.""" +from __future__ import annotations + +import datetime +import importlib.util +import inspect +import json +import logging +import os +import shutil +import subprocess +import sys +import threading +import time +import uuid +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Callable + +import httpx +from agentenv_protocol.client import find_extension_method, serves +from agentenv_protocol.types import METHOD_ADD, METHOD_RESET, RPC_PATH, WELL_KNOWN_PATH +from agentenv_protocol.website import CLOCK_EXTENSION_URI, WebsiteEnvironment +from agentenv_protocol.website.clock import iso, parse_time + +SITE_ENV = "AGENT_ENV_WEBSITE_DEV_SITE" +BOOT_HEADER = "X-Site-Boot" +POLL_QUERY = "agent-env-dev-poll" +_WEB_SKIPPED = {"node_modules", "dist"} + + +class DevError(Exception): + """Why the site can't be run as asked.""" + + +def site_class(path: Path) -> type[WebsiteEnvironment]: + """The WebsiteEnvironment ``path`` defines, imported under a name of its own (``site`` is the standard library's): + the one class, or the one of several that is declared with ``@website``. Its folder goes on ``sys.path`` first, + so it can import the files beside it.""" + folder = str(path.resolve().parent) + if folder not in sys.path: + sys.path.insert(0, folder) + spec = importlib.util.spec_from_file_location(f"agent_env_dev_site_{uuid.uuid4().hex[:8]}", path) + if spec is None or spec.loader is None: + raise DevError(f"can't import {path}") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + found = [v for v in vars(module).values() if inspect.isclass(v) and issubclass(v, WebsiteEnvironment) + and v.__module__ == spec.name] + declared = [cls for cls in found if "_aee_website" in vars(cls)] + chosen = declared if len(declared) == 1 else found + if len(chosen) != 1: + names = ", ".join(cls.__name__ for cls in found) or "none" + raise DevError(f"{path} defines {len(found)} WebsiteEnvironment classes ({names}); it needs exactly one, " + "or one declared with @website") + return chosen[0] + + +def create_app() -> Any: + """The app of the site ``SITE_ENV`` names, for uvicorn's ``--factory``: every answer carries this process's + ``BOOT_HEADER``.""" + path = Path(os.environ[SITE_ENV]).resolve() + logging.getLogger("uvicorn.access").addFilter(_unpolled) + return Booted(site_class(path)().create_app(), uuid.uuid4().hex) + + +def _unpolled(record: logging.LogRecord) -> bool: + """Whether an access log line is one of a request other than the supervisor's polls.""" + return not any(isinstance(arg, str) and POLL_QUERY in arg for arg in (record.args or ())) + + +class Booted: + """``app`` with ``BOOT_HEADER: boot`` on every HTTP answer.""" + + def __init__(self, app: Any, boot: str) -> None: + self.app, self.boot = app, boot + + async def __call__(self, scope: dict, receive: Any, send: Any) -> None: + if scope["type"] != "http": + return await self.app(scope, receive, send) + + async def stamped(message: dict) -> None: + if message["type"] == "http.response.start": + message = {**message, "headers": [*message.get("headers", []), + (BOOT_HEADER.lower().encode(), self.boot.encode())]} + await send(message) + + await self.app(scope, receive, stamped) + + +class DevClock: + """A local stand-in for the gateway's clock: ``GET /clock/time`` answers ``{"virtual_time", + "virtual_seconds_per_real_second"}`` as the gateway's does, ``GET /clock/state`` its state, ``PUT /clock/set-time`` takes ``{"virtual_time", + "virtual_seconds_per_real_second"}``, and ``PUT /clock/advance`` moves it forward to ``{"virtual_time"}`` or by + ``{"seconds"}``. It runs from ``start`` (wall time by default) at real speed.""" + + def __init__(self, start: datetime.datetime | None = None, *, host: str = "127.0.0.1", port: int = 0) -> None: + self._lock = threading.Lock() + self._virtual = start or datetime.datetime.now(datetime.timezone.utc) + self._real = time.monotonic() + self._rate = 1.0 + self._server = ThreadingHTTPServer((host, port), _clock_handler(self)) + self._thread: threading.Thread | None = None + self.url = f"http://{host}:{self._server.server_port}/clock/time" + + def start(self) -> DevClock: + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + return self + + def close(self) -> None: + self._server.shutdown() + self._server.server_close() + + def now(self) -> datetime.datetime: + with self._lock: + return self._virtual + datetime.timedelta(seconds=(time.monotonic() - self._real) * self._rate) + + def time(self) -> dict: + """The clock's time and its rate, read together, as ``/clock/time`` answers them.""" + with self._lock: + virtual, real, rate = self._virtual, self._real, self._rate + now = virtual + datetime.timedelta(seconds=(time.monotonic() - real) * rate) + return {"virtual_time": iso(now), "virtual_seconds_per_real_second": rate} + + def state(self) -> dict: + return {"armed": True, **self.time(), "env_get_time_url": self.url} + + def set_time(self, virtual_time: str, rate: float | None = None) -> dict: + moment = parse_time(virtual_time) + if rate is not None and rate < 0: + raise ValueError("virtual_seconds_per_real_second is not negative") + with self._lock: + self._virtual, self._real = moment, time.monotonic() + if rate is not None: + self._rate = float(rate) + return self.state() + + def advance(self, virtual_time: str | None = None, seconds: float | None = None) -> dict: + now = self.now() + if (virtual_time is None) == (seconds is None): + raise ValueError("advance takes virtual_time or seconds") + target = parse_time(virtual_time) if virtual_time is not None else now + datetime.timedelta(seconds=seconds) + if target < now: + raise ValueError(f"advance moves the clock forward, and {iso(target)} is before {iso(now)}") + with self._lock: + self._virtual, self._real = target, time.monotonic() + return self.state() + + +def _clock_handler(clock: DevClock) -> type[BaseHTTPRequestHandler]: + class Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + if self.path == "/clock/time": + self._answer(200, clock.time()) + elif self.path == "/clock/state": + self._answer(200, clock.state()) + else: + self._answer(404, {"ok": False, "error": "not found"}) + + def do_PUT(self) -> None: + try: + body = json.loads(self.rfile.read(int(self.headers.get("Content-Length") or 0)) or b"{}") + if not isinstance(body, dict): + raise ValueError("body must be a JSON object") + if self.path == "/clock/set-time": + self._answer(200, clock.set_time(body.get("virtual_time"), body.get("virtual_seconds_per_real_second"))) + elif self.path == "/clock/advance": + self._answer(200, clock.advance(body.get("virtual_time"), body.get("seconds"))) + else: + self._answer(404, {"ok": False, "error": "not found"}) + except (ValueError, TypeError, AttributeError) as e: + self._answer(400, {"ok": False, "error": str(e)}) + + def _answer(self, status: int, body: dict) -> None: + data = json.dumps(body).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def log_message(self, *args: Any) -> None: + return None + + return Handler + + +def web_stale(web: Path) -> bool: + """Whether ``web/dist`` is missing or older than a source file of ``web/``.""" + built = web / "dist" / "index.html" + if not built.is_file(): + return True + newest = max((p.stat().st_mtime for p in _web_sources(web)), default=0.0) + return newest > built.stat().st_mtime + + +def _web_sources(web: Path): + for path in web.rglob("*"): + if path.is_file() and not _WEB_SKIPPED & set(path.relative_to(web).parts): + yield path + + +def web_plan(web: Path, *, watch: bool) -> tuple[list[list[str]], list[str] | None]: + """What to run in ``web/`` before the site starts: an install when ``node_modules`` is missing (``npm ci`` with a + lockfile), and a build when ``dist`` is out of date; and with ``watch``, the build that then runs beside the site, + rebuilding as the sources change.""" + commands = [] + if not (web / "node_modules").is_dir(): + commands.append(["npm", "ci" if (web / "package-lock.json").is_file() else "install", "--no-audit", "--no-fund"]) + if web_stale(web): + commands.append(["npm", "run", "build"]) + return commands, (["npm", "run", "build", "--", "--watch"] if watch else None) + + +def prime(client: httpx.Client, base_url: str, clock_url: str, seed: Path | None) -> str | None: + """Sync the server at ``base_url`` to the clock at ``clock_url`` with ``clock/v1``'s ``sync_time``, and load + ``seed`` through the data plane in place of what it holds, at the card's ``url`` and refused when its declared + ``operations`` lack ``data/reset`` or ``data/add``; returns the server's boot.""" + card_answer = client.get(base_url + WELL_KNOWN_PATH) + card_answer.raise_for_status() + card = card_answer.json() + method = find_extension_method(card, CLOCK_EXTENSION_URI, "sync_time") + if method is None or not method.get("endpoint"): + raise DevError(f"the site's card advertises no {CLOCK_EXTENSION_URI} sync_time") + synced = client.request(method["method"], base_url + method["endpoint"], json={"env_get_time_url": clock_url}) + synced.raise_for_status() + if seed is not None: + data = json.loads(seed.read_text(encoding="utf-8")) + unserved = [m for m in (METHOD_RESET, METHOD_ADD) if serves(card, m) is False] + if unserved: + raise DevError(f"the site's card doesn't serve {', '.join(unserved)}, so its seed can't be loaded") + rpc_url = data_plane_url(base_url, card) + _rpc(client, rpc_url, METHOD_RESET, {}) + _rpc(client, rpc_url, METHOD_ADD, {"parts": [{"kind": "data", "data": data}]}) + return card_answer.headers.get(BOOT_HEADER) + + +def data_plane_url(base_url: str, card: dict) -> str: + """Where the site whose ``card`` is read from ``base_url`` answers the data plane: the card's ``url``, a path + joined onto ``base_url`` or a full URL as it is, and ``RPC_PATH`` there for a card naming none.""" + url = card.get("url") or RPC_PATH + return url if "://" in url else f"{base_url.rstrip('/')}{url}" + + +def _rpc(client: httpx.Client, rpc_url: str, method: str, params: dict) -> dict: + answer = client.post(rpc_url, json={"jsonrpc": "2.0", "id": method, "method": method, "params": params}) + answer.raise_for_status() + body = answer.json() + if "error" in body: + raise DevError(f"{method} answered {body['error']}") + return body.get("result") or {} + + +class Supervisor: + """Watches the server at ``base_url`` and primes each server a reload starts; after the first, runs the + conformance check once (and primes again, since the check leaves the server on a clock of its own). A check a + reload cuts off is reported rather than raised, so the supervisor goes on priming later reloads. A server it + couldn't prime, before or after the check and for a seed it couldn't read too, counts as unprimed, so the next + poll tries again.""" + + def __init__(self, base_url: str, clock_url: str, seed: Path | None, *, check: bool = True, + echo: Callable[[str], None] = print, client: httpx.Client | None = None, interval: float = 1.0) -> None: + self.base_url, self.clock_url, self.seed, self.check = base_url, clock_url, seed, check + self.echo, self.interval = echo, interval + self.client = client or httpx.Client(timeout=10) + self.primed: str | None = None + self.checked = False + self.stopped = threading.Event() + + def step(self) -> bool: + """Prime the server if it is one not yet primed; whether it answered.""" + try: + boot = self.client.get(f"{self.base_url}{WELL_KNOWN_PATH}?{POLL_QUERY}").headers.get(BOOT_HEADER) + except httpx.TransportError: + return False + if boot is not None and boot == self.primed: + return True + try: + self.primed = prime(self.client, self.base_url, self.clock_url, self.seed) + except (httpx.HTTPError, DevError, ValueError, OSError) as e: + self.echo(f"[dev] couldn't sync the site and load its seed: {e}") + return True + self.echo(f"[dev] synced the site to {self.clock_url}" + (f" and loaded {self.seed}" if self.seed else "")) + if self.check and not self.checked: + self.checked = True + self._conformance() + return True + + def _conformance(self) -> None: + from agentenv_protocol.website import conformance + + try: + results = conformance.check(self.base_url, client=self.client) + for result in results: + mark = {"passed": "PASS", "failed": "FAIL", "warning": "WARN", "skipped": "----"}[result.status] + self.echo(f"[dev] {mark} {result.name}{': ' + result.detail if result.detail else ''}") + failed = sum(not r.passed for r in results) + self.echo(f"[dev] conformance: {len(results) - failed} passed or skipped, {failed} failed") + except httpx.TransportError as e: + self.echo(f"[dev] the conformance check lost the site, likely to a reload: {e}") + finally: + try: + self.primed = prime(self.client, self.base_url, self.clock_url, self.seed) + except (httpx.HTTPError, DevError, ValueError, OSError) as e: + self.primed = None + self.echo(f"[dev] couldn't sync the site again after the check: {e}") + + def run(self) -> None: + while not self.stopped.is_set(): + self.step() + self.stopped.wait(self.interval) + + def start(self) -> Supervisor: + threading.Thread(target=self.run, daemon=True).start() + return self + + def stop(self) -> None: + self.stopped.set() + + +def npm_available() -> bool: + return shutil.which("npm") is not None + + +def run_npm(command: list[str], web: Path, *, background: bool = False) -> subprocess.Popen | None: + """Run ``command`` in ``web``; in the background it is returned running, else waited for (raising on failure).""" + if background: + return subprocess.Popen(command, cwd=web) + subprocess.run(command, cwd=web, check=True) + return None diff --git a/src/agent_env/cli/env/website_scaffold.py b/src/agent_env/cli/env/website_scaffold.py new file mode 100644 index 000000000..bde88c23d --- /dev/null +++ b/src/agent_env/cli/env/website_scaffold.py @@ -0,0 +1,534 @@ +"""What ``agent-env env website new`` writes: a site like the Notes example, from the templates beside this module, and +optionally from an MCP server's tools, one stub ``@api`` route each. A server's tools are read without running it: from +the ``@tool`` and ``@.tool`` methods of its Python source (agentenv_protocol's and FastMCP's alike), or from a +JSON file of them, as ``tools/list`` answers, as a list, or as an environment card's ``capabilities.tools``.""" +from __future__ import annotations + +import ast +import inspect +import json +import keyword +import re +from importlib import metadata +from pathlib import Path +from typing import Any + +import agentenv_protocol.website as website_sdk +from agentenv_protocol.website import SESSION_PATH, STATE_PATH, WebsiteEnvironment +from pydantic import BaseModel + +TEMPLATE_DIR = Path(__file__).resolve().parent / "website_template" +CLIENT_DIR = Path(website_sdk.__file__).resolve().parent / "client" +NAME = re.compile(r"^[a-z][a-z0-9-]*$") +_TOKEN = re.compile(r"{{(\w+)}}") +_SKIPPED_DIRS = {"node_modules", ".venv", "venv", "__pycache__", ".git", "dist", "build"} +_CONTEXT_TYPES = {"ToolContext", "Context"} +_READ_VERBS = ("get", "list", "search", "find", "read", "fetch", "query", "lookup", "show", "count") +_VERBS = _READ_VERBS + ("create", "add", "new", "update", "edit", "set", "delete", "remove", "send", "post", "put", + "patch", "make", "write", "cancel", "book", "submit") +_BUILTIN_PATHS = {"/api/users", "/api/sign-in", "/healthz", SESSION_PATH, STATE_PATH} +_JSON_TYPES = {"string": "str", "integer": "int", "number": "float", "boolean": "bool", "object": "dict", + "array": "list", "null": "None"} +_PY_TYPES = {"str": "string", "int": "integer", "float": "number", "bool": "boolean", "dict": "object", + "list": "array", "Dict": "object", "List": "array", "tuple": "array", "Tuple": "array", "set": "array"} +_BUILTIN_ROUTES = ''' + @api.get("/api/users", name="list_users") + def list_users(self) -> list: + """The users a browser can sign in as.""" + return self.store[USERS] + + @api.post("/api/sign-in", name="sign_in") + def sign_in(self, body: SignIn, ctx: ApiContext) -> dict: + """Sign the browser in as one of the site's users.""" + if self.store.get(USERS, body.user) is None: + raise NotFound(f"no user {body.user!r}") + ctx.sign_in(body.user) + return {"user": ctx.user} +''' + + +class ScaffoldError(Exception): + """Why a site can't be scaffolded as asked.""" + + +def scaffold(target: Path, name: str, *, domain: str | None = None, tools: list[dict] | None = None, + source: str | None = None, force: bool = False) -> list[Path]: + """Write the site ``name`` into ``target`` and return the files written. With ``tools``, its routes are their + stubs; ``source`` says where they came from. A non-empty ``target`` is refused unless ``force``.""" + if not NAME.match(name): + raise ScaffoldError(f"a site's name is lowercase letters, digits and hyphens, starting with a letter, not {name!r}") + if target.exists() and not target.is_dir(): + raise ScaffoldError(f"{target} is a file") + if target.is_dir() and any(target.iterdir()) and not force: + raise ScaffoldError(f"{target} is not empty; pass --force to write into it") + values = {"name": name, "title": _title(name), "class_name": _class_name(name), "domain": domain or f"{name}.test", + "protocol": _protocol_requirement(), "source": source or "an MCP server", + "source_note": f" from the tools of {source}" if tools is not None else ""} + files: dict[str, str] = {} + for out, template in (("data/seed.json", "data/seed.json.tmpl"), ("web/index.html", "web/index.html.tmpl"), + ("web/src/style.css", "web/src/style.css.tmpl"), ("web/package.json", "web/package.json.tmpl"), + ("web/vite.config.js", "web/vite.config.js.tmpl"), ("Dockerfile", "Dockerfile.tmpl"), + ("Dockerfile.dockerignore", "Dockerfile.dockerignore.tmpl"), ("env.toml", "env.toml.tmpl"), + ("README.md", "README.md.tmpl"), (".gitignore", "gitignore.tmpl")): + files[out] = _render(template, values) + if tools is None: + files["site.py"] = _render("site.py.tmpl", values) + files["web/src/main.js"] = _render("web/src/main.js.tmpl", values) + else: + if not tools: + raise ScaffoldError("there are no tools to make routes of") + stubs = _stubs(tools) + files["site.py"] = _render("site-tools.py.tmpl", {**values, **_site_parts(stubs)}) + files["data/seed.json"] = json.dumps(seed_for(tools), indent=2) + "\n" + routes = [{"name": s["name"], "method": s["method"], "path": s["path"].lstrip("/"), + "description": s["summary"], "example": s["example"]} for s in stubs] + files["web/src/main.js"] = _render("web/src/main-tools.js.tmpl", + {**values, "routes_json": json.dumps(routes, indent=2)}) + for client in ("site-client.js", "site-client.d.ts"): + files[f"web/src/{client}"] = (CLIENT_DIR / client).read_text(encoding="utf-8") + written = [] + for relative, text in files.items(): + path = target / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + written.append(path) + return sorted(written) + + +def tools_from_mcp(path: Path) -> list[dict]: + """The tools of the MCP server at ``path``, a folder or a ``.py`` file of its source, or a tools JSON file, each + as ``tools/list`` lists one: ``{name, description, inputSchema}``.""" + if path.is_file() and path.suffix == ".json": + return tools_from_json(path) + if path.is_file(): + sources = [path] + elif path.is_dir(): + sources = sorted(p for p in path.rglob("*.py") if not _SKIPPED_DIRS & set(p.relative_to(path).parts) + and not p.name.startswith("test_") and not p.name.endswith("_test.py")) + else: + raise ScaffoldError(f"no MCP server at {path}") + tools: dict[str, dict] = {} + for source in sources: + try: + tree = ast.parse(source.read_text(encoding="utf-8"), filename=str(source)) + except (SyntaxError, UnicodeDecodeError, OSError) as e: + raise ScaffoldError(f"can't read {source}: {e}") from e + for found in _tools_in(tree): + tools.setdefault(found["name"], found) + if not tools: + raise ScaffoldError(f"found no @tool methods in {path}; pass the server's tools as JSON with --from-tools-json") + return list(tools.values()) + + +def tools_from_json(path: Path) -> list[dict]: + """The tools a JSON file lists: ``tools/list``'s answer (or its JSON-RPC response), a list of tools, or an + environment card with ``capabilities.tools``.""" + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as e: + raise ScaffoldError(f"can't read tools from {path}: {e}") from e + if isinstance(data, dict) and isinstance(data.get("result"), dict): + data = data["result"] + if isinstance(data, dict) and isinstance((data.get("capabilities") or {}).get("tools"), list): + data = data["capabilities"]["tools"] + elif isinstance(data, dict) and isinstance(data.get("tools"), list): + data = data["tools"] + if not isinstance(data, list) or not all(isinstance(t, dict) and isinstance(t.get("name"), str) for t in data): + raise ScaffoldError(f"{path} lists no tools: expected tools/list's {{\"tools\": [...]}}, a list of " + "{name, description, inputSchema}, or an environment card") + return [{"name": t["name"], "description": t.get("description") or "", + "inputSchema": t.get("inputSchema") or t.get("input_schema") or {"type": "object", "properties": {}}} + for t in data] + + +def seed_for(tools: list[dict]) -> dict: + """A seed for a site made from ``tools``: an acting user, the users to sign in as, and an empty collection for + what each tool's name is about (``search_emails`` and ``send_email`` both ``emails``).""" + seed: dict = {"current_user": "user", "users": [{"id": "user", "name": "Sample User"}]} + for tool in tools: + collection = _collection(tool["name"]) + if collection and collection not in seed: + seed[collection] = [] + return seed + + +def _render(template: str, values: dict[str, str]) -> str: + text = (TEMPLATE_DIR / template).read_text(encoding="utf-8") + + def value(match: re.Match) -> str: + if match.group(1) not in values: + raise KeyError(f"{template} names {match.group(0)}, which nothing fills") + return values[match.group(1)] + + return _TOKEN.sub(value, text) + + +def _title(name: str) -> str: + return " ".join(part.capitalize() for part in name.split("-") if part) + + +def _class_name(name: str) -> str: + parts = [part.capitalize() for part in name.split("-") if part] + return "".join(parts) + ("" if parts[-1] == "Site" else "Site") + + +def _protocol_requirement() -> str: + """The protocol package the site's image installs: at least the version this CLI has, which has the website SDK.""" + try: + return f"agentenv-framework-protocol>={metadata.version('agentenv-framework-protocol')}" + except metadata.PackageNotFoundError: + return "agentenv-framework-protocol" + + +def _words(name: str) -> list[str]: + return [w for w in re.split(r"[^a-z0-9]+", re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", name).lower()) if w] + + +def _collection(name: str) -> str | None: + words = _words(name) + if words and words[0] in _VERBS: + words = words[1:] + if not words: + return None + if not words[-1].endswith("s"): + words[-1] += "s" + return "_".join(words) + + +def _identifier(name: str, taken: set[str]) -> str: + """A Python name for ``name``, not a keyword, a BaseModel attribute or one of ``taken``.""" + ident = "_".join(_words(name)) or "value" + if ident[0].isdigit(): + ident = f"n_{ident}" + while keyword.iskeyword(ident) or hasattr(BaseModel, ident) or ident in taken or ident in ("self", "ctx"): + ident += "_" + taken.add(ident) + return ident + + +def _py_type(schema: Any, typing: set[str]) -> str: + """The Python annotation of a JSON schema, adding the ``typing`` names it uses to ``typing``.""" + if not isinstance(schema, dict): + typing.add("Any") + return "Any" + options = schema.get("anyOf") or schema.get("oneOf") + if isinstance(options, list): + kinds = [o for o in options if not (isinstance(o, dict) and o.get("type") == "null")] + inner = _py_type(kinds[0], typing) if len(kinds) == 1 else _any(typing) + if len(kinds) < len(options): + typing.add("Optional") + return f"Optional[{inner}]" + return inner + if isinstance(schema.get("enum"), list) and schema["enum"] and all(isinstance(v, (str, int)) for v in schema["enum"]): + typing.add("Literal") + return f"Literal[{', '.join(repr(v) for v in schema['enum'])}]" + kind = schema.get("type") + if isinstance(kind, list): + real = [k for k in kind if k != "null"] + inner = _py_type({**schema, "type": real[0]}, typing) if len(real) == 1 else _any(typing) + if "null" in kind: + typing.add("Optional") + return f"Optional[{inner}]" + return inner + if kind == "array": + return f"list[{_py_type(schema['items'], typing)}]" if isinstance(schema.get("items"), dict) else "list" + if kind in _JSON_TYPES and kind != "null": + return _JSON_TYPES[kind] + return _any(typing) + + +def _any(typing: set[str]) -> str: + typing.add("Any") + return "Any" + + +def _scalar(schema: Any) -> bool: + if not isinstance(schema, dict): + return False + options = schema.get("anyOf") or schema.get("oneOf") + if isinstance(options, list): + return all(_scalar(o) or (isinstance(o, dict) and o.get("type") == "null") for o in options) + if isinstance(schema.get("enum"), list): + return True + kind = schema.get("type") + kinds = kind if isinstance(kind, list) else [kind] + return bool(kinds) and all(k in ("string", "integer", "number", "boolean", "null") for k in kinds) + + +def _literal(value: Any) -> str | None: + """``value`` as Python source, when it is plain JSON.""" + try: + source = repr(value) + return source if ast.literal_eval(source) == value else None + except (ValueError, SyntaxError): + return None + + +def _stubs(tools: list[dict]) -> list[dict]: + """Each tool's route: its function and model names, verb, path, parameters and the example the page sends. A + function is never named like the site class's own methods or a lowercase name ``site-tools.py.tmpl`` imports, + such as ``api``, which a method of that name would shadow for the routes declared after it.""" + functions: set[str] = {"api", "ready_check", "website", "environment_card", "seeded", "list_users", "sign_in", + *dir(WebsiteEnvironment)} + names: set[str] = {"list_users", "sign_in"} + models: set[str] = {"SignIn"} + paths: set[str] = set(_BUILTIN_PATHS) + stubs = [] + for tool in tools: + schema = tool.get("inputSchema") if isinstance(tool.get("inputSchema"), dict) else {} + properties = schema.get("properties") if isinstance(schema.get("properties"), dict) else {} + required = set(schema.get("required") or []) + words = _words(tool["name"]) + segment = "-".join(words) or "tool" + path = f"/api/{segment}" + while path in paths: + path += "-tool" + paths.add(path) + function = _identifier(tool["name"], functions) + model = "".join(w.capitalize() for w in words) + "Request" + if model[0].isdigit(): + model = f"N{model}" + while model in models: + model += "_" + models.add(model) + taken: set[str] = set() + params = [] + for key, prop in properties.items(): + prop = prop if isinstance(prop, dict) else {} + params.append({"key": key, "ident": _identifier(key, taken), "schema": prop, + "required": key in required, "description": str(prop.get("description") or "")}) + params.sort(key=lambda p: not p["required"]) + reads = bool(words) and words[0] in _READ_VERBS and all(_scalar(p["schema"]) and p["ident"] == p["key"] + for p in params) + route_name = tool["name"] + while route_name in names: + route_name += "_tool" + names.add(route_name) + example = {p["key"]: _example(p["schema"]) for p in params if p["required"]} + stubs.append({"tool": tool["name"], "name": route_name, "function": function, "model": model, "path": path, + "method": "GET" if reads else "POST", "params": params, "example": example, + "description": inspect.cleandoc(str(tool.get("description") or "")), + "summary": inspect.cleandoc(str(tool.get("description") or "")).split("\n\n")[0]}) + return stubs + + +def _example(schema: dict) -> Any: + if "default" in schema: + return schema["default"] + if isinstance(schema.get("enum"), list) and schema["enum"]: + return schema["enum"][0] + kind = schema.get("type") + kind = next((k for k in kind if k != "null"), None) if isinstance(kind, list) else kind + return {"string": "", "integer": 0, "number": 0, "boolean": False, "array": [], "object": {}}.get(kind, None) + + +def _site_parts(stubs: list[dict]) -> dict[str, str]: + """The ``{{models}}``, ``{{routes}}``, ``{{builtin_routes}}`` and ``{{typing_import}}`` of ``site-tools.py.tmpl``.""" + typing: set[str] = set() + models, routes = [], [] + for stub in stubs: + doc = _docstring(stub["description"] or f"The {stub['tool']} tool.", 8) + if stub["method"] == "GET": + args = ["self", "ctx: ApiContext"] + for p in stub["params"]: + annotation = _py_type(p["schema"], typing) + if p["required"]: + args.append(f"{p['ident']}: {annotation}") + else: + default = _literal(p["schema"].get("default")) if "default" in p["schema"] else None + if default is None or default == "None": + if not annotation.startswith("Optional["): + typing.add("Optional") + annotation = f"Optional[{annotation}]" + default = "None" + args.append(f"{p['ident']}: {annotation} = {default}") + notes = [f"``{p['ident']}``: {p['description']}" for p in stub["params"] if p["description"]] + if notes: + doc = _docstring((stub["description"] or f"The {stub['tool']} tool.") + "\n\n" + "\n".join(notes), 8) + signature = ", ".join(args) + else: + fields = [] + for p in stub["params"]: + annotation = _py_type(p["schema"], typing) + options = [] + if not p["required"]: + default = _literal(p["schema"].get("default")) if "default" in p["schema"] else None + if default is None or default == "None": + if not annotation.startswith("Optional["): + typing.add("Optional") + annotation = f"Optional[{annotation}]" + default = "None" + options.append(f"default={default}") + if p["ident"] != p["key"]: + options.append(f"alias={json.dumps(p['key'])}") + if p["description"]: + options.append(f"description={json.dumps(p['description'])}") + value = f" = Field({', '.join(options)})" if options else "" + fields.append(f" {p['ident']}: {annotation}{value}") + model_doc = _docstring(f"What {stub['tool']} is called with.", 4) + models.append(f"\n\nclass {stub['model']}(BaseModel):\n{model_doc}\n" + "\n".join(fields)) + signature = f"self, body: {stub['model']}, ctx: ApiContext" + routes.append(f"\n @api.{stub['method'].lower()}({json.dumps(stub['path'])}, name={json.dumps(stub['name'])})\n" + f" def {stub['function']}({signature}) -> Any:\n{doc}\n" + f" raise _stub({json.dumps(stub['tool'])})\n") + typing.add("Any") + return {"models": "".join(models) + ("\n" if models else ""), "routes": "".join(routes).rstrip("\n") + "\n", + "builtin_routes": _BUILTIN_ROUTES, + "typing_import": f"from typing import {', '.join(sorted(typing))}\n\n"} + + +def _docstring(text: str, indent: int) -> str: + pad = " " * indent + text = text.replace("\\", "\\\\").replace('"', '\\"').strip() + lines = text.splitlines() or [""] + if len(lines) == 1: + return f'{pad}"""{lines[0]}"""' + body = "\n".join(f"{pad}{line}" if line else "" for line in lines[1:]) + return f'{pad}"""{lines[0]}\n{body}"""' + + +def _tools_in(tree: ast.Module) -> list[dict]: + """The tools ``tree`` declares: each function under a ``tool`` decorator, named by the decorator's ``name`` (or + its first argument) else the function's name, described by its ``description`` else the docstring.""" + tools = [] + for node in ast.walk(tree): + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + decorator = next((d for d in node.decorator_list if _is_tool(d)), None) + if decorator is None: + continue + name, description = node.name, None + if isinstance(decorator, ast.Call): + given = next((k.value for k in decorator.keywords if k.arg == "name"), None) + if given is None and decorator.args: + given = decorator.args[0] + if isinstance(given, ast.Constant) and isinstance(given.value, str): + name = given.value + described = next((k.value for k in decorator.keywords if k.arg == "description"), None) + if isinstance(described, ast.Constant) and isinstance(described.value, str): + description = described.value + tools.append({"name": name, "description": description or ast.get_docstring(node) or "", + "inputSchema": _input_schema(node.args)}) + return tools + + +def _is_tool(decorator: ast.expr) -> bool: + target = decorator.func if isinstance(decorator, ast.Call) else decorator + return (isinstance(target, ast.Name) and target.id == "tool") or ( + isinstance(target, ast.Attribute) and target.attr == "tool") + + +def _input_schema(args: ast.arguments) -> dict: + """The JSON schema a tool's parameters ask for, read from their annotations, ``Field``\\ s and defaults; ``self``, + ``cls`` and ``ToolContext``/``Context`` parameters are not the caller's.""" + positional = [*args.posonlyargs, *args.args] + defaults: dict[str, ast.expr | None] = dict.fromkeys(a.arg for a in [*positional, *args.kwonlyargs]) + for arg, default in zip(positional[len(positional) - len(args.defaults):], args.defaults): + defaults[arg.arg] = default + for arg, default in zip(args.kwonlyargs, args.kw_defaults): + defaults[arg.arg] = default + properties: dict[str, dict] = {} + required = [] + for arg in [*positional, *args.kwonlyargs]: + if arg.arg in ("self", "cls") or _is_context(arg.annotation): + continue + schema = _annotation_schema(arg.annotation) + default = defaults[arg.arg] + has_default = default is not None + if isinstance(default, ast.Call) and _call_name(default) == "Field": + schema.update(_field_options(default)) + has_default = "default" in schema + elif default is not None: + try: + schema["default"] = ast.literal_eval(default) + except (ValueError, TypeError, SyntaxError): + pass + if "_required" in schema: + has_default = not schema.pop("_required") + if "default" in schema: + has_default = True + properties[arg.arg] = schema + if not has_default: + required.append(arg.arg) + return {"type": "object", "properties": properties, "required": required} + + +def _is_context(annotation: ast.expr | None) -> bool: + if annotation is None: + return False + names = {n.id for n in ast.walk(annotation) if isinstance(n, ast.Name)} | { + n.attr for n in ast.walk(annotation) if isinstance(n, ast.Attribute)} + return bool(names & _CONTEXT_TYPES) + + +def _call_name(call: ast.Call) -> str | None: + if isinstance(call.func, ast.Name): + return call.func.id + return call.func.attr if isinstance(call.func, ast.Attribute) else None + + +def _field_options(call: ast.Call) -> dict: + """What a ``Field(...)`` says of its parameter: its description, and its default or that it has none.""" + options: dict = {} + for k in call.keywords: + if k.arg == "description" and isinstance(k.value, ast.Constant) and isinstance(k.value.value, str): + options["description"] = k.value.value + given = next((k.value for k in call.keywords if k.arg == "default"), call.args[0] if call.args else None) + if given is not None: + if isinstance(given, ast.Constant) and given.value is Ellipsis: + options["_required"] = True + else: + try: + options["default"] = ast.literal_eval(given) + except (ValueError, TypeError, SyntaxError): + options["_required"] = False + elif any(k.arg == "default_factory" for k in call.keywords): + options["_required"] = False + return options + + +def _annotation_schema(node: ast.expr | None) -> dict: + """The JSON schema of a parameter's annotation, as far as the source shows it.""" + if node is None: + return {} + if isinstance(node, ast.Constant) and isinstance(node.value, str): + try: + return _annotation_schema(ast.parse(node.value, mode="eval").body) + except SyntaxError: + return {} + if isinstance(node, ast.Constant) and node.value is None: + return {"type": "null"} + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.BitOr): + return _union([_annotation_schema(node.left), _annotation_schema(node.right)]) + if isinstance(node, (ast.Name, ast.Attribute)): + name = node.id if isinstance(node, ast.Name) else node.attr + return {"type": _PY_TYPES[name]} if name in _PY_TYPES else {} + if isinstance(node, ast.Subscript): + outer = node.value.id if isinstance(node.value, ast.Name) else getattr(node.value, "attr", "") + inner = node.slice + items = list(inner.elts) if isinstance(inner, ast.Tuple) else [inner] + if outer == "Annotated": + schema = _annotation_schema(items[0]) + for extra in items[1:]: + if isinstance(extra, ast.Call) and _call_name(extra) == "Field": + schema.update(_field_options(extra)) + return schema + if outer == "Optional": + return _union([_annotation_schema(items[0]), {"type": "null"}]) + if outer == "Union": + return _union([_annotation_schema(i) for i in items]) + if outer == "Literal": + values = [i.value for i in items if isinstance(i, ast.Constant)] + return {"enum": values} if values else {} + if outer in _PY_TYPES and _PY_TYPES[outer] == "array": + return {"type": "array", "items": _annotation_schema(items[0])} + if outer in _PY_TYPES: + return {"type": _PY_TYPES[outer]} + return {} + + +def _union(options: list[dict]) -> dict: + real = [o for o in options if o.get("type") != "null"] + if len(real) == len(options): + return {"anyOf": options} if len(options) > 1 else options[0] + return {"anyOf": [*real, {"type": "null"}]} if real else {"type": "null"} diff --git a/src/agent_env/cli/env/website_template/Dockerfile.dockerignore.tmpl b/src/agent_env/cli/env/website_template/Dockerfile.dockerignore.tmpl new file mode 100644 index 000000000..10ae9270c --- /dev/null +++ b/src/agent_env/cli/env/website_template/Dockerfile.dockerignore.tmpl @@ -0,0 +1,6 @@ +**/node_modules +**/dist +**/__pycache__ +**/.pytest_cache +**/.venv +.git diff --git a/src/agent_env/cli/env/website_template/Dockerfile.tmpl b/src/agent_env/cli/env/website_template/Dockerfile.tmpl new file mode 100644 index 000000000..dee19c0de --- /dev/null +++ b/src/agent_env/cli/env/website_template/Dockerfile.tmpl @@ -0,0 +1,22 @@ +FROM mirror.gcr.io/library/node:22-slim@sha256:4b150b48b4116e2a22f2c39cc27025fbf4f153e2078fd3b14d159fdb5c7279da AS web +WORKDIR /web +COPY web/package*.json ./ +RUN if [ -f package-lock.json ]; then npm ci --no-audit --no-fund; else npm install --no-audit --no-fund; fi +COPY web/ ./ +RUN npm run build + +FROM mirror.gcr.io/library/python:3.12-slim@sha256:423ed6ab25b1921a477529254bfeeabf5855151dc2c3141699a1bfc852199fbf +ARG PROTOCOL="{{protocol}}" +WORKDIR /app +RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/* +RUN pip install --no-cache-dir "${PROTOCOL}" "uvicorn>=0.30" +COPY site.py /app/site.py +COPY data /app/data +COPY --from=web /web/dist /app/web/dist +ENV PORT=8000 +EXPOSE 8000 +RUN useradd --create-home --uid 10001 appuser +USER appuser +HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \ + CMD python -c "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:%s/healthz' % os.environ.get('PORT', '8000'), timeout=2)" +CMD ["python", "site.py"] diff --git a/src/agent_env/cli/env/website_template/README.md.tmpl b/src/agent_env/cli/env/website_template/README.md.tmpl new file mode 100644 index 000000000..c17de7b80 --- /dev/null +++ b/src/agent_env/cli/env/website_template/README.md.tmpl @@ -0,0 +1,40 @@ +# {{title}} + +{{title}} is a website environment on `agentenv_protocol.website.WebsiteEnvironment`, made by +`agent-env env website new`{{source_note}}. It meets the `urn:agentenv:website/v1` server contract. + +| File | What it is | +| --- | --- | +| `site.py` | The server: the `{{class_name}}` WebsiteEnvironment, its `@api` routes and its `@ready_check`, and `serve()` on `PORT` (8000). | +| `data/seed.json` | The seed data the store starts with: `current_user`, the acting user, the `users` a browser can sign in as, and the site's collections. | +| `web/` | A minimal Vite app. Its assets (`base: "./"`) and API calls are relative to the page, so it works at `/` and under the gateway's `/website/{{name}}/` alike. `src/site-client.js` is the SDK's browser client: `api()`, `universeNow()`, `followUniverseClock()`, `session()`, `signIn()` and `signOut()`. | +| `Dockerfile` | The server image, built with this folder as its context: Node builds `web/`, then Python installs `agentenv-framework-protocol` with uvicorn. | +| `env.toml` | The website env as a bundle entry. | + +## Develop it + +```bash +agent-env env website dev +``` + +It builds `web/` when it is out of date (and rebuilds it as it changes), serves `site.py` on http://localhost:8000 with +reload, syncs the site to a local clock and loads `data/seed.json` through the data plane, then runs the conformance +check once. The local clock answers `/clock/time` as the gateway's does, and `PUT /clock/set-time` +(`{"virtual_time": "..."}`) and `PUT /clock/advance` (`{"virtual_time": "..."}` or `{"seconds": 3600}`) move it. +Ctrl-C stops it all. For Vite's own dev server, run `npm run dev` in `web/` while the site runs; it proxies `/api` to +`SITE_URL` (http://127.0.0.1:8000). + +To check a running server against the contract: + +```bash +python -m agentenv_protocol.website.conformance http://localhost:8000 +``` + +## Register it + +```bash +agent-env env website put --id {{name}} --server-dockerfile Dockerfile +``` + +`environment_name` comes from the card (`@environment_card(name="{{name}}")`). The image installs +`{{protocol}}`; pass `--build-arg PROTOCOL=...` to `docker build` for another source of the package. diff --git a/src/agent_env/cli/env/website_template/data/seed.json.tmpl b/src/agent_env/cli/env/website_template/data/seed.json.tmpl new file mode 100644 index 000000000..1bf9af645 --- /dev/null +++ b/src/agent_env/cli/env/website_template/data/seed.json.tmpl @@ -0,0 +1,11 @@ +{ + "current_user": "ada", + "users": [ + {"id": "ada", "name": "Ada Park"}, + {"id": "grace", "name": "Grace Hall"} + ], + "items": [ + {"id": 1, "owner": "ada", "title": "Try the site", "created_at": "2026-01-05T09:00:00.000Z"}, + {"id": 2, "owner": "grace", "title": "Write the first task", "created_at": "2026-01-04T15:30:00.000Z"} + ] +} diff --git a/src/agent_env/cli/env/website_template/env.toml.tmpl b/src/agent_env/cli/env/website_template/env.toml.tmpl new file mode 100644 index 000000000..9e45ca364 --- /dev/null +++ b/src/agent_env/cli/env/website_template/env.toml.tmpl @@ -0,0 +1,3 @@ +type = "website" +environment_name = "{{name}}" +backend_image = { dockerfile = "Dockerfile" } diff --git a/src/agent_env/cli/env/website_template/gitignore.tmpl b/src/agent_env/cli/env/website_template/gitignore.tmpl new file mode 100644 index 000000000..03a6d6223 --- /dev/null +++ b/src/agent_env/cli/env/website_template/gitignore.tmpl @@ -0,0 +1,3 @@ +web/node_modules/ +web/dist/ +__pycache__/ diff --git a/src/agent_env/cli/env/website_template/site-tools.py.tmpl b/src/agent_env/cli/env/website_template/site-tools.py.tmpl new file mode 100644 index 000000000..abfa54c79 --- /dev/null +++ b/src/agent_env/cli/env/website_template/site-tools.py.tmpl @@ -0,0 +1,36 @@ +"""{{title}}: a site on ``agentenv_protocol.website.WebsiteEnvironment``, made by ``agent-env env website new`` from +the tools of {{source}}. Each tool is a stub ``@api`` route of the same name, which answers 501 until it is written; the +seed in ``data/seed.json`` has a collection for each of them to fill. A browser with no session is signed in as the +acting user, the seed's ``current_user``, and can sign in as another seeded user. + +Run it with ``agent-env env website dev``, or with ``python site.py`` after building ``web/`` into ``web/dist``; it +serves on ``PORT`` (8000).""" +from __future__ import annotations + +{{typing_import}}from pydantic import BaseModel, Field + +from agentenv_protocol import environment_card +from agentenv_protocol.website import ApiContext, ApiError, NotFound, WebsiteEnvironment, api, ready_check, website + +USERS = "users" + + +class SignIn(BaseModel): + user: str +{{models}} + +@environment_card(name="{{name}}") +@website(domains=["{{domain}}"], static="web/dist", health="/healthz", data="data/seed.json") +class {{class_name}}(WebsiteEnvironment): + @ready_check + def seeded(self) -> bool: + """The site is usable once its store knows who may sign in.""" + return bool(self.store[USERS]) +{{builtin_routes}}{{routes}} + +def _stub(tool: str) -> ApiError: + return ApiError(f"{tool} is a stub: write what it does in site.py", status=501, code="not_implemented") + + +if __name__ == "__main__": + {{class_name}}().serve() diff --git a/src/agent_env/cli/env/website_template/site.py.tmpl b/src/agent_env/cli/env/website_template/site.py.tmpl new file mode 100644 index 000000000..dd285c355 --- /dev/null +++ b/src/agent_env/cli/env/website_template/site.py.tmpl @@ -0,0 +1,78 @@ +"""{{title}}: a site on ``agentenv_protocol.website.WebsiteEnvironment``, made by ``agent-env env website new``. Each +user keeps a list of items; a browser with no session is signed in as the acting user, the seed's ``current_user``, and +can sign in as another seeded user. The site keeps its state in the default store, which the data plane loads, exports +and resets, and stamps each item with the universe's time. + +Run it with ``agent-env env website dev``, or with ``python site.py`` after building ``web/`` into ``web/dist``; it +serves on ``PORT`` (8000).""" +from __future__ import annotations + +from pydantic import BaseModel + +from agentenv_protocol import environment_card +from agentenv_protocol.website import ( + ApiContext, + BadRequest, + NotFound, + Unauthorized, + WebsiteEnvironment, + api, + ready_check, + website, +) +from agentenv_protocol.website.clock import iso + +ITEMS = "items" +USERS = "users" + + +class NewItem(BaseModel): + title: str + + +class SignIn(BaseModel): + user: str + + +@environment_card(name="{{name}}") +@website(domains=["{{domain}}"], static="web/dist", health="/healthz", data="data/seed.json") +class {{class_name}}(WebsiteEnvironment): + @ready_check + def seeded(self) -> bool: + """The site is usable once its store knows who may sign in.""" + return bool(self.store[USERS]) + + @api.get("/api/users", name="list_users") + def list_users(self) -> list: + """The users a browser can sign in as.""" + return self.store[USERS] + + @api.post("/api/sign-in", name="sign_in") + def sign_in(self, body: SignIn, ctx: ApiContext) -> dict: + """Sign the browser in as one of the site's users.""" + if self.store.get(USERS, body.user) is None: + raise NotFound(f"no user {body.user!r}") + ctx.sign_in(body.user) + return {"user": ctx.user} + + @api.get("/api/items", name="list_items") + def list_items(self, ctx: ApiContext) -> list: + """The signed-in user's items, newest first.""" + return sorted(self.store.find(ITEMS, owner=self._user(ctx)), key=lambda item: item["created_at"], reverse=True) + + @api.post("/api/items", name="create_item", status=201) + def create_item(self, item: NewItem, ctx: ApiContext) -> dict: + """Add an item for the signed-in user, created now.""" + if not item.title.strip(): + raise BadRequest("an item needs a title", code="empty_title") + return self.store.insert(ITEMS, {"owner": self._user(ctx), "title": item.title.strip(), + "created_at": iso(self.now())}) + + def _user(self, ctx: ApiContext) -> str: + if ctx.user is None: + raise Unauthorized("sign in first") + return ctx.user + + +if __name__ == "__main__": + {{class_name}}().serve() diff --git a/src/agent_env/cli/env/website_template/web/index.html.tmpl b/src/agent_env/cli/env/website_template/web/index.html.tmpl new file mode 100644 index 000000000..d4119cdc4 --- /dev/null +++ b/src/agent_env/cli/env/website_template/web/index.html.tmpl @@ -0,0 +1,24 @@ + + + + + + {{title}} + + +
+

{{title}}

+
+ + +
+
+
+ +
+
+ + + diff --git a/src/agent_env/cli/env/website_template/web/package.json.tmpl b/src/agent_env/cli/env/website_template/web/package.json.tmpl new file mode 100644 index 000000000..a4af53d1d --- /dev/null +++ b/src/agent_env/cli/env/website_template/web/package.json.tmpl @@ -0,0 +1,14 @@ +{ + "name": "{{name}}-site-web", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "preview": "vite preview" + }, + "devDependencies": { + "vite": "5.4.10" + } +} diff --git a/src/agent_env/cli/env/website_template/web/src/main-tools.js.tmpl b/src/agent_env/cli/env/website_template/web/src/main-tools.js.tmpl new file mode 100644 index 000000000..d0e5926fd --- /dev/null +++ b/src/agent_env/cli/env/website_template/web/src/main-tools.js.tmpl @@ -0,0 +1,68 @@ +import "./style.css"; +import { api, followUniverseClock, session, signIn, universeNow } from "./site-client.js"; + +// The routes `agent-env env website new` made from the MCP server's tools. Each answers 501 until site.py writes it. +const ROUTES = {{routes_json}}; + +const $ = (id) => document.getElementById(id); + +function showError(error) { + $("error").textContent = error ? error.error || error.message : ""; + $("error").hidden = !error; +} + +function tick() { + const now = universeNow(); + $("universe-now").textContent = `${now.toLocaleString(undefined, { dateStyle: "medium", timeStyle: "short", timeZone: "UTC" })} UTC`; + $("universe-now").dateTime = now.toISOString(); +} + +function routeItem(route) { + const item = document.createElement("li"); + item.className = "card"; + const title = document.createElement("code"); + title.textContent = `${route.method} ${route.path}`; + const about = document.createElement("p"); + about.textContent = route.description || route.name; + const params = document.createElement("textarea"); + params.rows = 2; + params.value = JSON.stringify(route.example, null, 0); + params.setAttribute("aria-label", `${route.name} parameters`); + const call = document.createElement("button"); + call.textContent = "Call"; + const out = document.createElement("pre"); + call.addEventListener("click", async () => { + try { + const values = JSON.parse(params.value || "{}"); + const query = new URLSearchParams(Object.entries(values).map(([k, v]) => [k, String(v)])).toString(); + const answer = route.method === "GET" + ? await api(query ? `${route.path}?${query}` : route.path) + : await api(route.path, { method: route.method, body: values }); + out.textContent = JSON.stringify(answer, null, 2); + showError(null); + } catch (error) { + out.textContent = ""; + showError(error); + } + }); + item.append(title, about, params, call, out); + return item; +} + +async function start() { + const [signedIn, users] = await Promise.all([session(), api("api/users")]); + $("user").replaceChildren(...users.map((user) => new Option(user.name || user.id, user.id))); + $("user").value = signedIn.user ?? ""; + $("user").addEventListener("change", (event) => signIn(event.target.value).then(() => showError(null), showError)); + + const list = document.createElement("ul"); + list.id = "routes"; + list.append(...ROUTES.map(routeItem)); + $("app").append(list); + + tick(); + setInterval(tick, 1000); + followUniverseClock({ path: "healthz" }); +} + +start().catch(showError); diff --git a/src/agent_env/cli/env/website_template/web/src/main.js.tmpl b/src/agent_env/cli/env/website_template/web/src/main.js.tmpl new file mode 100644 index 000000000..06c34f466 --- /dev/null +++ b/src/agent_env/cli/env/website_template/web/src/main.js.tmpl @@ -0,0 +1,85 @@ +import "./style.css"; +import { api, followUniverseClock, session, signIn, universeNow } from "./site-client.js"; + +const $ = (id) => document.getElementById(id); + +function showError(error) { + $("error").textContent = error ? error.error || error.message : ""; + $("error").hidden = !error; +} + +function formatTime(value) { + return new Date(value).toLocaleString(undefined, { dateStyle: "medium", timeStyle: "short", timeZone: "UTC" }); +} + +function tick() { + $("universe-now").textContent = `${formatTime(universeNow())} UTC`; + $("universe-now").dateTime = universeNow().toISOString(); +} + +async function render() { + const items = await api("api/items"); + const list = document.createElement("ul"); + list.id = "items"; + for (const item of items) { + const row = document.createElement("li"); + row.className = "card"; + row.textContent = item.title; + const meta = document.createElement("small"); + meta.textContent = ` ยท ${formatTime(item.created_at)}`; + row.append(meta); + list.append(row); + } + if (!items.length) { + const empty = document.createElement("li"); + empty.className = "empty"; + empty.textContent = "Nothing here yet."; + list.append(empty); + } + $("items")?.remove(); + $("app").append(list); +} + +async function run(action) { + try { + await action(); + showError(null); + await render(); + } catch (error) { + showError(error); + } +} + +async function start() { + const [signedIn, users] = await Promise.all([session(), api("api/users")]); + $("user").replaceChildren(...users.map((user) => new Option(user.name || user.id, user.id))); + $("user").value = signedIn.user ?? ""; + $("user").addEventListener("change", (event) => run(() => signIn(event.target.value))); + + const form = document.createElement("form"); + form.id = "new-item"; + form.className = "card"; + const title = document.createElement("input"); + title.placeholder = "New item"; + title.required = true; + title.setAttribute("aria-label", "New item"); + const add = document.createElement("button"); + add.type = "submit"; + add.textContent = "Add"; + form.append(title, add); + form.addEventListener("submit", (event) => { + event.preventDefault(); + run(async () => { + await api("api/items", { method: "POST", body: { title: title.value } }); + form.reset(); + }); + }); + $("app").append(form); + + tick(); + setInterval(tick, 1000); + followUniverseClock({ path: "healthz" }); + await render(); +} + +start().catch(showError); diff --git a/src/agent_env/cli/env/website_template/web/src/style.css.tmpl b/src/agent_env/cli/env/website_template/web/src/style.css.tmpl new file mode 100644 index 000000000..ba2c7a4fa --- /dev/null +++ b/src/agent_env/cli/env/website_template/web/src/style.css.tmpl @@ -0,0 +1,46 @@ +:root { + --bg: #f6f6f3; + --card: #ffffff; + --text: #1d1d1b; + --muted: #6b6b66; + --line: #deded8; + --accent: #2f5bd3; + font-family: system-ui, -apple-system, "Segoe UI", sans-serif; + color: var(--text); + background: var(--bg); +} + +* { box-sizing: border-box; } +body { margin: 0; } + +.bar { + display: flex; + flex-wrap: wrap; + gap: 12px; + align-items: center; + justify-content: space-between; + padding: 12px 16px; + border-bottom: 1px solid var(--line); + background: var(--card); +} +.bar h1 { margin: 0; font-size: 1.25rem; } +.who { display: flex; flex-wrap: wrap; gap: 12px; align-items: center; color: var(--muted); font-size: 0.9rem; } +.who time { font-variant-numeric: tabular-nums; } + +main { max-width: 720px; margin: 0 auto; padding: 16px; } + +.card { background: var(--card); border: 1px solid var(--line); border-radius: 8px; padding: 12px; } + +input, textarea, select, button { font: inherit; color: inherit; } +input, textarea { width: 100%; border: 1px solid var(--line); border-radius: 6px; padding: 6px 8px; background: transparent; } +button { border: 1px solid var(--line); border-radius: 6px; padding: 6px 12px; background: var(--card); cursor: pointer; } +button[type="submit"] { background: var(--accent); border-color: var(--accent); color: #fff; } + +form.card { display: flex; gap: 8px; margin-bottom: 16px; } +ul { list-style: none; margin: 0; padding: 0; display: grid; gap: 8px; } +li small { color: var(--muted); } +li pre { margin: 8px 0 0; white-space: pre-wrap; font-size: 0.85rem; } +li textarea { margin: 8px 0; font-family: ui-monospace, monospace; font-size: 0.85rem; } +.empty { color: var(--muted); text-align: center; padding: 24px; } + +#error { color: #b3261e; margin: 0 0 12px; } diff --git a/src/agent_env/cli/env/website_template/web/vite.config.js.tmpl b/src/agent_env/cli/env/website_template/web/vite.config.js.tmpl new file mode 100644 index 000000000..79f723c86 --- /dev/null +++ b/src/agent_env/cli/env/website_template/web/vite.config.js.tmpl @@ -0,0 +1,11 @@ +import { defineConfig } from "vite"; + +export default defineConfig({ + base: "./", + server: { + proxy: { + "/api": process.env.SITE_URL || "http://127.0.0.1:8000", + "/healthz": process.env.SITE_URL || "http://127.0.0.1:8000", + }, + }, +}); diff --git a/tst/unit/cli/website_dev_cli_test.py b/tst/unit/cli/website_dev_cli_test.py new file mode 100644 index 000000000..0fdcd7bb9 --- /dev/null +++ b/tst/unit/cli/website_dev_cli_test.py @@ -0,0 +1,310 @@ +"""agent-env env website dev's pieces, each without a long-running process: the site class it serves, the boot every +answer of a server carries, the local clock standing in for the gateway's, when web/ is rebuilt, priming a server +(clock/v1's sync_time and the seed through the data plane) once per boot with one conformance check, and the command +wiring uvicorn's reload, the clock and the supervisor together.""" + +import datetime +import json +import logging +import os +import sys +from pathlib import Path +from unittest.mock import MagicMock, patch + +import httpx +import pytest +from click.testing import CliRunner +from starlette.testclient import TestClient + +from agent_env.cli.env import website_dev +from agent_env.cli.env.website import website +from agent_env.cli.env.website_scaffold import scaffold +from agentenv_protocol.types import WELL_KNOWN_PATH +from agentenv_protocol.website import UNIVERSE_NOW_HEADER, conformance, website as declare +from agentenv_protocol.website.clock import parse_time + +TWO_SITES = ''' +from agentenv_protocol.website import WebsiteEnvironment, website + + +class Base(WebsiteEnvironment): + pass + + +@website(domains=["a.test"]) +class Site(Base): + pass +''' + + +@pytest.fixture +def site_dir(tmp_path: Path) -> Path: + scaffold(tmp_path / "shop", "shop") + page = tmp_path / "shop" / "web" / "dist" + page.mkdir() + (page / "index.html").write_text("shop") + return tmp_path / "shop" + + +@pytest.fixture +def clock(socket_enabled): + started = website_dev.DevClock(datetime.datetime(2030, 1, 1, 9, tzinfo=datetime.timezone.utc)).start() + yield started + started.close() + + +def _app(site_dir: Path, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv(website_dev.SITE_ENV, str(site_dir / "site.py")) + return website_dev.create_app() + + +def test_the_site_class_is_the_one_its_file_defines_or_the_one_declared_with_website(tmp_path, site_dir): + assert website_dev.site_class(site_dir / "site.py").__name__ == "ShopSite" + (tmp_path / "two.py").write_text(TWO_SITES) + assert website_dev.site_class(tmp_path / "two.py").__name__ == "Site" + (tmp_path / "none.py").write_text("x = 1\n") + with pytest.raises(website_dev.DevError, match="defines 0 WebsiteEnvironment classes"): + website_dev.site_class(tmp_path / "none.py") + + +def test_a_site_imports_the_files_beside_it(tmp_path, monkeypatch): + monkeypatch.setattr(sys, "path", list(sys.path)) + (tmp_path / "shop").mkdir() + (tmp_path / "shop" / "shop_models_beside.py").write_text("DOMAIN = 'shop.test'\n") + (tmp_path / "shop" / "site.py").write_text( + "from agentenv_protocol.website import WebsiteEnvironment, website\n" + "from shop_models_beside import DOMAIN\n\n\n@website(domains=[DOMAIN])\nclass Shop(WebsiteEnvironment):\n pass\n") + monkeypatch.chdir(tmp_path) + assert website_dev.site_class(Path("shop") / "site.py").__name__ == "Shop" + + +def test_every_answer_carries_the_servers_boot_and_the_supervisors_polls_leave_the_access_log(site_dir, monkeypatch): + first, second = TestClient(_app(site_dir, monkeypatch)), TestClient(_app(site_dir, monkeypatch)) + boot = first.get("/healthz").headers[website_dev.BOOT_HEADER] + assert first.get("/api/items").headers[website_dev.BOOT_HEADER] == boot + assert second.get("/healthz").headers[website_dev.BOOT_HEADER] != boot + + def line(path): + return logging.LogRecord("uvicorn.access", logging.INFO, "", 0, '%s - "%s %s HTTP/%s" %d', + ("127.0.0.1:1", "GET", path, "1.1", 200), None) + assert not website_dev._unpolled(line(f"/.well-known/agent-env.json?{website_dev.POLL_QUERY}")) + assert website_dev._unpolled(line("/api/items")) + + +def test_the_local_clock_answers_as_the_gateways_and_is_set_and_advanced(clock): + base = clock.url.rsplit("/clock/", 1)[0] + assert parse_time(httpx.get(clock.url).json()["virtual_time"]) - parse_time("2030-01-01T09:00:00Z") < datetime.timedelta(seconds=5) + state = httpx.put(base + "/clock/set-time", json={"virtual_time": "2031-02-03T04:05:06Z", + "virtual_seconds_per_real_second": 0}).json() + assert (state["virtual_time"], state["virtual_seconds_per_real_second"]) == ("2031-02-03T04:05:06.000Z", 0.0) + assert httpx.put(base + "/clock/advance", json={"seconds": 3600}).json()["virtual_time"] == "2031-02-03T05:05:06.000Z" + assert httpx.put(base + "/clock/advance", json={"virtual_time": "2031-02-04T00:00:00Z"}).status_code == 200 + assert httpx.get(clock.url).json() == {"virtual_time": "2031-02-04T00:00:00.000Z", + "virtual_seconds_per_real_second": 0.0} + assert httpx.get(base + "/clock/state").json()["env_get_time_url"] == clock.url + backwards = httpx.put(base + "/clock/advance", json={"virtual_time": "2031-01-01T00:00:00Z"}) + assert backwards.status_code == 400 and "forward" in backwards.json()["error"] + assert httpx.put(base + "/clock/advance", json={}).status_code == 400 + assert httpx.put(base + "/clock/set-time", content=b"[1]").status_code == 400 + assert httpx.get(base + "/clock/other").status_code == 404 + + +def test_web_is_installed_when_it_has_no_node_modules_and_built_when_dist_is_older_than_a_source(tmp_path): + web = tmp_path / "web" + (web / "src").mkdir(parents=True) + (web / "package.json").write_text("{}") + (web / "src" / "main.js").write_text("") + assert website_dev.web_plan(web, watch=False) == ( + [["npm", "install", "--no-audit", "--no-fund"], ["npm", "run", "build"]], None) + (web / "package-lock.json").write_text("{}") + (web / "node_modules" / "vite").mkdir(parents=True) + (web / "dist").mkdir() + (web / "dist" / "index.html").write_text("") + for path in [*web.rglob("*")]: + os.utime(path, (1000, 1000)) + os.utime(web / "dist" / "index.html", (2000, 2000)) + (web / "node_modules" / "vite" / "new.js").write_text("") + assert not website_dev.web_stale(web) + assert website_dev.web_plan(web, watch=True) == ([], ["npm", "run", "build", "--", "--watch"]) + os.utime(web / "src" / "main.js", (3000, 3000)) + assert website_dev.web_stale(web) + assert website_dev.web_plan(web, watch=False)[0] == [["npm", "run", "build"]] + (web / "node_modules").rename(web / "gone") + assert website_dev.web_plan(web, watch=False)[0][0] == ["npm", "ci", "--no-audit", "--no-fund"] + + +def test_priming_syncs_the_site_to_the_clock_and_loads_the_seed_in_place_of_its_state(site_dir, clock, monkeypatch): + client = TestClient(_app(site_dir, monkeypatch)) + client.post("/api/items", json={"title": "made before priming"}) + seed = site_dir / "data" / "seed.json" + boot = website_dev.prime(client, "http://testserver", clock.url, seed) + assert boot == client.get("/healthz").headers[website_dev.BOOT_HEADER] + assert client.get("/healthz").headers[UNIVERSE_NOW_HEADER].startswith("2030-01-01T09:0") + assert [i["title"] for i in client.get("/api/items").json()] == ["Try the site"] + assert client.post("/api/items", json={"title": "now"}).json()["created_at"].startswith("2030-01-01T09:0") + + +def _card_site(card: dict, sent: list) -> httpx.Client: + """A client of a site that serves ``card`` and records each request it is sent, answering every other one.""" + def answer(request: httpx.Request) -> httpx.Response: + sent.append(request) + if request.url.path == WELL_KNOWN_PATH: + return httpx.Response(200, json=card) + return httpx.Response(200, json={"jsonrpc": "2.0", "id": 1, "result": {}}) + + return httpx.Client(transport=httpx.MockTransport(answer)) + + +def test_priming_loads_the_seed_at_the_data_plane_the_card_names_and_refuses_one_it_doesnt_serve(site_dir): + seed = site_dir / "data" / "seed.json" + clock_ext = {"uri": website_dev.CLOCK_EXTENSION_URI, + "params": {"methods": {"sync_time": {"endpoint": "/agentenv/ext/clock/sync"}}}} + card = {"name": "shop", "url": "/svc/shop/rpc", "capabilities": {"extensions": [clock_ext], + "operations": ["data/reset", "data/add"]}} + sent: list[httpx.Request] = [] + website_dev.prime(_card_site(card, sent), "http://site", "http://clock", seed) + rpcs = [(str(r.url), json.loads(r.content)["method"]) for r in sent + if r.url.path not in (WELL_KNOWN_PATH, "/agentenv/ext/clock/sync")] + assert rpcs == [("http://site/svc/shop/rpc", "data/reset"), ("http://site/svc/shop/rpc", "data/add")] + + sent.clear() + website_dev.prime(_card_site({**card, "url": "http://elsewhere/rpc"}, sent), "http://site", "http://clock", seed) + assert {str(r.url) for r in sent if r.method == "POST" and "clock" not in r.url.path} == {"http://elsewhere/rpc"} + + sent.clear() + unserving = {**card, "capabilities": {**card["capabilities"], "operations": ["data/get"]}} + with pytest.raises(website_dev.DevError, match="doesn't serve data/reset, data/add"): + website_dev.prime(_card_site(unserving, sent), "http://site", "http://clock", seed) + assert not any("rpc" in r.url.path for r in sent) + + +def test_the_supervisor_primes_each_boot_once_and_runs_the_conformance_check_after_the_first(site_dir, clock, + monkeypatch): + first = TestClient(_app(site_dir, monkeypatch)) + said = [] + supervisor = website_dev.Supervisor("http://testserver", clock.url, site_dir / "data" / "seed.json", + echo=said.append, client=first) + with patch.object(website_dev, "prime", wraps=website_dev.prime) as primed: + assert supervisor.step() + assert "[dev] conformance: 9 passed or skipped, 0 failed" in said and primed.call_count == 2 + assert first.get("/healthz").headers[UNIVERSE_NOW_HEADER].startswith("2030-01-01T09:0") + assert supervisor.step() and primed.call_count == 2 + + reloaded = TestClient(_app(site_dir, monkeypatch)) + supervisor.client = reloaded + assert supervisor.step() and primed.call_count == 3 + assert sum("conformance:" in line for line in said) == 1 + assert reloaded.get("/healthz").headers[UNIVERSE_NOW_HEADER].startswith("2030-01-01T09:0") + + +def test_the_supervisor_shows_a_warning_and_syncs_the_site_again_even_when_the_check_raises(site_dir, clock, + monkeypatch): + client = TestClient(_app(site_dir, monkeypatch)) + said = [] + supervisor = website_dev.Supervisor("http://testserver", clock.url, None, echo=said.append, client=client) + warned = conformance.Result("clock", "warning", "interrupted by a reload", False) + with patch.object(conformance, "check", return_value=[warned]), \ + patch.object(website_dev, "prime", wraps=website_dev.prime) as primed: + assert supervisor.step() + assert "[dev] WARN clock: interrupted by a reload" in said and primed.call_count == 2 + + supervisor.checked = False + with patch.object(conformance, "check", side_effect=RuntimeError("boom")), \ + patch.object(website_dev, "prime", wraps=website_dev.prime) as primed, pytest.raises(RuntimeError): + supervisor._conformance() + assert primed.call_count == 1 + + with patch.object(conformance, "check", side_effect=httpx.RemoteProtocolError("reloaded")), \ + patch.object(website_dev, "prime", wraps=website_dev.prime) as primed: + supervisor._conformance() + assert "[dev] the conformance check lost the site, likely to a reload: reloaded" in said and primed.call_count == 1 + + +def test_the_supervisor_primes_again_on_the_next_poll_when_priming_after_the_check_fails(site_dir, clock, + monkeypatch): + client = TestClient(_app(site_dir, monkeypatch)) + said = [] + supervisor = website_dev.Supervisor("http://testserver", clock.url, None, echo=said.append, client=client) + real_prime = website_dev.prime + with patch.object(conformance, "check", return_value=[]), \ + patch.object(website_dev, "prime", side_effect=[real_prime(client, "http://testserver", clock.url, None), + httpx.ConnectError("refused")]): + assert supervisor.step() + assert "[dev] couldn't sync the site again after the check: refused" in said and supervisor.primed is None + with patch.object(website_dev, "prime", wraps=website_dev.prime) as primed: + assert supervisor.step() and primed.call_count == 1 + assert supervisor.primed == client.get("/healthz").headers[website_dev.BOOT_HEADER] + + +def test_the_supervisor_reports_a_seed_it_cant_read_and_primes_once_the_seed_is_back(site_dir, clock, monkeypatch): + client = TestClient(_app(site_dir, monkeypatch)) + said = [] + seed = site_dir / "data" / "seed.json" + kept = seed.read_text(encoding="utf-8") + seed.unlink() + supervisor = website_dev.Supervisor("http://testserver", clock.url, seed, echo=said.append, client=client) + with patch.object(conformance, "check", return_value=[]): + assert supervisor.step() + assert "couldn't sync the site and load its seed" in said[-1] and supervisor.primed is None + + seed.write_text(kept, encoding="utf-8") + real_prime = website_dev.prime + with patch.object(website_dev, "prime", side_effect=[real_prime(client, "http://testserver", clock.url, seed), + FileNotFoundError(str(seed))]): + assert supervisor.step() + assert "couldn't sync the site again after the check" in said[-1] and supervisor.primed is None + + assert supervisor.step() + assert supervisor.primed == client.get("/healthz").headers[website_dev.BOOT_HEADER] + assert [i["title"] for i in client.get("/api/items").json()] == ["Try the site"] + + +def test_the_supervisor_waits_for_a_server_that_isnt_up_and_reports_one_it_cant_prime(): + down = MagicMock(spec=httpx.Client) + down.get.side_effect = httpx.ConnectError("refused") + assert not website_dev.Supervisor("http://127.0.0.1:1", "http://clock", None, client=down).step() + + app = declare(domains=["x.test"])(type("Bare", (website_dev.WebsiteEnvironment,), {}))().create_app() + said = [] + supervisor = website_dev.Supervisor("http://testserver", "http://clock", None, client=TestClient(app), + echo=said.append, check=False) + with patch.object(website_dev, "find_extension_method", return_value=None): + assert supervisor.step() + assert "couldn't sync the site" in said[0] + + +def test_dev_runs_uvicorn_with_reload_beside_the_clock_and_the_supervisor(site_dir, monkeypatch): + monkeypatch.setenv(website_dev.SITE_ENV, "unset") + clock, supervisor = MagicMock(url="http://127.0.0.1:5555/clock/time"), MagicMock() + supervisor.start.return_value = supervisor + clock.start.return_value = clock + with patch.object(website_dev, "DevClock", return_value=clock) as made_clock, \ + patch.object(website_dev, "Supervisor", return_value=supervisor) as made_supervisor, \ + patch.object(website_dev, "npm_available", return_value=True), \ + patch.object(website_dev, "run_npm") as npm, \ + patch("uvicorn.run") as run: + result = CliRunner().invoke(website, ["dev", "--dir", str(site_dir), "--port", "8123", "--no-watch", + "--clock-time", "2030-01-01T09:00:00Z"]) + assert os.environ[website_dev.SITE_ENV] == str(site_dir.resolve() / "site.py") + assert result.exit_code == 0, result.output + assert [c.args[0] for c in npm.call_args_list] == [["npm", "install", "--no-audit", "--no-fund"]] + assert made_clock.call_args.args[0] == parse_time("2030-01-01T09:00:00Z") + assert made_supervisor.call_args.args == ("http://127.0.0.1:8123", clock.url, site_dir.resolve() / "data" / "seed.json") + assert run.call_args.args == ("agent_env.cli.env.website_dev:create_app",) + options = run.call_args.kwargs + assert (options["factory"], options["reload"], options["port"]) == (True, True, 8123) + assert options["reload_dirs"] == [str(site_dir.resolve())] + assert options["reload_excludes"] == [str(site_dir.resolve() / "web")] + assert "http://127.0.0.1:8123/" in result.output and "/clock/advance" in result.output + supervisor.stop.assert_called_once() + clock.close.assert_called_once() + + +def test_dev_refuses_a_folder_without_its_site_and_a_clock_time_that_isnt_one(tmp_path, site_dir): + missing = CliRunner().invoke(website, ["dev", "--dir", str(tmp_path)]) + assert missing.exit_code == 1 and "no site.py" in missing.output + bad = CliRunner().invoke(website, ["dev", "--dir", str(site_dir), "--clock-time", "soon"]) + assert bad.exit_code == 1 and "not an RFC3339 time" in bad.output + (tmp_path / "site.py").write_text("raise RuntimeError('broken')\n") + broken = CliRunner().invoke(website, ["dev", "--dir", str(tmp_path)]) + assert broken.exit_code == 1 and "broken" in broken.output diff --git a/tst/unit/cli/website_new_cli_test.py b/tst/unit/cli/website_new_cli_test.py new file mode 100644 index 000000000..383f18547 --- /dev/null +++ b/tst/unit/cli/website_new_cli_test.py @@ -0,0 +1,252 @@ +"""agent-env env website new: a site like the Notes example, which imports and passes the website/v1 conformance +check as it is written; refused over a non-empty folder without --force; and, from an MCP server's source or a tools +JSON, one stub @api route per tool with a seed collection for each.""" + +import importlib.util +import json +import sys +import uuid +from pathlib import Path + +import pytest +from click.testing import CliRunner +from starlette.testclient import TestClient + +from agent_env.cli.env.website import website +from agent_env.cli.env.website_scaffold import CLIENT_DIR, tools_from_json, tools_from_mcp +from agentenv_protocol.website import SESSION_PATH, WebsiteEnvironment, conformance +from agentenv_protocol.website import website as declare + +MAIL_SERVER = Path(__file__).resolve().parents[3] / "src" / "agent_env" / "examples" / "mail-mcp" / "envs" / "mail" +FILES = {".gitignore", "Dockerfile", "Dockerfile.dockerignore", "README.md", "env.toml", "site.py", "data/seed.json", + "web/index.html", "web/package.json", "web/vite.config.js", "web/src/main.js", "web/src/style.css", + "web/src/site-client.js", "web/src/site-client.d.ts"} +FASTMCP_SERVER = ''' +from typing import Annotated, Literal, Optional +from mcp.server.fastmcp import Context, FastMCP +from pydantic import Field + +mcp = FastMCP("calendar") + + +@mcp.tool(name="list_events") +async def list_events(day: str, limit: int = 10, ctx: Context = None) -> list: + """The events of a day.""" + + +@mcp.tool() +def create_event( + title: Annotated[str, Field(description="What the event is.")], + attendees: list[str], + kind: Literal["meeting", "focus"] = "meeting", + notes: Optional[str] = None, + room: str = Field(None, description="Where."), +) -> dict: + """Add an event.""" + + +def helper(x: int) -> int: + return x +''' + + +def _new(*args): + return CliRunner().invoke(website, ["new", *args]) + + +def _load(site_dir: Path) -> type[WebsiteEnvironment]: + """The generated ``site.py``'s site, serving a stand-in page instead of ``web/dist``, which only a Vite build makes.""" + spec = importlib.util.spec_from_file_location(f"generated_site_{uuid.uuid4().hex}", site_dir / "site.py") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + cls = next(v for v in vars(module).values() if isinstance(v, type) and issubclass(v, WebsiteEnvironment) + and v is not WebsiteEnvironment) + page = site_dir / "stand-in" + page.mkdir() + (page / "index.html").write_text("site") + declared = cls.declaration()._replace(static=str(page), data=str(site_dir / "data" / "seed.json")) + return declare(**declared._asdict())(type(cls.__name__, (cls,), {})) + + +def _passes_conformance(client: TestClient) -> None: + """The conformance check, whose clock check serves a clock on 127.0.0.1 for the site to read.""" + results = conformance.check("http://testserver", client=client) + assert {r.name: r.status for r in results} == dict.fromkeys( + ["card", "any_host", "health", "universe_now", "reserved_paths", "data_plane", "clock", "forwarded", "cookies"], "passed"), results + + +def test_a_new_site_has_notes_layout_and_the_sdks_browser_client(tmp_path): + result = _new("shop", "--dir", str(tmp_path / "shop"), "--domain", "shop.example") + assert result.exit_code == 0, result.output + assert "agent-env env website dev" in result.output and "--health-path" not in result.output + site = tmp_path / "shop" + assert {str(p.relative_to(site)) for p in site.rglob("*") if p.is_file()} == FILES + assert (site / "env.toml").read_text() == ('type = "website"\nenvironment_name = "shop"\n' + 'backend_image = { dockerfile = "Dockerfile" }\n') + assert "--health-path" not in (site / "README.md").read_text() + package = json.loads((site / "web" / "package.json").read_text()) + assert package["devDependencies"] == {"vite": "5.4.10"} and "dependencies" not in package + vite = (site / "web" / "vite.config.js").read_text() + assert 'base: "./"' in vite and '"/api":' in vite and '"/healthz":' in vite + assert 'from "./site-client.js"' in (site / "web" / "src" / "main.js").read_text() + for client in ("site-client.js", "site-client.d.ts"): + assert (site / "web" / "src" / client).read_text() == (CLIENT_DIR / client).read_text() + assert "{{" not in "".join(p.read_text() for p in site.rglob("*") if p.is_file()) + assert 'domains=["shop.example"]' in (site / "site.py").read_text() + assert "COPY site.py /app/site.py" in (site / "Dockerfile").read_text() + + +def test_the_new_site_imports_and_passes_the_conformance_check(tmp_path, socket_enabled): + assert _new("shop", "--dir", str(tmp_path / "shop")).exit_code == 0 + app = _load(tmp_path / "shop")().create_app() + client = TestClient(app) + _passes_conformance(client) + card = client.get("/.well-known/agent-env.json").json() + site = next(e for e in card["capabilities"]["extensions"] if e["uri"] == "urn:agentenv:website/v1")["params"] + assert (card["name"], site["domains"], site["health"]) == ("shop", ["shop.test"], "/healthz") + assert client.get("/api/session").json()["user"] == "ada" + created = client.post("/api/items", json={"title": " Pack "}) + assert created.status_code == 201 and created.json()["title"] == "Pack" + assert [i["title"] for i in client.get("/api/items").json()] == ["Pack", "Try the site"] + assert client.post("/api/sign-in", json={"user": "grace"}).json() == {"user": "grace"} + assert [i["title"] for i in client.get("/api/items").json()] == ["Write the first task"] + + +def test_a_non_empty_folder_is_refused_unless_forced_and_a_bad_name_always(tmp_path): + (tmp_path / "keep.txt").write_text("mine") + refused = _new("shop", "--dir", str(tmp_path)) + assert refused.exit_code == 1 and "not empty; pass --force" in refused.output + assert not (tmp_path / "site.py").exists() + forced = _new("shop", "--dir", str(tmp_path), "--force") + assert forced.exit_code == 0, forced.output + assert (tmp_path / "site.py").is_file() and (tmp_path / "keep.txt").read_text() == "mine" + bad = _new("My Shop", "--dir", str(tmp_path / "other")) + assert bad.exit_code == 1 and "lowercase letters, digits and hyphens" in bad.output + both = _new("shop", "--dir", str(tmp_path / "x"), "--from-mcp", str(MAIL_SERVER), + "--from-tools-json", str(tmp_path / "keep.txt")) + assert both.exit_code == 1 and "mutually exclusive" in both.output + + +def test_an_mcp_servers_tools_are_read_from_its_source_without_running_it(tmp_path): + assert tools_from_mcp(MAIL_SERVER) == [{ + "name": "search_emails", + "description": "Search the inbox: the emails holding every word of the query, ignoring case, as a JSON list.", + "inputSchema": {"type": "object", "required": ["query"], "properties": {"query": { + "type": "string", "description": "Words each email returned holds, in its sender, subject or body."}}}}] + (tmp_path / "server.py").write_text(FASTMCP_SERVER) + listed, created = tools_from_mcp(tmp_path) + assert listed["name"] == "list_events" and listed["description"] == "The events of a day." + assert listed["inputSchema"] == {"type": "object", "required": ["day"], "properties": { + "day": {"type": "string"}, "limit": {"type": "integer", "default": 10}}} + assert created["name"] == "create_event" + assert created["inputSchema"]["required"] == ["title", "attendees"] + assert created["inputSchema"]["properties"] == { + "title": {"type": "string", "description": "What the event is."}, + "attendees": {"type": "array", "items": {"type": "string"}}, + "kind": {"enum": ["meeting", "focus"], "default": "meeting"}, + "notes": {"anyOf": [{"type": "string"}, {"type": "null"}], "default": None}, + "room": {"type": "string", "description": "Where.", "default": None}} + empty = tmp_path / "empty" + empty.mkdir() + (empty / "x.py").write_text("def f():\n pass\n") + refused = _new("cal", "--dir", str(tmp_path / "cal"), "--from-mcp", str(empty)) + assert refused.exit_code == 1 and "--from-tools-json" in refused.output + + +def test_a_site_from_an_mcp_server_has_a_stub_route_per_tool_and_passes_the_check(tmp_path, socket_enabled): + (tmp_path / "mcp").mkdir() + (tmp_path / "mcp" / "server.py").write_text(FASTMCP_SERVER) + result = _new("calendar", "--dir", str(tmp_path / "site"), "--from-mcp", str(tmp_path / "mcp")) + assert result.exit_code == 0, result.output + assert "2 stub route(s)" in result.output + seed = json.loads((tmp_path / "site" / "data" / "seed.json").read_text()) + assert seed == {"current_user": "user", "users": [{"id": "user", "name": "Sample User"}], "events": []} + cls = _load(tmp_path / "site") + assert [(r.method, r.path, r.name) for r in cls.routes()] == [ + ("GET", "/api/users", "list_users"), ("POST", "/api/sign-in", "sign_in"), + ("GET", "/api/list-events", "list_events"), ("POST", "/api/create-event", "create_event")] + client = TestClient(cls().create_app()) + _passes_conformance(client) + listed = client.get("/api/list-events", params={"day": "2026-01-05"}) + assert (listed.status_code, listed.json()["code"]) == (501, "not_implemented") + assert client.get("/api/list-events", params={"limit": "x"}).status_code == 400 + created = client.post("/api/create-event", json={"title": "Standup", "attendees": ["ann"], "kind": "focus"}) + assert created.status_code == 501, created.text + assert client.post("/api/create-event", json={"title": "Standup", "kind": "party"}).status_code == 400 + main = (tmp_path / "site" / "web" / "src" / "main.js").read_text() + assert '"path": "api/create-event"' in main and '"name": "list_events"' in main + + +@pytest.mark.parametrize("shape", ["tools/list", "rpc", "list", "card"]) +def test_tools_json_in_any_of_its_shapes_makes_stub_routes(tmp_path, shape, socket_enabled): + tools = [{"name": "send_email", "description": "Send an email.", "inputSchema": {"type": "object", "properties": { + "to": {"type": "array", "items": {"type": "string"}}, "from": {"type": "string"}, + "urgent": {"type": "boolean", "default": False}}, "required": ["to"]}}, + {"name": "list_users", "description": "A tool named like a built-in route.", "inputSchema": {}}] + data = {"tools/list": {"tools": tools}, "rpc": {"jsonrpc": "2.0", "id": 1, "result": {"tools": tools}}, + "list": tools, "card": {"name": "mail", "capabilities": {"tools": tools}}}[shape] + (tmp_path / "tools.json").write_text(json.dumps(data)) + assert [t["name"] for t in tools_from_json(tmp_path / "tools.json")] == ["send_email", "list_users"] + result = _new("mail", "--dir", str(tmp_path / "site"), "--from-tools-json", str(tmp_path / "tools.json")) + assert result.exit_code == 0, result.output + cls = _load(tmp_path / "site") + assert [r.name for r in cls.routes()] == ["list_users", "sign_in", "send_email", "list_users_tool"] + client = TestClient(cls().create_app()) + _passes_conformance(client) + assert client.post("/api/send-email", json={"to": ["a@b.c"], "from": "me@b.c"}).status_code == 501 + assert client.post("/api/send-email", json={}).status_code == 400 + assert client.get("/api/list-users").status_code == 501 + assert client.get("/api/users").json() == [{"id": "user", "name": "Sample User"}] + + +def test_a_tool_named_like_a_path_the_sdk_serves_gets_a_path_of_its_own(tmp_path): + (tmp_path / "tools.json").write_text(json.dumps([{"name": "session", "inputSchema": {}}])) + result = _new("mail", "--dir", str(tmp_path / "site"), "--from-tools-json", str(tmp_path / "tools.json")) + assert result.exit_code == 0, result.output + stub = next(r for r in _load(tmp_path / "site").routes() if r.name == "session") + assert SESSION_PATH.startswith("/api/") and stub.path == f"{SESSION_PATH}-tool" + + +def test_a_tool_named_with_a_leading_digit_makes_a_site_that_loads(tmp_path): + (tmp_path / "tools.json").write_text(json.dumps([{"name": "3d_render", "inputSchema": {}}])) + result = _new("render", "--dir", str(tmp_path / "site"), "--from-tools-json", str(tmp_path / "tools.json")) + assert result.exit_code == 0, result.output + assert "class N3dRenderRequest(BaseModel)" in (tmp_path / "site" / "site.py").read_text() + assert [r.name for r in _load(tmp_path / "site").routes()][-1] == "3d_render" + + +def test_a_tool_description_ending_in_a_quote_or_a_backslash_makes_a_site_that_loads(tmp_path): + tools = [{"name": "set_status", "description": 'Set status to "done"', "inputSchema": {}}, + {"name": "read_path", "description": "Read C:\\temp\\", "inputSchema": {}}, + {"name": "find_note", "description": 'Find a "note"\n\nBy its "title"', "inputSchema": + {"type": "object", "properties": {"q": {"type": "string", "description": 'like "x"'}}}}] + (tmp_path / "tools.json").write_text(json.dumps(tools)) + result = _new("status", "--dir", str(tmp_path / "site"), "--from-tools-json", str(tmp_path / "tools.json")) + assert result.exit_code == 0, result.output + cls = _load(tmp_path / "site") + assert cls.set_status.__doc__ == 'Set status to "done"' and cls.read_path.__doc__ == "Read C:\\temp\\" + + +def test_a_tools_json_that_lists_no_tools_is_refused(tmp_path): + (tmp_path / "tools.json").write_text(json.dumps({"tools": "nope"})) + refused = _new("mail", "--dir", str(tmp_path / "site"), "--from-tools-json", str(tmp_path / "tools.json")) + assert refused.exit_code == 1 and "lists no tools" in refused.output + assert not (tmp_path / "site").exists() + + +def test_tools_named_after_what_the_site_module_uses_make_a_site_that_loads(tmp_path): + names = ["api", "ready_check", "website", "environment_card", "ApiContext", "ApiError", "NotFound", "BaseModel", + "Field", "Optional", "Any", "Literal", "SignIn", "USERS", "list", "dict", "str", "int", "float", "bool", + "after_all"] + (tmp_path / "tools.json").write_text(json.dumps([{"name": n, "inputSchema": {}} for n in names])) + result = _new("shadow", "--dir", str(tmp_path / "site"), "--from-tools-json", str(tmp_path / "tools.json")) + assert result.exit_code == 0, result.output + cls = _load(tmp_path / "site") + routes = cls.routes() + assert [r.name for r in routes] == ["list_users", "sign_in", *names] + client = TestClient(cls().create_app()) + assert client.get("/healthz").status_code == 200 + for route in routes[2:]: + answer = client.request(route.method, route.path, **({} if route.method == "GET" else {"json": {}})) + assert (answer.status_code, answer.json()["code"]) == (501, "not_implemented"), route diff --git a/tst/unit/cli/website_server_image_cli_test.py b/tst/unit/cli/website_server_image_cli_test.py index 1a04d38c9..8db97945a 100644 --- a/tst/unit/cli/website_server_image_cli_test.py +++ b/tst/unit/cli/website_server_image_cli_test.py @@ -10,6 +10,7 @@ from agent_env.cli.env.website import website from agent_env.env import Env +from agent_env.providers.env_providers.constants import AGENT_ENV_WEBSITE_HEALTH_PATH def _stored(id, **_): @@ -27,14 +28,22 @@ def test_a_site_is_built_from_its_server_dockerfile_alone_with_its_build_context with patch("agent_env.cli.env.website.build_image", side_effect=lambda *a, **k: built.append(a[2])), \ patch("agent_env.artifact.artifacts.docker_image.DockerImageArtifact.put", side_effect=lambda **k: _stored(k["id"])) as stored: - result = _put("--server-dockerfile", str(tmp_path / "Dockerfile"), "--environment-name", "mail", - "--health-path", "/healthz") + result = _put("--server-dockerfile", str(tmp_path / "Dockerfile"), "--environment-name", "mail") assert result.exit_code == 0, result.output assert len(built) == 1 and "deprecated" not in result.output assert stored.call_args.kwargs["build_context_path"] == str(tmp_path) assert stored.call_args.kwargs["dockerfile_path"] == str(tmp_path / "Dockerfile") env = Env.get("mail-site") - assert (env.frontend_docker_image_artifact, env.health_path) == (None, "/healthz") + assert (env.frontend_docker_image_artifact, env.health_path) == (None, AGENT_ENV_WEBSITE_HEALTH_PATH) + + +def test_a_health_path_is_kept_for_a_site_without_ready_and_warned_of_as_deprecated(local_stores): + with patch("agent_env.cli.env.website.DockerImageArtifact.put_ref", side_effect=lambda id, **k: _stored(id)): + result = _put("--server-image-ref", "ghcr.io/acme/mail:3", "--environment-name", "mail", + "--health-path", "/healthz") + assert result.exit_code == 0, result.output + assert "--health-path is deprecated" in result.output + assert Env.get("mail-site").health_path == "/healthz" def test_a_site_registers_a_server_image_already_in_a_registry(local_stores): diff --git a/tst/unit/config/test_explain_and_sources.py b/tst/unit/config/test_explain_and_sources.py index 07d55ad1f..809adaf91 100644 --- a/tst/unit/config/test_explain_and_sources.py +++ b/tst/unit/config/test_explain_and_sources.py @@ -163,7 +163,8 @@ def test_sources_says_how_the_file_was_found(config_file, tmp_path, monkeypatch) # Read for identity (USER), or set from [sandbox.providers.sail_vm.config] for the Sail SDK to read # (SAIL_*), never configuration agent-env takes, so none is a layer. # AGENT_ENV_INSTALLED_REQUIRES marks a command run again after installing what a bundle requires, so it never asks twice. -_NOT_CONFIGURATION = {"USER", "DISPLAY", "SAIL_API_KEY", "SAIL_RUNTIME_THREADS", "AGENT_ENV_INSTALLED_REQUIRES"} +_NOT_CONFIGURATION = {"USER", "DISPLAY", "SAIL_API_KEY", "SAIL_RUNTIME_THREADS", "AGENT_ENV_INSTALLED_REQUIRES", + "AGENT_ENV_WEBSITE_DEV_SITE"} def _env_vars_read_by(root): From d7413aaf017a187db5ccee045d3f41deb2a94a8d Mon Sep 17 00:00:00 2001 From: William Loo Date: Tue, 3 Nov 2026 15:00:05 -0800 Subject: [PATCH 2/6] fix(website): env website dev reloads on website.toml changes, and names its timeouts (FD-4683) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/agent_env/cli/env/website.py | 4 ++-- src/agent_env/cli/env/website_dev.py | 6 +++++- tst/unit/cli/website_dev_cli_test.py | 1 + 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/agent_env/cli/env/website.py b/src/agent_env/cli/env/website.py index de7d75aa0..6ce3e302e 100644 --- a/src/agent_env/cli/env/website.py +++ b/src/agent_env/cli/env/website.py @@ -146,14 +146,14 @@ def dev(directory: Path, site_file: str, host: str, port: int, clock_port: int, os.environ[website_dev.SITE_ENV] = str(site_path) try: uvicorn.run("agent_env.cli.env.website_dev:create_app", factory=True, host=host, port=port, reload=True, - reload_dirs=[str(site_dir)], reload_includes=["*.py", "*.json"], reload_excludes=excludes or None) + reload_dirs=[str(site_dir)], reload_includes=website_dev.RELOAD_INCLUDES, reload_excludes=excludes or None) finally: supervisor.stop() clock.close() if watcher is not None: watcher.terminate() try: - watcher.wait(timeout=5) + watcher.wait(timeout=website_dev.WATCHER_STOP_TIMEOUT_S) except subprocess.TimeoutExpired: watcher.kill() diff --git a/src/agent_env/cli/env/website_dev.py b/src/agent_env/cli/env/website_dev.py index e5af25a4e..499a8393a 100644 --- a/src/agent_env/cli/env/website_dev.py +++ b/src/agent_env/cli/env/website_dev.py @@ -30,6 +30,8 @@ SITE_ENV = "AGENT_ENV_WEBSITE_DEV_SITE" BOOT_HEADER = "X-Site-Boot" POLL_QUERY = "agent-env-dev-poll" +RELOAD_INCLUDES = ["*.py", "*.json", "*.toml"] +WATCHER_STOP_TIMEOUT_S = 5.0 _WEB_SKIPPED = {"node_modules", "dist"} @@ -264,11 +266,13 @@ class Supervisor: couldn't prime, before or after the check and for a seed it couldn't read too, counts as unprimed, so the next poll tries again.""" + HTTP_TIMEOUT_S = 10.0 + def __init__(self, base_url: str, clock_url: str, seed: Path | None, *, check: bool = True, echo: Callable[[str], None] = print, client: httpx.Client | None = None, interval: float = 1.0) -> None: self.base_url, self.clock_url, self.seed, self.check = base_url, clock_url, seed, check self.echo, self.interval = echo, interval - self.client = client or httpx.Client(timeout=10) + self.client = client or httpx.Client(timeout=self.HTTP_TIMEOUT_S) self.primed: str | None = None self.checked = False self.stopped = threading.Event() diff --git a/tst/unit/cli/website_dev_cli_test.py b/tst/unit/cli/website_dev_cli_test.py index 0fdcd7bb9..e62a338f0 100644 --- a/tst/unit/cli/website_dev_cli_test.py +++ b/tst/unit/cli/website_dev_cli_test.py @@ -294,6 +294,7 @@ def test_dev_runs_uvicorn_with_reload_beside_the_clock_and_the_supervisor(site_d options = run.call_args.kwargs assert (options["factory"], options["reload"], options["port"]) == (True, True, 8123) assert options["reload_dirs"] == [str(site_dir.resolve())] + assert options["reload_includes"] == ["*.py", "*.json", "*.toml"] assert options["reload_excludes"] == [str(site_dir.resolve() / "web")] assert "http://127.0.0.1:8123/" in result.output and "/clock/advance" in result.output supervisor.stop.assert_called_once() From 9716e6b9fc6b4c0385731d75c03d5dd41da0ec8f Mon Sep 17 00:00:00 2001 From: William Loo Date: Tue, 3 Nov 2026 16:37:56 -0800 Subject: [PATCH 3/6] fix(website): a new site's page reloads itself at its slash-ending address, as Notes does (FD-4683) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/agent_env/cli/env/website_template/README.md.tmpl | 2 +- src/agent_env/cli/env/website_template/web/index.html.tmpl | 3 +++ tst/unit/cli/website_new_cli_test.py | 2 ++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/agent_env/cli/env/website_template/README.md.tmpl b/src/agent_env/cli/env/website_template/README.md.tmpl index c17de7b80..e9296dca7 100644 --- a/src/agent_env/cli/env/website_template/README.md.tmpl +++ b/src/agent_env/cli/env/website_template/README.md.tmpl @@ -7,7 +7,7 @@ | --- | --- | | `site.py` | The server: the `{{class_name}}` WebsiteEnvironment, its `@api` routes and its `@ready_check`, and `serve()` on `PORT` (8000). | | `data/seed.json` | The seed data the store starts with: `current_user`, the acting user, the `users` a browser can sign in as, and the site's collections. | -| `web/` | A minimal Vite app. Its assets (`base: "./"`) and API calls are relative to the page, so it works at `/` and under the gateway's `/website/{{name}}/` alike. `src/site-client.js` is the SDK's browser client: `api()`, `universeNow()`, `followUniverseClock()`, `session()`, `signIn()` and `signOut()`. | +| `web/` | A minimal Vite app. Its assets (`base: "./"`) and API calls are relative to the page, so it works at `/` and under the gateway's `/website/{{name}}/` alike; opened at a path without its final slash, such as `/website/{{name}}`, the page first reloads itself with it. `src/site-client.js` is the SDK's browser client: `api()`, `universeNow()`, `followUniverseClock()`, `session()`, `signIn()` and `signOut()`. | | `Dockerfile` | The server image, built with this folder as its context: Node builds `web/`, then Python installs `agentenv-framework-protocol` with uvicorn. | | `env.toml` | The website env as a bundle entry. | diff --git a/src/agent_env/cli/env/website_template/web/index.html.tmpl b/src/agent_env/cli/env/website_template/web/index.html.tmpl index d4119cdc4..fca23f871 100644 --- a/src/agent_env/cli/env/website_template/web/index.html.tmpl +++ b/src/agent_env/cli/env/website_template/web/index.html.tmpl @@ -4,6 +4,9 @@ {{title}} +
diff --git a/tst/unit/cli/website_new_cli_test.py b/tst/unit/cli/website_new_cli_test.py index 383f18547..309407977 100644 --- a/tst/unit/cli/website_new_cli_test.py +++ b/tst/unit/cli/website_new_cli_test.py @@ -90,6 +90,8 @@ def test_a_new_site_has_notes_layout_and_the_sdks_browser_client(tmp_path): vite = (site / "web" / "vite.config.js").read_text() assert 'base: "./"' in vite and '"/api":' in vite and '"/healthz":' in vite assert 'from "./site-client.js"' in (site / "web" / "src" / "main.js").read_text() + page = (site / "web" / "index.html").read_text() + assert page.index('location.pathname.endsWith("/")') < page.index(' From 7c6718508609ae35e1a52a6120e233ba82a333ba Mon Sep 17 00:00:00 2001 From: William Loo Date: Tue, 3 Nov 2026 21:09:22 -0800 Subject: [PATCH 6/6] fix(website): the browser client reports a body lost after its headers as a network ApiError (FD-4683) Co-Authored-By: Claude Opus 5.5 (1M context) --- .../website/notes/web/src/site-client.js | 7 ++++++- .../website/client/site-client.js | 7 ++++++- .../tests/test_website_notes_example.py | 21 +++++++++++++++++++ 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/packages/agentenv-protocol/examples/website/notes/web/src/site-client.js b/packages/agentenv-protocol/examples/website/notes/web/src/site-client.js index 2d5b65a39..853cb12c4 100644 --- a/packages/agentenv-protocol/examples/website/notes/web/src/site-client.js +++ b/packages/agentenv-protocol/examples/website/notes/web/src/site-client.js @@ -53,7 +53,12 @@ export async function api(path, { method = "GET", body, headers = {} } = {}) { throw new ApiError(0, "network_error", `${method} ${url}: ${cause?.message || cause}`, null); } follow(response); - const text = await response.text(); + let text; + try { + text = await response.text(); + } catch (cause) { + throw new ApiError(response.status, "network_error", `${method} ${url}: ${cause?.message || cause}`, null); + } let data = null; try { data = text ? JSON.parse(text) : null; diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/client/site-client.js b/packages/agentenv-protocol/src/agentenv_protocol/website/client/site-client.js index 2d5b65a39..853cb12c4 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/client/site-client.js +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/client/site-client.js @@ -53,7 +53,12 @@ export async function api(path, { method = "GET", body, headers = {} } = {}) { throw new ApiError(0, "network_error", `${method} ${url}: ${cause?.message || cause}`, null); } follow(response); - const text = await response.text(); + let text; + try { + text = await response.text(); + } catch (cause) { + throw new ApiError(response.status, "network_error", `${method} ${url}: ${cause?.message || cause}`, null); + } let data = null; try { data = text ? JSON.parse(text) : null; diff --git a/packages/agentenv-protocol/tests/test_website_notes_example.py b/packages/agentenv-protocol/tests/test_website_notes_example.py index 2b61707d2..736f99e44 100644 --- a/packages/agentenv-protocol/tests/test_website_notes_example.py +++ b/packages/agentenv-protocol/tests/test_website_notes_example.py @@ -8,6 +8,8 @@ import importlib.util import json import re +import shutil +import subprocess import sys import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer @@ -243,3 +245,22 @@ def test_the_page_uses_the_sdks_browser_client_as_the_package_ships_it() -> None shipped = Path(website.__code__.co_filename).parent / "client" / "site-client.js" assert (NOTES_DIR / "web" / "src" / "site-client.js").read_text(encoding="utf-8") == shipped.read_text(encoding="utf-8") assert 'from "./site-client.js"' in (NOTES_DIR / "web" / "src" / "main.js").read_text(encoding="utf-8") + + +DROPPED_BODY = """ +globalThis.location = {href: "http://site.test/"}; +globalThis.fetch = async () => ({ok: false, status: 502, headers: {get: () => null}, + text: async () => { throw new Error("connection reset"); }}); +const {api, ApiError} = await import(process.argv[1]); +try { await api("api/notes"); } catch (error) { + console.log(JSON.stringify({isApiError: error instanceof ApiError, status: error.status, code: error.code})); +} +""" + + +@pytest.mark.skipif(shutil.which("node") is None, reason="needs node") +def test_the_browser_client_reports_a_body_lost_after_its_headers_as_a_network_api_error() -> None: + shipped = Path(website.__code__.co_filename).parent / "client" / "site-client.js" + out = subprocess.run(["node", "--input-type=module", "-e", DROPPED_BODY, shipped.as_uri()], + capture_output=True, text=True, check=True).stdout + assert json.loads(out) == {"isApiError": True, "status": 502, "code": "network_error"}