diff --git a/CURRENT-STATUS.md b/CURRENT-STATUS.md index 1b4ce08f..fa0d0a63 100644 --- a/CURRENT-STATUS.md +++ b/CURRENT-STATUS.md @@ -9,6 +9,7 @@ coordination branch. Detailed state belongs in each linked status file. | Name | Started | Goal | State | Branch association | Status file | |---|---|---|---|---|---| +| `presentations` | 2026-11-16 | Deliver agent-driven E2E recording and presentation from Markdown or inline English in adopter DevCapsules | paused; PR #43 reviewed; awaits owner merge to release 0.3.1 | `ws-presentations/0.3.1-authoring` | [workstream status](engineering-docs/wip/2026-11-16-presentations/CURRENT-STATUS.md) | | `github-auth` | 2026-11-16 | GitHub SSH authentication through a launcher-owned host proxy | paused; Pair Review #44 merged; PR #47 targets release-0.3.1; two inherited gate failures remain | `ws-github-auth/ssh-proxy` | [workstream status](engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md) | | `workflow-improvements` | 2026-08-09 | Improve the multiple-stream workflow from concrete dogfood findings during the recursive E2E cycle | active 2026-10-30; Pair Review Loop naming implemented and packaging verified for 0.3.0; mainline and release integration pending | `ws-workflow-improvements/v1` | [workstream status](engineering-docs/wip/2026-08-09-workflow-improvements/CURRENT-STATUS.md) | | `project-management` | 2026-08-09 | Maintain project-wide priorities, sequencing, dependencies, and lifecycle coordination | paused 2026-10-11 at owner direction; consent and forced configuration before RC work; remaining diagnostics/image scope and release cut decisions retained | `ws-project-management/coordination` | [workstream status](engineering-docs/wip/2026-08-09-project-management/CURRENT-STATUS.md) | diff --git a/README.md b/README.md index bca996a2..b734efd1 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,11 @@ default. You decide what additional access to grant. See a sample of a full graphical IDE running inside DevCapsule's isolated Docker container. More scenario illustrations coming soon. +To record an E2E scenario in your own project, ask your agent to run it with recording +and presentation. `devcapsule presentations guide` provides the installed instructions; +`devcapsule presentations export ` supplies the Python/Playwright helpers. +See the [agent recording guide](devcapsule-src/devcapsule/presentation_tools/README.md). + ## Why switch to DevCapsule? diff --git a/devcapsule-src/devcapsule/commands/presentations.py b/devcapsule-src/devcapsule/commands/presentations.py new file mode 100644 index 00000000..522e005e --- /dev/null +++ b/devcapsule-src/devcapsule/commands/presentations.py @@ -0,0 +1,89 @@ +"""Entry points for the packaged agent recording instructions and Python helpers.""" +import argparse +from importlib.resources import files +import os +from pathlib import Path +from typing import Mapping + +from devcapsule.commands.framework import Command, Group +from devcapsule.compat import CliError +from devcapsule.presentation_authoring import export_tools, presentation_link + +__all__ = ["COMMAND"] + +PACKAGE = "devcapsule.presentation_tools" +ASSETS = ("__init__.py", "producer.py", "recording.py", "README.md", "example.py") + + +class GuideCommand(Command): + name = "guide" + help = "Read the agent instructions for English-specified E2E recordings." + + @classmethod + def run(cls, arguments: argparse.Namespace, context: object | None) -> int: + print(files(PACKAGE).joinpath("README.md").read_text(encoding="utf-8")) + return 0 + + +class ExportCommand(Command): + name = "export" + help = "Export the instructions and standalone Python helpers to a new directory." + + @classmethod + def configure(cls, parser: argparse.ArgumentParser) -> None: + parser.add_argument("directory", type=Path, help="New directory; existing directories are preserved.") + + @classmethod + def run(cls, arguments: argparse.Namespace, context: object | None) -> int: + assets = {name: files(PACKAGE).joinpath(name).read_bytes() for name in ASSETS} + try: + package = export_tools(arguments.directory, assets) + except (OSError, ValueError) as error: + raise CliError(f"cannot export presentation tools: {error}") from error + print(f"Presentation tools: {package}\nAgent instructions: {package / 'README.md'}") + return 0 + + +class HandOffCommand(Command): + name = "hand-off" + help = "Print the recorded presentation's console link for the engineer." + + @classmethod + def configure(cls, parser: argparse.ArgumentParser) -> None: + parser.add_argument("folder", type=Path) + parser.add_argument("--project", type=Path, help="Console project root; defaults to PROJECT_PATH or cwd.") + parser.add_argument("--console-url", help="Defaults to DEVCAPSULE_CONSOLE_URL.") + parser.add_argument("--token-file", type=Path, help="Defaults to the capsule console token file.") + + @classmethod + def run(cls, arguments: argparse.Namespace, context: object | None) -> int: + root = (arguments.project or Path(os.environ.get("PROJECT_PATH") or ".")).resolve() + folder = arguments.folder.resolve() + origin = arguments.console_url or os.environ.get("DEVCAPSULE_CONSOLE_URL", "") + token_path = arguments.token_file or Path(os.environ.get("DEVCAPSULE_CONSOLE_TOKEN_FILE") or "/run/devcapsule-console-token") + try: + token = token_path.read_text(encoding="utf-8").strip() + except (OSError, UnicodeError): + token = "" + try: + if not (folder / "presentation.json").is_file(): + raise ValueError(f"no presentation.json in {folder}") + link = presentation_link(folder, project=root, console_url=origin, token=token) + except (OSError, ValueError) as error: + raise CliError(str(error)) from error + if not token: + print("Open the console first to establish a browser session; no readable run token was found.") + print(link) + return 0 + + +class PresentationsCommand(Group): + name = "presentations" + help = "Record E2E scenarios and make presentations: agent guide, Python toolkit and console link." + + @classmethod + def subcommands(cls) -> Mapping[str, type[Command] | type[Group]]: + return {command.name: command for command in (GuideCommand, ExportCommand, HandOffCommand)} + + +COMMAND = PresentationsCommand diff --git a/devcapsule-src/devcapsule/presentation_authoring.py b/devcapsule-src/devcapsule/presentation_authoring.py new file mode 100644 index 00000000..9258448d --- /dev/null +++ b/devcapsule-src/devcapsule/presentation_authoring.py @@ -0,0 +1,47 @@ +"""Export the portable authoring kit and form console links from explicit inputs. + +Public operations: ``export_tools`` and ``presentation_link``. +""" +from collections.abc import Mapping +from pathlib import Path +from urllib.parse import quote, urlencode, urlsplit, urlunsplit + +__all__ = ["export_tools", "presentation_link"] + + +def export_tools(destination: Path, assets: Mapping[str, bytes]) -> Path: + """Create a new directory with the supplied kit; never overwrite a prior kit.""" + if any(Path(name).name != name or name in ("", ".", "..") for name in assets): + raise ValueError("kit assets must be simple filenames") + destination.mkdir(parents=True, exist_ok=False) + package = destination / "presentation_tools" + package.mkdir() + for name, data in assets.items(): + (package / name).write_bytes(data) + return package + + +def presentation_link(folder: Path, *, project: Path, console_url: str, token: str) -> str: + """Link a visible descendant of the console project, preserving no URL query. + + Paths are absolute and resolved by the caller. The token is optional; without + it the user's browser must already have a console session. Never save a tokened + link in the evidence or project records. + """ + relative = folder.relative_to(project) + if not project.is_absolute() or not relative.parts or any( + part in ("..", "node_modules") or part.startswith(".") for part in relative.parts + ): + raise ValueError("presentation must be in a visible subdirectory of the console project") + if not console_url or any(ch.isspace() for ch in console_url): + raise ValueError("supply the capsule's HTTP(S) console URL") + origin = urlsplit(console_url) + try: + _ = origin.port + except ValueError as error: + raise ValueError("supply the capsule's HTTP(S) console URL") from error + if (origin.scheme not in ("http", "https") or not origin.hostname + or origin.username is not None or origin.password is not None): + raise ValueError("supply the capsule's HTTP(S) console URL") + path = "/presentations/" + quote(relative.as_posix(), safe="/") + return urlunsplit((origin.scheme, origin.netloc, path, urlencode({"token": token}) if token else "", "")) diff --git a/devcapsule-src/devcapsule/presentation_tools/README.md b/devcapsule-src/devcapsule/presentation_tools/README.md new file mode 100644 index 00000000..44868692 --- /dev/null +++ b/devcapsule-src/devcapsule/presentation_tools/README.md @@ -0,0 +1,104 @@ +# Run an E2E scenario with recording and presentation + +These instructions are for the agent in the engineer's project. Accept the +scenario from the engineer's existing Markdown file or their inline English. +Read it yourself; do not require the engineer to translate it into a schema. +Execute the requested test, check its observations, and give the engineer the +result and a link to the recorded presentation in their DevCapsule web console. + +## Get the tools + +`devcapsule presentations guide` prints this guide from the installed executable. +No DevCapsule source checkout is needed. Export the supporting Python package: + +```sh +devcapsule presentations export /var/tmp/e2e-authoring +``` + +The destination must be new. It contains `presentation_tools/` with this guide, +the producer, recorder and a runnable example. Add the export directory to +`PYTHONPATH` when running the project's driver. The helpers use only the Python +standard library and the Playwright page you supply. They import no test harness. + +Use the capsule's `browser-automation` capability, supplied by the `playwright` +component. Its interpreter is `$DEVCAPSULE_PLAYWRIGHT_PYTHON`, normally +`/opt/playwright/venv/bin/python`, and `$PLAYWRIGHT_BROWSERS_PATH` names its +browser directory. The directory's `ffmpeg-*/ffmpeg-linux` is the video encoder +on Linux. Locate the installed executable and pass its absolute path to the +driver. A full system `ffmpeg` also works. Do not download browsers or change +the capsule configuration silently. If the capability is missing, report it +and use the project's normal configuration/relaunch procedure with the engineer. + +## Execute the engineer's scenario + +1. Read the supplied scenario and the project's run/test instructions. Identify + the real application URL, required data, actions and observable assertions. + Ask only about a missing decision that prevents executing the test. +2. Export the tools. Adapt `presentation_tools/example.py` into a driver in the + project's normal E2E location. Translate the English into `Script` and + `ScriptStep` values in that driver. The engineer's Markdown stays unchanged. +3. Use ordinary synchronous Playwright operations for this project's UI. A + browser page can be the application itself or the capsule's noVNC desktop + when testing an IDE. The helper does not provision applications, databases + or capsules; use the project's established setup and teardown. +4. Create `Recording` with a **new visible evidence subdirectory beneath the + console's project root**: for example `evidence/login-success/run-001`. + Hidden directories, `node_modules`, paths outside the project, and the project + root itself do not appear in the presentation list. Do not overwrite attempts. +5. Call `recording.action(page, step_number, caption, callback)` around each small + real interaction and its assertions. Step numbers start at 1 and proceed in + order; several actions may belong to one step. Captions describe the action. + The callback can use locators, keyboard/mouse operations and project checks. + Assert expected observations independently of the screenshots; a recording + is evidence of a run, not proof the test passed. +6. Keep `record_video_dir=str(recording.evidence / "raw")` on the Playwright + context. The helper captures bursts before and after each action, following + the existing recorder's sparse-frame approach. Its presentation movie excludes + the agent's thinking pauses; the native video preserves motion during the + action. Split long callbacks into smaller recorded actions when viewers need + to follow intermediate UI states. Neither file is a generated explainer video. +7. Call `recording.finish()` while cleaning up, including after a test assertion + fails. Close the browser context to flush its native video. Keep the failure + exit code. Failed/incomplete runs receive labelled presentations containing + only steps reached. If capture or encoding prevents a deck, report that failure + and preserve the logs; do not claim a presentation exists. + +The helper writes `action-log.json`, `scenario-result.json`, JPEG frame data, +the encoded `scenario-video.webm`, `poster.png`, step screenshots and finally +`presentation.json`. Chapters and captions come from the recorded frame counts. +Do not estimate timestamps or hand-edit the manifest to hide a failed step. +The existing webconsole schema and viewer are used unchanged. + +## Verify and hand off + +Read `scenario-result.json` and the driver's exit status. Open the presentation +in the webconsole with Playwright; check its title, overview, screenshots and +chapter playback, including the last chapter. A failing assertion is a failed +test even when its presentation renders correctly. Report failed and unexecuted +steps explicitly. Keep evidence only under the project's normal retention rules; +avoid committing recordings or captured secrets automatically. + +Print the engineer's link with the installed CLI: + +```sh +devcapsule presentations hand-off evidence/login-success/run-001 --project "$PROJECT_PATH" +``` + +Run this from the console project root: a relative evidence folder resolves +from the current working directory, and `--project` supplies the console root +used to form the relative link. It uses `$DEVCAPSULE_CONSOLE_URL` and the +mounted console token; `--console-url` and `--token-file` supply explicit values. Keep tokened links in chat only, never +in a source file, log or committed record. If no token can be read, the link +requires a browser session that already opened the console. If there is no +console URL, report that missing environment input rather than inventing a URL. + +Finish the response with the test outcome, any limitation, and the clickable +presentation link. The engineer should not need to reconstruct a path themselves. + +## Existing replay scripts + +The exported `producer` also supports the existing structured replay format: +`parse_script`, `plan_presentation`, `PresentationPlan.timed` and +`write_presentation`. `plan_for_scenario(scenario, root)` reads the scenario's +script relative to the supplied project directory. This adapter is for agents +with existing structured scripts; it is not an input requirement for engineers. diff --git a/devcapsule-src/devcapsule/presentation_tools/__init__.py b/devcapsule-src/devcapsule/presentation_tools/__init__.py new file mode 100644 index 00000000..ae020953 --- /dev/null +++ b/devcapsule-src/devcapsule/presentation_tools/__init__.py @@ -0,0 +1 @@ +"""Portable E2E presentation tools; also exported for the Playwright interpreter.""" diff --git a/devcapsule-src/devcapsule/presentation_tools/example.py b/devcapsule-src/devcapsule/presentation_tools/example.py new file mode 100644 index 00000000..cd57bcbc --- /dev/null +++ b/devcapsule-src/devcapsule/presentation_tools/example.py @@ -0,0 +1,65 @@ +"""Adapt this two-step browser test to the engineer's English scenario. + +Run with the Playwright interpreter, from the exported kit's parent directory: +python -m presentation_tools.example --url http://127.0.0.1:8000 \ + --expected-text 'Welcome' --evidence /project/evidence/welcome/run-001 \ + --ffmpeg /opt/playwright/browsers/ffmpeg-1011/ffmpeg-linux + +The ffmpeg revision above is illustrative; locate the installed encoder. +Public operation: ``main``. All host discovery stays at this entry point. +""" +import argparse +import importlib +from pathlib import Path + +from .producer import Script, ScriptStep +from .recording import Recording + +__all__ = ["main"] + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--url", required=True) + parser.add_argument("--expected-text", required=True) + parser.add_argument("--evidence", type=Path, required=True) + parser.add_argument("--ffmpeg", required=True) + args = parser.parse_args() + playwright = importlib.import_module("playwright.sync_api") + script = Script( + goal="Verify the application's welcome page survives a reload.", + stack=(("Browser", "Chromium driven by Playwright"),), illustrates=(), + steps=( + ScriptStep("Open the application", "Navigate to the application.", args.expected_text), + ScriptStep("Reload", "Reload the page.", args.expected_text), + ), + ) + recording = Recording(args.evidence, script=script, scenario_id="welcome", + title="Welcome page", summary=script.goal, ffmpeg=(args.ffmpeg,)) + with playwright.sync_playwright() as runtime: + browser = runtime.chromium.launch(headless=True) + context = browser.new_context(viewport={"width": 1280, "height": 800}, + record_video_dir=str(args.evidence / "raw")) + page = context.new_page() + + def open_application() -> None: + page.goto(args.url) + playwright.expect(page.get_by_text(args.expected_text, exact=True)).to_be_visible() + + def reload_application() -> None: + page.reload() + playwright.expect(page.get_by_text(args.expected_text, exact=True)).to_be_visible() + + try: + recording.action(page, 1, "Open the application and check its welcome message", open_application) + recording.action(page, 2, "Reload and check the message remains visible", reload_application) + finally: + try: + recording.finish() + finally: + context.close() + browser.close() + + +if __name__ == "__main__": + main() diff --git a/devcapsule-src/devcapsule/presentation_tools/producer.py b/devcapsule-src/devcapsule/presentation_tools/producer.py new file mode 100644 index 00000000..16718f0a --- /dev/null +++ b/devcapsule-src/devcapsule/presentation_tools/producer.py @@ -0,0 +1,392 @@ +"""The presentation of a recorded scenario: ``presentation.json`` beside its recording. + +The web console shows a folder that holds a ``presentation.json`` as slides +beside the recording: an overview, then one slide per step with the step's +text, its screenshot, and its part of the video with captions. The document's +contract is ``devcapsule-webconsole/PRESENTATIONS.md``. This module writes +that document from what a recorded scenario already has, so that a recording +needs no hand-made data to be presented: + +- the scenario's JSON: ``id``, ``title``, ``description``, ``script`` (the + path of its script below the caller-supplied root) and ``actions``; +- the script, a Markdown file in the form + ``engineering-docs/development/running-e2e-scenario-recordings.md`` gives: + the goal, the stack, what the scenario illustrates, and the numbered steps + with the user's action and the expected observation; +- the count of frames the recorder had taken after each action. + +An action belongs to the step whose number starts its ``reason`` ("3. Save +the file"). A step's captions come from its actions: when none of them has a +``caption``, every action's reason, without the number, is its caption; when +some have, a caption starts at each action that has one and lasts until the +next, and the step's first action must have one. + +Public operations: ``parse_script``, ``plan_presentation``, +``PresentationPlan.timed``, ``plan_for_scenario`` and ``write_presentation``. +They raise ``PresentationError`` for a script or a scenario that cannot be +presented; the message names what to correct. +""" +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +import json +from pathlib import Path +import re +from typing import Any + +__all__ = [ + "PRESENTATION_FILE", + "PresentationError", + "PresentationPlan", + "Script", + "ScriptStep", + "parse_script", + "plan_for_scenario", + "plan_presentation", + "write_presentation", +] + +PRESENTATION_FILE = "presentation.json" +SCHEMA_VERSION = 1 + +_SECTIONS = ("Goal", "Stack And Capabilities", "What It Illustrates", "Steps") +_HEADING = re.compile(r"^## (?P.+?)\s*$") +_STEP = re.compile(r"^(?P\d+)\. \*\*(?P.+?)\*\*\s*$") +_FIELD = re.compile(r"^\s+- \*(?P<name>User action|Expected observation):\*\s+(?P<text>.+?)\s*$") +_STACK_ROW = re.compile(r"^- \*\*(?P<name>.+?):\*\*\s+(?P<detail>.+?)\s*$") +_BULLET = re.compile(r"^- (?P<text>.+?)\s*$") +_CONTINUATION = re.compile(r"^\s+(?P<text>\S.*?)\s*$") +_STEP_NUMBER = re.compile(r"^(?P<number>\d+)\.\s+(?P<text>\S.*)$", re.DOTALL) + + +class PresentationError(ValueError): + """A script or a scenario cannot be presented; the message names what to correct.""" + + +@dataclass(frozen=True) +class ScriptStep: + """One numbered step of a script.""" + + title: str + action: str + expected: str + + +@dataclass(frozen=True) +class Script: + """What a scenario's script says, section by section.""" + + goal: str + stack: tuple[tuple[str, str], ...] + illustrates: tuple[str, ...] + steps: tuple[ScriptStep, ...] + + +def parse_script(text: str) -> Script: + """The script a Markdown ``text`` in the script form states. + + The form: ``## Goal``, one paragraph; ``## Stack And Capabilities``, + optional, bullets ``- **Name:** detail``; ``## What It Illustrates``, + optional, bullets; ``## Steps``, items ``N. **Title.**`` numbered from 1, + each with the sub-bullets ``- *User action:* …`` and + ``- *Expected observation:* …``. A bullet may continue on indented lines. + Text before the first ``##`` heading is not read. Any other ``##`` + heading, a missing ``Goal`` or ``Steps``, a step out of order or without + both sub-bullets, and a line that fits nowhere are refused. + """ + sections: dict[str, list[tuple[int, str]]] = {} + current: list[tuple[int, str]] | None = None + for number, line in enumerate(text.splitlines(), start=1): + heading = _HEADING.match(line) + if heading is not None: + name = heading.group("name") + if name not in _SECTIONS: + raise PresentationError( + f"script line {number}: section {name!r} is not one of {', '.join(_SECTIONS)}" + ) + if name in sections: + raise PresentationError(f"script line {number}: section {name!r} appears twice") + current = sections[name] = [] + elif current is not None and line.strip(): + current.append((number, line)) + for required in ("Goal", "Steps"): + if not sections.get(required): + raise PresentationError(f"script: section {required!r} is missing or empty") + goal = " ".join(line.strip() for _, line in sections["Goal"]) + stack = tuple( + _stack_row(number, item) for number, item in _bullets(sections.get("Stack And Capabilities", []), "Stack And Capabilities") + ) + illustrates = tuple(item for _, item in _bullets(sections.get("What It Illustrates", []), "What It Illustrates")) + return Script(goal=goal, stack=stack, illustrates=illustrates, steps=_steps(sections["Steps"])) + + +def _bullets(lines: Sequence[tuple[int, str]], section: str) -> list[tuple[int, str]]: + """The bullets of a section, each with its first line's number, continuations joined.""" + items: list[tuple[int, str]] = [] + for number, line in lines: + bullet = _BULLET.match(line) + continuation = _CONTINUATION.match(line) + if bullet is not None: + items.append((number, bullet.group("text"))) + elif continuation is not None and items: + first, so_far = items[-1] + items[-1] = (first, f"{so_far} {continuation.group('text')}") + else: + raise PresentationError(f"script line {number}: {section!r} takes bullets, '- text'") + return items + + +def _stack_row(number: int, item: str) -> tuple[str, str]: + row = _STACK_ROW.match(f"- {item}") + if row is None: + raise PresentationError(f"script line {number}: a stack bullet is '- **Name:** detail'") + return row.group("name"), row.group("detail") + + +def _steps(lines: Sequence[tuple[int, str]]) -> tuple[ScriptStep, ...]: + steps: list[dict[str, str]] = [] + field: str | None = None + for number, line in lines: + step = _STEP.match(line) + named = _FIELD.match(line) + continuation = _CONTINUATION.match(line) + if step is not None: + if int(step.group("number")) != len(steps) + 1: + raise PresentationError(f"script line {number}: expected step {len(steps) + 1}, found {step.group('number')}") + steps.append({"title": step.group("title").rstrip(".").strip()}) + field = None + elif named is not None and steps: + field = named.group("name") + if field in steps[-1]: + raise PresentationError(f"script line {number}: step {len(steps)} states {field!r} twice") + steps[-1][field] = named.group("text") + elif continuation is not None and steps and field is not None: + steps[-1][field] = f"{steps[-1][field]} {continuation.group('text')}" + else: + raise PresentationError( + f"script line {number}: a step is 'N. **Title.**' with '- *User action:* …' and '- *Expected observation:* …'" + ) + for index, step_fields in enumerate(steps, start=1): + for required in ("User action", "Expected observation"): + if required not in step_fields: + raise PresentationError(f"script: step {index} has no {required!r}") + if not step_fields["title"]: + raise PresentationError(f"script: step {index} has no title") + return tuple( + ScriptStep(title=item["title"], action=item["User action"], expected=item["Expected observation"]) for item in steps + ) + + +@dataclass(frozen=True) +class _Caption: + """A caption and the actions it lasts for, as indexes into the scenario's actions.""" + + first_action: int + last_action: int + text: str + + +@dataclass(frozen=True) +class _PlannedStep: + step: ScriptStep + first_action: int + last_action: int + captions: tuple[_Caption, ...] + + +@dataclass(frozen=True) +class PresentationPlan: + """Everything of a presentation that is known before the scenario runs. + + ``plan_presentation`` makes it; ``timed`` adds what only a run knows. + """ + + scenario_id: str + title: str + summary: str + script: Script + action_count: int + steps: tuple[_PlannedStep, ...] + + def timed( + self, + frames_after_actions: Sequence[int], + total_frames: int, + frames_per_second: int, + *, + video: str, + poster: str, + screenshots: Sequence[str], + ) -> dict[str, Any]: + """The presentation document of one recorded run of the planned scenario. + + ``frames_after_actions[i]`` is the count of frames recorded when + action ``i`` had finished, ``total_frames`` the count in the video, + which plays ``frames_per_second`` of them, and ``screenshots[i]`` the + file taken after action ``i``. A step runs from the end of the step + before it, the first from the start of the video, to the end of its + last action, the last to the end of the video; its screenshot is its + last action's. Refuses counts that do not match the plan, that + decrease, that exceed the total, or that leave a step or a caption + without a frame. + """ + if len(frames_after_actions) != self.action_count or len(screenshots) != self.action_count: + raise PresentationError( + f"the run has {len(frames_after_actions)} frame counts and {len(screenshots)} screenshots " + f"for {self.action_count} actions" + ) + if frames_per_second <= 0: + raise PresentationError(f"frames per second must be positive, not {frames_per_second}") + counts = list(frames_after_actions) + if any(later < earlier for earlier, later in zip([0, *counts], [*counts, total_frames])): + raise PresentationError("the frame counts decrease or exceed the total number of frames") + last_action = self.action_count - 1 + + def seconds(frames: int) -> float: + return round(frames / frames_per_second, 3) + + def starts(action: int) -> int: + return 0 if action == 0 else counts[action - 1] + + def ends(action: int) -> int: + return total_frames if action == last_action else counts[action] + + steps = [] + for number, planned in enumerate(self.steps, start=1): + if any(starts(caption.first_action) >= ends(caption.last_action) for caption in planned.captions): + raise PresentationError(f"step {number} has a caption that lasts no frame") + captions = [ + {"start": seconds(starts(caption.first_action)), "end": seconds(ends(caption.last_action)), "text": caption.text} + for caption in planned.captions + ] + steps.append({ + "title": planned.step.title, + "action": planned.step.action, + "expected": planned.step.expected, + "screenshot": screenshots[planned.last_action], + "start": seconds(starts(planned.first_action)), + "end": seconds(ends(planned.last_action)), + "captions": captions, + }) + overview: dict[str, Any] = {"summary": self.summary, "intent": self.script.goal} + if self.script.illustrates: + overview["illustrates"] = list(self.script.illustrates) + if self.script.stack: + overview["stack"] = [{"name": name, "detail": detail} for name, detail in self.script.stack] + return { + "schema_version": SCHEMA_VERSION, + "scenario_id": self.scenario_id, + "title": self.title, + "overview": overview, + "video": {"file": video, "poster": poster, "duration": seconds(total_frames)}, + "steps": steps, + } + + +def plan_presentation(scenario: Mapping[str, Any], script: Script) -> PresentationPlan: + """Join a scenario's JSON document to its parsed script, or say why they do not join. + + The scenario needs a non-empty ``id``, ``title`` and ``description``. Each + action's ``reason`` starts with the number of its step and a period; the + numbers never decrease and every step of the script has at least one + action. ``caption``, where an action has it, is a non-empty string, and a + step with any caption has one on its first action. + """ + texts = {} + for key in ("id", "title", "description"): + value = scenario.get(key) + if not isinstance(value, str) or not value.strip(): + raise PresentationError(f"the scenario has no {key}") + texts[key] = value.strip() + if not isinstance(script.goal, str) or not script.goal.strip(): + raise PresentationError("the script has no goal") + if not script.steps: + raise PresentationError("the script has no steps") + for index, step in enumerate(script.steps, start=1): + for field_name, field_value in (("title", step.title), ("action", step.action), ("expected", step.expected)): + if not isinstance(field_value, str) or not field_value.strip(): + raise PresentationError(f"script step {index} has no {field_name}") + if any(not isinstance(item, str) or not item.strip() for item in script.illustrates): + raise PresentationError("script illustrations must be non-empty texts") + if any(not isinstance(name, str) or not name.strip() or not isinstance(detail, str) or not detail.strip() + for name, detail in script.stack): + raise PresentationError("script stack entries must have non-empty name and detail") + actions = scenario.get("actions") + if not isinstance(actions, list) or not actions: + raise PresentationError("the scenario has no actions") + numbers: list[int] = [] + reasons: list[str] = [] + captions: list[str | None] = [] + for position, action in enumerate(actions, start=1): + if not isinstance(action, Mapping): + raise PresentationError(f"action {position} is not an object") + reason = action.get("reason") + numbered = _STEP_NUMBER.match(reason) if isinstance(reason, str) else None + if numbered is None: + raise PresentationError(f"action {position}: its reason must start with the number of its step, as in '3. Save the file'") + number = int(numbered.group("number")) + if not 1 <= number <= len(script.steps): + raise PresentationError(f"action {position} names step {number}; the script has steps 1 to {len(script.steps)}") + if numbers and number < numbers[-1]: + raise PresentationError(f"action {position} names step {number} after an action of step {numbers[-1]}") + caption = action.get("caption") + if caption is not None and (not isinstance(caption, str) or not caption.strip()): + raise PresentationError(f"action {position}: its caption must be a non-empty text") + numbers.append(number) + reasons.append(numbered.group("text").strip()) + captions.append(caption.strip() if isinstance(caption, str) else None) + planned: list[_PlannedStep] = [] + for number, step in enumerate(script.steps, start=1): + members = [index for index, owner in enumerate(numbers) if owner == number] + if not members: + raise PresentationError(f"step {number} of the script has no action") + planned.append(_PlannedStep( + step=step, first_action=members[0], last_action=members[-1], captions=_captions(number, members, reasons, captions), + )) + return PresentationPlan( + scenario_id=texts["id"], title=texts["title"], summary=texts["description"], script=script, + action_count=len(actions), steps=tuple(planned), + ) + + +def _captions(number: int, members: Sequence[int], reasons: Sequence[str], captions: Sequence[str | None]) -> tuple[_Caption, ...]: + """The captions of one step's actions, by the rule in the module's documentation.""" + written = [index for index in members if captions[index] is not None] + if not written: + return tuple(_Caption(first_action=index, last_action=index, text=reasons[index]) for index in members) + if written[0] != members[0]: + raise PresentationError( + f"step {number} has captions, so its first action, action {members[0] + 1}, needs one" + ) + ends = [*(index - 1 for index in written[1:]), members[-1]] + return tuple( + _Caption(first_action=first, last_action=last, text=str(captions[first])) for first, last in zip(written, ends) + ) + + +def plan_for_scenario(scenario: Mapping[str, Any], root: Path) -> PresentationPlan | None: + """The plan of a scenario that names its ``script``, read below ``root``; None for one that names none. + + A scenario without a script is recorded without a presentation. A script + that is named and cannot be read, or that does not join the scenario, is + an error: call this before the scenario runs, so the recording is not + made in vain. + """ + named = scenario.get("script") + if named is None: + return None + if not isinstance(named, str) or not named.strip() or Path(named).is_absolute() or ".." in Path(named).parts: + raise PresentationError(f"the scenario's script {named!r} is not a path below {root}") + try: + text = (root / named).read_text(encoding="utf-8") + except OSError as error: + raise PresentationError(f"the scenario's script {named!r} cannot be read below {root}: {error}") from error + return plan_presentation(scenario, parse_script(text)) + + +def write_presentation(evidence: Path, document: Mapping[str, Any]) -> Path: + """Write ``document`` as ``presentation.json`` in the ``evidence`` folder and return its path.""" + path = evidence / PRESENTATION_FILE + path.write_text(json.dumps(document, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + return path diff --git a/devcapsule-src/devcapsule/presentation_tools/recording.py b/devcapsule-src/devcapsule/presentation_tools/recording.py new file mode 100644 index 00000000..4b418c88 --- /dev/null +++ b/devcapsule-src/devcapsule/presentation_tools/recording.py @@ -0,0 +1,143 @@ +"""Record real Playwright actions and derive presentation timing from their frames. + +Public surface: ``Recording``. The caller supplies its page, scenario, evidence +directory and ffmpeg command. No DevCapsule installation or test harness is +needed by the exported package. Assertions belong in the supplied action. +""" +from __future__ import annotations + +from collections.abc import Callable, Sequence +import json +from pathlib import Path +import subprocess +from typing import Any + +from .producer import PresentationError, Script, plan_presentation, write_presentation + +__all__ = ["Recording"] + + +class Recording: + """Capture bursts around actions, excluding time the agent spends thinking. + + Each action has a one-based script step and a caption. Repeated actions may + belong to the same step. Steps must run in order. ``finish`` keeps a failed + or incomplete run visibly failed; it never infers success from screenshots. + Use a new evidence directory for each attempt; existing runs are refused. + """ + + def __init__( + self, evidence: Path, *, script: Script, scenario_id: str, title: str, + summary: str, ffmpeg: Sequence[str], + ) -> None: + # Check the complete scenario before opening a browser or making files. + scenario: dict[str, Any] = {"id": scenario_id, "title": title, "description": summary, + "actions": [{"reason": f"{n}. {step.action}"} + for n, step in enumerate(script.steps, 1)]} + plan_presentation(scenario, script) + if isinstance(ffmpeg, (str, bytes)) or not ffmpeg or not all(isinstance(part, str) and part for part in ffmpeg): + raise PresentationError("supply the ffmpeg command from the browser-automation environment") + evidence.mkdir(parents=True, exist_ok=False) + self.evidence = evidence + self.script = script + self.scenario = scenario + self.ffmpeg = tuple(ffmpeg) + self.actions: list[dict[str, Any]] = [] + self.frames_after_actions: list[int] = [] + self.screenshots: list[str] = [] + self.frame_count = 0 + self.last_step = 0 + self.failure = "" + self.finished = False + self.frames = evidence / "frames.mjpeg" + + def _capture(self, page: Any, count: int) -> None: + # Playwright's bundled ffmpeg decodes MJPEG, but need not decode PNG. + with self.frames.open("ab") as output: + for _ in range(count): + output.write(page.screenshot(type="jpeg", quality=90)) + self.frame_count += 1 + page.wait_for_timeout(100) + + def action(self, page: Any, step: int, caption: str, perform: Callable[[], object]) -> None: + """Record an action and its assertions; retain failure evidence and re-raise. + + The supplied callback uses ordinary Playwright selectors, mouse/keyboard + operations or project checks. Keep actions small enough that their before + and after frames show the interaction; native Playwright video may also be + kept in ``evidence/raw`` to preserve uninterrupted motion. + """ + if self.finished or self.failure: + raise PresentationError("this run has finished or failed; start a new run") + if (isinstance(step, bool) or not isinstance(step, int) + or not 1 <= step <= len(self.script.steps) + or step not in (self.last_step, self.last_step + 1)): + raise PresentationError("actions must visit script steps in order, starting at 1") + if not isinstance(caption, str) or not caption.strip(): + raise PresentationError("each recorded action needs a non-empty caption") + self.last_step = step + entry: dict[str, Any] = {"reason": f"{step}. {caption}", "status": "passed"} + self.actions.append(entry) + try: + self._capture(page, 2) + perform() + self._capture(page, 6) + screenshot = f"step-{len(self.actions):03d}.png" + page.screenshot(path=str(self.evidence / screenshot)) + self.screenshots.append(screenshot) + except BaseException as error: + self.failure = f"{type(error).__name__}: {error}" + entry.update(status="failed", error=self.failure) + # A disconnected browser must not replace the original test failure. + try: + self._capture(page, 1) + screenshot = f"step-{len(self.actions):03d}.png" + page.screenshot(path=str(self.evidence / screenshot)) + self.screenshots.append(screenshot) + except Exception: + pass + raise + finally: + self.frames_after_actions.append(self.frame_count) + entry["frames_after"] = self.frame_count + self._write_result() + + def _write_result(self) -> None: + complete = self.last_step == len(self.script.steps) and not self.failure + result = {"status": "failed" if self.failure else "passed" if self.finished and complete else "incomplete", + "error": self.failure, "steps_reached": self.last_step, + "steps_planned": len(self.script.steps)} + for name, value in (("scenario-result.json", result), ("action-log.json", self.actions)): + (self.evidence / name).write_text(json.dumps(value, indent=2) + "\n", encoding="utf-8") + + def finish(self) -> Path: + """Encode frames and publish the deck last; refuse missing visual evidence. + + Logs survive encoding failure. A failed test produces a labelled deck + containing only the steps it reached, and keeps its failed result. + """ + if self.finished: + raise PresentationError("this recording has already been finished") + self.finished = True + self._write_result() + if not self.actions or len(self.screenshots) != len(self.actions): + raise PresentationError("no complete visual evidence; inspect scenario-result.json and action-log.json") + complete = self.last_step == len(self.script.steps) and not self.failure + suffix = "" if complete else " (FAILED)" if self.failure else " (INCOMPLETE)" + scenario = dict(self.scenario, title=self.scenario["title"].strip() + suffix, actions=self.actions) + reached = Script(self.script.goal, self.script.stack, self.script.illustrates, + self.script.steps[:self.last_step]) + plan = plan_presentation(scenario, reached) + document = plan.timed(self.frames_after_actions, self.frame_count, 12, + video="scenario-video.webm", poster="poster.png", screenshots=self.screenshots) + video = self.evidence / "scenario-video.webm" + with self.frames.open("rb") as frames, (self.evidence / "ffmpeg.log").open("wb") as log: + subprocess.run( + [*self.ffmpeg, "-y", "-f", "image2pipe", "-vcodec", "mjpeg", "-r", "12", + "-i", "pipe:0", "-c:v", "libvpx", "-b:v", "3M", "-crf", "10", str(video)], + stdin=frames, stdout=log, stderr=log, check=True, timeout=120, + ) + if not video.is_file() or video.stat().st_size == 0: + raise PresentationError("video encoding produced no scenario-video.webm; inspect ffmpeg.log") + (self.evidence / "poster.png").write_bytes((self.evidence / self.screenshots[0]).read_bytes()) + return write_presentation(self.evidence, document) diff --git a/devcapsule-src/pyproject.toml b/devcapsule-src/pyproject.toml index 37dc4aba..60159b91 100644 --- a/devcapsule-src/pyproject.toml +++ b/devcapsule-src/pyproject.toml @@ -47,6 +47,7 @@ where = ["."] include = ["devcapsule*"] [tool.setuptools.package-data] +"devcapsule.presentation_tools" = ["README.md"] "devcapsule" = [ "_build_info.json", "_compatibility-v1.json", diff --git a/devcapsule-src/tests/e2e/scenario_presentation.py b/devcapsule-src/tests/e2e/scenario_presentation.py index 254439a3..b9f7ae96 100644 --- a/devcapsule-src/tests/e2e/scenario_presentation.py +++ b/devcapsule-src/tests/e2e/scenario_presentation.py @@ -1,382 +1,27 @@ -"""The presentation of a recorded scenario: ``presentation.json`` beside its recording. +"""Repository scenario adapter for the shipped presentation producer. -The web console shows a folder that holds a ``presentation.json`` as slides -beside the recording: an overview, then one slide per step with the step's -text, its screenshot, and its part of the video with captions. The document's -contract is ``devcapsule-webconsole/PRESENTATIONS.md``. This module writes -that document from what a recorded scenario already has, so that a recording -needs no hand-made data to be presented: - -- the scenario's JSON: ``id``, ``title``, ``description``, ``script`` (the - path of its script below ``tests/e2e``) and ``actions``; -- the script, a Markdown file in the form - ``engineering-docs/development/running-e2e-scenario-recordings.md`` gives: - the goal, the stack, what the scenario illustrates, and the numbered steps - with the user's action and the expected observation; -- the count of frames the recorder had taken after each action. - -An action belongs to the step whose number starts its ``reason`` ("3. Save -the file"). A step's captions come from its actions: when none of them has a -``caption``, every action's reason, without the number, is its caption; when -some have, a caption starts at each action that has one and lasts until the -next, and the step's first action must have one. - -Public operations: ``parse_script``, ``plan_presentation``, -``PresentationPlan.timed``, ``plan_for_scenario`` and ``write_presentation``. -They raise ``PresentationError`` for a script or a scenario that cannot be -presented; the message names what to correct. +Public operations are the producer exports and ``plan_for_scenario`` with +this test harness's script root. """ -from __future__ import annotations - -from collections.abc import Mapping, Sequence -from dataclasses import dataclass -import json +from collections.abc import Mapping from pathlib import Path -import re from typing import Any -__all__ = [ - "E2E_ROOT", - "PRESENTATION_FILE", - "PresentationError", - "PresentationPlan", - "Script", - "ScriptStep", - "parse_script", - "plan_for_scenario", - "plan_presentation", - "write_presentation", -] +from devcapsule.presentation_tools.producer import ( + PRESENTATION_FILE as PRESENTATION_FILE, + PresentationError as PresentationError, + PresentationPlan as PresentationPlan, + Script as Script, + ScriptStep as ScriptStep, + parse_script as parse_script, + plan_for_scenario as _plan_for_scenario, + plan_presentation as plan_presentation, + write_presentation as write_presentation, +) E2E_ROOT = Path(__file__).resolve().parent -"""A scenario's ``script`` path is relative to this directory.""" -PRESENTATION_FILE = "presentation.json" -SCHEMA_VERSION = 1 - -_SECTIONS = ("Goal", "Stack And Capabilities", "What It Illustrates", "Steps") -_HEADING = re.compile(r"^## (?P<name>.+?)\s*$") -_STEP = re.compile(r"^(?P<number>\d+)\. \*\*(?P<title>.+?)\*\*\s*$") -_FIELD = re.compile(r"^\s+- \*(?P<name>User action|Expected observation):\*\s+(?P<text>.+?)\s*$") -_STACK_ROW = re.compile(r"^- \*\*(?P<name>.+?):\*\*\s+(?P<detail>.+?)\s*$") -_BULLET = re.compile(r"^- (?P<text>.+?)\s*$") -_CONTINUATION = re.compile(r"^\s+(?P<text>\S.*?)\s*$") -_STEP_NUMBER = re.compile(r"^(?P<number>\d+)\.\s+(?P<text>\S.*)$", re.DOTALL) - - -class PresentationError(ValueError): - """A script or a scenario cannot be presented; the message names what to correct.""" - - -@dataclass(frozen=True) -class ScriptStep: - """One numbered step of a script.""" - - title: str - action: str - expected: str - - -@dataclass(frozen=True) -class Script: - """What a scenario's script says, section by section.""" - - goal: str - stack: tuple[tuple[str, str], ...] - illustrates: tuple[str, ...] - steps: tuple[ScriptStep, ...] - - -def parse_script(text: str) -> Script: - """The script a Markdown ``text`` in the script form states. - - The form: ``## Goal``, one paragraph; ``## Stack And Capabilities``, - optional, bullets ``- **Name:** detail``; ``## What It Illustrates``, - optional, bullets; ``## Steps``, items ``N. **Title.**`` numbered from 1, - each with the sub-bullets ``- *User action:* …`` and - ``- *Expected observation:* …``. A bullet may continue on indented lines. - Text before the first ``##`` heading is not read. Any other ``##`` - heading, a missing ``Goal`` or ``Steps``, a step out of order or without - both sub-bullets, and a line that fits nowhere are refused. - """ - sections: dict[str, list[tuple[int, str]]] = {} - current: list[tuple[int, str]] | None = None - for number, line in enumerate(text.splitlines(), start=1): - heading = _HEADING.match(line) - if heading is not None: - name = heading.group("name") - if name not in _SECTIONS: - raise PresentationError( - f"script line {number}: section {name!r} is not one of {', '.join(_SECTIONS)}" - ) - if name in sections: - raise PresentationError(f"script line {number}: section {name!r} appears twice") - current = sections[name] = [] - elif current is not None and line.strip(): - current.append((number, line)) - for required in ("Goal", "Steps"): - if not sections.get(required): - raise PresentationError(f"script: section {required!r} is missing or empty") - goal = " ".join(line.strip() for _, line in sections["Goal"]) - stack = tuple( - _stack_row(number, item) for number, item in _bullets(sections.get("Stack And Capabilities", []), "Stack And Capabilities") - ) - illustrates = tuple(item for _, item in _bullets(sections.get("What It Illustrates", []), "What It Illustrates")) - return Script(goal=goal, stack=stack, illustrates=illustrates, steps=_steps(sections["Steps"])) - - -def _bullets(lines: Sequence[tuple[int, str]], section: str) -> list[tuple[int, str]]: - """The bullets of a section, each with its first line's number, continuations joined.""" - items: list[tuple[int, str]] = [] - for number, line in lines: - bullet = _BULLET.match(line) - continuation = _CONTINUATION.match(line) - if bullet is not None: - items.append((number, bullet.group("text"))) - elif continuation is not None and items: - first, so_far = items[-1] - items[-1] = (first, f"{so_far} {continuation.group('text')}") - else: - raise PresentationError(f"script line {number}: {section!r} takes bullets, '- text'") - return items - - -def _stack_row(number: int, item: str) -> tuple[str, str]: - row = _STACK_ROW.match(f"- {item}") - if row is None: - raise PresentationError(f"script line {number}: a stack bullet is '- **Name:** detail'") - return row.group("name"), row.group("detail") - - -def _steps(lines: Sequence[tuple[int, str]]) -> tuple[ScriptStep, ...]: - steps: list[dict[str, str]] = [] - field: str | None = None - for number, line in lines: - step = _STEP.match(line) - named = _FIELD.match(line) - continuation = _CONTINUATION.match(line) - if step is not None: - if int(step.group("number")) != len(steps) + 1: - raise PresentationError(f"script line {number}: expected step {len(steps) + 1}, found {step.group('number')}") - steps.append({"title": step.group("title").rstrip(".").strip()}) - field = None - elif named is not None and steps: - field = named.group("name") - if field in steps[-1]: - raise PresentationError(f"script line {number}: step {len(steps)} states {field!r} twice") - steps[-1][field] = named.group("text") - elif continuation is not None and steps and field is not None: - steps[-1][field] = f"{steps[-1][field]} {continuation.group('text')}" - else: - raise PresentationError( - f"script line {number}: a step is 'N. **Title.**' with '- *User action:* …' and '- *Expected observation:* …'" - ) - for index, step_fields in enumerate(steps, start=1): - for required in ("User action", "Expected observation"): - if required not in step_fields: - raise PresentationError(f"script: step {index} has no {required!r}") - if not step_fields["title"]: - raise PresentationError(f"script: step {index} has no title") - return tuple( - ScriptStep(title=item["title"], action=item["User action"], expected=item["Expected observation"]) for item in steps - ) - - -@dataclass(frozen=True) -class _Caption: - """A caption and the actions it lasts for, as indexes into the scenario's actions.""" - - first_action: int - last_action: int - text: str - - -@dataclass(frozen=True) -class _PlannedStep: - step: ScriptStep - first_action: int - last_action: int - captions: tuple[_Caption, ...] - - -@dataclass(frozen=True) -class PresentationPlan: - """Everything of a presentation that is known before the scenario runs. - - ``plan_presentation`` makes it; ``timed`` adds what only a run knows. - """ - - scenario_id: str - title: str - summary: str - script: Script - action_count: int - steps: tuple[_PlannedStep, ...] - - def timed( - self, - frames_after_actions: Sequence[int], - total_frames: int, - frames_per_second: int, - *, - video: str, - poster: str, - screenshots: Sequence[str], - ) -> dict[str, Any]: - """The presentation document of one recorded run of the planned scenario. - - ``frames_after_actions[i]`` is the count of frames recorded when - action ``i`` had finished, ``total_frames`` the count in the video, - which plays ``frames_per_second`` of them, and ``screenshots[i]`` the - file taken after action ``i``. A step runs from the end of the step - before it, the first from the start of the video, to the end of its - last action, the last to the end of the video; its screenshot is its - last action's. Refuses counts that do not match the plan, that - decrease, that exceed the total, or that leave a step or a caption - without a frame. - """ - if len(frames_after_actions) != self.action_count or len(screenshots) != self.action_count: - raise PresentationError( - f"the run has {len(frames_after_actions)} frame counts and {len(screenshots)} screenshots " - f"for {self.action_count} actions" - ) - if frames_per_second <= 0: - raise PresentationError(f"frames per second must be positive, not {frames_per_second}") - counts = list(frames_after_actions) - if any(later < earlier for earlier, later in zip([0, *counts], [*counts, total_frames])): - raise PresentationError("the frame counts decrease or exceed the total number of frames") - last_action = self.action_count - 1 - - def seconds(frames: int) -> float: - return round(frames / frames_per_second, 3) - - def starts(action: int) -> int: - return 0 if action == 0 else counts[action - 1] - - def ends(action: int) -> int: - return total_frames if action == last_action else counts[action] - - steps = [] - for number, planned in enumerate(self.steps, start=1): - if any(starts(caption.first_action) >= ends(caption.last_action) for caption in planned.captions): - raise PresentationError(f"step {number} has a caption that lasts no frame") - captions = [ - {"start": seconds(starts(caption.first_action)), "end": seconds(ends(caption.last_action)), "text": caption.text} - for caption in planned.captions - ] - steps.append({ - "title": planned.step.title, - "action": planned.step.action, - "expected": planned.step.expected, - "screenshot": screenshots[planned.last_action], - "start": seconds(starts(planned.first_action)), - "end": seconds(ends(planned.last_action)), - "captions": captions, - }) - overview: dict[str, Any] = {"summary": self.summary, "intent": self.script.goal} - if self.script.illustrates: - overview["illustrates"] = list(self.script.illustrates) - if self.script.stack: - overview["stack"] = [{"name": name, "detail": detail} for name, detail in self.script.stack] - return { - "schema_version": SCHEMA_VERSION, - "scenario_id": self.scenario_id, - "title": self.title, - "overview": overview, - "video": {"file": video, "poster": poster, "duration": seconds(total_frames)}, - "steps": steps, - } - - -def plan_presentation(scenario: Mapping[str, Any], script: Script) -> PresentationPlan: - """Join a scenario's JSON document to its parsed script, or say why they do not join. - - The scenario needs a non-empty ``id``, ``title`` and ``description``. Each - action's ``reason`` starts with the number of its step and a period; the - numbers never decrease and every step of the script has at least one - action. ``caption``, where an action has it, is a non-empty string, and a - step with any caption has one on its first action. - """ - texts = {} - for key in ("id", "title", "description"): - value = scenario.get(key) - if not isinstance(value, str) or not value.strip(): - raise PresentationError(f"the scenario has no {key}") - texts[key] = value.strip() - actions = scenario.get("actions") - if not isinstance(actions, list) or not actions: - raise PresentationError("the scenario has no actions") - numbers: list[int] = [] - reasons: list[str] = [] - captions: list[str | None] = [] - for position, action in enumerate(actions, start=1): - if not isinstance(action, Mapping): - raise PresentationError(f"action {position} is not an object") - reason = action.get("reason") - numbered = _STEP_NUMBER.match(reason) if isinstance(reason, str) else None - if numbered is None: - raise PresentationError(f"action {position}: its reason must start with the number of its step, as in '3. Save the file'") - number = int(numbered.group("number")) - if not 1 <= number <= len(script.steps): - raise PresentationError(f"action {position} names step {number}; the script has steps 1 to {len(script.steps)}") - if numbers and number < numbers[-1]: - raise PresentationError(f"action {position} names step {number} after an action of step {numbers[-1]}") - caption = action.get("caption") - if caption is not None and (not isinstance(caption, str) or not caption.strip()): - raise PresentationError(f"action {position}: its caption must be a non-empty text") - numbers.append(number) - reasons.append(numbered.group("text").strip()) - captions.append(caption.strip() if isinstance(caption, str) else None) - planned: list[_PlannedStep] = [] - for number, step in enumerate(script.steps, start=1): - members = [index for index, owner in enumerate(numbers) if owner == number] - if not members: - raise PresentationError(f"step {number} of the script has no action") - planned.append(_PlannedStep( - step=step, first_action=members[0], last_action=members[-1], captions=_captions(number, members, reasons, captions), - )) - return PresentationPlan( - scenario_id=texts["id"], title=texts["title"], summary=texts["description"], script=script, - action_count=len(actions), steps=tuple(planned), - ) - - -def _captions(number: int, members: Sequence[int], reasons: Sequence[str], captions: Sequence[str | None]) -> tuple[_Caption, ...]: - """The captions of one step's actions, by the rule in the module's documentation.""" - written = [index for index in members if captions[index] is not None] - if not written: - return tuple(_Caption(first_action=index, last_action=index, text=reasons[index]) for index in members) - if written[0] != members[0]: - raise PresentationError( - f"step {number} has captions, so its first action, action {members[0] + 1}, needs one" - ) - ends = [*(index - 1 for index in written[1:]), members[-1]] - return tuple( - _Caption(first_action=first, last_action=last, text=str(captions[first])) for first, last in zip(written, ends) - ) def plan_for_scenario(scenario: Mapping[str, Any], root: Path = E2E_ROOT) -> PresentationPlan | None: - """The plan of a scenario that names its ``script``, read below ``root``; None for one that names none. - - A scenario without a script is recorded without a presentation. A script - that is named and cannot be read, or that does not join the scenario, is - an error: call this before the scenario runs, so the recording is not - made in vain. - """ - named = scenario.get("script") - if named is None: - return None - if not isinstance(named, str) or not named.strip() or Path(named).is_absolute() or ".." in Path(named).parts: - raise PresentationError(f"the scenario's script {named!r} is not a path below {root}") - try: - text = (root / named).read_text(encoding="utf-8") - except OSError as error: - raise PresentationError(f"the scenario's script {named!r} cannot be read below {root}: {error}") from error - return plan_presentation(scenario, parse_script(text)) - - -def write_presentation(evidence: Path, document: Mapping[str, Any]) -> Path: - """Write ``document`` as ``presentation.json`` in the ``evidence`` folder and return its path.""" - path = evidence / PRESENTATION_FILE - path.write_text(json.dumps(document, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") - return path + """Read a repository scenario's script relative to its E2E directory.""" + return _plan_for_scenario(scenario, root) diff --git a/devcapsule-src/tests/integration/test_pex_runtime.py b/devcapsule-src/tests/integration/test_pex_runtime.py index 78f4b45e..10e64d5d 100644 --- a/devcapsule-src/tests/integration/test_pex_runtime.py +++ b/devcapsule-src/tests/integration/test_pex_runtime.py @@ -288,3 +288,44 @@ def invoke(*arguments: str) -> subprocess.CompletedProcess[str]: saved = invoke("run", "--ide", "codium", "--force", "--save", "--print-command") assert saved.returncode == 2 and "cannot be combined with --save" in saved.stderr assert {path: path.read_bytes() for path in before} == before, "no refusal writes anything" + + +def test_built_pex_exports_standalone_presentation_tools( + built_pex: Path, tmp_path: Path, environment: dict[str, str], +) -> None: + """The built executable prints the guide and exports a standalone ``presentation_tools`` kit.""" + guide = subprocess.run( + [str(built_pex), "presentations", "guide"], + cwd=tmp_path, env=environment, text=True, capture_output=True, check=False, + ) + assert guide.returncode == 0, guide.stderr + assert "# Run an E2E scenario with recording and presentation" in guide.stdout + + kit = tmp_path / "exported-kit" + exported = subprocess.run( + [str(built_pex), "presentations", "export", str(kit)], + cwd=tmp_path, env=environment, text=True, capture_output=True, check=False, + ) + assert exported.returncode == 0, exported.stderr + package = kit / "presentation_tools" + assert (package / "README.md").read_text(encoding="utf-8") + "\n" == guide.stdout + assert {path.name for path in package.iterdir()} == { + "__init__.py", "README.md", "example.py", "producer.py", "recording.py", + } + + isolated = subprocess.run( + [ + sys.executable, "-I", "-c", + ( + "import sys; sys.path.insert(0, sys.argv[1]); " + "from presentation_tools.producer import Script, ScriptStep; " + "from presentation_tools.recording import Recording; " + "import presentation_tools.example; " + "assert 'devcapsule' not in sys.modules and 'tests' not in sys.modules" + ), + str(kit), + ], + cwd=tmp_path, env={"PATH": os.defpath, "PYTHONDONTWRITEBYTECODE": "1"}, + text=True, capture_output=True, check=False, + ) + assert isolated.returncode == 0, isolated.stderr diff --git a/devcapsule-src/tests/integration/test_presentations_cli.py b/devcapsule-src/tests/integration/test_presentations_cli.py new file mode 100644 index 00000000..db382efe --- /dev/null +++ b/devcapsule-src/tests/integration/test_presentations_cli.py @@ -0,0 +1,107 @@ +"""Integration tests of ``devcapsule presentations``: real CLI subprocesses with explicit environments.""" +from __future__ import annotations + +import os +from pathlib import Path +import subprocess +import sys + +import pytest + +pytestmark = pytest.mark.integration + + +def _run_presentations( + tmp_path: Path, *arguments: str, extra_env: dict[str, str] | None = None, +) -> subprocess.CompletedProcess[str]: + environment = { + "PATH": os.defpath, + "PYTHONDONTWRITEBYTECODE": "1", + "HOME": str(tmp_path / "home"), + "DEVCAPSULE_CONFIG_HOME": str(tmp_path / "config"), + } + if extra_env: + environment.update(extra_env) + return subprocess.run( + [sys.executable, "-m", "devcapsule", "presentations", *arguments], + cwd=tmp_path, env=environment, capture_output=True, text=True, timeout=120, check=False, + ) + + +def test_guide_and_export_supply_an_importable_standalone_kit(tmp_path: Path) -> None: + """``presentations guide`` and ``export`` deliver the guide and a stdlib-only kit, refusing overwrites.""" + guide = _run_presentations(tmp_path, "guide") + assert guide.returncode == 0, guide.stderr + assert "# Run an E2E scenario with recording and presentation" in guide.stdout + + kit = tmp_path / "kit" + exported = _run_presentations(tmp_path, "export", str(kit)) + assert exported.returncode == 0, exported.stderr + assert f"Presentation tools: {kit / 'presentation_tools'}" in exported.stdout + assert (kit / "presentation_tools" / "README.md").read_text(encoding="utf-8") + "\n" == guide.stdout + + isolated = subprocess.run( + [ + sys.executable, "-I", "-c", + ( + "import sys; sys.path.insert(0, sys.argv[1]); " + "from presentation_tools.producer import Script; " + "from presentation_tools.recording import Recording; " + "import presentation_tools.example; " + "assert 'devcapsule' not in sys.modules and 'tests' not in sys.modules" + ), + str(kit), + ], + cwd=tmp_path, env={"PATH": os.defpath, "PYTHONDONTWRITEBYTECODE": "1"}, + capture_output=True, text=True, timeout=120, check=False, + ) + assert isolated.returncode == 0, isolated.stderr + + refused = _run_presentations(tmp_path, "export", str(kit)) + assert refused.returncode == 2 + assert "cannot export presentation tools" in refused.stderr + + +def test_hand_off_reads_environment_and_handles_token_states(tmp_path: Path) -> None: + """``presentations hand-off`` prints a tokenized link, falls back cleanly on missing or non-UTF-8 tokens, and refuses missing decks or origins.""" + project = tmp_path / "project" + evidence = project / "evidence" / "run-001" + evidence.mkdir(parents=True) + (evidence / "presentation.json").write_text("{}", encoding="utf-8") + token_file = tmp_path / "console-token" + token_file.write_text("secret-token\n", encoding="utf-8") + + with_token = _run_presentations( + tmp_path, "hand-off", str(evidence), + extra_env={ + "PROJECT_PATH": str(project), + "DEVCAPSULE_CONSOLE_URL": "http://127.0.0.1:8787", + "DEVCAPSULE_CONSOLE_TOKEN_FILE": str(token_file), + }, + ) + assert with_token.returncode == 0, with_token.stderr + assert with_token.stdout.strip() == "http://127.0.0.1:8787/presentations/evidence/run-001?token=secret-token" + + token_file.write_bytes(b"\xff\xfe\xfd") + corrupt_token = _run_presentations( + tmp_path, "hand-off", str(evidence), + "--project", str(project), "--console-url", "http://127.0.0.1:8787", "--token-file", str(token_file), + ) + assert corrupt_token.returncode == 0, corrupt_token.stderr + assert "no readable run token was found" in corrupt_token.stdout + assert corrupt_token.stdout.splitlines()[-1] == "http://127.0.0.1:8787/presentations/evidence/run-001" + + missing_url = _run_presentations( + tmp_path, "hand-off", str(evidence), "--project", str(project), "--token-file", str(tmp_path / "absent"), + ) + assert missing_url.returncode == 2 + assert "HTTP(S) console URL" in missing_url.stderr + + empty_dir = project / "evidence" / "empty" + empty_dir.mkdir(parents=True) + missing_deck = _run_presentations( + tmp_path, "hand-off", str(empty_dir), + "--project", str(project), "--console-url", "http://127.0.0.1:8787", + ) + assert missing_deck.returncode == 2 + assert "no presentation.json" in missing_deck.stderr diff --git a/devcapsule-src/tests/unit/tests/e2e/test_scenario_presentation.py b/devcapsule-src/tests/unit/devcapsule/presentation_tools/test_producer.py similarity index 90% rename from devcapsule-src/tests/unit/tests/e2e/test_scenario_presentation.py rename to devcapsule-src/tests/unit/devcapsule/presentation_tools/test_producer.py index f6f5c471..2643f304 100644 --- a/devcapsule-src/tests/unit/tests/e2e/test_scenario_presentation.py +++ b/devcapsule-src/tests/unit/devcapsule/presentation_tools/test_producer.py @@ -1,4 +1,4 @@ -"""Unit tests for ``tests.e2e.scenario_presentation``.""" +"""Unit tests for ``devcapsule.presentation_tools.producer``.""" from __future__ import annotations import json @@ -8,7 +8,7 @@ import pytest -from tests.e2e.scenario_presentation import ( +from devcapsule.presentation_tools.producer import ( PresentationError, Script, ScriptStep, @@ -216,6 +216,24 @@ def test_plan_presentation_refuses_a_scenario_that_does_not_join_its_script(docu plan_presentation(document, SCRIPT) +@pytest.mark.parametrize( + ("bad_script", "message"), + [ + (Script("", (), (), SCRIPT.steps), "the script has no goal"), + (Script("Goal", (), (), ()), "the script has no steps"), + (Script("Goal", (), (), (ScriptStep(" ", "Open", "Visible"),)), "script step 1 has no title"), + (Script("Goal", (), (), (ScriptStep("Open", " ", "Visible"),)), "script step 1 has no action"), + (Script("Goal", (), (), (ScriptStep("Open", "Open", ""),)), "script step 1 has no expected"), + (Script("Goal", (), (" ",), SCRIPT.steps[:1]), "script illustrations must be non-empty texts"), + (Script("Goal", (("Editor", ""),), (), SCRIPT.steps[:1]), "script stack entries must have non-empty name and detail"), + ], +) +def test_plan_presentation_refuses_incomplete_python_script(bad_script: Script, message: str) -> None: + """``plan_presentation`` refuses Python-constructed scripts with missing goal, steps, step fields, stack or illustrations.""" + with pytest.raises(PresentationError, match=re.escape(message)): + plan_presentation(scenario(action("1. One")), bad_script) + + def test_plan_for_scenario_reads_the_named_script_below_the_root(tmp_path: Path) -> None: """A scenario that names its script is planned from the file at that path below the root.""" (tmp_path / "scripts").mkdir() diff --git a/devcapsule-src/tests/unit/devcapsule/presentation_tools/test_recording.py b/devcapsule-src/tests/unit/devcapsule/presentation_tools/test_recording.py new file mode 100644 index 00000000..300e9c6f --- /dev/null +++ b/devcapsule-src/tests/unit/devcapsule/presentation_tools/test_recording.py @@ -0,0 +1,333 @@ +"""Unit tests for ``devcapsule.presentation_tools.recording``.""" +from __future__ import annotations + +import json +from pathlib import Path +import subprocess +import sys + +import pytest + +from devcapsule.presentation_tools.producer import PresentationError, Script, ScriptStep +from devcapsule.presentation_tools.recording import Recording + +SCRIPT = Script( + goal="Verify the application flow.", + stack=(("Browser", "Chromium"),), + illustrates=("A passing and failing check.",), + steps=( + ScriptStep("Open", "Open the app", "The heading appears"), + ScriptStep("Reload", "Reload the page", "The heading stays"), + ), +) + + +class _StubPage: + """In-memory Playwright page double recording captured bytes and wait calls.""" + + def __init__(self, *, fail_jpeg_at: int | None = None, fail_png_at: int | None = None) -> None: + self.fail_jpeg_at = fail_jpeg_at + self.fail_png_at = fail_png_at + self.jpeg_calls = 0 + self.png_calls = 0 + self.captured_jpeg_bytes = bytearray() + self.waits: list[int] = [] + + def screenshot(self, *, type: str = "png", quality: int | None = None, path: str | None = None) -> bytes: + del quality + if path is None: + self.jpeg_calls += 1 + if self.fail_jpeg_at is not None and self.jpeg_calls >= self.fail_jpeg_at: + raise RuntimeError("page disconnected during frame capture") + assert type == "jpeg" + chunk = f"FRAME-{self.jpeg_calls}:".encode("ascii") + self.captured_jpeg_bytes.extend(chunk) + return chunk + self.png_calls += 1 + if self.fail_png_at is not None and self.png_calls >= self.fail_png_at: + raise RuntimeError("page disconnected during step screenshot") + Path(path).write_bytes(f"PNG-{self.png_calls}".encode("ascii")) + return b"" + + def wait_for_timeout(self, ms: int) -> None: + self.waits.append(ms) + + +def _encoder_command(*, exit_code: int = 0, write_video: bool = True) -> tuple[str, ...]: + program = ( + "import pathlib, sys; " + "data = sys.stdin.buffer.read(); " + "sys.stderr.write(f'bytes={len(data)}\\n'); " + f"pathlib.Path(sys.argv[-1]).write_bytes(b'WEBM:' + data) if {write_video!r} else None; " + f"sys.exit({exit_code})" + ) + return (sys.executable, "-I", "-S", "-c", program) + + +@pytest.mark.parametrize( + ("steps", "ffmpeg"), + [ + ((), ("encoder",)), + ((ScriptStep("Open", "Open the app", "The heading appears"),), ()), + ((ScriptStep("Open", "Open the app", "The heading appears"),), "ffmpeg"), + ], +) +def test_recording_rejects_invalid_setup_before_creating_evidence( + tmp_path: Path, steps: tuple[ScriptStep, ...], ffmpeg: tuple[str, ...] | str, +) -> None: + """An empty script, absent encoder or unsplit string encoder is refused before an evidence folder exists.""" + with pytest.raises(PresentationError): + Recording(tmp_path / "run", script=Script("Goal", (), (), steps), scenario_id="example", + title="Example", summary="Example run", ffmpeg=ffmpeg) + + assert not (tmp_path / "run").exists() + + +def test_recording_preserves_existing_evidence(tmp_path: Path) -> None: + """A recording cannot overwrite a prior attempt, even if that directory is empty.""" + with pytest.raises(FileExistsError): + Recording(tmp_path, script=SCRIPT, scenario_id="example", title="Example", + summary="Example run", ffmpeg=("encoder",)) + + assert list(tmp_path.iterdir()) == [] + + +def test_recording_finish_writes_aligned_deck_and_media_for_passing_run(tmp_path: Path) -> None: + """Finishing a complete multi-action run encodes captured bytes and aligns chapters to frame boundaries.""" + # Precondition: record a complete two-step run (two actions in step 1, one in step 2). + evidence = tmp_path / "run-pass" + page = _StubPage() + recording = Recording(evidence, script=SCRIPT, scenario_id="example", title=" Example ", + summary="Example run", ffmpeg=_encoder_command()) + recording.action(page, 1, "Navigate to page", lambda: None) + recording.action(page, 1, "Confirm heading", lambda: None) + recording.action(page, 2, "Reload page", lambda: None) + + # Action: finalize the recording. + manifest = recording.finish() + + # Postcondition: logs record increasing frame boundaries, deck aligns to them, and encoder gets captured bytes. + result = json.loads((evidence / "scenario-result.json").read_text(encoding="utf-8")) + actions = json.loads((evidence / "action-log.json").read_text(encoding="utf-8")) + boundaries = [entry["frames_after"] for entry in actions] + deck = json.loads(manifest.read_text(encoding="utf-8")) + assert result == {"status": "passed", "error": "", "steps_reached": 2, "steps_planned": 2} + assert len(boundaries) == 3 and 0 < boundaries[0] < boundaries[1] < boundaries[2] + assert deck["title"] == "Example" + assert deck["video"]["file"] == "scenario-video.webm" + assert deck["video"]["poster"] == "poster.png" + assert deck["video"]["duration"] > 0 + step1, step2 = deck["steps"] + assert (step1["screenshot"], step2["screenshot"]) == ("step-002.png", "step-003.png") + assert step1["start"] == 0.0 + assert 0.0 < step1["end"] == step2["start"] < step2["end"] == deck["video"]["duration"] + assert [c["text"] for c in step1["captions"]] == ["Navigate to page", "Confirm heading"] + assert [c["text"] for c in step2["captions"]] == ["Reload page"] + assert step1["captions"][0]["start"] == step1["start"] + assert step1["captions"][0]["end"] == step1["captions"][1]["start"] + assert step1["captions"][1]["end"] == step1["end"] + assert step2["captions"][0]["start"] == step2["start"] and step2["captions"][0]["end"] == step2["end"] + assert (evidence / "poster.png").read_bytes() == (evidence / "step-001.png").read_bytes() + assert (evidence / "scenario-video.webm").read_bytes() == b"WEBM:" + bytes(page.captured_jpeg_bytes) + assert f"bytes={len(page.captured_jpeg_bytes)}\n" in (evidence / "ffmpeg.log").read_text(encoding="utf-8") + + +def test_recording_action_reraises_callback_failure_and_records_failed_state(tmp_path: Path) -> None: + """A failing action callback propagates its exception while preserving failed logs and screenshot.""" + # Precondition: step 1 succeeded. + evidence = tmp_path / "run-fail-action" + page = _StubPage() + recording = Recording(evidence, script=SCRIPT, scenario_id="example", title="Example", + summary="Example run", ffmpeg=_encoder_command()) + recording.action(page, 1, "Open app", lambda: None) + + def fail_reload() -> None: + raise AssertionError("heading missing") + + # Action: execute a failing callback in step 2. + with pytest.raises(AssertionError, match="heading missing"): + recording.action(page, 2, "Reload app", fail_reload) + + # Postcondition: failure state, action log and failure screenshot are recorded on disk. + result = json.loads((evidence / "scenario-result.json").read_text(encoding="utf-8")) + actions = json.loads((evidence / "action-log.json").read_text(encoding="utf-8")) + assert result == { + "status": "failed", + "error": "AssertionError: heading missing", + "steps_reached": 2, + "steps_planned": 2, + } + assert actions[1]["status"] == "failed" and actions[1]["error"] == "AssertionError: heading missing" + assert (evidence / "step-002.png").is_file() + + +def test_recording_action_refuses_calls_after_failure(tmp_path: Path) -> None: + """Once an action fails, any subsequent action call is refused before capturing frames.""" + # Precondition: step 1 failed. + page = _StubPage() + recording = Recording(tmp_path / "run-after-fail", script=SCRIPT, scenario_id="example", title="Example", + summary="Example run", ffmpeg=_encoder_command()) + with pytest.raises(AssertionError): + recording.action(page, 1, "Open app", lambda: (_ for _ in ()).throw(AssertionError("boom"))) + calls_before = page.jpeg_calls + + # Action: attempt another action on the failed recording. + with pytest.raises(PresentationError, match="finished or failed"): + recording.action(page, 2, "Reload app", lambda: None) + + # Postcondition: no further frames were captured. + assert page.jpeg_calls == calls_before + + +def test_recording_finish_writes_failed_deck_for_reached_steps(tmp_path: Path) -> None: + """Finishing a failed run keeps the failed result and labels the presentation FAILED.""" + # Precondition: step 1 passed and step 2 failed with visual evidence intact. + evidence = tmp_path / "run-finish-fail" + page = _StubPage() + recording = Recording(evidence, script=SCRIPT, scenario_id="example", title=" Example ", + summary="Example run", ffmpeg=_encoder_command()) + recording.action(page, 1, "Open app", lambda: None) + with pytest.raises(AssertionError): + recording.action(page, 2, "Reload app", lambda: (_ for _ in ()).throw(AssertionError("heading missing"))) + + # Action: finalize the failed recording. + manifest = recording.finish() + + # Postcondition: result remains failed and presentation is labelled FAILED for the two reached steps. + result = json.loads((evidence / "scenario-result.json").read_text(encoding="utf-8")) + deck = json.loads(manifest.read_text(encoding="utf-8")) + assert result["status"] == "failed" and result["error"] == "AssertionError: heading missing" + assert deck["title"] == "Example (FAILED)" + assert len(deck["steps"]) == 2 + + +def test_recording_labels_incomplete_run_with_only_reached_steps(tmp_path: Path) -> None: + """Finishing before all script steps run keeps an incomplete status and an INCOMPLETE deck.""" + # Precondition: only step 1 of 2 ran. + evidence = tmp_path / "run-incomplete" + page = _StubPage() + recording = Recording(evidence, script=SCRIPT, scenario_id="example", title="Example", + summary="Example run", ffmpeg=_encoder_command()) + recording.action(page, 1, "Open app", lambda: None) + + # Action: finalize the partial recording. + manifest = recording.finish() + + # Postcondition: result stays incomplete and deck contains only the reached step. + result = json.loads((evidence / "scenario-result.json").read_text(encoding="utf-8")) + deck = json.loads(manifest.read_text(encoding="utf-8")) + assert result == {"status": "incomplete", "error": "", "steps_reached": 1, "steps_planned": 2} + assert deck["title"] == "Example (INCOMPLETE)" + assert len(deck["steps"]) == 1 + + +@pytest.mark.parametrize(("step", "caption"), [(0, "Open"), (True, "Open"), (2, "Skipped"), (1, " ")]) +def test_recording_refuses_invalid_step_order_or_empty_caption( + tmp_path: Path, step: int | bool, caption: str, +) -> None: + """Out-of-order steps, boolean steps and blank captions are refused before capturing frames.""" + page = _StubPage() + recording = Recording(tmp_path / "run", script=SCRIPT, scenario_id="example", title="Example", + summary="Example run", ffmpeg=_encoder_command()) + + with pytest.raises(PresentationError): + recording.action(page, step, caption, lambda: None) + + assert page.jpeg_calls == 0 + + +def test_recording_finish_refuses_second_call(tmp_path: Path) -> None: + """Finishing an already finished recording raises PresentationError.""" + # Precondition: recording is already finished. + page = _StubPage() + recording = Recording(tmp_path / "run", script=SCRIPT, scenario_id="example", title="Example", + summary="Example run", ffmpeg=_encoder_command()) + recording.action(page, 1, "Open", lambda: None) + recording.finish() + + # Action: call finish a second time. + with pytest.raises(PresentationError, match="already been finished"): + recording.finish() + + +def test_recording_action_refuses_calls_after_finish(tmp_path: Path) -> None: + """Calling action after finish raises PresentationError without capturing frames.""" + # Precondition: recording is already finished. + page = _StubPage() + recording = Recording(tmp_path / "run", script=SCRIPT, scenario_id="example", title="Example", + summary="Example run", ffmpeg=_encoder_command()) + recording.action(page, 1, "Open", lambda: None) + recording.finish() + calls_before = page.jpeg_calls + + # Action: attempt another action after finish. + with pytest.raises(PresentationError, match="finished or failed"): + recording.action(page, 2, "Reload", lambda: None) + + assert page.jpeg_calls == calls_before + + +def test_recording_finish_refuses_run_with_no_actions(tmp_path: Path) -> None: + """Finishing with zero actions writes an incomplete result and refuses writing a presentation.""" + evidence = tmp_path / "empty" + recording = Recording(evidence, script=SCRIPT, scenario_id="example", title="Example", + summary="Example run", ffmpeg=_encoder_command()) + + with pytest.raises(PresentationError, match="no complete visual evidence"): + recording.finish() + + assert json.loads((evidence / "scenario-result.json").read_text(encoding="utf-8"))["status"] == "incomplete" + assert not (evidence / "presentation.json").exists() + + +def test_recording_finish_refuses_run_with_missing_step_screenshot(tmp_path: Path) -> None: + """Finishing when a failed action could not save a screenshot keeps the failed result and writes no deck.""" + # Precondition: step 1 callback failed and screenshot capture also failed. + evidence = tmp_path / "broken-capture" + page = _StubPage(fail_png_at=1) + recording = Recording(evidence, script=SCRIPT, scenario_id="example", title="Example", + summary="Example run", ffmpeg=_encoder_command()) + with pytest.raises(AssertionError, match="original test assertion"): + recording.action(page, 1, "Open", lambda: (_ for _ in ()).throw(AssertionError("original test assertion"))) + + # Action: finalize the recording without complete screenshots. + with pytest.raises(PresentationError, match="no complete visual evidence"): + recording.finish() + + assert json.loads((evidence / "scenario-result.json").read_text(encoding="utf-8"))["status"] == "failed" + assert not (evidence / "presentation.json").exists() + + +def test_recording_finish_preserves_logs_when_encoder_fails(tmp_path: Path) -> None: + """A non-zero encoder exit leaves result and encoder logs intact and writes neither poster nor presentation.""" + # Precondition: step 1 succeeded with an encoder configured to exit 1. + evidence = tmp_path / "encode-fail" + recording = Recording(evidence, script=SCRIPT, scenario_id="example", title="Example", + summary="Example run", ffmpeg=_encoder_command(exit_code=1, write_video=False)) + recording.action(_StubPage(), 1, "Open", lambda: None) + + # Action: finalize the recording. + with pytest.raises(subprocess.CalledProcessError): + recording.finish() + + assert json.loads((evidence / "scenario-result.json").read_text(encoding="utf-8"))["status"] == "incomplete" + assert (evidence / "ffmpeg.log").is_file() + assert not (evidence / "presentation.json").exists() + assert not (evidence / "poster.png").exists() + + +def test_recording_finish_withholds_deck_when_encoder_writes_no_video(tmp_path: Path) -> None: + """An encoder that exits zero without writing scenario-video.webm raises PresentationError and writes no deck.""" + # Precondition: step 1 succeeded with an encoder that exits 0 without producing video output. + evidence = tmp_path / "encode-empty" + recording = Recording(evidence, script=SCRIPT, scenario_id="example", title="Example", + summary="Example run", ffmpeg=_encoder_command(exit_code=0, write_video=False)) + recording.action(_StubPage(), 1, "Open", lambda: None) + + # Action: finalize the recording. + with pytest.raises(PresentationError, match="scenario-video.webm"): + recording.finish() + + assert (evidence / "ffmpeg.log").is_file() + assert not (evidence / "presentation.json").exists() + assert not (evidence / "poster.png").exists() diff --git a/devcapsule-src/tests/unit/devcapsule/test_presentation_authoring.py b/devcapsule-src/tests/unit/devcapsule/test_presentation_authoring.py new file mode 100644 index 00000000..ac2e3c8b --- /dev/null +++ b/devcapsule-src/tests/unit/devcapsule/test_presentation_authoring.py @@ -0,0 +1,80 @@ +"""Export and hand-off contracts independent of the installed capsule.""" +from pathlib import Path + +import pytest + +from devcapsule.presentation_authoring import export_tools, presentation_link + + +def test_export_keeps_the_supplied_bytes(tmp_path: Path) -> None: + """The kit preserves the supplied instructions and Python source verbatim.""" + assets = {"README.md": b"agent instructions\n", "recording.py": b"# recorder\n"} + + package = export_tools(tmp_path / "kit", assets) + + assert {file.name: file.read_bytes() for file in package.iterdir()} == assets + + +def test_export_refuses_an_existing_directory(tmp_path: Path) -> None: + """An existing destination remains unchanged, including user-edited helpers.""" + existing = tmp_path / "README.md" + existing.write_text("engineer's changes") + + with pytest.raises(FileExistsError): + export_tools(tmp_path, {"README.md": b"replacement"}) + + assert existing.read_text() == "engineer's changes" + assert not (tmp_path / "presentation_tools").exists() + + +@pytest.mark.parametrize("name", ["../outside.py", "/absolute.py", ".", "nested/source.py"]) +def test_export_refuses_asset_paths(tmp_path: Path, name: str) -> None: + """Malformed asset names cannot write outside the package directory.""" + with pytest.raises(ValueError, match="filenames"): + export_tools(tmp_path / "kit", {name: b"untrusted"}) + + assert not (tmp_path / "kit").exists() + + +def test_link_encodes_folder_and_run_token() -> None: + """The link addresses the project-relative folder and safely carries the token.""" + link = presentation_link(Path("/project/evidence/My run #1"), project=Path("/project"), + console_url="https://console.example:9443/?stale=query", token="a&b #") + + assert link == "https://console.example:9443/presentations/evidence/My%20run%20%231?token=a%26b+%23" + + +def test_link_without_token_uses_existing_browser_session() -> None: + """No empty authentication query is added when the token is unavailable.""" + link = presentation_link(Path("/project/evidence/run"), project=Path("/project"), + console_url="http://localhost:8000", token="") + + assert link == "http://localhost:8000/presentations/evidence/run" + + +@pytest.mark.parametrize("folder", ["/outside/run", "/project", "/project/.hidden/run", + "/project/node_modules/run", "/project/evidence/../run"]) +def test_link_refuses_folders_the_console_cannot_list(folder: str) -> None: + """A hand-off never knowingly points to a hidden, outside or ambiguous folder.""" + with pytest.raises(ValueError): + presentation_link(Path(folder), project=Path("/project"), + console_url="http://localhost:8000", token="") + + +@pytest.mark.parametrize( + "url", + [ + "", + "file:///tmp", + "https:///missing-host", + "http://:8000", + "http://@example.org", + "http://localhost:bad", + "http://localhost:8000 ", + "https://user:secret@example.org", + ], +) +def test_link_refuses_invalid_console_origins(url: str) -> None: + """The console origin must be HTTP(S) and must not embed user credentials.""" + with pytest.raises(ValueError): + presentation_link(Path("/project/evidence/run"), project=Path("/project"), console_url=url, token="") diff --git a/engineering-docs/development/review-partner-loop.md b/engineering-docs/development/review-partner-loop.md index 008ddc63..a4a45da3 100644 --- a/engineering-docs/development/review-partner-loop.md +++ b/engineering-docs/development/review-partner-loop.md @@ -305,6 +305,7 @@ when the workstream pauses, and say so in its open threads. | Date | Workstream | Implementor | Review partner | Slices and reviews | Outcome | |---|---|---|---|---|---| +| 2026-11-16 | presentations, portable E2E authoring for 0.3.1 | GPT-6 Codex pair | Antigravity, Gemini 4 Hydrogen High pair (`gemini-4-hydrogen-high`, `--effort high`) | #43/#46 | accepted at `e331693f` after one correction round, merged at `d4564313`; guarded direct Script inputs, encoder command/output and console URLs, retained undecodable-token fallback, and added recorder/CLI/PEX coverage. Round 1 split multi-operation unit tests and removed fixed-cadence assertions. Reviewer: 148 focused, 11 integration, 474 console tests; 353 mypy files; full gate stopped at inherited Docker-group failure; docs contract skipped in review worktree. Implementor final checks recorded in the presentations validation record. Both invocations streamed events; the `-c` continuation retained conversation `861df986-5407-4636-8a5d-3a2d26da53fc`, model and high effort; both exited 0. Briefs are committed under the workstream evidence directory. | | 2026-11-16 | github-auth, launcher-owned GitHub SSH proxy for release 0.3.1 | GPT-6 Codex | Antigravity, Gemini 4 Hydrogen High (`gemini-4-hydrogen-high`, high effort) | #47/#44 | Accepted after round 2 at `455d3acc`, merged at `9638cb11`; bounded handshake and cleanup, diagnostics/disclosure and parser/authorization/process coverage. 64 focused tests; 3320 unit passes with two inherited Docker gid-zero failures; types/conventions, PEX/smoke, docs and 474 console tests passed. Rebuilt PEX authenticated read and missing-socket control passed. [Final record](../wip/2026-11-16-github-auth/evidence/2026-11-16-final-review.md) states limits and follow-ups. | | 2026-11-12 | maintenance, the rewrite of PR #26 with host inputs as values and no patch in a touched test (`0.3.1`) | Claude Code, Claude Fable 5.2 pair | Codex, gpt-6-astra (high) pair | #26/#28 | accepted at `04770142` without a round, merged at `83d238b8`; the first loop whose brief holds the code to *Host Inputs* and asks the reviewer to review the rule itself, as an agent that must follow it with no other context; the reviewer restored 0.3.0's order of launch file and launch variable, replaced two integration oracles that repeated the implementation, proved the real owner change by passing the caller's uid as a value, and closed three of the implementor's five uncaught faults with integration tests and no patch; its "On the rule" table named seven places where two agents would read the rule differently, recorded in the `maintenance` status for the owner's ruling; 12 planted faults of its own, 4 caught by mirror tests, 1 by a legacy unit test, 5 by integration tests, 2 by nothing; the capsule's 17 standing failures unchanged; brief committed under `evidence/2026-11-12-host-inputs-as-values/review-brief.md` | | 2026-11-11 | maintenance, PR #18's test-only parameters removed and the tests set the host at its source (follow-up to #18, `0.3.1`) | Claude Code, Claude Fable 5.2 pair | Codex, gpt-6-astra (high) pair | #26/#27 | accepted at `1b5d3fc7` without a round, merged at `60ad2c5a`; the first loop whose brief asks for the hosted check's result and names the capsule's standing unit failures, and whose review started only on a green hosted run; the reviewer corrected a docstring that claimed more than the launcher enforces, ended each process-wide test patch with its action, read the session environment when the supervisor is made, replaced an expected value derived from the constant under test, and added a test of the default build-record lookup that the implementor had recorded as impossible; 13 planted faults of its own and 25 of the implementor's 26 caught by the unit files, the last by the integration file; GitHub runs no check on a pull request into a slice branch, so the hosted run of #26 after the merge is the one that counts; the capsule's 17 standing failures keep `nox -s build` from passing as one session, its parts were run one by one; four proposals recorded in the `maintenance` status; brief committed under `evidence/2026-11-11-host-inputs-at-source/review-brief.md` | diff --git a/engineering-docs/development/running-e2e-scenario-recordings.md b/engineering-docs/development/running-e2e-scenario-recordings.md index 638750cb..e38b8431 100644 --- a/engineering-docs/development/running-e2e-scenario-recordings.md +++ b/engineering-docs/development/running-e2e-scenario-recordings.md @@ -1,5 +1,11 @@ # Running E2E Scenario Recordings From A Shell With Nox +For another project, use the shipped `devcapsule presentations guide` and +`devcapsule presentations export <new-directory>` commands. The +[portable kit](../../devcapsule-src/devcapsule/presentation_tools/README.md) +contains the shared producer and Playwright helper without this repository's +IDE harness. This document covers the repository-specific replay commands. + [Development](README.md) · [End-to-end tests](e2e-tests.md) · [Implementation note](../implementation-notes/devcapsule/2026-10-21-e2e-scenario-video-recorder.md) This note shows how to run a recorded IDE scenario—or author a new one with an diff --git a/engineering-docs/wip/2026-11-16-presentations/2026-11-16-record-authoring-validation.md b/engineering-docs/wip/2026-11-16-presentations/2026-11-16-record-authoring-validation.md new file mode 100644 index 00000000..47ae5006 --- /dev/null +++ b/engineering-docs/wip/2026-11-16-presentations/2026-11-16-record-authoring-validation.md @@ -0,0 +1,122 @@ +# Authoring toolkit validation, 2026-11-16 + +Target: `release-0.3.1`, starting at `10e7fda3`. The implementation branch is +`ws-presentations/0.3.1-authoring`. Main's next-development version was not +carried onto this branch. The original registration branch remains intact. + +## Contract and implementation + +The engineer supplies Markdown or inline English. The agent translates that +scenario into the ordinary Playwright actions and assertions for the project, +using shipped instructions and helpers. The toolkit does not interpret natural +language itself. It does not require a DevCapsule source clone or test imports. + +The installed CLI exposes `presentations guide`, `export <new-directory>` and +`hand-off <evidence-folder>`. Export supplies a standard-library Python package +for the capsule's Playwright interpreter, plus the guide and an adaptable driver. +The existing producer moved into that package; the repository replay adapter +keeps its former script-root default. Its 39 existing tests moved with it. + +The recorder captures bursts around small Playwright callbacks. It keeps native +video when the caller configures it, derives times from frame counts and writes +the presentation last. The project's assertions determine test success. Failed +and incomplete attempts stay labelled. The existing viewer and schema are unchanged. + +## Checks + +Commands run from `devcapsule-src` with its existing `.venv`: + +```sh +.venv/bin/python -m pytest -q --no-cov tests/test_scenario_recorder.py tests/unit/devcapsule/presentation_tools tests/unit/devcapsule/test_presentation_authoring.py +DEVCAPSULE_DIFF_BASE=upstream/release-0.3.1 PYTEST_ADDOPTS='--basetemp=/var/tmp/presentations-gate/pytest --maxfail=1' .venv/bin/python -m nox -s build +DEVCAPSULE_DIFF_BASE=upstream/release-0.3.1 PYTEST_ADDOPTS='--basetemp=/var/tmp/presentations-gate/console' .venv/bin/python -m nox -s pex docs-contract webconsole +``` + +The scratch parent was created first. `/opt/devcapsule-gate` is read-only here. + +| Check | Result | +| --- | --- | +| Focused unit tests | 59 passed; final run including the repository recorder: 121 passed | +| Version and syntax | passed; package remains 0.3.1 | +| Mypy | 352 source files, no errors | +| Conventions | 99 legacy files / 1315 legacy tests; 64 conforming files; 0 breaches; 3 files in command-line debt | +| Full gate's unit phase | 1438 passed, 1 skipped, 175 deselected, stopped at 1 failure | +| Remaining PEX build and smoke | passed | +| Documentation contract | 249 pages including 18 aliases, passed | +| Webconsole | 474 tests passed; type check passed | + +The failure is the previously observed +`test_public_dry_run_is_unique_redacted_and_cleans_owned_staging`: +`host Docker group ID must be a positive integer`. No attempt was made to +change this unrelated fixture. The full gate is not green. An initial invocation +from the wrong directory, and a missing scratch parent, were corrected before +the results above; neither is counted as a product failure. + +## Installed artifact and real browser + +Built `dist/devcapsule-local.pex` with the repository's local-source policy; +SHA-256 `ca6ee98a3f7af039729fa42c34f05e3b9ad299acc1a83ab2b03b8adced1e2aea`. +This is not a published release candidate. From `/var/tmp`, the executable +printed its packaged guide and exported the kit into an unrelated scratch +directory. Its helpers imported without importing `devcapsule` or `tests`. + +The scratch project was a small shopping-list application served on loopback. +Its English scenario was: open the application, add milk, reload and verify +milk remains. The agent wrote a driver using only the exported kit and ordinary +Playwright. The cached Chromium and Playwright ffmpeg ran locally; no image, +capsule or browser download was needed. + +| Run | Assertions/result | Presentation | +| --- | --- | --- | +| Passing | add and reload observations passed | 3 steps; 2 seconds | +| Deliberately failing | wrong expected item raised AssertionError; result stayed failed | 2 reached steps; FAILED title; 0.917 seconds | +| Incomplete | stopped after opening; result stayed incomplete | 1 reached step; INCOMPLETE title; 0.667 seconds | + +Each retained a native Playwright video and the generated movie/screenshots. +The viewer was extracted from the built executable and served with the existing +webconsole environment against the scratch project. Links came from the built +`presentations hand-off`, including its token-file handling. All six chapters +played, all step screenshots loaded, and the native media duration agreed with +each manifest to within 0.1 second. No viewer schema failure appeared. +The temporary services were stopped. Raw files and verification scripts remain +only under `/var/tmp/presentations-acceptance`; no tokened URL is committed. + +The exported example driver also ran successfully against the scratch application's file URL, checking its heading before and after reload. + +These checks prove packaging and compatibility with the accepted viewer. They +do not replace the owner's prior human acceptance or the requested independent +Pair Review. The model catalog lists `gemini-4-hydrogen-high` and CLI help +supports `--effort high`; no reviewer invocation has occurred. The prescribed +GitHub CLI is absent, so PR creation is an owner action under WORKFLOW-LOCAL. + + +## Final implementation after Pair Review + +Antigravity `gemini-4-hydrogen-high --effort high` delivered review PR #46. +The implementor read its diff and requested one correction round for unit-test +contracts and the guide's relative-path wording. Final review commit `e331693f` +was accepted and merged into the implementation branch at `d4564313`. +Both reviewer invocations exited 0. Their briefs are retained under `evidence/`; +the assessment and acceptance are comments on PR #46. + +The merged branch passed 148 focused/recorder-compatibility tests and 11 real +CLI/PEX integration tests. The full gate passed version/syntax, mypy (353 files) +and conventions (65 conforming files, 0 breaches), then stopped at the same +inherited Docker-group failure after 1438 passes, 1 skip and 178 deselections. +Separate PEX build/smoke, docs contract (249 pages) and webconsole checks +(474 tests) all passed. The reviewer worktree lacked website contents, so its +docs check skipped; the implementor's docs check ran and passed. + +Final local-source executable SHA-256: +`da1f2d894f0ca53c26f36a962c1313baf8f92090aade9d528f9d39c218cd5c41`. +The real Chromium acceptance was rerun from the rebuilt executable, exporting +into `/var/tmp/presentations-final-acceptance`. Passing, failing and incomplete +scenarios again produced 3, 2 and 1 chapters respectively. All six chapters +played in the executable's embedded viewer, with screenshots loaded and media +durations matching the manifest within 0.1 second. Both temporary services +were stopped. The kit did not import `devcapsule` or `tests` in the driver. + +PR #43 targets `release-0.3.1`; the human performs that merge. Its checks are +available on the live PR and should be reverified before integration. The +reviewer proposal to add more nox smoke commands was not adopted because the +new installed-PEX integration test directly covers the delivery boundary. diff --git a/engineering-docs/wip/2026-11-16-presentations/2026-11-16-record-pr-draft.md b/engineering-docs/wip/2026-11-16-presentations/2026-11-16-record-pr-draft.md new file mode 100644 index 00000000..c5252bc6 --- /dev/null +++ b/engineering-docs/wip/2026-11-16-presentations/2026-11-16-record-pr-draft.md @@ -0,0 +1,30 @@ +# Package E2E recording and presentation tools for adopter agents + +Base: `release-0.3.1` +Head: `ws-presentations/0.3.1-authoring` + +An agent in another project currently needs DevCapsule's source tree to turn +an English E2E scenario into a recorded presentation. Ship the instructions +and Python helpers in the installed executable: `presentations guide` explains +the flow, `export` supplies a standalone Playwright kit, and `hand-off` prints +the console link. The engineer can supply an existing Markdown scenario or +inline English; the agent performs the project-specific actions and assertions. + +Move the existing presentation producer into the shipped package, retaining +the repository replay adapter. Add a recorder that derives timing from action +frames and labels failed or incomplete attempts. Preserve the accepted viewer. + +Validation: 148 focused/recorder-compatibility tests and 11 integration tests passed; syntax, mypy and test conventions passed; +PEX build/smoke and the 249-page documentation contract passed; 474 webconsole +tests passed. A real browser exercised the exported kit in an unrelated scratch +project and played every chapter of passing, failed and incomplete presentations +using the viewer embedded in the executable. + +The full gate still stops on the inherited recursive Docker-group failure +after 1438 passes. Pair Review with Antigravity +`gemini-4-hydrogen-high`, `--effort high`, is complete. Review PR #46 was +accepted after one correction round and merged at `d4564313`. The final +recording/playback checks passed again after that merge. Check the live hosted +results before the owner merges into the release branch. The hosted gate +does not exercise this project's real application scenario or independently +establish presentation quality. diff --git a/engineering-docs/wip/2026-11-16-presentations/2026-11-16-record-use-case-handoff.md b/engineering-docs/wip/2026-11-16-presentations/2026-11-16-record-use-case-handoff.md new file mode 100644 index 00000000..b73e1c80 --- /dev/null +++ b/engineering-docs/wip/2026-11-16-presentations/2026-11-16-record-use-case-handoff.md @@ -0,0 +1,72 @@ +# Owner-directed presentation-authoring use case + +From: project-management +To: presentations +Date: 2026-11-16 + +The owner requested a presentations workstream if none already exists. +No standalone workstream exists; the earlier viewer slice belonged to capsule-webconsole and is merged. +This hands over the defined use case and the two authoring messages for the new workstream's goal. +Creation does not start design or implementation; the owner explicitly deferred that work while defining this use case. +No release target or packaging interface is assigned. + +## Agreed use case + + + +An engineer works with an LLM agent in a DevCapsule on their own codebase. +The engineer wants the agent to execute an E2E scenario and present the recorded result in that capsule's web console. + +## Request + +The engineer tells the agent: + +> Run E2E with recording and presentation along the scenario described in `<scenario.md>`. + +Or: + +> Run E2E with recording and presentation along the scenario described here: `<plain-English scenario>`. + +The Markdown file already belongs to the engineer, or the engineer describes the steps inline. +The engineer supplies the scenario in ordinary language, not a recording-tool-specific script or metadata format. + +## What the agent does + +1. Read the specified Markdown file or the inline scenario. +2. Run the described E2E test in the project's environment, recording the journey. +3. Produce the presentation from that run, using the capability already demonstrated in DevCapsule. +4. Make the presentation available in that project's existing DevCapsule web console. +5. Report the test result and show the engineer the presentation link when finished. + +The agent uses the provided instructions and supporting code to handle the recording and presentation mechanics. +The engineer does not need to direct it to DevCapsule's public source or reconstruct the process with it. + +## Successful outcome + +The engineer opens the returned link in their web console and views the recorded scenario as a presentation. +The experience is the one already demonstrated by DevCapsule's web-console presentations. + +## Evidence and scope + +The owner reports that an agent already recorded an English-specified E2E scenario in another project after being pointed at this repository. +That establishes demonstrated reuse; this is not a request to invent a new process. +The owner confirms the existing viewer's human acceptance, including Florin's test and enthusiastic response. +Both engineering and prospective-user/tutorial audiences remain served by the existing presentation experience. + +The delivery need is to make the existing Markdown instructions and supporting Python/Playwright apparatus available to adopter agents. +Package layout, interfaces, installation, and implementation ownership are later work. +Do not start that design work in this use-case definition or make selecting a new example project a prerequisite. + + +## Original correspondence and source revision + +Both original messages remain available at project-management commit `cac3861e699641836dec5ba96de29fc453571784`: + +- `engineering-docs/wip/2026-08-09-project-management/intake/2026-11-16-github-auth-presentation-authoring-handoff.md` +- `engineering-docs/wip/2026-08-09-project-management/intake/2026-11-16-presentation-authoring-portability.md` + +The full use case at that revision is `engineering-docs/wip/2026-08-09-project-management/2026-11-16-design-presentation-excellence.md`. +Read the consolidated handoff for the installed-artifact audit, prior recording evidence, and existing source locations. +Current mainline `dac6b9d4` contains the recorder, producer, console contract and recording guide. +Accepting this item makes the defined use case this workstream's goal and retains the original evidence as input. +The viewer remains the accepted baseline; the deliverable is making the proven instructions and Python/Playwright support usable in other projects. diff --git a/engineering-docs/wip/2026-11-16-presentations/CURRENT-STATUS.md b/engineering-docs/wip/2026-11-16-presentations/CURRENT-STATUS.md new file mode 100644 index 00000000..53978723 --- /dev/null +++ b/engineering-docs/wip/2026-11-16-presentations/CURRENT-STATUS.md @@ -0,0 +1,115 @@ +# Workstream Current Status: Presentations + +Mnemonic: `presentations` +Start date: 2026-11-16 +State: paused; Pair Review complete; PR #43 awaits owner merge to release 0.3.1 +Branch association: `ws-presentations/0.3.1-authoring` +Branch prefix: `ws-presentations/` +Integration target: `release-0.3.1` +Delivery method: pull request; records published on coordination + +Definition read: WORKFLOW.md@196d3760f99e, WORKFLOW-LOCAL.md@94a29cb6eedb + +## Goal and agreed use case + +Make the proven E2E recording and presentation capability available to LLM agents in other DevCapsule projects. +Provide the existing Markdown instructions and supporting Python/Playwright apparatus without requiring a DevCapsule source clone. + +The engineer asks their agent to run E2E with recording and presentation along a scenario: + +- In an existing Markdown file supplied by the engineer; or +- Described inline in plain English. + +The agent reads the scenario, executes the E2E test, records it and creates the presentation. +When finished, it reports the result and returns a link the engineer opens in that project's DevCapsule web console. +The engineer does not need to point it at this repository or reconstruct the recording process. + +## Current state and authority + +Created at the owner's request after checking the live list, canonical mainline, open/archive directories and remote refs. +No standalone presentations workstream existed. The earlier `ws-capsule-webconsole/presentations` slice is merged. +The owner confirms that the current viewer is already human-validated, including Florin's enthusiastic test. +An agent already recorded an English-specified scenario in another project after the owner pointed it at this source. +Both engineering and prospective-user/tutorial audiences remain in scope; the existing viewer is the accepted baseline. + +The owner authorized implementation on 2026-11-16 and selected release 0.3.1. +Pair reviewer: Antigravity `gemini-4-hydrogen-high`, with `--effort high`, verified in `antigravity models`. +The existing viewer and its schema remain the contract. +The broader reusable-processes idea is not assigned wholesale to this workstream. +Project-management delivered the use case and references to both original authoring messages by coordination mail. +Acknowledged that handoff as the goal above; its full text is preserved in the handoff record. +The mailbox and intake are empty after receipt and acknowledgement. +Original correspondence is preserved on `ws-project-management/coordination` at `cac3861e` under its intake directory. +The self-contained handoff names the exact paths and evidence. + +## Planned next step + +The scoped implementation and Pair Review are complete. The owner merges +[PR #43](https://github.com/mycodespace-ai/devcapsule/pull/43) into `release-0.3.1`. +Consult the PR's live hosted checks before merging. Only the human merges the release target. +After the owner reports the merge, fetch upstream and verify the release contains this tree. +Maintenance owns carrying the release to main; do not conclude this workstream before main contains it. +Do not restart a viewer redesign, audience study or general process invention. + +## Validation and checkout state + +The implementation branch starts at `upstream/release-0.3.1` (`10e7fda3`), carrying only this workstream's registration commits from the former `ws-presentations/authoring`. +Do not synchronize with main: its development-version changes do not belong in this release-targeted slice. +The workflow text differs only in its source-version stamp; its rules were read. +The owner installed the prescribed GitHub CLI. Authentication passed as `ccozianu` on resumption. +Re-fetched upstream: release is unchanged and main differs in workflow only by the development-version stamp. +Keep the release baseline; do not merge next-cycle development-version changes. +The required pre-PR unit session again stopped at the known Docker-group failure after 1438 passes. +No open bug is owned by presentations at registration. +Product changes: `presentations guide`, `export` and `hand-off`; the existing producer is now shipped +and the repository replay harness imports it through an adapter. The portable recording helper accepts +ordinary Playwright callbacks and derives chapters/captions from captured frames. Failed and incomplete +runs keep their outcome and have labelled decks. The viewer is unchanged. + +Final validation after merging review PR #46: 148 focused/compatibility tests and 11 integration tests passed; +syntax, mypy (353 files) and conventions (0 breaches) passed. The full gate stopped at the inherited Docker-group failure after 1438 passes and 1 skip (178 deselected). +Separate PEX build/smoke, docs contract (249 pages) and webconsole checks (474 tests) passed. +An unrelated scratch project used only the final exported kit and produced passing, failed and incomplete +recordings; all six chapters played in the viewer extracted from the rebuilt PEX. See the validation record. +The exported example driver also passed its two-step browser run. +Maintenance was notified by coordination mail of the owner-directed 0.3.1 dependency. +No Docker image or capsule was launched. Temporary loopback application and console services were stopped. +The local artifact is a development validation build, not a published release candidate. + +## Pair Review + +Antigravity `gemini-4-hydrogen-high` with `--effort high` reviewed in its own worktree. +Conversation `861df986-5407-4636-8a5d-3a2d26da53fc`; initial invocation and `-c` correction round both exited 0. +Review PR [#46](https://github.com/mycodespace-ai/devcapsule/pull/46) delivered input/output guards, +recorder lifecycle tests and installed-CLI/PEX coverage. Round 1 split multi-operation unit tests, +removed fixed-cadence assertions, isolated encoder interpreter startup and clarified cwd-relative paths. +Accepted final review commit `e331693f` after reading its actual diff; merged at `d4564313`. +The assessment and acceptance comments are on PR #46. Final implementor checks are recorded below and +in the validation record. Hosted checks passed on the pre-review implementation; PR #43 holds the final live results. + +Reviewer proposals: the relative-path clarification is included. The optional nox smoke expansion was +not adopted because the new PEX integration test covers the installed delivery boundary. The requested +Chromium check and the docs contract both passed again on the merged implementation. + +## Open Threads + +- Implementation authorized; release target and reviewer settled. +- PR #43 awaits owner merge to release-0.3.1, followed by maintenance's release-to-main integration. +- Preserve the recorded viewer acceptance and demonstrated external-project recording reuse. +- Local full gate still fails on the known recursive Docker-group test; no unrelated fixture fix is included. +- Automated checks establish the kit and existing viewer contract, not a second human acceptance of presentation quality. +- Final raw acceptance files and logs are in `/var/tmp/presentations-final-acceptance`, + `/var/tmp/presentations-antigravity` and `/var/tmp/presentations-*.log`; + they are disposable and not committed. Durable outcomes are in the validation record. +- No transcript or tokened link is preserved in Git. + +## Workstream Document Index + +- [Validation record](2026-11-16-record-authoring-validation.md): installed-artifact and browser checks, counts and limits. +- [Implementation PR draft](2026-11-16-record-pr-draft.md): description prepared for PR #43, targeting release 0.3.1. +- [Round 1 brief](evidence/2026-11-16-review-round-1.md): accepted test-contract corrections and hand-off path clarification. +- [Pair Review brief](evidence/2026-11-16-review-brief.md): PR #43; exact Antigravity model and high effort. +- [Full use-case handoff](2026-11-16-record-use-case-handoff.md): owner-defined flow, constraints and original correspondence references. + +- [Intake](intake/README.md): handoff and correspondence awaiting acknowledgement. +- [Intake decisions](intake-dispositions.md): append-only record of acknowledged work. diff --git a/engineering-docs/wip/2026-11-16-presentations/evidence/2026-11-16-review-brief.md b/engineering-docs/wip/2026-11-16-presentations/evidence/2026-11-16-review-brief.md new file mode 100644 index 00000000..0470d4a0 --- /dev/null +++ b/engineering-docs/wip/2026-11-16-presentations/evidence/2026-11-16-review-brief.md @@ -0,0 +1,140 @@ +# Pair Review brief: Antigravity, Gemini 4 Hydrogen (High) + +Implementation PR: https://github.com/mycodespace-ai/devcapsule/pull/43 +Base: `release-0.3.1`. Head: `ws-presentations/0.3.1-authoring`. +The prescribed GitHub CLI is now installed and the implementor's authentication +check passed. Check authentication once in your own session as directed below. +The PR is a draft pending this review and its hosted checks. + +## Goal and checkout + +Review `ws-presentations/0.3.1-authoring` against `upstream/release-0.3.1`. +The implementor pair is GPT-6 Codex. This is the first review of the slice. +Deliver any improvements on `ws-presentations/0.3.1-authoring-review`, forked +from the implementation branch at review start, in +`~/worktrees/presentations-review`. Work only in that worktree. Do not touch +the implementor checkout `/workspace/301e4208ef81-ChatGPT_Codex`. + +Read first, in order: + +1. `AGENTS.md`, `README.md`, `DEVELOPING.md`, `.devcapsule/devcapsule.toml`, + `WORKFLOW.md` and `WORKFLOW-LOCAL.md`. +2. `engineering-docs/wip/2026-11-16-presentations/CURRENT-STATUS.md` and + `2026-11-16-record-use-case-handoff.md` in the same directory. +3. `2026-11-16-record-authoring-validation.md` in that directory. +4. `devcapsule-src/devcapsule/presentation_tools/README.md` and + `devcapsule-webconsole/PRESENTATIONS.md`. + +The engineer supplies an existing Markdown scenario or inline English. The +agent translates that into project-specific Playwright actions and assertions, +records them and returns the existing viewer's link. The viewer is already +human-accepted. Do not redesign it, invent an English parser or require a new +user-facing scenario format. The scope is packaging the proven producer and +recording support so an adopter agent does not need this source checkout. + +## Changes and review priorities + +Read the branch diff. Review correctness, then clarity, then coverage of the +changed lines. New modules: `commands/presentations.py`, +`presentation_authoring.py`, and `presentation_tools/`. The latter contains +the existing producer moved from `tests/e2e/scenario_presentation.py`, a new +portable recorder, shipped guide and example. The old path is a compatibility +adapter; its producer tests moved to the corresponding production mirror. +Package-data, README, recording documentation and the index expose the kit. + +Check especially that an installed PEX can export complete, importable tools +without `tests` or `devcapsule` in the adopter's Playwright interpreter; that +timing, step ownership, screenshots and captions satisfy the viewer contract; +that failure/incompletion never becomes a passing result; and that existing +evidence is not overwritten. Check callback failure, capture failure, encoder +failure, repeated steps and missing steps. Check console path/token handling +without persisting tokened links. The helper intentionally uses sparse bursts +around small actions, with native video kept by the driver. + +Do not expand into a generic process framework, application provisioning, +viewer redesign, natural-language parsing, unrelated legacy test repairs or +changes to the workflow definition. Propose substantive product choices to +the implementor rather than choosing them in this review. + +For every new or changed test, verify Unit Tests rules 1–4 and 7: a module +mirror; its own explicit precondition, one public action and a promised +postcondition; a plain-English contract docstring; and a declared public +surface. Name what could make each fail for a reason that is not a bug. +Verify Host Inputs rules 1–6 on code and tests: host reads only at entry; +required explicit values thereafter; shared host values in frozen data; +decisions separate from effects; no monkeypatch, unittest.mock, environment +mutation or module/class attribute replacement; short entry sequencing tested +through a real program in an integration test. Respect legacy-test retirement +and command-line test restrictions. Do not weaken tests to obtain a green gate. + +## Environment and validation + +The implementor has prepared a separate `devcapsule-src/.venv` in your worktree +using the local workflow's validation instructions. Your branch starts at +`78f512c2`; later implementor commits only update review records. As the review +partner, do not run workflow claim, mail or publish commands; the Pair Review +exception governs over the ordinary session-start instructions. Run from `devcapsule-src`: + +```sh +mkdir -p /var/tmp/presentations-antigravity +DEVCAPSULE_DIFF_BASE=upstream/release-0.3.1 .venv/bin/python -m pytest -q --no-cov tests/unit/devcapsule/presentation_tools tests/unit/devcapsule/test_presentation_authoring.py +DEVCAPSULE_DIFF_BASE=upstream/release-0.3.1 PYTEST_ADDOPTS='--basetemp=/var/tmp/presentations-antigravity/pytest --maxfail=1' .venv/bin/python -m nox -s build +``` + +Use that separate scratch because `/opt/devcapsule-gate` is read-only on this +host; Validation Commands permits another directory with room. The known +inherited Docker-group failure may stop the full gate; report it and do not +repair that unrelated fixture. The implementor's separate PEX, docs and +webconsole checks passed. Its real Chromium checks covered passing, failing +and incomplete recordings and playback of every generated chapter. If code +changes invalidate this evidence, request a focused rerun from the implementor. +Do not build images, launch capsules, install browsers or provision services. +Report what the hosted gate does not catch. + +## Authorship and delivery + +Check `git var GIT_AUTHOR_IDENT`. Preserve the human author +`Costin Cozianu <ccozianu@gmail.com>`. Credit the actual reviewer engine/model +using `Co-authored-by: Antigravity (Gemini 4 Hydrogen High) <noreply@google.com>`, +consistent with existing Antigravity commits; verify the actual model before crediting it. + +Detect GitHub CLI only at `/opt/xtras/miniconda3/envs/gh/bin/gh`. In each shell +using it, activate it first: + +```sh +source /opt/xtras/miniconda3/etc/profile.d/conda.sh && conda activate gh +``` + +Check `gh auth status -h github.com` once before first use. On auth failure, +stop and report; no retries or alternate credentials. If absent, stop at the +owner-operated PR handoff prescribed in WORKFLOW-LOCAL. Use ordinary Git SSH +to push the review branch to `upstream`. + +After required checks, create the review PR against +`ws-presentations/0.3.1-authoring`, never against the release or main. Its body +lists findings (problem, location, change, evidence), what was considered and +left alone with reasons, unimplemented proposals, and validation counts/limits. +If nothing changes, report that and create no empty PR. Do not merge, rebase, +force-push, change workstreams, edit status/intake/published state, or send mail. +At most three review rounds; the implementor assesses the actual diff. + +Finish with exactly one of these forms, filling the result: + +`PAIR REVIEW COMPLETE: <review PR URL or no changes>; <counts and limits>` + +`PAIR REVIEW BLOCKED: <specific blocker and next human action>` + +## Invocation for the implementor + +The model catalog was checked on 2026-11-16. Use the exact listed ID and the +explicit effort setting; do not substitute a model: + +```sh +antigravity --model gemini-4-hydrogen-high --effort high --dangerously-skip-permissions --output-format stream-json -p 'Follow the Pair Review brief on standard input.' < <brief-path> +``` + +Run detached in the review worktree, capture its streaming output in session +scratch and append its exit status. Relay start, commits, gate results and end +to the human. Keep this brief as the reviewer's only task context. Record the +actual invocation and outcome in the Pair Review development document after +the loop runs; this brief alone is not evidence of a completed review. diff --git a/engineering-docs/wip/2026-11-16-presentations/evidence/2026-11-16-review-round-1.md b/engineering-docs/wip/2026-11-16-presentations/evidence/2026-11-16-review-round-1.md new file mode 100644 index 00000000..e8e9bb7a --- /dev/null +++ b/engineering-docs/wip/2026-11-16-presentations/evidence/2026-11-16-review-round-1.md @@ -0,0 +1,52 @@ +# Round 1: test-contract corrections for PR #46 + +Continue the same Antigravity Gemini 4 Hydrogen High review of PR #43. +I read the actual diff of PR #46 at `e40d16a5`. The product fixes are accepted +in principle; make only the following corrections before integration. Do not +repeat the broad investigation or widen product behavior. + +1. Make the new recorder unit tests follow WORKFLOW-LOCAL Unit Tests rule 2: + one public action under test, with setup stated separately. In particular: + - `test_recording_keeps_failed_outcome_and_reached_steps_when_callback_fails` + tests the callback exception, rejection of a later action and finalization. + Split those into distinct contracts/tests, constructing each precondition. + - Split repeated finish from action-after-finish. + - Split or parameterize empty evidence versus missing screenshot evidence. + - Split or parameterize encoder nonzero exit versus successful exit without + output. Each case should call `finish` once as its action under test. + - The passing-run test may prepare its run through `action` calls, then test + `finish`; make that separation explicit. Its numeric `[8,16,24]`, fixed + duration and fixed repeated frame bytes encode the current burst cadence, + which the public contract does not promise. Assert the contract instead: + logs have increasing frame boundaries, chapters/captions align to those + boundaries and the video duration, and encoder input equals captured data. + Do not declare a new public cadence guarantee merely to keep the assertions. + - Keep the controlled encoder subprocess (running the encoder is part of + `finish`'s contract), but isolate its interpreter invocation from ambient + Python startup customization with `-I -S`. + Correct the PR body's blanket claim that every new test already has one + public action. Report concrete non-bug failure conditions for the final tests. +2. Clarify the shipped guide's hand-off example: run it from the console project + root; a relative evidence folder resolves from cwd, and `--project` supplies + the root used to form the console-relative link. No resolution behavior change. + +Do not add the proposed nox smoke expansion: the new PEX integration test already +covers this delivery boundary. The implementor will rerun the real Chromium +acceptance and docs contract on the merged branch. No browser/service/container +work in your worktree. Do not edit workstream records or published state. + +Run from your worktree's `devcapsule-src`: + +```sh +DEVCAPSULE_DIFF_BASE=upstream/release-0.3.1 .venv/bin/python -m pytest -q --no-cov tests/test_scenario_recorder.py tests/unit/devcapsule/presentation_tools tests/unit/devcapsule/test_presentation_authoring.py +DEVCAPSULE_DIFF_BASE=upstream/release-0.3.1 .venv/bin/python scripts/validate_unittests_follow_conventions.py +DEVCAPSULE_DIFF_BASE=upstream/release-0.3.1 PYTEST_ADDOPTS='--basetemp=/var/tmp/presentations-antigravity/round-1 --maxfail=1' .venv/bin/python -m nox -s build +``` + +The known Docker-group limitation remains disclosed. Do not repeat the other +unchanged suites merely to accumulate checks. Commit on your existing review +branch with the established authorship, push via upstream SSH and update PR #46 +using the prescribed Conda gh environment. Preserve previous findings and limits, +update the unit counts, and mark this response to Round 1. No merge/rebase/force push. +Finish with `PAIR REVIEW COMPLETE: <PR URL>; <counts and limits>` or the established +blocked form if necessary. diff --git a/engineering-docs/wip/2026-11-16-presentations/intake-dispositions.md b/engineering-docs/wip/2026-11-16-presentations/intake-dispositions.md new file mode 100644 index 00000000..7751f3e6 --- /dev/null +++ b/engineering-docs/wip/2026-11-16-presentations/intake-dispositions.md @@ -0,0 +1,5 @@ +# Presentations intake decisions + +| Item | Dispositioned | Outcome | Note | +|---|---|---|---| +| `2026-11-16-project-management-presentation-authoring.md` | 2026-11-16 | acknowledged | Accepted as this workstream's goal; full handoff preserved in `2026-11-16-record-use-case-handoff.md`; design and implementation remain deferred. | diff --git a/engineering-docs/wip/2026-11-16-presentations/intake/README.md b/engineering-docs/wip/2026-11-16-presentations/intake/README.md new file mode 100644 index 00000000..f8c86d73 --- /dev/null +++ b/engineering-docs/wip/2026-11-16-presentations/intake/README.md @@ -0,0 +1,4 @@ +# Presentations intake + +Incoming work awaiting acknowledgement under WORKFLOW.md topic 8. +The workstream's goal and current task are in [its status file](../CURRENT-STATUS.md). diff --git a/index.md b/index.md index 7a8d4aeb..c4dd5ac5 100644 --- a/index.md +++ b/index.md @@ -44,6 +44,7 @@ status file; internal WIP/archive documents use the local index in that status. - [Unit tests](engineering-docs/development/unit-tests.md) - [Integration tests](engineering-docs/development/integration-tests.md) - [End-to-end tests](engineering-docs/development/e2e-tests.md) +- [Packaged agent instructions for E2E recording and presentation](devcapsule-src/devcapsule/presentation_tools/README.md) - [Running E2E scenario recordings from a shell with nox](engineering-docs/development/running-e2e-scenario-recordings.md) - [Web console presentations: discovery, storage and generation](devcapsule-webconsole/PRESENTATIONS.md) - [E2E scenario script: GitHub SSH through the host proxy](devcapsule-src/tests/e2e/scripts/github-ssh-proxy.md) @@ -182,6 +183,8 @@ status file; internal WIP/archive documents use the local index in that status. ## Workstream Status +- [Presentations](engineering-docs/wip/2026-11-16-presentations/CURRENT-STATUS.md): agent-driven E2E recordings and presentations in adopter projects. + - [Multiple-stream workflow successful archive](engineering-docs/archive/2026-08-08-multi-workflow/CURRENT-STATUS.md) - [Recursive dogfood E2E successful archive](engineering-docs/archive/2026-08-06-recursive-e2e/CURRENT-STATUS.md) - [Eclipse interactive surface, concluded 2026-10-01 and folded into component-catalog](engineering-docs/archive/2026-09-09-eclipse-surface/CURRENT-STATUS.md)