diff --git a/flows/aot/__init__.py b/flows/aot/__init__.py new file mode 100644 index 0000000..a1507f8 --- /dev/null +++ b/flows/aot/__init__.py @@ -0,0 +1,533 @@ +"""AOT -- the flow that writes a flow: a description in, a checked and proved flow out. + +hmz exec -f official/aot -a claude/MODEL:high -a codex/MODEL:high \\ + "two agents take turns on the task until a reviewer says it is done" + +The writer reads the description against a briefing of what this installed humanize serves, +and says what the flow is to be first -- the places, the settings, the ways it ends, and what +it needs of the interface. What it needs is checked against the catalogue before anything is +written: a description that asks for what nothing here serves is refused at compile time, +with the person at the prompt asked whether to narrow it, rather than compiled into a flow +that fails at hour three. + +Then the writer writes the flow, in a scratch directory of its own, and the flow is held to +three gates before anybody keeps it. The checker reads it without running it. The stubs drive +it against the worst worlds there are -- the reviewer that never says done, the turn that +always fails -- in a subprocess held to a clock, so a loop that cannot end is caught in +milliseconds. And a critic that shares nothing with the writer reads it fresh against the +spec. Whatever any gate refuses goes back to the writer's own session, word for word, for as +many repairs as the config allows. + +One rule is the compiler's own and not the checker's: a generated loop is bounded, always. +The checker only warns about a loop whose every way out waits on an agent's verdict; here +that warning is a refusal, whatever `strict` says, so the flow that lands ends even when its +reviewer never says done. What lands is copied whole into the flows of your own -- atomically, +the way `fork` copies one -- and the compile ends with a report: what it is called, what it +drives, how every loop ends, and the line that runs it. +""" + +import os +import shutil +import tempfile +from collections.abc import Sequence +from pathlib import Path +from typing import Literal, NamedTuple + +from _aot import prompts +from hmz.flows import ( + ALWAYS_DONE, + ENTRY, + EVERYWHERE, + MINE, + NEVER_DONE, + SILENT, + Agent, + Finding, + Person, + Scenario, + Session, + briefed, + catalogue, + checked, + flow, + proved, +) +from pydantic import BaseModel, Field + + +class Compiling(NamedTuple): + """The three the compile drives: two agents that share nothing, and the person. + + The writer holds the whole compile in one session -- the spec it drew, the drafts it + wrote, every refusal it was handed -- because repair is a conversation. The critic + arrives fresh each round and reads only the draft against the spec, which is the one + reviewer arrangement that catches what the writer has talked itself into. The person + is the gate for what a compiler must not decide alone: an ask nothing serves, a name + already taken, a draft the repairs ran out on. + """ + + writer: Agent + critic: Agent + human: Person + + +class Config(BaseModel): + """What a compile takes.""" + + model_config = {"frozen": True} + + name: str = Field( + default="", + description="what to call the flow that lands, or '' to take the name the spec " + "derives from the description", + ) + into: Literal["local", "user"] = Field( + default="local", + description="where it lands: `local` is this project's .humanize/flows, `user` " + "the one in your home directory", + ) + repairs: int = Field( + default=3, + ge=0, + le=6, + description="how many rounds of repair the writer is given after its first draft", + ) + strict: bool = Field( + default=False, + description="whether every warning sends a draft back, rather than only what " + "blocks -- an unbounded loop always blocks, whatever this says", + ) + seconds: float = Field( + default=60.0, + gt=0, + description="the clock each stub-driven proof of a draft is held to, a scenario " + "apiece", + ) + + +class Seat(BaseModel): + """One agent the compiled flow will drive. + + Every field required, here and in every shaped answer of this flow: a backend that + holds a model to a strict schema refuses one whose fields have defaults, and the + compiler must compile on any backend that shapes. + """ + + model_config = {"extra": "forbid"} + + name: str = Field(description="what the flow calls it, snake_case, for what it does") + person: bool = Field( + description="true only for the person at the prompt, whom nobody configures", + ) + moments: list[str] = Field( + description="moments this seat hangs hooks on beyond the ones every backend " + "runs, each by the name the briefing uses -- e.g. PermissionRequest -- and [] " + "for a seat that needs none", + ) + goal: bool = Field( + description="true only if this seat runs under the backend's own goal feature", + ) + + +class Setting(BaseModel): + """One knob the compiled flow can be set up with.""" + + model_config = {"extra": "forbid"} + + name: str = Field(description="the field's name, snake_case") + kind: Literal["number", "text", "switch"] = Field(description="what it holds") + default: str = Field(description="the default, written out -- '10.0', 'off'") + about: str = Field(description="one line saying what it does, for whoever sets it") + + +class Ending(BaseModel): + """One way the compiled flow ends.""" + + model_config = {"extra": "forbid"} + + by: Literal["budget", "rounds", "verdict"] = Field( + description="what ends it: output tokens spent, a cap on the rounds, or an " + "agent's shaped verdict -- which always travels with a budget or a cap besides" + ) + bound: str = Field( + description="the bound, written out -- '10 million output tokens', '6 rounds', " + "'reviewer says done, under a 10M budget'" + ) + + +class Spec(BaseModel): + """What the flow is to be, drawn from the description before anything is written.""" + + model_config = {"extra": "forbid"} + + about: str = Field(description="one line saying what the flow does") + name: str = Field(description="what to call it, snake_case") + seats: list[Seat] = Field(description="every agent it drives, the person included " + "if it talks to one") + settings: list[Setting] = Field( + description="the knobs it takes, [] for a flow that takes none" + ) + endings: list[Ending] = Field( + description="every way it ends -- at least one, and never a verdict alone" + ) + needs: list[str] = Field( + description="what it needs of the interface, each named exactly as the briefing " + "names a capability -- e.g. shapes, pursue, moment:PermissionRequest -- and " + "nothing the briefing does not name" + ) + plan: str = Field(description="how the flow will work, a short paragraph") + + +class Review(BaseModel): + """What the critic answers, having read the draft fresh against the spec.""" + + model_config = {"extra": "forbid"} + + approved: bool = Field( + description="true only if the draft does what the spec says, keeps to the " + "writing-flows contract, and you would run it on a repository of your own" + ) + notes: str = Field( + description="what to tell the writer: what is wrong or missing and what to do " + "about it, citing files and lines -- passed on word for word. when approved, " + "one line on what convinced you." + ) + + +class Going(BaseModel): + """A yes or no the compiler must not answer for itself.""" + + model_config = {"extra": "forbid"} + + proceed: bool = Field(description="yes to go on as asked, no to stop the compile") + + +class Renamed(BaseModel): + """Another name, where the one the spec chose is already taken.""" + + model_config = {"extra": "forbid"} + + name: str = Field(description="another name for the flow, or '' to stop the compile") + + +@flow +def run(agents: Compiling, task: str, config: Config | None = None) -> None: + held = config or Config() + # A scratch directory of the compile's own: drafts are proved there, and only what + # passed every gate is copied out. Taken away however the compile ends, so a refused + # draft is nowhere. + scratch = tempfile.mkdtemp(prefix=".aot.") + try: + _compiled(agents, task, held, Path(scratch)) + finally: + shutil.rmtree(scratch, ignore_errors=True) + + +def _compiled(agents: Compiling, task: str, held: Config, scratch: Path) -> None: + # One session for the whole compile: the spec it drew and every refusal it was handed + # are the context its repairs are made of. + writing = agents.writer.new(cwd=scratch) + spec = _drafted(writing, task) + if spec is None: + print("hmz: aot: the writer could not draw a spec from the description; nothing " + "was written") + return + unserved, limited = _unserved(spec) + if unserved: + # The writer's own round first: an unserved ask is more often the description's + # words taken for a capability -- writing a file, reading the repository -- than + # a real hole, and the writer can restate the spec in the catalogue's vocabulary + # before anybody is asked to build less. + resaid = writing( + prompts.RESAID.format( + unserved="\n".join(f"- {one}" for one in unserved) + ), + suppress=True, + schema=Spec, + ) + if resaid is not None: + spec = resaid + unserved, limited = _unserved(spec) + if unserved and not _narrowed(agents.human, unserved): + print("hmz: aot: cannot compile -- nothing was written") + return + name = _named(held.name or spec.name) + draft = scratch / name + feedback = "" + landed = False + for attempt in range(held.repairs + 1): + asked = ( + prompts.WRITE.format(spec=spec.model_dump_json(indent=2), draft=draft, + name=name) + if attempt == 0 + else prompts.REPAIR.format(draft=draft, refused=feedback) + ) + writing(asked, suppress=True) + feedback = _refused(draft, spec, held, agents) + if not feedback: + landed = True + break + print(f"hmz: aot: draft {attempt + 1} refused --") + print(feedback) + if not landed: + if not (draft / ENTRY).is_file(): + # There is nothing to take: the writer never landed a draft at all. + print("hmz: aot: the repairs ran out with no draft to show; nothing was " + "written") + return + if not _taken(agents.human, feedback): + print("hmz: aot: the repairs ran out and nobody took the draft as it " + "stands; nothing was written") + return + at = _landed(draft, name, held.into, agents.human) + if at is None: + return + _reported(at, spec, held, limited) + + +def _drafted(writing: Session, task: str) -> Spec | None: + """The spec, drawn from the description against the briefing -- with one more try. + + Args: + writing: The writer's session, whose first turn this is. + task: The description, as it was given. + + Returns: + The spec, or None for a writer that would not answer in shape twice. + """ + asked = prompts.SPEC.format(briefing=briefed(), task=task) + spec = writing(asked, suppress=True, schema=Spec) + if spec is None: + spec = writing(prompts.SPEC_AGAIN, suppress=True, schema=Spec) + return spec + + +def _named(said: str) -> str: + """A flow's name as a directory may be called: snake_case, and never empty.""" + held = "".join(one if one.isalnum() else "_" for one in said.strip().lower()) + held = "_".join(part for part in held.split("_") if part) + return held or "compiled_flow" + + +def _unserved(spec: Spec) -> tuple[list[str], list[str]]: + """The spec's needs, checked against the catalogue of what is actually served. + + Args: + spec: What the flow is to be. + + Returns: + What nothing here serves -- each one a reason not to compile -- and what only some + backends serve, which is compiled and said in the report. + """ + served = {one.name: one for one in catalogue()} + everywhere = {one.value for one in EVERYWHERE} + unserved: list[str] = [] + limited: list[str] = [] + for need in spec.needs: + if need not in served: + unserved.append(need) + elif served[need].backends: + limited.append( + f"{need} -- runs only on: {', '.join(sorted(served[need].backends))}" + ) + for seat in spec.seats: + for moment in seat.moments: + if moment in everywhere: + continue + key = f"moment:{moment}" + if key not in served: + unserved.append(f"a hook on {moment!r}") + else: + limited.append( + f"{seat.name} needs Moment.{moment} -- runs only on: " + f"{', '.join(sorted(served[key].backends))}" + ) + return unserved, limited + + +def _narrowed(human: Person, unserved: list[str]) -> bool: + """Puts an ask nothing serves to the person: narrow the flow, or stop here. + + Args: + human: The person at the prompt, who answers nothing when nobody is there. + unserved: What was asked for that nothing here serves. + + Returns: + Whether to compile the rest. Nobody there is no: a compiler must not decide alone + to build less than what was asked for. + """ + for one in unserved: + print(f"hmz: aot: cannot compile -- asks for {one}, which nothing here serves") + going = human( + prompts.NARROW.format(unserved="\n".join(f"- {one}" for one in unserved)), + suppress=True, + schema=Going, + ) + return going is not None and going.proceed + + +def _refused(draft: Path, spec: Spec, held: Config, agents: Compiling) -> str: + """The three gates, in their order, and what the first to refuse said. + + Args: + draft: Where the writer was told to put the flow. + spec: What it is to be. + held: The compile's config. + agents: For the critic, who reads the draft fresh. + + Returns: + What to hand the writer, or "" for a draft every gate let through. + """ + if not (draft / ENTRY).is_file(): + return ( + f"nothing landed at {draft} -- write the flow there: a directory of that " + f"name holding the {ENTRY} that is the flow" + ) + found = checked(draft) + blocking = [one for one in found if _blocks(one, strict=held.strict)] + if blocking: + return "the checker refused it:\n" + _said(blocking) + proof = proved(draft, scenarios=_worlds(held.seconds)) + if proof.findings: + return "loading it was refused:\n" + _said(proof.findings) + stalled = [one for one in proof.outcomes if not one.finished] + if stalled: + return "driven by stubs, it did not end:\n" + "\n".join( + f"- under {one.scenario}: {one.said}" for one in stalled + ) + review = agents.critic( + prompts.REVIEW.format(spec=spec.model_dump_json(indent=2), draft=draft), + suppress=True, + schema=Review, + cwd=draft.parent, + ) + if review is None: + return ( + "the critic's turn failed, so nothing has read the draft fresh -- hold it " + "tighter to the spec and to the writing-flows contract, and it will be " + "read again" + ) + if not review.approved: + return review.notes or "the critic did not approve it, and said nothing more" + return "" + + +def _blocks(one: Finding, *, strict: bool) -> bool: + """Whether one finding sends a draft back. + + Every error does. `unbounded-loop` does whatever `strict` says: it is the compiler's + own rule that a generated loop is bounded, since the flow that lands must end even + when its reviewer never says done. The rest of the warnings block only under + `strict`, and are said in the report either way. + """ + return one.severity == "error" or one.code == "unbounded-loop" or strict + + +def _worlds(seconds: float) -> tuple[Scenario, ...]: + """The scenarios every draft is driven against, at the compile's own clock.""" + return tuple( + one._replace(seconds=seconds) for one in (NEVER_DONE, ALWAYS_DONE, SILENT) + ) + + +def _said(findings: Sequence[Finding]) -> str: + """Findings as the writer is handed them, one a line.""" + return "\n".join( + f"- {one.where.name}:{one.line}: {one.severity}: {one.code}: {one.said}" + for one in findings + ) + + +def _taken(human: Person, feedback: str) -> bool: + """The last gate: take the draft with what is still wrong with it, or stop. + + Args: + human: The person at the prompt. + feedback: The last refusal, which is what they would be taking. + + Returns: + Whether to keep it anyway. Nobody there is no. + """ + going = human( + prompts.TAKEN.format(refused=feedback), suppress=True, schema=Going + ) + return going is not None and going.proceed + + +def _landed(draft: Path, name: str, into: str, human: Person) -> str | None: + """Copies the draft into the flows of your own, whole and atomically. + + The way `fork` lands one: copied beside and then moved into place, so a copy that + fails partway leaves no half a flow under the name. A name already taken is the + person's to change, not the compiler's to write over. + + Args: + draft: The draft that passed the gates. + name: What it is to be called. + into: Which of the two places of your own, `local` or `user`. + human: The person, for a name already taken. + + Returns: + The directory it landed in, or None for a landing refused. + """ + mine = os.path.expanduser(MINE[into]) + for _ in range(2): + at = os.path.join(mine, name) + stem = at.removesuffix(".py") + if not (os.path.exists(stem) or os.path.exists(stem + ".py")): + break + asked = human(prompts.RENAME.format(name=name), suppress=True, schema=Renamed) + if asked is None or not asked.name.strip(): + print(f"hmz: aot: there is already a flow called {name!r} in {mine}, and " + "nobody gave another name; nothing was written") + return None + name = _named(asked.name) + else: + print(f"hmz: aot: there is already a flow called {name!r} in {mine}; nothing " + "was written") + return None + os.makedirs(mine, exist_ok=True) + holding = tempfile.mkdtemp(dir=mine, prefix=f".{name}.") + try: + kept = os.path.join(holding, name) + shutil.copytree(draft, kept) + os.replace(kept, os.path.join(mine, name)) + finally: + shutil.rmtree(holding, ignore_errors=True) + return os.path.join(mine, name) + + +def _reported(at: str, spec: Spec, held: Config, limited: list[str]) -> None: + """Says what was compiled: what it is, what it drives, how it ends, how to run it. + + Args: + at: Where the flow landed. + spec: What it was compiled to be. + held: The compile's config. + limited: What it builds on that only some backends serve. + """ + name = os.path.basename(at) + print(f"\ncompiled: {name} -- {spec.about}") + print(f"landed: {at}") + for seat in spec.seats: + what = "the person at the prompt" if seat.person else "an agent" + extras = [f"Moment.{one}" for one in seat.moments] + if seat.goal: + extras.append("a goal feature") + needs = f" (needs {', '.join(extras)})" if extras else "" + print(f"drives: {seat.name} -- {what}{needs}") + for setting in spec.settings: + print(f"takes: {setting.name} = {setting.default} -- {setting.about}") + for ending in spec.endings: + print(f"ends: by {ending.by} -- {ending.bound}") + waived = [ + one for one in checked(at) if not _blocks(one, strict=held.strict) + ] + if waived: + print("waived:") + print(_said(waived)) + for one in limited: + print(f"only on: {one}") + chosen = sum(1 for seat in spec.seats if not seat.person) + line = " ".join(["-a CLI/MODEL:EFFORT"] * chosen) + print(f'\nhmz exec -f {held.into}/{name} {line} "the task"') diff --git a/flows/aot/_aot/__init__.py b/flows/aot/_aot/__init__.py new file mode 100644 index 0000000..b297dab --- /dev/null +++ b/flows/aot/_aot/__init__.py @@ -0,0 +1 @@ +"""What the aot flow imports beside itself: the prompts its turns are made of.""" diff --git a/flows/aot/_aot/prompts.py b/flows/aot/_aot/prompts.py new file mode 100644 index 0000000..7f401c1 --- /dev/null +++ b/flows/aot/_aot/prompts.py @@ -0,0 +1,124 @@ +"""The prompts of the compile, one constant per turn the flow takes. + +Kept beside the flow rather than inline, so a fork that wants its compiler to speak +differently edits this file and runs. What each turn is *for* is the flow's own docstrings; +what is said to get it is here. +""" + +#: The writer's first turn: the description read against the briefing, answered as a Spec. +SPEC = """You are the writer half of a compiler that turns a description into a humanize \ +flow. Below is a briefing of what this installed humanize actually serves, and then the \ +description. Read both and answer with the spec of the flow to be written -- do not write \ +any code yet. + +Ground rules for the spec: +- `needs` names only capabilities the briefing names, spelled exactly as it spells them. \ +If the description asks for something the briefing does not serve, still list it in \ +`needs` as the description asked for it: the compiler refuses it honestly rather than \ +building around it silently. +- `seats` is every agent the flow drives, the person at the prompt included only if the \ +flow talks to them. A seat's `moments` names only moments from the briefing's \ +"only some backends" list that the seat truly hangs hooks on. +- `endings` must hold at least one, and a `verdict` ending never stands alone: it travels \ +with a budget or a round cap, because an agent may never say the verdict. +- `name` is snake_case, short, and says what the flow does. + +The briefing: + +{briefing} + +The description: + +{task}""" + +#: One more try at the spec, for a first answer that was not in shape. +SPEC_AGAIN = """Your last answer did not fit the shape asked for. Answer again with the \ +spec alone, exactly in the shape: every field, nothing outside it.""" + +#: The writer's second turn: the spec written out as a flow, in the scratch directory. +WRITE = """Now write the flow the spec describes. You are working in a scratch directory; \ +create the flow at exactly this path: + + {draft} + +as a directory called `{name}` holding the `__init__.py` that is the flow -- plus whatever \ +it imports beside itself (an underscore-named sibling module or package inside the flow's \ +own directory), and a `skills/` directory only if the flow brings skills. + +Follow the writing-flows skill you carry: it is the contract this draft will be held to. \ +The compiler will read the draft without running it, drive it with stubs against the worst \ +worlds there are -- a reviewer that never says done, turns that always fail -- and hand it \ +to a fresh critic. Three rules decide most refusals: + +- Every loop is bounded. Even where the spec ends by verdict, the loop also ends by budget \ +(`agent.spent().output` against a limit) or by a round cap (`for` over a `range`). A loop \ +whose only way out is an agent's verdict is refused, always. +- Every shaped answer is guarded. A turn taken with `suppress=True, schema=...` answers \ +None when it fails; test it before reading a field off it. +- One import of humanize's: `from hmz.flows import ...`, and only names it offers. + +The spec: + +{spec} + +Write the files now, and end by saying what you wrote where.""" + +#: A refused draft handed back, word for word, to the session that wrote it. +REPAIR = """The draft at {draft} was refused. Here is everything found, exactly as the \ +gates said it: + +{refused} + +Fix the draft in place -- edit the files at {draft} -- addressing every line above. Keep to \ +the writing-flows contract: every loop bounded, every shaped answer guarded, one import of \ +humanize's. End by saying what you changed.""" + +#: The critic's whole turn: fresh eyes, the spec, and the draft on disk. +REVIEW = """You are the critic half of a compiler that turns a description into a humanize \ +flow. A writer you share nothing with has produced a draft; it has already passed a static \ +checker and been driven to completion by stubs, so what is left is what only reading can \ +catch: does it do what the spec says, and would you run it? + +The draft is the directory `{draft}` in your working directory -- read every file in it \ +with your tools. Judge it against this spec: + +{spec} + +Hold it to the writing-flows contract: every loop bounded even where an ending is a \ +verdict; every shaped answer guarded before a field is read; settings as a frozen or \ +extra-forbidding pydantic model with a described field apiece; a docstring whose first \ +line says what the flow does, with the `hmz exec` line under it; prints that say where a \ +long run has got to. Approve only what you would run on a repository of your own. Answer \ +in the shape.""" + +#: The person's gate for an ask nothing serves: narrow the flow, or stop the compile. +NARROW = """This description asks for things nothing in this humanize serves: + +{unserved} + +Compile the rest without them? Answering no -- or nothing -- stops the compile.""" + +#: The person's last gate: the repairs ran out, and the draft stands as it is. +TAKEN = """The repairs ran out. The last refusal was: + +{refused} + +Keep the draft anyway, as it stands? It will land with the findings above still in it. \ +Answering no -- or nothing -- stops the compile and keeps nothing.""" + +#: The person's gate for a name already taken at the destination. +RENAME = """There is already a flow called {name!r} where this one is to land, and a \ +compiler does not write over what somebody keeps. Give another name for the compiled \ +flow, or answer nothing to stop the compile.""" + +#: The writer's own round on an ask nothing serves, before the person is troubled with it. +RESAID = """Some of the spec's `needs` name nothing the briefing serves: + +{unserved} + +A need is one of the briefing's capability names, spelled exactly as the briefing spells \ +it, and nothing else belongs in `needs`. An ordinary ability -- writing files, reading the \ +repository, taking turns, printing -- is not a capability to declare: every agent has it, \ +so drop it from `needs`. Only if the description truly requires something the briefing \ +does not serve should you keep it listed, exactly as the description asks, and the compile \ +will stop honestly. Answer with the whole corrected spec, in the shape.""" diff --git a/flows/aot/skills/writing-flows/SKILL.md b/flows/aot/skills/writing-flows/SKILL.md new file mode 100644 index 0000000..448cf1b --- /dev/null +++ b/flows/aot/skills/writing-flows/SKILL.md @@ -0,0 +1,140 @@ +--- +name: writing-flows +description: The contract a humanize flow is written to. Use when writing or repairing a flow -- its shape on disk, its one import, how its loops end, how its answers are guarded, and how it says what it is. +--- + +# Writing a flow + +A flow is a Python function driving coding agents, and everything below is what a checker, +a stub-driven proof and a fresh critic will hold your draft to. Write to it the first time; +every refusal costs a repair round. + +## The shape on disk + +A flow is a directory named for the flow, holding the `__init__.py` that is the flow. +Everything it needs lives inside that directory: helper code as an underscore-named sibling +module or package (`_myflow/`), skills it brings as `skills//SKILL.md`. A flow whose +parts are elsewhere is a flow with a hole in it wherever it is copied to. + +## One import + +The whole of humanize a flow imports is `hmz.flows`, and only names it offers: + +```python +from hmz.flows import Agent, Person, flow +``` + +Plus the standard library and `pydantic`. Never `hmz.agents`, `hmz.backends` or any other +module of humanize's own -- those move, and a flow is somebody else's repository. + +## The entry point + +```python +from typing import Any, NamedTuple + +from hmz.flows import Agent, Person, flow +from pydantic import BaseModel, Field + + +class Agents(NamedTuple): + actor: Agent # one field per agent, named for what it is for + human: Person # only if the flow talks to the person at the prompt + + +@flow +def run(agents: Agents, task: str, config: Config | None = None) -> None: ... +``` + +- The first parameter is the agents, annotated with a NamedTuple of them (or a fixed-length + `tuple[Agent, ...spelled out...]`). Never `tuple[Agent, ...]` with an ellipsis: how many + agents the flow drives is the one thing a command line cannot otherwise know. +- The second is the task. A config, if the flow takes one, is third and defaults to `None`. +- A resumable flow says `@flow(resumable=True)` and takes a `state: dict[str, Any]` as its + last parameter: what it wrote there last time. Write only the handful of things the next + run needs (a round counter, spent tokens), and `state.clear()` when the run is over -- + what is over is not picked up. + +## Every loop is bounded + +The rule refusals come from most. A loop must be endable by something inside it, and a +verdict alone is not a bound -- an agent may never say it. So every `while True:` carries a +backstop besides any verdict exit: + +```python +while True: + agent(task, suppress=True) + kept["output"] = spent = before + agent.spent().output + if held.budget and spent >= held.budget * 1_000_000: + print(f"stopping: {spent / 1e6:.2f}M output tokens") + return + time.sleep(5) +``` + +- A budget reads `agent.spent().output` against a limit from the config. Default the budget + to at most 10 million output tokens -- a run that wants more says so. +- A round cap is `for round_ in range(held.rounds):` -- a `for` over a `range` is bounded by + construction. +- A loop that only sleeps, or has no `break`/`return`/`raise` at all, is refused outright. + +## Every shaped answer is guarded + +A turn held to a shape answers `None` when it fails, and `suppress=True` is how a loop +survives a failed turn instead of dying on it: + +```python +review = agents.reviewer(prompt, suppress=True, schema=Review) +if review is not None and review.done: + return +``` + +Never read a field off a shaped answer something has not tested. A plain turn under +`suppress=True` answers `""` on failure -- test it too, and take the round again rather +than advancing past a turn that never landed. + +A shaped answer's model declares every field required -- no defaults. Some backends hold +the model to a strict schema that refuses a field with a default, and a flow's shapes must +work on any backend that shapes. (A *config* model is the opposite: every field carries a +default, since a flow runs unset.) + +## Settings + +A flow that takes settings declares a pydantic model: + +```python +class Config(BaseModel): + model_config = {"extra": "forbid"} + + budget: float = Field( + default=10.0, + ge=0, + description="millions of output tokens the loop may spend before it stops", + ) +``` + +`extra: "forbid"` (or `frozen: True`), and every field carries a `Field(description=...)` +-- the descriptions are what whoever sets the flow up is shown. + +## What the flow says about itself + +The module docstring's first line is what every list of flows shows; under it goes the +`hmz exec` line that runs it, then prose saying how the flow works and what ends it. +`print()` progress as the loop goes -- which round, what has been spent -- because a run is +watched from its transcript. + +## What each agent is + +- A session held across turns remembers; `agent(task)` alone is a fresh session per turn + and remembers nothing. Choose deliberately per seat. +- A reviewer that must arrive fresh gets a new session (or a bare `agent(...)` call) each + round, so it reads the repository rather than its own last review. +- The person at the prompt is a `Person` seat. Asking them stops the turn until they + answer; run where nobody is, they answer nothing -- so a flow written to stop on nothing + stops. Never loop on asking them without a bound. +- A capability only some backends serve -- a moment, the goal feature -- is declared on the + seat (`Annotated[Agent, Moment.PERMISSION_REQUEST]`, `Annotated[Agent, Goal]`), so an + unfit agent is refused before the first turn. + +## Rest between rounds + +`time.sleep(5)` at the foot of a loop, so a loop that is spinning on failures does not +hammer anything. The proof's world sleeps for free, so this costs the proof nothing. diff --git a/tests/test_aot.py b/tests/test_aot.py new file mode 100644 index 0000000..1e6258b --- /dev/null +++ b/tests/test_aot.py @@ -0,0 +1,387 @@ +"""The compiler flow, driven by scripted stand-ins: every gate shown to gate. + +The writer is a stub that answers a canned spec and then writes a scripted source tree per +attempt; the critic answers canned reviews; the person is the real HumanAgent, absent by +default and answering only where a test says. What is asserted is the compile around them: +a good draft lands whole, a refused one is handed back word for word, an ask nothing serves +is refused before anything is written, and a name already taken is not written over. +""" + +# ruff: noqa: D103, PLR2004, S101 + +from __future__ import annotations + +import json +import sys +import textwrap +from pathlib import Path +from typing import TYPE_CHECKING, ClassVar + +from hmz.agents import AgentBase, AgentConfig, Event, HumanAgent, SessionBase +from hmz.flows import checked, configures, drives, resumes +from hmz.flows.skills import brought + +ROOT = Path(__file__).parents[1] +FLOW = ROOT / "flows" / "aot" +sys.path[:0] = [str(FLOW), str(FLOW.parent)] + +import aot # noqa: E402 + +if TYPE_CHECKING: + import os + from collections.abc import Iterator, Mapping + + import pytest + from pydantic import BaseModel + +CONFIG = AgentConfig(model="test-model", effort="high") + + +def spec(name: str = "pair_loop", needs: tuple[str, ...] = ()) -> dict[str, object]: + return { + "about": "two agents take turns until a reviewer says it is done", + "name": name, + "seats": [ + {"name": "actor", "person": False, "moments": [], "goal": False}, + {"name": "reviewer", "person": False, "moments": [], "goal": False}, + ], + "settings": [ + { + "name": "budget", + "kind": "number", + "default": "1.0", + "about": "millions of output tokens before the loop stops", + } + ], + "endings": [ + {"by": "verdict", "bound": "the reviewer says done, under the budget"} + ], + "needs": list(needs), + "plan": "the actor works, the reviewer reads it fresh, the budget backstops", + } + + +#: A draft that passes every gate: a verdict exit with a budget backstop beside it. +GOOD = { + "pair_loop/__init__.py": ''' + """Two agents take turns until a reviewer says it is done. + + hmz exec -f local/pair_loop -a claude/MODEL:high -a codex/MODEL:high "the task" + + The actor works in its own turn and a fresh reviewer reads the repository; what ends + the loop is the reviewer saying so, and the budget is the backstop for a reviewer + that never does. + """ + + import time + from typing import NamedTuple + + from hmz.flows import Agent, flow + from pydantic import BaseModel, Field + + + class Agents(NamedTuple): + actor: Agent + reviewer: Agent + + + class Config(BaseModel): + model_config = {"extra": "forbid"} + + budget: float = Field( + default=1.0, + ge=0, + description="millions of output tokens before the loop stops", + ) + + + class Review(BaseModel): + model_config = {"extra": "forbid"} + + done: bool = Field(description="whether the task is completely done") + + + @flow + def run(agents: Agents, task: str, config: Config | None = None) -> None: + held = config or Config() + while True: + agents.actor(task, suppress=True) + review = agents.reviewer(task, suppress=True, schema=Review) + if review is not None and review.done: + print("the reviewer says it is done") + return + if held.budget and agents.actor.spent().output >= held.budget * 1_000_000: + print("stopping: the budget is spent") + return + time.sleep(5) + ''', +} + +#: A first draft the checker refuses outright: a loop nothing inside can end. +DEAD = { + "pair_loop/__init__.py": ''' + """A loop nothing can end.""" + + from hmz.flows import Agent, flow + + + @flow + def run(agents: tuple[Agent, Agent], task: str) -> None: + while True: + agents[0](task, suppress=True) + ''', +} + +#: A draft the static reading trusts and the stubs catch: its one exit can never be taken. +STALLING = { + "pair_loop/__init__.py": ''' + """A loop whose bound is no bound at all.""" + + from hmz.flows import Agent, flow + + + @flow + def run(agents: tuple[Agent, Agent], task: str) -> None: + while True: + agents[0](task, suppress=True) + if agents[0].spent().output < 0: + return + ''', +} + + +class WriterSession(SessionBase): + """Answers the canned spec, then writes the next scripted tree into its cwd.""" + + shapes: ClassVar[bool] = True + + def _stream( + self, prompt: str, *, schema: type[BaseModel] | None = None + ) -> Iterator[Event]: + agent = self._agent + assert isinstance(agent, WriterAgent) + if self._id is None: + self._adopt(f"writer-{id(self)}") + if schema is not None and schema.__name__ == "Spec": + held = ( + agent.blueprints.pop(0) + if len(agent.blueprints) > 1 + else agent.blueprints[0] + ) + yield Event(kind="result", text=json.dumps(held)) + return + agent.asked.append(prompt) + if agent.trees: + tree = agent.trees.pop(0) + for rel, source in tree.items(): + at = Path(self._cwd or ".") / rel + at.parent.mkdir(parents=True, exist_ok=True) + at.write_text(textwrap.dedent(source).strip() + "\n") + yield Event(kind="result", text="written") + + +class WriterAgent(AgentBase): + def __init__( + self, + spec_: dict[str, object] | list[dict[str, object]], + trees: list[Mapping[str, str]], + ) -> None: + super().__init__(CONFIG, name="writer") + #: The spec answers, in order; the last is answered again where more are asked. + self.blueprints = list(spec_) if isinstance(spec_, list) else [spec_] + self.trees = list(trees) + #: Every write or repair prompt, in order -- what the gates handed back. + self.asked: list[str] = [] + + def new(self, cwd: str | os.PathLike[str] | None = None) -> WriterSession: + return WriterSession(self, cwd) + + +class CriticSession(SessionBase): + """Answers the next canned review, approving where the script ran out.""" + + shapes: ClassVar[bool] = True + + def _stream( + self, prompt: str, *, schema: type[BaseModel] | None = None + ) -> Iterator[Event]: + del prompt, schema + agent = self._agent + assert isinstance(agent, CriticAgent) + if self._id is None: + self._adopt(f"critic-{id(self)}") + said = ( + agent.reviews.pop(0) + if agent.reviews + else {"approved": True, "notes": "sound"} + ) + yield Event(kind="result", text=json.dumps(said)) + + +class CriticAgent(AgentBase): + def __init__(self, reviews: list[dict[str, object]] | None = None) -> None: + super().__init__(CONFIG, name="critic") + self.reviews = list(reviews or []) + + def new(self, cwd: str | os.PathLike[str] | None = None) -> CriticSession: + return CriticSession(self, cwd) + + +def compiled( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + *, + spec_: dict[str, object] | list[dict[str, object]] | None = None, + trees: list[Mapping[str, str]] | None = None, + reviews: list[dict[str, object]] | None = None, + human: HumanAgent | None = None, + config: aot.Config | None = None, + task: str = "two agents take turns until a reviewer says it is done", +) -> WriterAgent: + """One compile, in a temporary working directory, and the writer to read back.""" + monkeypatch.chdir(tmp_path) + writer = WriterAgent(spec_ or spec(), trees if trees is not None else [GOOD]) + agents = aot.Compiling( + writer=writer, critic=CriticAgent(reviews), human=human or HumanAgent() + ) + aot.run(agents, task, config or aot.Config(seconds=30.0)) + return writer + + +def test_a_good_draft_lands_whole( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + writer = compiled(tmp_path, monkeypatch) + landed = tmp_path / ".humanize" / "flows" / "pair_loop" + assert (landed / "__init__.py").is_file() + # What landed is a flow: it declares its agents, and reads clean. + assert drives(landed / "__init__.py") == ("actor", "reviewer") + assert checked(landed) == () + # One write turn was enough, and the report says what to run. + assert len(writer.asked) == 1 + out = capsys.readouterr().out + assert "compiled: pair_loop" in out + assert 'hmz exec -f local/pair_loop -a CLI/MODEL:EFFORT -a CLI/MODEL:EFFORT' in out + assert "ends: by verdict" in out + + +def test_a_refused_draft_is_handed_back_word_for_word( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + writer = compiled(tmp_path, monkeypatch, trees=[DEAD, GOOD]) + assert (tmp_path / ".humanize" / "flows" / "pair_loop" / "__init__.py").is_file() + assert len(writer.asked) == 2 + # The second prompt is a repair, carrying the checker's own finding. + assert "dead-loop" in writer.asked[1] + assert "cannot end" in writer.asked[1] + + +def test_the_stubs_catch_what_the_static_reading_trusts( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + writer = compiled(tmp_path, monkeypatch, trees=[STALLING, GOOD]) + assert (tmp_path / ".humanize" / "flows" / "pair_loop" / "__init__.py").is_file() + assert len(writer.asked) == 2 + assert "did not end" in writer.asked[1] + assert "never-done" in writer.asked[1] + + +def test_the_critics_veto_is_a_repair_round( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + writer = compiled( + tmp_path, + monkeypatch, + trees=[GOOD, GOOD], + reviews=[{"approved": False, "notes": "the budget knob wants a ceiling"}], + ) + assert (tmp_path / ".humanize" / "flows" / "pair_loop" / "__init__.py").is_file() + assert len(writer.asked) == 2 + assert "the budget knob wants a ceiling" in writer.asked[1] + + +def test_an_ask_nothing_serves_is_refused_before_anything_is_written( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + writer = compiled( + tmp_path, + monkeypatch, + spec_=spec(needs=("interrupting a turn mid-stream",)), + ) + # Nobody at the prompt to narrow it: the compile stops, and nothing was written. + assert not (tmp_path / ".humanize").exists() + assert writer.asked == [] + out = capsys.readouterr().out + assert ( + "cannot compile -- asks for interrupting a turn mid-stream, which nothing " + "here serves" in out + ) + + +def test_a_mis_worded_need_is_the_writers_to_restate( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + # The first spec claims an ordinary ability as a need; the writer's own round -- not + # the person's -- restates it, and the compile goes on to land. + writer = compiled( + tmp_path, + monkeypatch, + spec_=[spec(needs=("plan file",)), spec()], + ) + assert (tmp_path / ".humanize" / "flows" / "pair_loop" / "__init__.py").is_file() + assert len(writer.asked) == 1 + assert "cannot compile" not in capsys.readouterr().out + + +def test_a_name_already_taken_is_not_written_over( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + kept = tmp_path / ".humanize" / "flows" / "pair_loop" + kept.mkdir(parents=True) + (kept / "__init__.py").write_text('"""Somebody\'s own flow."""\n') + compiled(tmp_path, monkeypatch) + # The flow that was there is exactly the flow that is there. + assert (kept / "__init__.py").read_text() == '"""Somebody\'s own flow."""\n' + assert "already a flow called 'pair_loop'" in capsys.readouterr().out + + +def test_a_person_may_take_the_draft_the_repairs_ran_out_on( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + person = HumanAgent() + person.ask = lambda question: "yes" # noqa: ARG005 -- every gate answered yes + writer = compiled( + tmp_path, + monkeypatch, + trees=[DEAD], + human=person, + config=aot.Config(repairs=0, seconds=30.0), + ) + # The draft lands as it stands, dead loop and all: the person said so. + landed = tmp_path / ".humanize" / "flows" / "pair_loop" + assert (landed / "__init__.py").is_file() + assert [one.code for one in checked(landed)] == ["dead-loop"] + assert len(writer.asked) == 1 + + +def test_the_compiler_passes_its_own_gates() -> None: + """Dogfood: the flow that holds drafts to the contract holds to it itself.""" + assert checked(FLOW) == () + + +def test_what_the_compiler_declares() -> None: + entry = FLOW / "__init__.py" + assert drives(entry) == ("writer", "critic") + assert not resumes(entry) + config = configures(entry) + assert config is not None + assert set(config.model_fields) == {"name", "into", "repairs", "strict", "seconds"} + assert [skill.name for skill in brought(FLOW)] == ["writing-flows"] diff --git a/tests/test_aot_golden.py b/tests/test_aot_golden.py new file mode 100644 index 0000000..25907ad --- /dev/null +++ b/tests/test_aot_golden.py @@ -0,0 +1,183 @@ +"""Compile the official flows from their descriptions, with real agents -- when asked. + +Off by default: a compile is minutes of a real coding agent's turns, and CI has no agent. +Set `AOT_WRITER` -- and, to differ, `AOT_CRITIC` -- to `cli/model:effort` to run the golden +compiles, and `AOT_SMOKE=1` besides to also run the compiled review loop once on a toy +repository with the same agents. + +What is asserted is structural equivalence with the flow each description describes, never +text: the compiled flow loads, drives as many agents as the description says, can be set up, +reads clean under the checker, and -- the point of the whole compiler -- ends under the +reviewer that never says done. +""" + +# ruff: noqa: D103, PLR2004, S101 + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest + +from hmz.agents import HumanAgent, driver +from hmz.flows import NEVER_DONE, carries, checked, configures, load, proved, wanted + +ROOT = Path(__file__).parents[1] +FLOW = ROOT / "flows" / "aot" +sys.path[:0] = [str(FLOW), str(FLOW.parent)] + +import aot # noqa: E402 + +if TYPE_CHECKING: + from hmz.agents import AgentBase + +WRITER = os.environ.get("AOT_WRITER", "") +CRITIC = os.environ.get("AOT_CRITIC", "") or WRITER + +pytestmark = pytest.mark.skipif( + not WRITER, + reason="a compile is minutes of a real agent; AOT_WRITER=cli/model:effort runs these", +) + + +def agent_of(spec: str) -> AgentBase: + """One real agent off `cli/model:effort`, built the way `-a` builds one.""" + cli, _, rest = spec.partition("/") + model, _, effort = rest.rpartition(":") + agent, config = driver(cli) + return agent(config(model=model, effort=effort)) + + +def compiled(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, task: str) -> Path: + """One real compile in a temporary project, answering with where the flow landed.""" + monkeypatch.chdir(tmp_path) + agents = aot.Compiling( + writer=agent_of(WRITER), critic=agent_of(CRITIC), human=HumanAgent() + ) + # What a run of the flow does before its first turn: the flow's own skills -- the + # writing-flows contract -- mounted onto every session these agents open. + carries(str(FLOW), list(agents)) + aot.run(agents, task) + landed = tmp_path / ".humanize" / "flows" + flows = [one for one in landed.iterdir() if (one / "__init__.py").is_file()] + assert len(flows) == 1, f"expected one compiled flow, found {flows}" + return flows[0] + + +def equivalent(at: Path, *, drives_count: int, person: bool, takes_config: bool) -> None: + """The structural bar every compiled flow is held to.""" + entry = at / "__init__.py" + places = wanted(entry) + assert len(places) == drives_count, places + chairs = [one for one in _all_places(entry) if one.person] + assert bool(chairs) == person + if takes_config: + assert configures(entry) is not None + found = checked(at) + assert not [one for one in found if one.severity == "error"], found + assert "unbounded-loop" not in {one.code for one in found}, found + proof = proved(at, scenarios=(NEVER_DONE,)) + assert proof.findings == (), proof.findings + assert proof.outcomes[0].finished, proof.outcomes + + +def _all_places(entry: Path): # noqa: ANN202 + from hmz.flows.driving import declares + + return declares(entry)[1] + + +def test_flame_chase_from_one_line( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + at = compiled( + tmp_path, + monkeypatch, + "two agents take turns on the same task, one after the other, until a budget " + "of output tokens is spent", + ) + equivalent(at, drives_count=2, person=False, takes_config=True) + # The golden's shape: both agents take turns, and the budget is what ends it. + source = (at / "__init__.py").read_text() + assert "spent()" in source + + +def test_gen_idea_from_its_description( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + at = compiled( + tmp_path, + monkeypatch, + "open a loose idea into a repository-grounded design draft: one agent reads " + "the repository, expands the idea into a draft with goals, constraints and " + "open questions, and writes it to a markdown file whose path it prints; one " + "pass, no loop", + ) + equivalent(at, drives_count=1, person=False, takes_config=False) + + +def test_gen_plan_from_its_description( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + at = compiled( + tmp_path, + monkeypatch, + "turn a design draft into an implementation plan two sides converge on: a " + "planner writes and revises the plan file, and an analyst who shares no " + "context with the planner reviews it fresh each round and answers whether it " + "is settled; the loop ends when the analyst says settled, and a cap on the " + "rounds backstops an analyst that never does", + ) + equivalent(at, drives_count=2, person=False, takes_config=True) + source = (at / "__init__.py").read_text() + assert "range(" in source or "spent()" in source # the backstop is real + + +def test_rlcr_from_its_description( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + at = compiled(tmp_path, monkeypatch, RLCR) + equivalent(at, drives_count=2, person=False, takes_config=True) + source = (at / "__init__.py").read_text() + assert "schema=" in source # the review is read off a shape, not a marker + assert "spent()" in source or "range(" in source + + +#: The rlcr loop, described the way somebody would describe it. +RLCR = ( + "a builder works through a task under review, in one session that remembers: each " + "round the builder builds, then a reviewer that shares no context with the builder " + "reads the repository fresh and answers two things in one shape -- whether there is " + "nothing left to do, and the findings to hand the builder next, written as its next " + "prompt with the important ones marked [P0] to [P9]; the findings go to the builder " + "word for word; the loop ends when the reviewer says there is nothing left, and a " + "budget of output tokens backstops a reviewer that never does" +) + + +@pytest.mark.skipif( + os.environ.get("AOT_SMOKE", "") != "1", + reason="the smoke drives the compiled loop with real agents; AOT_SMOKE=1 runs it", +) +def test_the_compiled_rlcr_runs_once_on_a_toy_repository( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + at = compiled(tmp_path, monkeypatch, RLCR) + equivalent(at, drives_count=2, person=False, takes_config=True) + # A toy repository for the loop to work in: the run is the flow's own directory's. + workshop = tmp_path / "workshop" + workshop.mkdir() + subprocess.run(["git", "init", "-q"], cwd=workshop, check=True) + (workshop / "README.md").write_text("# workshop\n\nA toy repository.\n") + monkeypatch.chdir(workshop) + run = load(str(at)) + run( + (agent_of(WRITER), agent_of(CRITIC)), + "create a file called hello.txt containing exactly the line `hello`, and " + "nothing else; the task is done when that file exists with that content", + ) + assert (workshop / "hello.txt").read_text().strip() == "hello"