From 9d4fc618ed15f3922eb1d7810d21190e279b4321 Mon Sep 17 00:00:00 2001 From: dongyun Date: Fri, 28 Aug 2026 21:16:22 +0000 Subject: [PATCH 1/8] feat(flows): add checking -- the static read of a flow's legality Pure ast over every file a flow holds, executing nothing, answering with findings rather than raising: ten errors for a flow that cannot run, cannot be answered or cannot end, and eight warnings for a run that may be regretted. surface() reads the flow-facing interfaces themselves -- lifted from the interface test, which now imports it -- and offered() reads the package's own tables, so the checker states the contract it checks rather than keeping a copy to drift. Every rule is the proof of an absence, one function at a time, and the sweep over the builtin and official flows pins the false-positive rate: nothing anywhere, except the one warning rlar has earned -- a loop only its reviewer can end. Co-Authored-By: Claude Fable 5 --- src/hmz/flows/SPEC.md | 53 +- src/hmz/flows/__init__.py | 3 + src/hmz/flows/checking.py | 1559 ++++++++++++++++++++++++++++++++++ tests/test_checking.py | 842 ++++++++++++++++++ tests/test_flow_interface.py | 27 +- 5 files changed, 2458 insertions(+), 26 deletions(-) create mode 100644 src/hmz/flows/checking.py create mode 100644 tests/test_checking.py diff --git a/src/hmz/flows/SPEC.md b/src/hmz/flows/SPEC.md index f163a2d..f2023d8 100644 --- a/src/hmz/flows/SPEC.md +++ b/src/hmz/flows/SPEC.md @@ -7,6 +7,7 @@ ├── __init__.py ├── agent.py ├── builtin +├── checking.py ├── driving.py ├── skills.py └── verses.py @@ -164,8 +165,9 @@ def __getattr__(name: str) -> object: ... flow -- one phase of a thing, an engine two flows share -- is a flow to call by name and not a flow to start, and one that appeared in the picker would be a line nobody can act on. - This module MUST be everything a flow imports, which is the interfaces beside it, the mark, - the finding, the calling, and what is written down in another layer handed through: the - vocabulary a turn is described in, the facts about the CLIs and what each of them runs, and + the finding, the calling, the checking, and what is written down in another layer handed + through: the vocabulary a turn is described in, the facts about the CLIs and what each of + them runs, and where humanize keeps what outlives a run. What is handed through MUST be the same object the layer it is written in holds, so that a flow and humanize are talking about one thing. - What is handed through MUST be fetched when it is asked for. Importing this module MUST cost @@ -428,6 +430,53 @@ run another. `hmz.runner` asks this and then opens a cycle around the answer. than take one as it comes: a flow is loaded by running its file, so the class it declared last time is a stranger to the class it declares this time, and what survives that is the fields. +## `checking.py` + +```python +class Finding(NamedTuple): + code: str + severity: Literal["error", "warning"] + where: Path + line: int + said: str + + +def checked(flow: str | os.PathLike[str]) -> tuple[Finding, ...]: ... + + +def surface(protocol: type) -> frozenset[str]: ... + + +def offered() -> frozenset[str]: ... +``` + +The static read of a flow's legality: what will not run, said before anything runs it. +`driving.py` refuses a flow as it loads it, and loading a flow means running its file -- so +this is the reading for a flow nobody has read yet, generated or fetched or forked, and it is +the first of two: what only running the file can show is `proving.py`'s, in a process of its +own. + +- `checked` MUST NOT import or execute anything of the flow it reads. It is pointed at + untrusted code -- that is what it is for -- and a checker that ran what it was checking + would be the attack it exists to catch. Every file the flow's directory holds MUST be read, + except what is under its `skills/`, which is content for the agents rather than code. +- It MUST answer with findings rather than raise, and every finding MUST carry a code, a + severity, a file and a line: a checker is asked so that everything wrong can be said at + once, and a finding that cannot say where it is is a finding nobody can act on. +- `error` MUST be kept for a flow that cannot run, cannot be answered, or cannot end -- + something no run of it survives -- and `warning` for a flow that runs and may be regretted. + A flow with no error findings MUST be one `driving.py` would load, as far as reading can + tell; nothing here MUST refuse a flow for style. +- Every rule MUST be the proof of an absence, worked out one function at a time: no exit in + this loop, no bound in this function, no guard on this name. Nothing MUST claim an exit + reachable or a bound tight, and nothing MUST follow a value through a call -- a flow that + keeps its loop in one function and its bound in another is a flow this reading trusts, + since a rule that guessed further would refuse flows that run. +- What an agent may be asked MUST be read off the interfaces in `agent.py` themselves, which + is `surface`, and what a flow may import MUST be read off this package's own tables, which + is `offered`: the checker states what the interface is, so a second copy of either would be + the drift it checks for. + ## `verses.py` ```python diff --git a/src/hmz/flows/__init__.py b/src/hmz/flows/__init__.py index 127ee74..115ee9c 100644 --- a/src/hmz/flows/__init__.py +++ b/src/hmz/flows/__init__.py @@ -52,6 +52,7 @@ def run(agents: tuple[Agent, Agent], task: str) -> None: from typing import TYPE_CHECKING, Any, NamedTuple, overload from .agent import Agent, Driven, Person, Session +from .checking import Finding, checked from .driving import ( NotAFlow, Place, @@ -127,6 +128,7 @@ def run(agents: tuple[Agent, Agent], task: str) -> None: "Driven", "Event", "Failed", + "Finding", "Flow", "Flowverse", "Goal", @@ -156,6 +158,7 @@ def run(agents: tuple[Agent, Agent], task: str) -> None: "at", "backends", "carries", + "checked", "configures", "container", "drives", diff --git a/src/hmz/flows/checking.py b/src/hmz/flows/checking.py new file mode 100644 index 0000000..edcbf91 --- /dev/null +++ b/src/hmz/flows/checking.py @@ -0,0 +1,1559 @@ +"""The static read of a flow's legality: what will not run, said before anything runs it. + +:mod:`hmz.flows.driving` refuses a flow as it loads it -- the wrong arity, a moment no agent +can run -- but loading a flow means running its file, and the flow most worth checking is one +nobody has read yet: generated, fetched, forked and edited. This is the reading that executes +nothing. Pure `ast` over every Python file the flow's directory holds, answering with findings +rather than raising, so that whatever asked can say everything that is wrong at once. + +What a severity means is one line apiece. An error is a flow that cannot run, cannot be +answered, or cannot end -- something no run of it survives. A warning is a flow that runs, +and a run of it that may be regretted: a loop with no bound of its own, a shaped answer read +without a guard, a config that takes anything. + +And what the reading does not claim is said as plainly. Every rule is the proof of an +absence -- no exit in this loop, no bound in this function, no guard on this name -- worked +out one function at a time. Nothing here proves an exit reachable or a bound tight, and +nothing follows a value through a call: a flow that keeps its loop in one function and its +bound in another is a flow this reading trusts, and a checker that guessed further would +refuse flows that run. +""" + +from __future__ import annotations + +import ast +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING, Literal, NamedTuple, Protocol + +from .agent import Agent, Driven, Person, Session + +if TYPE_CHECKING: + import os + from collections.abc import Iterator + +__all__ = ["Finding", "checked", "offered", "surface"] + + +class Finding(NamedTuple): + """One thing the reading of a flow found, where it found it. + + Attributes: + code: Which rule found it, as one hyphenated word -- `dead-loop`, `unknown-ask`. + severity: What finding it is. An error is a flow that cannot run, cannot be answered + or cannot end; a warning is a flow that runs and may be regretted. + where: The file it is in. + line: The line, 1-based, or 0 for a finding about the whole file. + said: What is wrong, said the way `NotAFlow` says it. + """ + + code: str + severity: Literal["error", "warning"] + where: Path + line: int + said: str + + +def surface(protocol: type) -> frozenset[str]: + """What one of the flow-facing interfaces asks for, by name. + + The one reading of a protocol's members, shared by the checker's own rules and by the + tests that hold the drivers to the same interfaces -- two readings of what an agent + answers to would be two readings to drift apart. + + Args: + protocol: The interface. + + Returns: + One name per member it declares, its own and whatever it is itself an interface of. + `__call__` is among them where it is declared, an agent and a session both being things + a flow calls; the rest of the dunders are Python's and are not part of the contract. + """ + said: set[str] = set() + for one in protocol.__mro__: + if one in (object, Protocol): + continue + held = set(vars(one)) | set(getattr(one, "__annotations__", {})) + said.update( + name for name in held if not name.startswith("_") or name == "__call__" + ) + return frozenset(said) + + +def offered() -> frozenset[str]: + """Every name a flow may import from `hmz.flows`, which is the whole of its vocabulary. + + What the module says it offers rather than what happens to be reachable on it: a name + that works today because a submodule leaked it is a name the next release takes away. + + Returns: + The names: what `__all__` declares, what is handed through by name from the layers a + flow does not import, and the two modules handed through whole. + """ + # The package's own tables, read by the package's own checker: private to every + # flow, and one copy rather than a second one kept here to drift. + from . import _ELSEWHERE, _MODULES # pyright: ignore[reportPrivateUsage] + from . import __all__ as declared + + return frozenset(declared) | frozenset(_ELSEWHERE) | frozenset(_MODULES) + + +def checked(flow: str | os.PathLike[str]) -> tuple[Finding, ...]: + """Reads a flow without running it, and answers with everything that reading found. + + Every Python file the flow's directory holds is read -- the entry point and whatever it + imports beside it -- except what is under its `skills/`, which is content for the agents + rather than code this process runs. Nothing is imported and nothing is executed, so this + is safe to point at a flow nobody has read: what running the file would refuse is the + second reading, :mod:`hmz.flows.proving`, which runs it in a process of its own. + + Args: + flow: The flow: its directory, or the Python file a single-file flow is. + + Returns: + One finding per thing found, in file order, and nothing at all for a flow this reading + has nothing to say about -- which is not a proof, only a reading with nothing to say. + """ + from . import ENTRY + + at = Path(flow) + if at.is_dir(): + entry = at / ENTRY + files = [ + one + for one in sorted(at.rglob("*.py")) + if "__pycache__" not in one.relative_to(at).parts + # The skills are content: one directory per skill, laid out the way every one + # of these CLIs reads a skill in, and nothing in one is imported by the flow. + and one.relative_to(at).parts[0] != "skills" + ] + else: + entry = at + files = [at] if at.is_file() else [] + if not entry.is_file(): + return ( + Finding( + "not-a-flow", + "error", + at, + 0, + "no flow to read: a flow is a directory with an __init__.py in it", + ), + ) + + found: list[Finding] = [] + read: list[_Read] = [] + for one in files: + held = _parsed(one) + if isinstance(held, Finding): + found.append(held) + else: + read.append(held) + + marked = any(one.marks for one in read) + entered = next((one for one in read if one.where == entry), None) + if entered is not None and not marked: + found.append( + Finding( + "not-a-flow", + "error", + entry, + 0, + "nothing in it is marked @flow() -- a flow is a function marked with it, " + "which is how a file says which of the functions in it is one", + ) + ) + if entered is not None and ast.get_docstring(entered.tree) is None: + found.append( + Finding( + "unsaid-flow", + "warning", + entry, + 0, + "the flow says nothing about itself -- the first line of this file's " + "docstring is what every list of flows shows for it", + ) + ) + + # What the flow declared about moments anywhere in its files, for the hooks it hangs: + # a place annotated in the entry point covers a hook hung in the module beside it. + declared = frozenset(moment for one in read for moment in one.moments_declared) + asks = _Asks() + for one in read: + found.extend(_imports(one)) + found.extend(_marks(one)) + found.extend(_hooks(one, declared)) + found.extend(_functions(one, asks)) + return tuple(found) + + +# --------------------------------------------------------------------------------------- +# Reading one file: what it imports, what it marks, and what it declares. +# --------------------------------------------------------------------------------------- + +#: The kinds of thing a flow drives, each the name of one flow-facing interface. What a +#: tracked name may be asked is read off the interface itself, so the checker and the +#: drivers are held to one surface. +_KINDS: dict[str, type] = { + "agent": Agent, + "person": Person, + "session": Session, + "driven": Driven, +} + +#: How the local names for those interfaces read where a flow imports them. +_PROTOCOLS = { + "Agent": "agent", + "Person": "person", + "Session": "session", + "Driven": "driven", +} + +#: What an element of a plain tuple of agents may still be asked by name: the tuple's own +#: two methods. A named place is a field of the NamedTuple the flow declared instead. +_OF_A_TUPLE = frozenset({"count", "index"}) + + +class _Mark(NamedTuple): + """One function a file marked with `@flow`, and what the mark said.""" + + node: ast.FunctionDef | ast.AsyncFunctionDef + name: str + resumable: bool + + +@dataclass +class _Read: + """One file, parsed, and what one pass over its top collected.""" + + where: Path + tree: ast.Module + #: The local names of :func:`hmz.flows.flow`, `Moment` and pydantic's `Field`. + flow_alias: set[str] = field(default_factory=set[str]) + moment_alias: set[str] = field(default_factory=set[str]) + field_alias: set[str] = field(default_factory=set[str]) + #: Local name -> which interface, for every flow-facing interface the file imports. + proto: dict[str, str] = field(default_factory=dict[str, str]) + #: The NamedTuple and pydantic model classes the file itself declares. + crews: dict[str, ast.ClassDef] = field(default_factory=dict[str, ast.ClassDef]) + models: dict[str, ast.ClassDef] = field(default_factory=dict[str, ast.ClassDef]) + #: Names bound only under `if TYPE_CHECKING:`, which a running flow cannot read. + unread: set[str] = field(default_factory=set[str]) + #: Every name the file binds at module level as it runs, which excuses the above. + bound: set[str] = field(default_factory=set[str]) + marks: list[_Mark] = field(default_factory=list["_Mark"]) + #: Every `Moment.X` written inside an annotation, which is a flow declaring a need. + moments_declared: set[str] = field(default_factory=set[str]) + + +def _parsed(where: Path) -> _Read | Finding: + """One file read into a tree, or the finding that it could not be. + + Args: + where: The file. + + Returns: + What was read, or an `unread` error: a file that will not parse is a flow that will + not load, said here rather than left for the loading to hit. + """ + try: + tree = ast.parse(where.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, SyntaxError, ValueError) as why: + line = getattr(why, "lineno", 0) or 0 + return Finding( + "unread", + "error", + where, + line, + f"nothing here can be read as Python -- {why}", + ) + read = _Read(where, tree) + _collected(read, tree.body, type_checking=False) + # Bound under TYPE_CHECKING and nowhere else: a name a type checker reads and a + # running flow cannot, which is the one thing `unread-annotation` is about. + read.unread -= read.bound + for node in ast.walk(tree): + if isinstance(node, (ast.AnnAssign, ast.arg)) and node.annotation is not None: + read.moments_declared.update(_moments_in(node.annotation, read)) + return read + + +def _collected(read: _Read, body: list[ast.stmt], *, type_checking: bool) -> None: + """Walks one file's statements for what the rules read off its top. + + Args: + read: What is being collected into. + body: The statements, at whatever depth the walk has reached. + type_checking: Whether these statements are under `if TYPE_CHECKING:`, where a name + is bound for a type checker and for nothing that runs. + """ + for node in body: + if isinstance(node, ast.ImportFrom) and node.module == "hmz.flows": + for alias in node.names: + bound = alias.asname or alias.name + if type_checking: + read.unread.add(bound) + continue + read.bound.add(bound) + if alias.name == "flow": + read.flow_alias.add(bound) + elif alias.name == "Moment": + read.moment_alias.add(bound) + elif alias.name in _PROTOCOLS: + read.proto[bound] = _PROTOCOLS[alias.name] + elif isinstance(node, (ast.Import, ast.ImportFrom)): + for alias in node.names: + bound = (alias.asname or alias.name).split(".")[0] + (read.unread if type_checking else read.bound).add(bound) + if ( + isinstance(node, ast.ImportFrom) + and node.module == "pydantic" + and alias.name == "Field" + and not type_checking + ): + read.field_alias.add(alias.asname or alias.name) + elif isinstance(node, (ast.Assign, ast.AnnAssign)) and not type_checking: + targets = node.targets if isinstance(node, ast.Assign) else [node.target] + read.bound.update(one.id for one in targets if isinstance(one, ast.Name)) + elif isinstance(node, ast.ClassDef) and not type_checking: + read.bound.add(node.name) + bases = {_root(base) for base in node.bases} + if "NamedTuple" in bases: + read.crews[node.name] = node + elif "BaseModel" in bases or bases & set(read.models): + read.models[node.name] = node + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + if not type_checking: + read.bound.add(node.name) + mark = _marked(node, read) + if mark is not None and not type_checking: + read.marks.append(mark) + elif isinstance(node, ast.If): + under = type_checking or _root(node.test) == "TYPE_CHECKING" + _collected(read, node.body, type_checking=under) + _collected(read, node.orelse, type_checking=type_checking) + elif isinstance(node, (ast.Try, ast.With)): + _collected(read, node.body, type_checking=type_checking) + + +def _marked(node: ast.FunctionDef | ast.AsyncFunctionDef, read: _Read) -> _Mark | None: + """The mark on one function, where it carries one. + + Args: + node: The function. + read: The file it is in, for what the decorator is called there. + + Returns: + The mark, or None for a function that is not a flow. + """ + for one in node.decorator_list: + if isinstance(one, ast.Name) and one.id in read.flow_alias: + return _Mark(node, "", resumable=False) + if ( + isinstance(one, ast.Call) + and isinstance(one.func, ast.Name) + and one.func.id in read.flow_alias + ): + name = "" + resumable = False + for said in one.keywords: + if said.arg == "name" and isinstance(said.value, ast.Constant): + name = str(said.value.value) + elif said.arg == "resumable" and isinstance(said.value, ast.Constant): + resumable = bool(said.value.value) + return _Mark(node=node, name=name, resumable=resumable) + return None + + +def _root(node: ast.expr) -> str: + """The name at the root of one expression, or "" where there is none.""" + while isinstance(node, ast.Attribute): + node = node.value + return node.id if isinstance(node, ast.Name) else "" + + +def _moments_in(annotation: ast.expr, read: _Read) -> set[str]: + """Every moment one annotation names, which is a place saying what it needs.""" + return { + node.attr + for node in ast.walk(annotation) + if isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Name) + and node.value.id in read.moment_alias + } + + +# --------------------------------------------------------------------------------------- +# The import rules: one import, and only the names it offers. +# --------------------------------------------------------------------------------------- + + +def _imports(read: _Read) -> Iterator[Finding]: + """What one file imports of humanize's, held to the one import a flow writes. + + Args: + read: The file. + + Yields: + A `foreign-import` error per import of a module of humanize's own that is not + `hmz.flows`, and an `unknown-name` error per name asked of `hmz.flows` that it does + not offer. + """ + offers: frozenset[str] | None = None + for node in ast.walk(read.tree): + if isinstance(node, ast.Import): + for alias in node.names: + if alias.name.split(".")[0] == "hmz" and alias.name != "hmz.flows": + yield Finding( + "foreign-import", + "error", + read.where, + node.lineno, + f"a flow imports hmz.flows and nothing else of humanize's -- " + f"{alias.name} is humanize's own business, and a flow that names " + "it breaks whenever humanize moves it", + ) + elif isinstance(node, ast.ImportFrom) and node.module: + said = node.module + if said.split(".")[0] != "hmz": + continue + if said != "hmz.flows": + yield Finding( + "foreign-import", + "error", + read.where, + node.lineno, + f"a flow imports hmz.flows and nothing else of humanize's -- " + f"{said} is humanize's own business, and a flow that names it " + "breaks whenever humanize moves it", + ) + continue + if offers is None: + offers = offered() + for alias in node.names: + if alias.name not in offers: + yield Finding( + "unknown-name", + "error", + read.where, + node.lineno, + f"hmz.flows does not offer {alias.name!r} -- what a flow may " + "import from it is what it hands through, and a name it does " + "not hold fails at the first run", + ) + + +# --------------------------------------------------------------------------------------- +# The mark rules: what the entry point declares, read as the loader will read it. +# --------------------------------------------------------------------------------------- + +#: How many arguments a resumable flow's entry point takes at the least: the agents, the +#: task, and somewhere for what the last run of it wrote to be handed back in. +_WITH_A_STATE = 3 + +#: The one sentence the loader says about a flow that does not state its arity, said here +#: too so that the two readings refuse the same flow in the same words. +_UNSIZED = ( + "a flow is a function marked @flow() taking (agents, task), whose agents are " + "annotated with a tuple of a fixed length -- how many agents the flow drives -- or " + "with a NamedTuple of them, which also says what each is for" +) + + +def _marks(read: _Read) -> Iterator[Finding]: + """What each flow one file marks says about itself, held to what a run needs said. + + Args: + read: The file. + + Yields: + `unsized-agents` and `unread-annotation` errors for an arity a run cannot read back, + `stateless-resume` for a flow that says it can be picked up and takes nothing to be + picked up with, `twice-named` for two flows under one name, `state-kept` for kept + state nothing ever clears, and the config findings for the model an entry declares. + """ + named: set[str] = set() + configured: set[str] = set() + for mark in read.marks: + if mark.name in named: + yield Finding( + "twice-named", + "warning", + read.where, + mark.node.lineno, + f"two flows here are both {_said_name(mark.name)} -- the first of them " + "wins, and the second can never be run", + ) + named.add(mark.name) + yield from _sized(mark, read) + yield from _resumes(mark, read) + for model in _settings(mark.node, read): + if model not in configured: + configured.add(model) + yield from _configured(read.models[model], read) + + +def _said_name(name: str) -> str: + """How a flow's name reads in a finding about two of them.""" + return f"called {name!r}" if name else "the one their file holds under its own name" + + +def _sized(mark: _Mark, read: _Read) -> Iterator[Finding]: + """Whether one flow states how many agents it drives where a run can read it back.""" + args = mark.node.args + params = [*args.posonlyargs, *args.args] + kind = params[0].annotation if params else None + if kind is None: + yield Finding("unsized-agents", "error", read.where, mark.node.lineno, _UNSIZED) + return + unread = sorted(_names_in(kind) & read.unread) + if unread: + yield Finding( + "unread-annotation", + "error", + read.where, + mark.node.lineno, + f"the flow's agents cannot be read here ({', '.join(unread)} is imported " + "under TYPE_CHECKING) -- import what the annotation names at runtime, so " + "the count it states can be checked", + ) + return + said = _unquoted(kind) + if said is None: + return + if isinstance(said, ast.Name) and said.id == "tuple": + yield Finding( + "unsized-agents", + "error", + read.where, + mark.node.lineno, + _UNSIZED, + ) + return + if ( + isinstance(said, ast.Subscript) + and _root(said.value) == "tuple" + and any( + isinstance(one, ast.Constant) and one.value is Ellipsis + for one in _elements(said.slice) + ) + ): + yield Finding( + "unsized-agents", + "error", + read.where, + mark.node.lineno, + _UNSIZED, + ) + + +def _resumes(mark: _Mark, read: _Read) -> Iterator[Finding]: + """Whether a flow that says it can be picked up takes anything to be picked up with.""" + if not mark.resumable: + return + args = mark.node.args + params = [*args.posonlyargs, *args.args] + taken = len(params) + third = params[2].annotation if taken >= _WITH_A_STATE else None + settles = third is not None and bool(_names_in(third) & set(read.models)) + if taken < _WITH_A_STATE or (taken == _WITH_A_STATE and settles): + yield Finding( + "stateless-resume", + "error", + read.where, + mark.node.lineno, + "the flow says it can be picked up, and takes nothing to be picked up with " + "-- a resumable flow is handed a dict as its last argument, holding what it " + "wrote there last time", + ) + return + yield from _kept(mark, read, params[-1].arg) + + +def _kept(mark: _Mark, read: _Read, state: str) -> Iterator[Finding]: + """Whether kept state something writes is state anything ever clears. + + Args: + mark: The resumable flow. + read: Its file. + state: What its entry point calls the dict it is handed. + """ + held = {state} + wrote = 0 + cleared = False + for node in ast.walk(mark.node): + if isinstance(node, ast.Assign) and len(node.targets) == 1: + target = node.targets[0] + if isinstance(target, ast.Name) and any( + isinstance(one, ast.Name) and one.id in held + for one in ast.walk(node.value) + ): + held.add(target.id) + elif isinstance(node, ast.Subscript) and isinstance(node.ctx, ast.Store): + if _root(node.value) in held: + wrote = wrote or node.lineno + elif ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and _root(node.func.value) in held + ): + if node.func.attr in {"update", "setdefault"}: + wrote = wrote or node.lineno + elif node.func.attr == "clear": + cleared = True + if wrote and not cleared: + yield Finding( + "state-kept", + "warning", + read.where, + wrote, + "the flow writes its kept state and never clears it -- a run that is over " + "leaves what the next run here opens on, so a loop that has ended clears " + "what it kept", + ) + + +def _settings( + node: ast.FunctionDef | ast.AsyncFunctionDef, read: _Read +) -> Iterator[str]: + """The config models one entry point declares, of the ones its own file holds.""" + args = node.args + for param in [*args.posonlyargs, *args.args][2:]: + if param.annotation is not None: + yield from ( + name for name in _names_in(param.annotation) if name in read.models + ) + + +def _configured(model: ast.ClassDef, read: _Read) -> Iterator[Finding]: + """One config model, held to refusing what it does not take and saying what it does. + + Args: + model: The model a flow's entry point says it can be set up with. + read: The file it is declared in. + + Yields: + A `loose-config` warning for one that takes anything, and an `unsaid-field` warning + per field that says nothing about itself -- the descriptions are what whoever sets + the flow up is shown, and a field without one is a question nobody can answer. + """ + if not _strict(model, read, set()): + yield Finding( + "loose-config", + "warning", + read.where, + model.lineno, + "the config takes anything -- set model_config to extra: forbid or frozen: " + "True, so a setting that is misspelled is refused rather than quietly " + "ignored", + ) + for node in model.body: + if not isinstance(node, ast.AnnAssign) or not isinstance(node.target, ast.Name): + continue + name = node.target.id + if name.startswith("_") or _root(node.annotation) == "ClassVar": + continue + bare = node.value is None + unsaid = ( + isinstance(node.value, ast.Call) + and _root(node.value.func) in read.field_alias + and not any(one.arg == "description" for one in node.value.keywords) + ) + if bare or unsaid: + yield Finding( + "unsaid-field", + "warning", + read.where, + node.lineno, + f"the config field {name!r} says nothing about itself -- give it a " + "Field(description=...), which is what whoever sets the flow up is " + "shown", + ) + + +def _strict(model: ast.ClassDef, read: _Read, seen: set[str]) -> bool: + """Whether one config model refuses what it does not take, its local bases included.""" + seen.add(model.name) + for node in model.body: + if ( + isinstance(node, ast.Assign) + and any( + isinstance(one, ast.Name) and one.id == "model_config" + for one in node.targets + ) + and isinstance(node.value, ast.Dict) + ): + said = { + key.value: value.value + for key, value in zip(node.value.keys, node.value.values, strict=True) + if isinstance(key, ast.Constant) and isinstance(value, ast.Constant) + } + if said.get("extra") == "forbid" or said.get("frozen") is True: + return True + return any( + _strict(read.models[base], read, seen) + for base in {_root(one) for one in model.bases} + if base in read.models and base not in seen + ) + + +# --------------------------------------------------------------------------------------- +# The hook rule: a moment only some backends run is a moment the flow says it needs. +# --------------------------------------------------------------------------------------- + + +def _hooks(read: _Read, declared: frozenset[str]) -> Iterator[Finding]: + """Every moment one file hangs a hook on, held to the moments the flow declared. + + Args: + read: The file. + declared: Every moment named in an annotation anywhere in the flow, which is how a + place says what the agent filling it has to run. + + Yields: + An `unsaid-moment` warning per hook hung on a moment only some backends reach that + no place declares: the run finds out from `Unhooked`, mid-flow, where a declaration + would have refused the agent before its first turn. + """ + everywhere: frozenset[str] | None = None + for node in ast.walk(read.tree): + if not ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr == "on" + and node.args + ): + continue + moment = node.args[0] + if not ( + isinstance(moment, ast.Attribute) + and isinstance(moment.value, ast.Name) + and moment.value.id in read.moment_alias + ): + continue + if everywhere is None: + # Read off the live enum rather than copied out of it, so that a moment + # humanize adds is a moment this rule already knows. Fetched here rather + # than imported with the module: the vocabulary lives beside the drivers. + from hmz.agents import EVERYWHERE + + everywhere = frozenset(one.name for one in EVERYWHERE) + if moment.attr in everywhere or moment.attr in declared: + continue + yield Finding( + "unsaid-moment", + "warning", + read.where, + node.lineno, + f"a hook is hung on Moment.{moment.attr}, which only some backends run, and " + "no place declares it -- write Annotated[Agent, Moment." + f"{moment.attr}] where the place is declared, so an agent that cannot run " + "it is refused before its first turn rather than hours in", + ) + + +# --------------------------------------------------------------------------------------- +# The function rules: what is asked of what the flow drives, and how its loops end. +# --------------------------------------------------------------------------------------- + + +class _Crew(NamedTuple): + """A tuple of agents as one function holds it: the places, or only the count. + + Attributes: + fields: One (name, kinds) pair per place for a NamedTuple of them, or None for a + plain tuple, which named nothing. Kinds of None is a place whose annotation this + file cannot read, which is tracked and asked nothing. + kinds: The kind of each element by position, for a plain tuple that said them. + """ + + fields: tuple[tuple[str, frozenset[str] | None], ...] | None + kinds: tuple[frozenset[str] | None, ...] = () + + def held(self) -> frozenset[str] | None: + """What one element of this crew is, where every place is the same thing.""" + each = ( + [kinds for _, kinds in self.fields] + if self.fields is not None + else list(self.kinds) + ) + if not each or any(not one for one in each): + return None + return frozenset(kind for one in each if one for kind in one) + + +class _Answer(NamedTuple): + """One name holding what a turn answered, and how the turn was taken.""" + + shaped: bool + suppressed: bool + line: int + + +@dataclass +class _Asks: + """What each kind of tracked thing may be asked, read once per checking.""" + + _surfaces: dict[str, frozenset[str]] = field( + default_factory=dict[str, frozenset[str]] + ) + + def allowed(self, kinds: frozenset[str]) -> frozenset[str]: + """Every name something of these kinds answers to.""" + held: frozenset[str] = frozenset() + for kind in kinds: + if kind not in self._surfaces: + self._surfaces[kind] = surface(_KINDS[kind]) + held |= self._surfaces[kind] + return held + + +@dataclass +class _Scope: + """One function being read: what is bound to what, and what was found so far.""" + + read: _Read + asks: _Asks + bindings: dict[str, frozenset[str] | _Crew] = field( + default_factory=dict[str, "frozenset[str] | _Crew"] + ) + answers: dict[str, _Answer] = field(default_factory=dict[str, "_Answer"]) + guarded: set[str] = field(default_factory=set[str]) + #: Attribute reads off a shaped, suppressed answer: (name, line) apiece. + reads: list[tuple[str, int]] = field(default_factory=list[tuple[str, int]]) + #: Whether the function holds a bound of its own -- a spent() call, a range(), an + #: ordering comparison against a number -- which is what excuses its loops. + bounded: bool = False + findings: list[Finding] = field(default_factory=list[Finding]) + + def forgot(self, name: str) -> None: + """Stops tracking one name, which is what any doubtful binding does to it.""" + self.bindings.pop(name, None) + self.answers.pop(name, None) + + +def _functions(read: _Read, asks: _Asks) -> Iterator[Finding]: + """Reads every function in one file for what it asks and how its loops end. + + Args: + read: The file. + asks: The interface surfaces, shared across the files of one checking. + + Yields: + The findings, function by function. + """ + for node in read.tree.body: + yield from _defined(node, read, asks, {}) + + +def _defined( + node: ast.stmt, + read: _Read, + asks: _Asks, + inherited: dict[str, frozenset[str] | _Crew], +) -> Iterator[Finding]: + """One top-level statement, read for the functions in it.""" + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + yield from _function(node, read, asks, inherited) + elif isinstance(node, ast.ClassDef): + for held in node.body: + yield from _defined(held, read, asks, inherited) + elif isinstance(node, ast.If): + for held in [*node.body, *node.orelse]: + yield from _defined(held, read, asks, inherited) + + +def _function( + node: ast.FunctionDef | ast.AsyncFunctionDef, + read: _Read, + asks: _Asks, + inherited: dict[str, frozenset[str] | _Crew], +) -> Iterator[Finding]: + """One function: bindings followed in order, then its loops read against them. + + Args: + node: The function. + read: Its file. + asks: The interface surfaces. + inherited: What the enclosing function had bound, which a closure reads. + """ + scope = _Scope(read, asks, bindings=dict(inherited)) + args = node.args + params = [*args.posonlyargs, *args.args, *args.kwonlyargs] + for param in params: + scope.forgot(param.arg) + kind = _annotated(param.annotation, read) + crew = _crewed(param.annotation, read) if not kind else None + if kind: + scope.bindings[param.arg] = kind + elif crew is not None: + scope.bindings[param.arg] = crew + for one in (args.vararg, args.kwarg): + if one is not None: + scope.forgot(one.arg) + _statements(node.body, scope, read, asks) + yield from scope.findings + for name, line in scope.reads: + if name not in scope.guarded: + yield Finding( + "unguarded-answer", + "warning", + read.where, + line, + f"{name} may be None here -- a suppressed turn held to a shape answers " + "with nothing when it fails, and a field read off nothing ends the run " + "where a guard would have taken the turn again", + ) + if not _yields(node): + yield from _loops(node, scope) + + +def _annotated(annotation: ast.expr | None, read: _Read) -> frozenset[str] | None: + """What kinds of driven thing one annotation says a name is, or None for no answer.""" + said = _unquoted(annotation) + if said is None: + return None + if isinstance(said, ast.Subscript) and _root(said.value) == "Annotated": + first = next(iter(_elements(said.slice)), None) + return _annotated(first, read) + if isinstance(said, ast.Subscript) and _root(said.value) == "Optional": + first = next(iter(_elements(said.slice)), None) + return _annotated(first, read) + if isinstance(said, ast.BinOp) and isinstance(said.op, ast.BitOr): + left = _annotated(said.left, read) + right = _annotated(said.right, read) + if left is None and right is None: + return None + # `Agent | None` is an agent to be guarded; `Agent | Session` is either, and is + # asked only what both answer to being too strict -- so it is asked the union. + return (left or frozenset()) | (right or frozenset()) + if isinstance(said, ast.Constant) and said.value is None: + return frozenset() + if isinstance(said, ast.Name) and said.id in read.proto: + return frozenset({read.proto[said.id]}) + return None + + +def _crewed(annotation: ast.expr | None, read: _Read) -> _Crew | None: + """The tuple of agents one annotation declares, where this file can read it. + + A crew it is not sure of is no crew at all: a NamedTuple with one place this file + cannot read as an interface is somebody's data rather than the agents, and tracking + it would find askings that are nobody's business here. + """ + said = _unquoted(annotation) + if said is None: + return None + if isinstance(said, ast.Name) and said.id in read.crews: + fields = [ + (node.target.id, _annotated(node.annotation, read)) + for node in read.crews[said.id].body + if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name) + ] + if fields and all(kinds for _, kinds in fields): + return _Crew(tuple(fields)) + return None + if isinstance(said, ast.Subscript) and _root(said.value) == "tuple": + kinds = tuple( + _annotated(one, read) + for one in _elements(said.slice) + if not (isinstance(one, ast.Constant) and one.value is Ellipsis) + ) + if kinds and all(kinds): + return _Crew(None, kinds) + return None + + +def _unquoted(annotation: ast.expr | None) -> ast.expr | None: + """One annotation with any quoting read through, since a string is still the words.""" + if isinstance(annotation, ast.Constant) and isinstance(annotation.value, str): + try: + return ast.parse(annotation.value, mode="eval").body + except (SyntaxError, ValueError): + return None + return annotation + + +def _elements(slice_: ast.expr) -> tuple[ast.expr, ...]: + """The elements of one subscript, one or many.""" + return tuple(slice_.elts) if isinstance(slice_, ast.Tuple) else (slice_,) + + +def _names_in(annotation: ast.expr) -> set[str]: + """Every plain name one annotation mentions, quoting and all.""" + said = _unquoted(annotation) + if said is None: + return set() + return {node.id for node in ast.walk(said) if isinstance(node, ast.Name)} + + +def _statements(body: list[ast.stmt], scope: _Scope, read: _Read, asks: _Asks) -> None: + """Walks statements in order, checking what they ask and following what they bind.""" + for node in body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + scope.findings.extend(_function(node, read, asks, dict(scope.bindings))) + scope.forgot(node.name) + elif isinstance(node, ast.ClassDef): + for held in node.body: + if isinstance(held, (ast.FunctionDef, ast.AsyncFunctionDef)): + scope.findings.extend(_function(held, read, asks, {})) + scope.forgot(node.name) + elif isinstance(node, ast.Assign): + _expression(node.value, scope) + _assigned(node.targets, node.value, scope) + elif isinstance(node, ast.AnnAssign): + if node.value is not None: + _expression(node.value, scope) + if isinstance(node.target, ast.Name): + scope.forgot(node.target.id) + kind = _annotated(node.annotation, scope.read) + if kind: + scope.bindings[node.target.id] = kind + elif node.value is not None: + _assigned([node.target], node.value, scope) + else: + _expression(node.target, scope) + elif isinstance(node, ast.AugAssign): + _expression(node.value, scope) + if isinstance(node.target, ast.Name): + scope.forgot(node.target.id) + elif isinstance(node, (ast.For, ast.AsyncFor)): + _expression(node.iter, scope) + _bound_target(node.target, _element_of(node.iter, scope), scope) + _statements(node.body, scope, read, asks) + _statements(node.orelse, scope, read, asks) + elif isinstance(node, (ast.While, ast.If)): + _guards(node.test, scope) + _expression(node.test, scope) + _statements(node.body, scope, read, asks) + _statements(node.orelse, scope, read, asks) + elif isinstance(node, (ast.With, ast.AsyncWith)): + for item in node.items: + _expression(item.context_expr, scope) + if item.optional_vars is not None: + _bound_target(item.optional_vars, None, scope) + _statements(node.body, scope, read, asks) + elif isinstance(node, ast.Try): + _statements(node.body, scope, read, asks) + for handler in node.handlers: + if handler.name: + scope.forgot(handler.name) + _statements(handler.body, scope, read, asks) + _statements(node.orelse, scope, read, asks) + _statements(node.finalbody, scope, read, asks) + elif isinstance(node, ast.Assert): + _guards(node.test, scope) + _expression(node.test, scope) + elif isinstance(node, ast.Delete): + for target in node.targets: + if isinstance(target, ast.Name): + scope.forgot(target.id) + elif isinstance(node, (ast.Import, ast.ImportFrom)): + for alias in node.names: + scope.forgot((alias.asname or alias.name).split(".")[0]) + elif isinstance(node, (ast.Return, ast.Expr)): + if node.value is not None: + _expression(node.value, scope) + elif isinstance(node, ast.Raise): + for held in (node.exc, node.cause): + if held is not None: + _expression(held, scope) + elif isinstance(node, ast.Match): + _expression(node.subject, scope) + for case in node.cases: + for name in _captured(case.pattern): + scope.forgot(name) + if case.guard is not None: + _expression(case.guard, scope) + _statements(case.body, scope, read, asks) + + +def _captured(pattern: ast.pattern) -> set[str]: + """Every name one match pattern binds, all of which stop being tracked.""" + said: set[str] = set() + for node in ast.walk(pattern): + if isinstance(node, (ast.MatchAs, ast.MatchStar)) and node.name: + said.add(node.name) + elif isinstance(node, ast.MatchMapping) and node.rest: + said.add(node.rest) + return said + + +def _assigned(targets: list[ast.expr], value: ast.expr, scope: _Scope) -> None: + """Follows one assignment: what the targets are now bound to, if anything tracked.""" + held = _valued(value, scope) + for target in targets: + if isinstance(target, ast.Name): + scope.forgot(target.id) + if isinstance(held, _Answer): + scope.answers[target.id] = held + elif held is not None: + scope.bindings[target.id] = held + elif isinstance(target, (ast.Tuple, ast.List)): + _unpacked(target, value, scope) + else: + _expression(target, scope) + + +def _unpacked(target: ast.Tuple | ast.List, value: ast.expr, scope: _Scope) -> None: + """Follows a tuple unpacking, which is how `(agent,) = agents` hands a place over.""" + crew = scope.bindings.get(value.id) if isinstance(value, ast.Name) else None + each = crew.held() if isinstance(crew, _Crew) else None + kinds: list[frozenset[str] | None] | None = None + if isinstance(crew, _Crew): + held = ( + [one for _, one in crew.fields] + if crew.fields is not None + else list(crew.kinds) + ) + kinds = held if len(held) == len(target.elts) else None + for at, one in enumerate(target.elts): + if isinstance(one, ast.Name): + scope.forgot(one.id) + bound = kinds[at] if kinds is not None else each + if isinstance(crew, _Crew) and bound: + scope.bindings[one.id] = bound + elif isinstance(one, (ast.Tuple, ast.List)): + _unpacked(one, value, scope) + elif isinstance(one, ast.Starred) and isinstance(one.value, ast.Name): + scope.forgot(one.value.id) + + +def _bound_target(target: ast.expr, kind: frozenset[str] | None, scope: _Scope) -> None: + """Binds a loop or with target: to the element kind where there is one, else to doubt.""" + if isinstance(target, ast.Name): + scope.forgot(target.id) + if kind: + scope.bindings[target.id] = kind + elif isinstance(target, (ast.Tuple, ast.List)): + for one in target.elts: + _bound_target(one, None, scope) + + +def _element_of(iterable: ast.expr, scope: _Scope) -> frozenset[str] | None: + """What iterating one expression yields, where it is a tracked crew.""" + if isinstance(iterable, ast.Name): + held = scope.bindings.get(iterable.id) + if isinstance(held, _Crew): + return held.held() + return None + + +def _valued(value: ast.expr, scope: _Scope) -> frozenset[str] | _Crew | _Answer | None: + """What one expression is, as far as the bindings can say. + + Args: + value: The expression on the right of an assignment. + scope: The function so far. + + Returns: + A kind for something driven, a crew for a tuple of them, an answer for what a turn + of one said, and None for anything this reading does not follow -- which stops the + target being tracked rather than mistracking it. + """ + if isinstance(value, ast.Name): + return scope.bindings.get(value.id) + if isinstance(value, ast.Await): + return _valued(value.value, scope) + if isinstance(value, ast.Attribute): + held = ( + scope.bindings.get(value.value.id) + if isinstance(value.value, ast.Name) + else None + ) + if isinstance(held, _Crew) and held.fields is not None: + return next( + (kinds for name, kinds in held.fields if name == value.attr), None + ) + return None + if isinstance(value, ast.Subscript): + held = ( + scope.bindings.get(value.value.id) + if isinstance(value.value, ast.Name) + else None + ) + return held.held() if isinstance(held, _Crew) else None + if isinstance(value, ast.Call): + return _called(value, scope) + return None + + +def _called(value: ast.Call, scope: _Scope) -> frozenset[str] | _Crew | _Answer | None: + """What calling one thing answers with, where what is called is tracked.""" + func = value.func + opened = isinstance(func, ast.Attribute) and func.attr in {"new", "clone"} + target = func.value if isinstance(func, ast.Attribute) and opened else func + held = _valued(target, scope) if opened else None + if isinstance(func, ast.Attribute) and func.attr == "new": + if isinstance(held, frozenset) and held: + return frozenset({"session"}) + return None + if isinstance(func, ast.Attribute) and func.attr == "clone": + return held if isinstance(held, frozenset) else None + spoken = _valued(func, scope) + if isinstance(spoken, frozenset) and spoken: + # Calling an agent or a session is a turn, and what it answers is an answer. + return _Answer( + shaped=any(one.arg == "schema" for one in value.keywords), + suppressed=any( + one.arg == "suppress" + and isinstance(one.value, ast.Constant) + and one.value.value is True + for one in value.keywords + ), + line=value.lineno, + ) + if isinstance(func, ast.Attribute) and func.attr in {"aturn", "pursue", "apursue"}: + asked = _valued(func.value, scope) + if isinstance(asked, frozenset) and asked: + return _Answer( + shaped=any(one.arg == "schema" for one in value.keywords), + suppressed=any( + one.arg == "suppress" + and isinstance(one.value, ast.Constant) + and one.value.value is True + for one in value.keywords + ), + line=value.lineno, + ) + return None + + +#: The comparisons that read as a bound: a count against something, ordered. +_ORDERED = (ast.Lt, ast.LtE, ast.Gt, ast.GtE) + + +def _expression(node: ast.expr, scope: _Scope) -> None: + """Checks one expression tree against the bindings, and notes what it holds. + + What is checked is every attribute asked of a tracked name; what is noted is every + guard on an answer, and whether the function holds a bound of its own. + """ + if isinstance(node, ast.Attribute): + _asked(node, scope) + _expression(node.value, scope) + return + if isinstance(node, ast.Call): + if isinstance(node.func, ast.Attribute) and node.func.attr == "spent": + scope.bounded = True + if isinstance(node.func, ast.Name) and node.func.id == "range": + scope.bounded = True + for one in [node.func, *node.args]: + _expression(one, scope) + for kw in node.keywords: + _expression(kw.value, scope) + return + if isinstance(node, ast.Compare): + if any(isinstance(op, _ORDERED) for op in node.ops) and any( + isinstance(side, ast.Constant) + and isinstance(side.value, (int, float)) + and not isinstance(side.value, bool) + for side in [node.left, *node.comparators] + ): + scope.bounded = True + _guards(node, scope) + for one in [node.left, *node.comparators]: + _expression(one, scope) + return + if isinstance(node, ast.BoolOp): + _guards(node, scope) + for one in node.values: + _expression(one, scope) + return + if isinstance(node, ast.IfExp): + _guards(node.test, scope) + for one in (node.test, node.body, node.orelse): + _expression(one, scope) + return + if isinstance(node, ast.NamedExpr): + _expression(node.value, scope) + scope.forgot(node.target.id) + held = _valued(node.value, scope) + if isinstance(held, _Answer): + scope.answers[node.target.id] = held + elif held is not None: + scope.bindings[node.target.id] = held + return + if isinstance(node, ast.Lambda): + return # its own scope, and nothing in it runs here + if isinstance(node, (ast.ListComp, ast.SetComp, ast.GeneratorExp, ast.DictComp)): + _comprehended(node, scope) + return + for one in ast.iter_child_nodes(node): + if isinstance(one, ast.expr): + _expression(one, scope) + + +def _comprehended( + node: ast.ListComp | ast.SetComp | ast.GeneratorExp | ast.DictComp, + scope: _Scope, +) -> None: + """One comprehension, read with its own targets bound for as long as it lasts.""" + before = dict(scope.bindings) + for gen in node.generators: + _expression(gen.iter, scope) + _bound_target(gen.target, _element_of(gen.iter, scope), scope) + for test in gen.ifs: + _guards(test, scope) + _expression(test, scope) + if isinstance(node, ast.DictComp): + _expression(node.key, scope) + _expression(node.value, scope) + else: + _expression(node.elt, scope) + scope.bindings = before + + +def _guards(test: ast.expr, scope: _Scope) -> None: + """Notes every answer one test stands guard over. + + A guard is the answer read as a truth -- `if worked:` -- or compared against None, + either way round. Noted wherever it appears rather than matched to a branch: what the + warning is for is the answer nobody tested at all. + """ + if isinstance(test, ast.Name): + scope.guarded.add(test.id) + elif isinstance(test, ast.UnaryOp) and isinstance(test.operand, ast.Name): + scope.guarded.add(test.operand.id) + elif isinstance(test, ast.BoolOp): + for one in test.values: + _guards(one, scope) + elif isinstance(test, ast.Compare): + sides = [test.left, *test.comparators] + against_none = any( + isinstance(side, ast.Constant) and side.value is None for side in sides + ) + if against_none: + for side in sides: + if isinstance(side, ast.Name): + scope.guarded.add(side.id) + + +def _asked(node: ast.Attribute, scope: _Scope) -> None: + """One attribute asked of a name, checked where the name is tracked.""" + if not isinstance(node.value, ast.Name) or node.attr.startswith("_"): + return + name = node.value.id + answer = scope.answers.get(name) + if answer is not None and answer.shaped and answer.suppressed: + scope.reads.append((name, node.lineno)) + return + held = scope.bindings.get(name) + if held is None: + return + if isinstance(held, _Crew): + places: frozenset[str] = ( + frozenset(field for field, _ in held.fields) + if held.fields is not None + else frozenset() + ) + if node.attr in places | _OF_A_TUPLE: + return + which = ( + ", ".join(field for field, _ in held.fields) + if held.fields + else "how many there are, and nothing else" + ) + scope.findings.append( + Finding( + "unknown-ask", + "error", + scope.read.where, + node.lineno, + f"the agents have no place called {node.attr!r} -- the flow declares " + f"{which}, and a place it does not declare is not one it was handed", + ) + ) + return + if node.attr in scope.asks.allowed(held): + return + what = " or ".join(f"hmz.flows.{kind.capitalize()}" for kind in sorted(held)) + scope.findings.append( + Finding( + "unknown-ask", + "error", + scope.read.where, + node.lineno, + f"nothing here answers to {node.attr!r} -- what a flow may ask is written " + f"on {what}, and a name that is not there fails at the first turn", + ) + ) + + +# --------------------------------------------------------------------------------------- +# The loop rules: a loop is legal when something inside it can end it. +# --------------------------------------------------------------------------------------- + + +def _yields(node: ast.FunctionDef | ast.AsyncFunctionDef) -> bool: + """Whether one function is a generator, whose loops end where their consumer stops.""" + waiting: list[ast.AST] = list(ast.iter_child_nodes(node)) + while waiting: + held = waiting.pop() + if isinstance(held, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)): + continue + if isinstance(held, (ast.Yield, ast.YieldFrom)): + return True + waiting.extend(ast.iter_child_nodes(held)) + return False + + +class _Exit(NamedTuple): + """One way out of a loop, and the conditions standing between the loop and it.""" + + line: int + conditions: tuple[ast.expr, ...] + + +def _loops( + node: ast.FunctionDef | ast.AsyncFunctionDef, scope: _Scope +) -> Iterator[Finding]: + """Every constant-true loop in one function, read for how it ends. + + Args: + node: The function, with its bindings already followed. + scope: What following them collected. + + Yields: + A `dead-loop` error for one nothing inside can end, a `sleeping-loop` error for one + that only sleeps -- alive from the outside and doing nothing -- and an + `unbounded-loop` warning for one whose every way out waits for an agent to say so, + in a function with no bound of its own. + """ + for loop in _whiles(node.body): + if not (isinstance(loop.test, ast.Constant) and loop.test.value): + continue + exits = _exits(loop.body, ()) + if not exits: + if _sleeps(loop.body): + code = "sleeping-loop" + said = ( + "this loop only sleeps -- from outside it looks alive, and each " + "round does nothing; a loop earns its keep by doing something " + "that can end it" + ) + else: + code = "dead-loop" + said = ( + "this loop cannot end -- no break, no return, no raise inside " + "it; a loop is legal when something inside it can end it" + ) + yield Finding(code, "error", scope.read.where, loop.lineno, said) + continue + if scope.bounded: + continue + shaped = {name for name, answer in scope.answers.items() if answer.shaped} + if all(_by_verdict(one, shaped) for one in exits): + yield Finding( + "unbounded-loop", + "warning", + scope.read.where, + loop.lineno, + "every way out of this loop waits for an agent to say so, and an agent " + "may never say it -- give the loop a bound of its own: a budget read " + "off spent(), a cap on the rounds, a range", + ) + + +def _whiles(body: list[ast.stmt]) -> Iterator[ast.While]: + """Every while loop in one function's own body, nested functions left to themselves.""" + for node in body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + continue + if isinstance(node, ast.While): + yield node + for held in _blocks(node): + yield from _whiles(held) + + +def _blocks(node: ast.stmt) -> Iterator[list[ast.stmt]]: + """The statement blocks one statement holds, whichever shape it is.""" + for name in ("body", "orelse", "finalbody"): + held = getattr(node, name, None) + if isinstance(held, list) and held and isinstance(held[0], ast.stmt): + yield held + for handler in getattr(node, "handlers", []): + yield handler.body + for case in getattr(node, "cases", []): + yield case.body + + +def _exits(body: list[ast.stmt], conditions: tuple[ast.expr, ...]) -> list[_Exit]: + """Every way out of a loop with this body, each with the conditions guarding it. + + A `break` of the loop's own, a `return`, a `raise`: anything that ends the loop or the + function around it. A `break` inside a nested loop ends that loop instead, and is not + one; a `return` inside one still ends the function, and is. + """ + found: list[_Exit] = [] + for node in body: + if isinstance(node, (ast.Break, ast.Return, ast.Raise)): + found.append(_Exit(node.lineno, conditions)) + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + continue + elif isinstance(node, ast.If): + found.extend(_exits(node.body, (*conditions, node.test))) + found.extend(_exits(node.orelse, (*conditions, node.test))) + elif isinstance(node, (ast.While, ast.For, ast.AsyncFor)): + # A break in there is that loop's; a return or a raise is still a way out. + for held in _blocks(node): + found.extend( + one + for one in _exits(held, conditions) + if not _breaks_at(node, one.line) + ) + else: + for held in _blocks(node): + found.extend(_exits(held, conditions)) + return found + + +def _breaks_at(loop: ast.stmt, line: int) -> bool: + """Whether the exit at this line is a break belonging to this nested loop.""" + return any( + isinstance(node, ast.Break) and node.lineno == line + for node in ast.walk(loop) + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + ) + + +def _by_verdict(exit_: _Exit, shaped: set[str]) -> bool: + """Whether one way out waits on a field of what an agent answered. + + The `review.done` shape: a condition reading an attribute off an answer the turn was + held to. A turn merely having landed -- `if said:` -- is not one, since a loop that + takes a failed turn again is bounded by the turns landing, not by what they say. + """ + for condition in exit_.conditions: + for node in ast.walk(condition): + if ( + isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Name) + and node.value.id in shaped + and not node.attr.startswith(("_", "model_")) + ): + return True + return False + + +def _sleeps(body: list[ast.stmt]) -> bool: + """Whether a loop body does nothing but wait: sleeps, passes, and constants.""" + for node in body: + if isinstance(node, (ast.Pass, ast.Continue)): + continue + if isinstance(node, ast.Expr): + value = node.value + if isinstance(value, ast.Constant): + continue + if isinstance(value, ast.Call) and ( + (isinstance(value.func, ast.Name) and value.func.id == "sleep") + or ( + isinstance(value.func, ast.Attribute) and value.func.attr == "sleep" + ) + ): + continue + return False + if isinstance(node, ast.Assign) and isinstance(node.value, ast.Constant): + continue + return False + return True diff --git a/tests/test_checking.py b/tests/test_checking.py new file mode 100644 index 0000000..d1e78ae --- /dev/null +++ b/tests/test_checking.py @@ -0,0 +1,842 @@ +"""The static read of a flow's legality, held to finding what it claims and nothing else. + +Two halves. A fixture or two per rule -- a flow that trips it, and a neighbour standing just +on the legal side -- so that every rule is shown to fire and shown to know where the edge is. +And a sweep over every flow humanize ships and every flow the official flowverse holds: the +rules were written against real flows, and the sweep is the alarm that goes off the day one +of them starts reading a good flow as a bad one. +""" + +from __future__ import annotations + +import textwrap +from typing import TYPE_CHECKING + +import pytest + +import hmz.flows +from hmz.flows import BUILTIN_AT, ENTRY, Agent, Driven, Person, Session, entry +from hmz.flows.checking import checked, offered, surface +from tests.stubs import written + +if TYPE_CHECKING: + from pathlib import Path + +#: What each code is, so that a rule quietly changing its severity is a failing test. +SEVERITY = { + "unread": "error", + "not-a-flow": "error", + "unsized-agents": "error", + "unread-annotation": "error", + "foreign-import": "error", + "unknown-name": "error", + "unknown-ask": "error", + "dead-loop": "error", + "sleeping-loop": "error", + "stateless-resume": "error", + "unbounded-loop": "warning", + "unguarded-answer": "warning", + "unsaid-moment": "warning", + "loose-config": "warning", + "unsaid-field": "warning", + "unsaid-flow": "warning", + "state-kept": "warning", + "twice-named": "warning", +} + +#: What a flow standing just on the legal side of a rule reads as: nothing at all. +CLEAN: set[str] = set() + +#: The one line every fixture flow says about itself, so `unsaid-flow` stays out of the way +#: of every rule but its own. +DOC = '"""A flow written for one rule of the checker."""\n' + +CASES = [ + pytest.param( + DOC + + """ +from hmz.flows import Agent, flow + +@flow +def run(agents: tuple[Agent], task: str) -> None: + (agent,) = agents + while True: + agent(task, suppress=True) +""", + {"dead-loop"}, + id="dead-loop", + ), + pytest.param( + DOC + + """ +from hmz.flows import Agent, flow + +@flow +def run(agents: tuple[Agent], task: str) -> None: + (agent,) = agents + while True: + worked = agent(task, suppress=True) + if not worked: + break +""", + CLEAN, + id="dead-loop-edge-a-break-inside", + ), + pytest.param( + DOC + + """ +import time + +from hmz.flows import Agent, flow + +@flow +def run(agents: tuple[Agent], task: str) -> None: + while True: + time.sleep(5) +""", + {"sleeping-loop"}, + id="sleeping-loop", + ), + pytest.param( + DOC + + """ +import time + +from hmz.flows import Agent, flow + +@flow +def run(agents: tuple[Agent], task: str) -> None: + while True: + time.sleep(5) + break +""", + CLEAN, + id="sleeping-loop-edge-it-can-end", + ), + pytest.param( + DOC + + """ +from hmz.flows import Agent, flow + +@flow +def run(agents: tuple[Agent], task: str) -> None: + (agent,) = agents + agent.launch(task) +""", + {"unknown-ask"}, + id="unknown-ask", + ), + pytest.param( + DOC + + """ +from hmz.flows import Agent, flow + +@flow +def run(agents: tuple[Agent], task: str) -> None: + (agent,) = agents + print(agent.spent().output) + session = agent.new() + session(task, suppress=True) + session.close() +""", + CLEAN, + id="unknown-ask-edge-the-interface", + ), + pytest.param( + DOC + + """ +from typing import NamedTuple + +from hmz.flows import Agent, Person, flow + +class Crew(NamedTuple): + actor: Agent + human: Person + +@flow +def run(agents: Crew, task: str) -> None: + agents.reviewer(task) +""", + {"unknown-ask"}, + id="unknown-ask-a-place-not-declared", + ), + pytest.param( + DOC + + """ +from typing import NamedTuple + +from hmz.flows import Agent, Person, flow + +class Crew(NamedTuple): + actor: Agent + human: Person + +@flow +def run(agents: Crew, task: str) -> None: + agents.actor(task, suppress=True) + agents.human.board.put("doing", task) +""", + CLEAN, + id="unknown-ask-edge-the-places-declared", + ), + pytest.param( + DOC + + """ +from hmz.flows import Agent, flow + +@flow +def run(agents: tuple[Agent], task: str) -> None: + session = agents[0].new() + session.rewind() +""", + {"unknown-ask"}, + id="unknown-ask-of-a-session", + ), + pytest.param( + DOC + + """ +from hmz.agents import Moment + +from hmz.flows import Agent, flow + +@flow +def run(agents: tuple[Agent], task: str) -> None: + agents[0](task) +""", + {"foreign-import"}, + id="foreign-import", + ), + pytest.param( + DOC + + """ +import hmz.backends + +from hmz.flows import Agent, flow + +@flow +def run(agents: tuple[Agent], task: str) -> None: + agents[0](task) +""", + {"foreign-import"}, + id="foreign-import-a-module", + ), + pytest.param( + DOC + + """ +from hmz.flows import Agent, Moment, Usage, backends, flow, home + +@flow +def run(agents: tuple[Agent], task: str) -> None: + agents[0](task) +""", + CLEAN, + id="foreign-import-edge-the-one-import", + ), + pytest.param( + DOC + + """ +from hmz.flows import Agent, flow, teleport + +@flow +def run(agents: tuple[Agent], task: str) -> None: + agents[0](task) +""", + {"unknown-name"}, + id="unknown-name", + ), + pytest.param( + DOC + + """ +from hmz.flows import flow + +@flow +def run(agents, task): + agents[0](task) +""", + {"unsized-agents"}, + id="unsized-agents-nothing-said", + ), + pytest.param( + DOC + + """ +from hmz.flows import Agent, flow + +@flow +def run(agents: tuple, task: str) -> None: + agents[0](task) +""", + {"unsized-agents"}, + id="unsized-agents-a-bare-tuple", + ), + pytest.param( + DOC + + """ +from hmz.flows import Agent, flow + +@flow +def run(agents: tuple[Agent, ...], task: str) -> None: + agents[0](task) +""", + {"unsized-agents"}, + id="unsized-agents-any-number", + ), + pytest.param( + DOC + + """ +from hmz.flows import Agent, flow + +@flow +def run(agents: tuple[Agent, Agent], task: str) -> None: + agents[0](task) + agents[1](task) +""", + CLEAN, + id="unsized-agents-edge-a-fixed-length", + ), + pytest.param( + DOC + + """ +from typing import TYPE_CHECKING + +from hmz.flows import flow + +if TYPE_CHECKING: + from hmz.flows import Agent + +@flow +def run(agents: "tuple[Agent]", task: str) -> None: + agents[0](task) +""", + {"unread-annotation"}, + id="unread-annotation", + ), + pytest.param( + DOC + + """ +from typing import TYPE_CHECKING + +from hmz.flows import Agent, flow + +if TYPE_CHECKING: + from hmz.flows import Agent + +@flow +def run(agents: "tuple[Agent]", task: str) -> None: + agents[0](task) +""", + CLEAN, + id="unread-annotation-edge-also-at-runtime", + ), + pytest.param( + DOC + + """ +from hmz.flows import Agent, flow + +@flow(resumable=True) +def run(agents: tuple[Agent], task: str) -> None: + agents[0](task) +""", + {"stateless-resume"}, + id="stateless-resume", + ), + pytest.param( + DOC + + """ +from hmz.flows import Agent, flow +from pydantic import BaseModel, Field + +class Config(BaseModel): + model_config = {"extra": "forbid"} + + budget: float = Field(default=1.0, description="the bound") + +@flow(resumable=True) +def run(agents: tuple[Agent], task: str, config: Config | None = None) -> None: + agents[0](task) +""", + {"stateless-resume"}, + id="stateless-resume-a-config-in-the-way", + ), + pytest.param( + DOC + + """ +from typing import Any + +from hmz.flows import Agent, flow + +@flow(resumable=True) +def run(agents: tuple[Agent], task: str, state: dict[str, Any]) -> None: + agents[0](task) +""", + CLEAN, + id="stateless-resume-edge-the-state-third", + ), + pytest.param( + DOC + + """ +import time + +from hmz.flows import Agent, flow +from pydantic import BaseModel, Field + +class Review(BaseModel): + model_config = {"extra": "forbid"} + + done: bool = Field(description="whether it is over") + +@flow +def run(agents: tuple[Agent, Agent], task: str) -> None: + working = agents[0].new() + prompt = task + while True: + worked = working(prompt, suppress=True) + if worked: + review = agents[1](task, suppress=True, schema=Review) + if review is not None and review.done: + return + time.sleep(5) +""", + {"unbounded-loop"}, + id="unbounded-loop", + ), + pytest.param( + DOC + + """ +import time + +from hmz.flows import Agent, flow +from pydantic import BaseModel, Field + +class Review(BaseModel): + model_config = {"extra": "forbid"} + + done: bool = Field(description="whether it is over") + +@flow +def run(agents: tuple[Agent, Agent], task: str) -> None: + working = agents[0].new() + while True: + working(task, suppress=True) + review = agents[1](task, suppress=True, schema=Review) + if review is not None and review.done: + return + if agents[0].spent().output > 1_000_000: + return + time.sleep(5) +""", + CLEAN, + id="unbounded-loop-edge-a-budget-bound", + ), + pytest.param( + DOC + + """ +import time + +from hmz.flows import Agent, flow + +@flow +def run(agents: tuple[Agent], task: str) -> None: + (agent,) = agents + while True: + said = agent(task, suppress=True) + if said: + return + time.sleep(5) +""", + CLEAN, + id="unbounded-loop-edge-a-turn-merely-landing", + ), + pytest.param( + DOC + + """ +from hmz.flows import Agent, flow +from pydantic import BaseModel, Field + +class Review(BaseModel): + model_config = {"extra": "forbid"} + + done: bool = Field(description="whether it is over") + +@flow +def run(agents: tuple[Agent], task: str) -> None: + review = agents[0](task, suppress=True, schema=Review) + print(review.done) +""", + {"unguarded-answer"}, + id="unguarded-answer", + ), + pytest.param( + DOC + + """ +from hmz.flows import Agent, flow +from pydantic import BaseModel, Field + +class Review(BaseModel): + model_config = {"extra": "forbid"} + + done: bool = Field(description="whether it is over") + +@flow +def run(agents: tuple[Agent], task: str) -> None: + review = agents[0](task, suppress=True, schema=Review) + if review is not None and review.done: + print("done") +""", + CLEAN, + id="unguarded-answer-edge-guarded", + ), + pytest.param( + DOC + + """ +from hmz.flows import Agent, Moment, flow + +@flow +def run(agents: tuple[Agent], task: str) -> None: + agents[0].hooks.on(Moment.PERMISSION_REQUEST, print) +""", + {"unsaid-moment"}, + id="unsaid-moment", + ), + pytest.param( + DOC + + """ +from typing import Annotated, NamedTuple + +from hmz.flows import Agent, Moment, flow + +class Crew(NamedTuple): + builder: Annotated[Agent, Moment.PERMISSION_REQUEST] + +@flow +def run(agents: Crew, task: str) -> None: + agents.builder.hooks.on(Moment.PERMISSION_REQUEST, print) +""", + CLEAN, + id="unsaid-moment-edge-the-place-declares-it", + ), + pytest.param( + DOC + + """ +from hmz.flows import Agent, Moment, flow + +@flow +def run(agents: tuple[Agent], task: str) -> None: + agents[0].hooks.on(Moment.STOP, print) +""", + CLEAN, + id="unsaid-moment-edge-a-moment-every-backend-runs", + ), + pytest.param( + DOC + + """ +from hmz.flows import Agent, flow +from pydantic import BaseModel, Field + +class Config(BaseModel): + budget: float = Field(default=1.0, description="the bound") + +@flow +def run(agents: tuple[Agent], task: str, config: Config | None = None) -> None: + agents[0](task) +""", + {"loose-config"}, + id="loose-config", + ), + pytest.param( + DOC + + """ +from hmz.flows import Agent, flow +from pydantic import BaseModel, Field + +class Config(BaseModel): + model_config = {"frozen": True} + + budget: float = Field(default=1.0, description="the bound") + +@flow +def run(agents: tuple[Agent], task: str, config: Config | None = None) -> None: + agents[0](task) +""", + CLEAN, + id="loose-config-edge-frozen", + ), + pytest.param( + DOC + + """ +from hmz.flows import Agent, flow +from pydantic import BaseModel + +class Config(BaseModel): + model_config = {"extra": "forbid"} + + budget: float + +@flow +def run(agents: tuple[Agent], task: str, config: Config | None = None) -> None: + agents[0](task) +""", + {"unsaid-field"}, + id="unsaid-field", + ), + pytest.param( + DOC + + """ +from hmz.flows import Agent, flow +from pydantic import BaseModel + +class Answer(BaseModel): + said: str + +@flow +def run(agents: tuple[Agent], task: str) -> None: + agents[0](task, suppress=True, schema=Answer) +""", + CLEAN, + id="unsaid-field-edge-a-schema-is-not-a-config", + ), + pytest.param( + """ +from hmz.flows import Agent, flow + +@flow +def run(agents: tuple[Agent], task: str) -> None: + agents[0](task) +""", + {"unsaid-flow"}, + id="unsaid-flow", + ), + pytest.param( + DOC + + """ +from typing import Any + +from hmz.flows import Agent, flow + +@flow(resumable=True) +def run(agents: tuple[Agent], task: str, state: dict[str, Any]) -> None: + kept = state + kept["rounds"] = kept.get("rounds", 0) + 1 + agents[0](task) +""", + {"state-kept"}, + id="state-kept", + ), + pytest.param( + DOC + + """ +from typing import Any + +from hmz.flows import Agent, flow + +@flow(resumable=True) +def run(agents: tuple[Agent], task: str, state: dict[str, Any]) -> None: + kept = state + kept["rounds"] = kept.get("rounds", 0) + 1 + agents[0](task) + kept.clear() +""", + CLEAN, + id="state-kept-edge-cleared", + ), + pytest.param( + DOC + + """ +from hmz.flows import Agent, flow + +@flow(name="draft") +def one(agents: tuple[Agent], task: str) -> None: + agents[0](task) + +@flow(name="draft") +def two(agents: tuple[Agent], task: str) -> None: + agents[0](task) +""", + {"twice-named"}, + id="twice-named", + ), + pytest.param( + DOC + + """ +from hmz.flows import Agent, flow + +@flow(name="draft") +def one(agents: tuple[Agent], task: str) -> None: + agents[0](task) + +@flow(name="check") +def two(agents: tuple[Agent], task: str) -> None: + agents[0](task) +""", + CLEAN, + id="twice-named-edge-two-names", + ), + pytest.param( + DOC + + """ +def run(agents, task): + return None +""", + {"not-a-flow"}, + id="not-a-flow", + ), + pytest.param( + "def run(:\n", + {"unread"}, + id="unread", + ), +] + + +@pytest.mark.parametrize(("source", "expected"), CASES) +def test_each_rule_fires_and_knows_the_edge( + tmp_path: Path, source: str, expected: set[str] +) -> None: + at = written(tmp_path, "one", textwrap.dedent(source)) + found = checked(at) + assert {one.code for one in found} == expected, found + for one in found: + assert one.severity == SEVERITY[one.code] + assert one.line >= 0 + + +def test_a_flow_that_is_not_there_is_not_a_flow(tmp_path: Path) -> None: + found = checked(tmp_path / "nowhere") + assert [one.code for one in found] == ["not-a-flow"] + + +def test_a_single_file_flow_is_read_as_one(tmp_path: Path) -> None: + at = tmp_path / "alone.py" + at.write_text( + textwrap.dedent( + ''' + """A flow that is one file.""" + + from hmz.flows import Agent, flow + + @flow + def run(agents: tuple[Agent], task: str) -> None: + while True: + agents[0](task, suppress=True) + ''' + ) + ) + assert [one.code for one in checked(at)] == ["dead-loop"] + + +def test_what_is_under_skills_is_not_read(tmp_path: Path) -> None: + """A skill may ship a helper script, and it is the agents' content, not the flow's.""" + at = written( + tmp_path, + "one", + DOC + + textwrap.dedent( + """ + from hmz.flows import Agent, flow + + @flow + def run(agents: tuple[Agent], task: str) -> None: + agents[0](task) + """ + ), + skills={"helping": "# How to help\n"}, + ) + beside = at / "skills" / "helping" / "helper.py" + beside.write_text("import hmz.backends\nwhile True:\n pass\n") + assert checked(at) == () + + +def test_a_finding_says_which_file_beside_the_entry_it_is_in(tmp_path: Path) -> None: + at = written( + tmp_path, + "one", + DOC + + textwrap.dedent( + """ + from hmz.flows import Agent, flow + + @flow + def run(agents: tuple[Agent], task: str) -> None: + agents[0](task) + """ + ), + ) + (at / "helper.py").write_text( + textwrap.dedent( + ''' + """What the flow imports beside itself.""" + + def churn() -> None: + while True: + print("round and round") + ''' + ) + ) + found = checked(at) + assert [(one.code, one.where.name) for one in found] == [ + ("dead-loop", "helper.py") + ] + + +def test_the_surface_is_the_interfaces_themselves() -> None: + """What an agent may be asked is read off `agent.py`, not kept as a second list.""" + assert {"new", "clone", "spent", "hooks", "cycle", "__call__"} <= surface(Agent) + assert "board" not in surface(Agent) + assert "board" in surface(Person) + assert {"loads", "close", "stream"} <= surface(Session) + assert "rename" not in surface(Agent) + assert "rename" in surface(Driven) + + +def test_everything_offered_is_reachable() -> None: + """`offered` is what `unknown-name` trusts, so a name in it nothing answers is a lie.""" + said = offered() + assert {"flow", "Agent", "Moment", "home", "models", "backends"} <= said + assert "ClaudeCodeAgent" not in said + for name in sorted(said): + assert getattr(hmz.flows, name, None) is not None, name + + +#: Every warning a flow humanize ships or the official flowverse holds is allowed to keep. +#: rlar's loop is ended by its reviewer alone, which is the flow's own documented shape -- +#: and exactly the shape the checker exists to point at, so the warning stands. +ALLOWED_WARNINGS = {"rlar": {"unbounded-loop"}} + + +def _swept() -> list[object]: + """One parameter per flow humanize ships or the official flowverse holds now.""" + places = [("builtin", BUILTIN_AT)] + for verse in hmz.flows.flowverses(): + if verse.name == hmz.flows.OFFICIAL and verse.fetched: + places.append((verse.name, hmz.flows.holds(verse))) + held: list[object] = [] + seen_official = False + for whose, under in places: + seen_official = seen_official or whose == hmz.flows.OFFICIAL + for name in hmz.flows.offered(under): + at = entry(under, name) + if at is None: + continue + target = at.parent if at.name == ENTRY else at + held.append(pytest.param(target, name, id=f"{whose}/{name}")) + if not seen_official: + held.append( + pytest.param( + None, + "official", + id="official/unfetched", + marks=pytest.mark.skip( + reason="the official flowverse has not been fetched here" + ), + ) + ) + return held + + +@pytest.mark.parametrize(("at", "name"), _swept()) +def test_every_flow_humanize_offers_reads_clean(at: Path, name: str) -> None: + """The false-positive alarm: real flows, and exactly the warnings they are allowed.""" + found = checked(at) + errors = [one for one in found if one.severity == "error"] + assert not errors, errors + warned = {one.code for one in found if one.severity == "warning"} + assert warned == ALLOWED_WARNINGS.get(name, set()), found diff --git a/tests/test_flow_interface.py b/tests/test_flow_interface.py index 574bd84..3878c64 100644 --- a/tests/test_flow_interface.py +++ b/tests/test_flow_interface.py @@ -11,7 +11,7 @@ from __future__ import annotations import ast -from typing import TYPE_CHECKING, Protocol +from typing import TYPE_CHECKING import pytest @@ -20,6 +20,7 @@ from hmz.agents import Unrecoverable as AgentUnrecoverable from hmz.flows import BUILTIN_AT, Agent, Person, Session from hmz.flows import Unrecoverable as FlowUnrecoverable +from hmz.flows.checking import surface if TYPE_CHECKING: from collections.abc import Iterator @@ -72,28 +73,6 @@ def test_an_unrecoverable_turn_is_the_same_exception_a_flow_can_catch() -> None: assert FlowUnrecoverable is AgentUnrecoverable -def _members(protocol: type) -> set[str]: - """What one interface asks for, by name. - - Args: - protocol: The interface. - - Returns: - One name per member it declares, its own and whatever it is itself an interface of. - `__call__` is among them where it is declared, an agent and a session both being things - a flow calls; the rest of the dunders are Python's and are not part of the contract. - """ - said: set[str] = set() - for one in protocol.__mro__: - if one in (object, Protocol): - continue - held = set(vars(one)) | set(getattr(one, "__annotations__", {})) - said.update( - name for name in held if not name.startswith("_") or name == "__call__" - ) - return said - - def _answers(driver: object) -> set[str]: """What one driver has, by name. @@ -121,7 +100,7 @@ def test_the_drivers_answer_to_what_a_flow_drives() -> None: (Person, person), (Session, person.new()), ): - missing = {one for one in _members(interface) if one not in _answers(driver)} + missing = {one for one in surface(interface) if one not in _answers(driver)} assert not missing, ( f"{type(driver).__name__} does not answer to {interface.__name__}: {missing}" ) From a09797355d6656aec1ae434fdf4647ecea5bb4cd Mon Sep 17 00:00:00 2001 From: dongyun Date: Fri, 28 Aug 2026 21:42:44 +0000 Subject: [PATCH 2/8] feat(flows): add proving -- a flow driven by stubs against a clock The second of the two readings: the flow loaded and driven for real, in a subprocess per scenario, by stubs that claim every capability over the real driver base classes -- so the hooks a flow hangs fire as they would, every turn lands at once, and each costs what the scenario says. NEVER_DONE is the reviewer that never says the work is done: a loop with a bound of its own walks to the end of it in milliseconds, and one without is caught by the turn cap or killed by the parent's clock -- the executable proof that a run can end. SILENT answers every turn with nothing, which is every guard tried at once. A refused load comes back as a finding rather than a raise, and the config rules run again on the model the loading actually resolved. The proof's world sleeps for free and works in a scratch directory taken away with the process. Co-Authored-By: Claude Fable 5 --- src/hmz/flows/SPEC.md | 67 ++++ src/hmz/flows/__init__.py | 8 + src/hmz/flows/proving.py | 628 ++++++++++++++++++++++++++++++++++++++ tests/test_checking.py | 12 +- tests/test_proving.py | 382 +++++++++++++++++++++++ 5 files changed, 1091 insertions(+), 6 deletions(-) create mode 100644 src/hmz/flows/proving.py create mode 100644 tests/test_proving.py diff --git a/src/hmz/flows/SPEC.md b/src/hmz/flows/SPEC.md index f2023d8..f4c2473 100644 --- a/src/hmz/flows/SPEC.md +++ b/src/hmz/flows/SPEC.md @@ -9,6 +9,7 @@ ├── builtin ├── checking.py ├── driving.py +├── proving.py ├── skills.py └── verses.py ``` @@ -477,6 +478,72 @@ own. is `offered`: the checker states what the interface is, so a second copy of either would be the drift it checks for. +## `proving.py` + +```python +class Scenario(NamedTuple): + name: str + verdict: bool | None + answer: str + climb: float = 100_000.0 + turns: int = 200 + seconds: float = 60.0 + + +NEVER_DONE: Scenario +ALWAYS_DONE: Scenario +SILENT: Scenario + + +class Outcome(NamedTuple): + scenario: str + finished: bool + turns: int + said: str + + +class Proof(NamedTuple): + findings: tuple[Finding, ...] + outcomes: tuple[Outcome, ...] + + +def proved( + flow: str | os.PathLike[str], + *, + name: str = "", + config: Mapping[str, object] | None = None, + scenarios: tuple[Scenario, ...] = (NEVER_DONE, ALWAYS_DONE), +) -> Proof: ... +``` + +The second of the two readings: the flow loaded and driven for real, by stubs, so that what +only running the file can show is shown -- and shown in milliseconds, since every turn lands +at once and costs what the scenario says. + +- The flow MUST be run in a process of its own, one per scenario, and the clock MUST be held + by the asking process: loading a flow means running its file, and a flow that hangs, spins + or corrupts what it touches must be able to be killed without taking the checker with it. + Nothing of the flow MUST execute in the asking process. +- Every proof MUST end. A flow that takes turns is ended by the cap on them, one that takes + none by the clock, and which of the two it was MUST be said in the outcome: they are the + two ways a flow fails to stop, and the fix is different. +- The stubs MUST claim every capability there is -- every moment, a goal feature, shapes, + tools -- since what is on trial is the flow and not the agents: refusing an agent that + cannot fill a place is the loading's job, done where the agents are real. Beneath the + claims they MUST be the real driver base classes, so that the hooks a flow hangs fire as + they would under a real backend, and a `Stop` hook that refuses is a counted turn. +- A scenario MUST answer deterministically, whatever it is asked: every boolean field of a + shaped answer says its verdict, every string field says its answer, and a verdict of None + is a turn that answers nothing -- which is what a failed turn answers, so the silent + scenario is every guard tried at once. `NEVER_DONE` MUST be among the default scenarios: + the reviewer that never says done is the question every loop must have an answer to. +- The world a proof runs in MUST sleep for free and MUST work in a scratch directory taken + away with the process: the rest a loop takes between rounds and the files it writes while + being proved are no part of its shape. +- A flow the loading refuses MUST come back as a `refused-load` finding rather than a raise, + and the config rules MUST be run again on the model the loading actually resolved: a model + built out of the static reading's sight is still the one whoever sets the flow up meets. + ## `verses.py` ```python diff --git a/src/hmz/flows/__init__.py b/src/hmz/flows/__init__.py index 115ee9c..cb3c25a 100644 --- a/src/hmz/flows/__init__.py +++ b/src/hmz/flows/__init__.py @@ -66,6 +66,7 @@ def run(agents: tuple[Agent, Agent], task: str) -> None: running, wanted, ) +from .proving import ALWAYS_DONE, NEVER_DONE, SILENT, Outcome, Proof, Scenario, proved from .verses import ( BUILTIN, FLOWS, @@ -111,14 +112,17 @@ def run(agents: tuple[Agent, Agent], task: str) -> None: from hmz.backends import Model, Profile __all__ = [ + "ALWAYS_DONE", "BUILTIN", "BUILTIN_AT", "ENTRY", "EVERYWHERE", "FLOWS", "LOCAL", + "NEVER_DONE", "OFFICIAL", "PERMISSIONS", + "SILENT", "SWARM", "USER", "WINDOW", @@ -142,12 +146,15 @@ def run(agents: tuple[Agent, Agent], task: str) -> None: "NotAFlow", "Occasion", "Offer", + "Outcome", "Person", "Place", "Profile", + "Proof", "Question", "Remote", "Running", + "Scenario", "Session", "Stopped", "Unhooked", @@ -178,6 +185,7 @@ def run(agents: tuple[Agent, Agent], task: str) -> None: "nearest", "offered", "offers", + "proved", "resumes", "running", "wanted", diff --git a/src/hmz/flows/proving.py b/src/hmz/flows/proving.py new file mode 100644 index 0000000..b2cc5d5 --- /dev/null +++ b/src/hmz/flows/proving.py @@ -0,0 +1,628 @@ +"""A flow driven by stubs against a clock: the reading only running the file can give. + +:mod:`hmz.flows.checking` reads a flow without running it, and some of what a flow is only +running can show -- the annotation built at runtime, the config model declared in a helper, +the loop that looks bounded and is not. This is that second reading. The flow is loaded and +driven for real, in a subprocess of its own, by agents that are stubs: every turn lands at +once, answers deterministically, and costs what the scenario says a turn costs -- so a loop +held to a budget walks to the end of it in milliseconds, and what is being proved is the +flow's own shape rather than any model's mood. + +The scenarios are the questions worth asking of a loop. `NEVER_DONE` is the reviewer that +never says the work is done: a flow with a bound of its own still ends, and one without is +caught by the turn cap or killed by the clock -- which is the executable proof that a run of +it can end. `ALWAYS_DONE` is the shortest road through. `SILENT` answers every turn with +nothing, which is what a turn that failed answers, so a flow that reads a field off an +unguarded answer falls over here rather than at hour three. + +A subprocess per scenario, because loading a flow means running its file: whatever it does as +it is read -- imports, prints, mistakes -- happens in a process built to be killed, the parent +holds the clock, and nothing of the flow outlives its own proof. +""" + +from __future__ import annotations + +import inspect +import json +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import ( + TYPE_CHECKING, + Any, + ClassVar, + Literal, + NamedTuple, + cast, + get_args, + get_origin, +) + +from .checking import Finding + +if TYPE_CHECKING: + import os + from collections.abc import Iterator, Mapping, Sequence + + from pydantic import BaseModel + + from hmz.agents import AgentBase, Event + + from .driving import Place + +__all__ = [ + "ALWAYS_DONE", + "NEVER_DONE", + "SILENT", + "Outcome", + "Proof", + "Scenario", + "proved", +] + + +class Scenario(NamedTuple): + """One way the world answers a flow, held constant for the length of a proof. + + Attributes: + name: What the scenario is called, which is what its outcome is filed under. + verdict: What every boolean field of a shaped answer says -- False is the reviewer + that never says done -- or None for a turn that answers with nothing at all, which + is what a failed turn answers. + answer: What a plain turn answers, and what every string field of a shaped one says. + climb: What each turn adds to what the agent has spent, in output tokens, so that a + loop held to a budget walks to the end of it in a handful of turns. + turns: How many turns the flow may take before it is read as one that does not stop. + seconds: How long the scenario's process may live before the clock kills it. + """ + + name: str + verdict: bool | None + answer: str + climb: float = 100_000.0 + turns: int = 200 + seconds: float = 60.0 + + +#: The reviewer that never says the work is done. A flow with a bound of its own -- a +#: budget, a cap on the rounds -- still ends here, and one that waits forever on a verdict +#: is caught by the turn cap: the executable proof that a run of it can end. +NEVER_DONE = Scenario("never-done", verdict=False, answer="did some of it") + +#: The shortest road through: every verdict is yes, so what is proved is that the flow can +#: end the way it means to. +ALWAYS_DONE = Scenario("always-done", verdict=True, answer="did it") + +#: Every turn answers with nothing, which is what a failed turn answers: a flow that reads +#: a field off an answer nobody guarded falls over here rather than hours into a run. +SILENT = Scenario("silent", verdict=None, answer="") + + +class Outcome(NamedTuple): + """How one scenario ended. + + Attributes: + scenario: Which scenario it was. + finished: Whether the flow ended on its own -- returned, or raised what it meant to. + turns: How many turns it took, as far as that was counted. + said: Why it did not finish, for one that did not: the clock, the turn cap, or the + tail of what it raised. "" for one that did. + """ + + scenario: str + finished: bool + turns: int + said: str + + +class Proof(NamedTuple): + """What driving one flow against the scenarios showed. + + Attributes: + findings: What loading it refused or the live reading found, in the same shape the + static reading answers with -- `refused-load` for a flow `driving.py` would not + take, and the config findings only the declared model itself can show. + outcomes: One per scenario, in the order they were asked. + """ + + findings: tuple[Finding, ...] + outcomes: tuple[Outcome, ...] + + +#: How long the load-only proof is given, there being no scenario to say. +_PATIENCE = 60.0 + +#: What the stubbed flow is driven with. Constant, so a proof is a proof of the flow: what +#: the task says cannot matter to agents that answer the same thing whatever they are told. +_TASK = "the task this proof drives the flow on" + + +def proved( + flow: str | os.PathLike[str], + *, + name: str = "", + config: Mapping[str, object] | None = None, + scenarios: tuple[Scenario, ...] = (NEVER_DONE, ALWAYS_DONE), +) -> Proof: + """Loads a flow in a subprocess and drives it with stubs, once per scenario. + + Args: + flow: The flow: its directory, its file, or the name `-f` takes. + name: Which of the flows the file holds, or "" for the one it holds under its own + name -- the half after the colon, for whoever has it separately. + config: What to set the flow up with, read back through the flow's own model exactly + as a run of it would, or None for a flow left to its defaults. + scenarios: The worlds to drive it against, each in a process of its own. Empty proves + only that it loads: the flow is declared and its config model read, and nothing + takes a turn. + + Returns: + The findings and one outcome per scenario. A finding is something to fix; an outcome + that did not finish is a flow that could not end in that world, said with why. + """ + from . import find, inside + + # Resolved here, where names still mean what the caller meant: the child runs in a + # scratch directory of its own, against which a relative path names nothing. + at = find(str(flow)) + wanted = name or inside(str(flow)) + where = Path(at) + findings: list[Finding] = [] + outcomes: list[Outcome] = [] + seen: set[tuple[str, str]] = set() + asked: tuple[Scenario | None, ...] = scenarios or (None,) + for scenario in asked: + told = _asked(at, wanted, config, scenario) + if isinstance(told, Outcome): + outcomes.append(told) + continue + for one in told.get("findings", ()): + key = (str(one["code"]), str(one["said"])) + if key not in seen: + seen.add(key) + severity: Literal["error", "warning"] = ( + "error" if one["severity"] == "error" else "warning" + ) + findings.append(Finding(key[0], severity, where, 0, key[1])) + refused = told.get("refused") + if refused is not None: + key = ("refused-load", str(refused)) + if key not in seen: + seen.add(key) + findings.append(Finding(key[0], "error", where, 0, key[1])) + if scenario is not None: + outcomes.append( + Outcome( + scenario.name, + finished=False, + turns=0, + said="nothing ran: the flow could not be loaded", + ) + ) + continue + if scenario is not None: + outcomes.append( + Outcome( + scenario.name, + finished=bool(told.get("finished")), + turns=int(told.get("turns", 0)), + said=str(told.get("said", "")), + ) + ) + return Proof(tuple(findings), tuple(outcomes)) + + +def _asked( + flow: str, + name: str, + config: Mapping[str, object] | None, + scenario: Scenario | None, +) -> dict[str, Any] | Outcome: + """One scenario, asked of a child process holding the clock over it. + + Args: + flow: The flow, as :func:`proved` was given it. + name: Which of the file's flows. + config: What to set it up with, or None. + scenario: The world to drive it in, or None to only load it. + + Returns: + What the child answered, or the outcome of a child that could not answer: one the + clock killed, or one that died without saying why in the one line this reads. + """ + called = scenario.name if scenario is not None else "" + spec = json.dumps( + { + "name": name, + "config": dict(config) if config is not None else None, + "scenario": scenario._asdict() if scenario is not None else None, + } + ) + patience = scenario.seconds if scenario is not None else _PATIENCE + # A scratch directory to work in, taken away with the process: what a flow writes while + # it is being proved is part of the proof, not part of anybody's repository. + with tempfile.TemporaryDirectory(prefix="hmz-proving-") as scratch: + try: + done = subprocess.run( + [sys.executable, "-m", "hmz.flows.proving", flow, spec], + capture_output=True, + text=True, + check=False, + timeout=patience, + cwd=scratch, + ) + except subprocess.TimeoutExpired: + return Outcome( + called, + finished=False, + turns=0, + said=f"still running after {patience:g}s -- nothing inside the flow " + "ended it, so the clock did", + ) + # The last line that is the child's: a flow prints whatever it prints, so the answer is + # found from the end rather than trusted to be alone. + for line in reversed(done.stdout.splitlines()): + try: + held = json.loads(line) + except ValueError: + continue + if isinstance(held, dict) and "proving" in held: + return cast("dict[str, Any]", held["proving"]) + tail = "\n".join(done.stderr.strip().splitlines()[-3:]) + return Outcome( + called, + finished=False, + turns=0, + said=f"the flow's process ended without answering -- {tail or 'and said nothing'}", + ) + + +# --------------------------------------------------------------------------------------- +# The child: loads the flow, builds the stubs, and drives it. Run as `-m hmz.flows.proving` +# with the flow and the scenario as its two arguments, and answers with one JSON line. +# --------------------------------------------------------------------------------------- + + +def _rested(seconds: float) -> None: + """A sleep that has already happened, which is what the stubs' world does with rests. + + Args: + seconds: How long the flow meant to wait, which the proof does not. + """ + del seconds + + +class _Enough(BaseException): + """The turn cap, raised past everything a flow catches: a proof is over when it is. + + A `BaseException`, so that a flow's own `except Exception` -- which is a fine thing for + a loop to write around a turn -- does not swallow the one thing that ends its proof. + """ + + +class _Steps: + """The turns taken so far, shared by every stub of one proof.""" + + def __init__(self, cap: int) -> None: + self.cap = cap + self.taken = 0 + + def step(self) -> None: + """Counts one turn, and ends the proof on the one past the cap.""" + self.taken += 1 + if self.taken > self.cap: + raise _Enough + + +def _driven(flow: str, spec: dict[str, Any]) -> dict[str, Any]: + """Loads one flow and, given a scenario, drives it with stubs to whatever end. + + Args: + flow: The flow, as the parent was given it. + spec: The parent's ask: the name inside the file, the config, and the scenario -- + or None for a proof that only loads. + + Returns: + What the parent folds into the proof: `refused` for a flow that would not load, + `findings` off the live config model, and how driving it went. + """ + import time + + from .driving import NotAFlow, declares, set_up + + # A proof's world sleeps for free. The rest a loop takes between rounds is part of its + # manners and no part of its shape, and it is the shape on trial: a loop that rests + # five seconds a round is not five hundred seconds more legal than one that does not. + # Patched before the flow is even loaded, so a `from time import sleep` reads this one. + time.sleep = _rested + + named = f"{flow}:{spec['name']}" if spec["name"] else flow + scenario = Scenario(**spec["scenario"]) if spec["scenario"] else None + try: + run, places, make, setting, mark = declares(named) + except NotAFlow as refused: + return {"refused": str(refused)} + except BaseException as raised: # noqa: BLE001 -- reported, in a process built for it + return {"refused": f"the flow's own file raised as it was read -- {raised}"} + answered: dict[str, Any] = {"findings": _styled(setting)} + given = None + if spec["config"] is not None: + try: + given = set_up(named, setting, spec["config"]) + except NotAFlow as refused: + answered["refused"] = str(refused) + return answered + if scenario is None: + return answered + steps = _Steps(scenario.turns) + settings = () if setting is None else (given,) + held: tuple[dict[str, Any], ...] = ({},) if mark.resumable else () + try: + out = run(make(_crewed(places, scenario, steps)), _TASK, *settings, *held) + if inspect.isawaitable(out): + import asyncio + + asyncio.run(_awaited(out)) + finished, turns, said = True, steps.taken, "" + except _Enough: + finished, turns = False, scenario.turns + said = ( + f"still going after {scenario.turns} turns -- nothing inside the flow " + "ended it, and a loop is legal when something inside it can end it" + ) + except BaseException: # noqa: BLE001 -- the flow's own crash is the outcome + import traceback + + tail = traceback.format_exc().strip().splitlines() + finished, turns, said = False, steps.taken, "\n".join(tail[-3:]) + answered.update(finished=finished, turns=turns, said=said) + return answered + + +async def _awaited(out: Any) -> None: + """One awaitable flow, awaited: what `asyncio.run` takes is a coroutine.""" + await out + + +def _styled(setting: type[BaseModel] | None) -> list[dict[str, str]]: + """The config findings only the live model can show, said as the static reading says. + + The model a flow declares may be built anywhere -- a helper module, a call -- and the + static reading only checks the ones written in plain sight. This is the same two rules + against the model `declares` actually resolved. + + Args: + setting: The model, or None for a flow that takes no setting up. + + Returns: + One finding per thing found, as plain values for the one JSON line home. + """ + if setting is None: + return [] + found: list[dict[str, str]] = [] + config = setting.model_config + if not (config.get("extra") == "forbid" or config.get("frozen") is True): + found.append( + { + "code": "loose-config", + "severity": "warning", + "said": "the config takes anything -- set model_config to extra: " + "forbid or frozen: True, so a setting that is misspelled is refused " + "rather than quietly ignored", + } + ) + for name, field in setting.model_fields.items(): + if not field.description: + found.append( + { + "code": "unsaid-field", + "severity": "warning", + "said": f"the config field {name!r} says nothing about itself -- " + "give it a Field(description=...), which is what whoever sets the " + "flow up is shown", + } + ) + return found + + +def _crewed( + places: Sequence[Place], scenario: Scenario, steps: _Steps +) -> list[AgentBase]: + """The stub agents for one flow's places, all of one scenario and one turn count. + + Built inside a function because the drivers are heavy and the parent half of this + module is imported by `hmz.flows` itself: only a child actually proving a flow pays + for them. + + The stubs claim every capability there is -- every moment, a goal feature, shapes, + tools -- because what is being proved is the flow and not the agents: a flow legal on + the widest backend is refused for a narrower one where the agents are chosen, which is + `driving.py`'s job and not this one's. Everything else is the real base classes, so + the hooks a flow hangs fire exactly as they would under a real backend -- a `Stop` + hook that refuses sends a stub on again, and that continuation is a counted turn. + + Args: + places: What the flow declared. + scenario: The world the stubs answer from. + steps: The shared turn count, whose cap ends a proof nothing else ends. + + Returns: + One agent per place, the person's included. + """ + from hmz.agents import ( + AgentBase, + AgentConfig, + Event, + HumanAgent, + Moment, + SessionBase, + Usage, + ) + from hmz.agents.human import HumanSession + + class StubSession(SessionBase): + """A turn that lands at once and answers what the scenario says.""" + + shapes: ClassVar[bool] = True + takes_tools: ClassVar[bool] = True + + def _stream( + self, prompt: str, *, schema: type[BaseModel] | None = None + ) -> Iterator[Event]: + del prompt + steps.step() + if self._id is None: + self._adopt(f"stub-{id(self)}-{steps.taken}") + spent = Usage(output=scenario.climb) + self._spends(spent) + yield Event(kind="result", text=_said(schema, scenario), spent=spent) + + def _pursue(self, objective: str) -> str: + del objective + steps.step() + self._spends(Usage(output=scenario.climb)) + return scenario.answer + + class StubAgent(AgentBase): + """An agent claiming every capability, so only the flow is on trial.""" + + moments: ClassVar[frozenset[Moment]] = frozenset(Moment) + pursues: ClassVar[bool] = True + + def new(self, cwd: str | os.PathLike[str] | None = None) -> StubSession: + return StubSession(self, cwd) + + class StubTalk(HumanSession): + """The person's answers, deterministic: the scenario's, not a prompt's.""" + + shapes: ClassVar[bool] = True + + def stream( + self, prompt: str, *, schema: type[BaseModel] | None = None + ) -> Iterator[Event]: + # Overridden whole, as the real person's session is: their turn is not one + # being watched, and not one bracketed by an agent's moments. It still counts + # against the cap -- a flow that loops on asking forever is a flow that does + # not stop, whoever it is asking. + del prompt + steps.step() + yield Event(kind="result", text=_said(schema, scenario)) + + class StubPerson(HumanAgent): + """The person at the prompt, answering as the scenario has them answer.""" + + def new(self, cwd: str | os.PathLike[str] | None = None) -> StubTalk: + return StubTalk(self, cwd) + + return [ + StubPerson() + if place.person + else StubAgent(AgentConfig(model="stub", effort=""), name=place.name or None) + for place in places + ] + + +def _said(schema: type[BaseModel] | None, scenario: Scenario) -> str: + """What one stub turn answers with, as the text the base classes read back. + + Args: + schema: The shape the turn was held to, or None for a plain turn. + scenario: The world answering. + + Returns: + The scenario's answer for a plain turn; for a shaped one, the fabricated model as + its own JSON -- or "", which the base reads back as no answer at all, for the silent + scenario and for a shape nothing can be fabricated for. + """ + if schema is None: + return scenario.answer + if scenario.verdict is None: + return "" + from pydantic import ValidationError + + try: + return schema.model_validate(_made(schema, scenario)).model_dump_json() + except ValidationError: + return "" + + +def _made(schema: type[BaseModel], scenario: Scenario) -> dict[str, Any]: + """A shaped answer fabricated field by field, deterministically. + + Every boolean says the scenario's verdict -- which is what makes `NEVER_DONE` the + reviewer that never says done, whatever the field is called -- and every string says + its answer. The rest is the quietest legal value: a default where the field has one, + the first of a literal's few, an empty list, a zero, a nested shape made the same way. + + Args: + schema: The shape. + scenario: The world answering. + + Returns: + The fields, ready to be read back through the model. + """ + return { + name: _filled(field.annotation, field, scenario) + for name, field in schema.model_fields.items() + } + + +def _unioned(kind: Any) -> tuple[Any, ...]: + """One annotation and, for a union, what it is a union of. + + Only a union: the arguments of a `list[str]` are what is inside it, not what the field + itself may be, and a list of strings answered as one string would be the confusion. + """ + import types + import typing + + if get_origin(kind) in (types.UnionType, typing.Union): + return (kind, *get_args(kind)) + return (kind,) + + +def _filled(kind: Any, field: Any, scenario: Scenario) -> Any: + """One field's value, off its annotation. + + Args: + kind: What the field was annotated with, unions unwrapped as they are met. + field: The field itself, for the default it may carry. + scenario: The world answering. + + Returns: + The value. + """ + from pydantic import BaseModel + + for said in _unioned(kind): + if said is bool: + return scenario.verdict + if said is str: + return scenario.answer + if get_origin(said) is Literal: + return get_args(said)[0] + if field is not None and not field.is_required(): + return field.get_default(call_default_factory=True) + for said in _unioned(kind): + if said in (int, float): + return 0 + if get_origin(said) in (list, tuple, set, frozenset): + return [] + if get_origin(said) is dict: + return {} + if isinstance(said, type) and issubclass(said, BaseModel): + return _made(said, scenario) + return None + + +def _main(argv: list[str]) -> None: + """The child's whole life: one flow, one spec, one JSON line back.""" + flow, spec = argv + said = json.dumps({"proving": _driven(flow, json.loads(spec))}) + sys.stdout.write(said + "\n") + sys.stdout.flush() + + +if __name__ == "__main__": + _main(sys.argv[1:]) diff --git a/tests/test_checking.py b/tests/test_checking.py index d1e78ae..7fe6da7 100644 --- a/tests/test_checking.py +++ b/tests/test_checking.py @@ -772,9 +772,7 @@ def churn() -> None: ) ) found = checked(at) - assert [(one.code, one.where.name) for one in found] == [ - ("dead-loop", "helper.py") - ] + assert [(one.code, one.where.name) for one in found] == [("dead-loop", "helper.py")] def test_the_surface_is_the_interfaces_themselves() -> None: @@ -805,9 +803,11 @@ def test_everything_offered_is_reachable() -> None: def _swept() -> list[object]: """One parameter per flow humanize ships or the official flowverse holds now.""" places = [("builtin", BUILTIN_AT)] - for verse in hmz.flows.flowverses(): - if verse.name == hmz.flows.OFFICIAL and verse.fetched: - places.append((verse.name, hmz.flows.holds(verse))) + places.extend( + (verse.name, hmz.flows.holds(verse)) + for verse in hmz.flows.flowverses() + if verse.name == hmz.flows.OFFICIAL and verse.fetched + ) held: list[object] = [] seen_official = False for whose, under in places: diff --git a/tests/test_proving.py b/tests/test_proving.py new file mode 100644 index 0000000..7c1178a --- /dev/null +++ b/tests/test_proving.py @@ -0,0 +1,382 @@ +"""A flow driven by stubs against a clock, held to ending every proof and saying why. + +The scenarios are the questions: a budget loop walks to the end of its budget when the +reviewer never says done, a loop whose only exit is that verdict is caught by the turn cap, +a flow that takes no turns at all is killed by the clock, and the silent world -- every turn +answering nothing -- is every guard tried at once. Each proof is a subprocess, so these +tests are also the test that the child half answers the parent at all. +""" + +from __future__ import annotations + +import textwrap +from typing import TYPE_CHECKING, Literal + +from pydantic import BaseModel, Field + +from hmz.flows.proving import ( + ALWAYS_DONE, + NEVER_DONE, + SILENT, + Scenario, + _made, + _said, + proved, +) +from tests.stubs import written + +if TYPE_CHECKING: + from pathlib import Path + +#: A world that says no quickly: few turns allowed, and a short clock, so a flow that +#: cannot end is caught in test time rather than in a minute apiece. +QUICKLY = Scenario("never-done", verdict=False, answer="did some of it", turns=6, seconds=20.0) + +#: A budget loop in miniature: each stub turn climbs 100k output tokens, so three turns +#: spend the 250k this flow holds itself to. +BUDGETED = ''' +"""A loop held to a budget of its own.""" + +import time + +from hmz.flows import Agent, flow + + +@flow +def run(agents: tuple[Agent], task: str) -> None: + (agent,) = agents + while True: + agent(task, suppress=True) + if agent.spent().output >= 250_000: + return + time.sleep(5) +''' + +#: A loop whose one way out is the reviewer's verdict, which is rlar's shape. +VERDICT_ONLY = ''' +"""A loop only its reviewer can end.""" + +from hmz.flows import Agent, flow +from pydantic import BaseModel, Field + + +class Review(BaseModel): + model_config = {"extra": "forbid"} + + done: bool = Field(description="whether it is over") + + +@flow +def run(agents: tuple[Agent, Agent], task: str) -> None: + working = agents[0].new() + while True: + working(task, suppress=True) + review = agents[1](task, suppress=True, schema=Review) + if review is not None and review.done: + return +''' + + +def test_a_budget_loop_ends_when_the_reviewer_never_says_done(tmp_path: Path) -> None: + """The point of the whole module: bounded flows end in the worst world there is.""" + at = written(tmp_path, "one", textwrap.dedent(BUDGETED)) + proof = proved(at, scenarios=(NEVER_DONE,)) + assert proof.findings == () + (outcome,) = proof.outcomes + assert outcome.finished + assert outcome.turns == 3 + assert outcome.said == "" + + +def test_a_verdict_only_loop_is_caught_by_the_turn_cap(tmp_path: Path) -> None: + at = written(tmp_path, "one", textwrap.dedent(VERDICT_ONLY)) + proof = proved(at, scenarios=(QUICKLY, ALWAYS_DONE)) + never, always = proof.outcomes + assert not never.finished + assert never.turns == QUICKLY.turns + assert f"{QUICKLY.turns} turns" in never.said + # The same loop under a reviewer that says yes ends the way it means to. + assert always.finished + assert always.turns == 2 + + +def test_a_flow_that_takes_no_turns_is_killed_by_the_clock(tmp_path: Path) -> None: + at = written( + tmp_path, + "one", + textwrap.dedent( + ''' + """A loop with no turns for the cap to count.""" + + from hmz.flows import Agent, flow + + + @flow + def run(agents: tuple[Agent], task: str) -> None: + held = 0 + while True: + held += 1 + ''' + ), + ) + proof = proved(at, scenarios=(Scenario("spin", None, "", seconds=3.0),)) + (outcome,) = proof.outcomes + assert not outcome.finished + assert "still running after 3s" in outcome.said + + +def test_a_crash_is_the_outcome_with_its_last_words(tmp_path: Path) -> None: + at = written( + tmp_path, + "one", + textwrap.dedent( + ''' + """A flow that falls over.""" + + from hmz.flows import Agent, flow + + + @flow + def run(agents: tuple[Agent], task: str) -> None: + raise RuntimeError("the kettle is on fire") + ''' + ), + ) + proof = proved(at, scenarios=(ALWAYS_DONE,)) + (outcome,) = proof.outcomes + assert not outcome.finished + assert "the kettle is on fire" in outcome.said + + +def test_what_is_not_a_flow_is_a_refused_load(tmp_path: Path) -> None: + at = written(tmp_path, "one", '"""Not a flow."""\n\nx = 1\n') + proof = proved(at, scenarios=(ALWAYS_DONE,)) + assert [one.code for one in proof.findings] == ["refused-load"] + assert proof.findings[0].severity == "error" + assert "marked @flow()" in proof.findings[0].said + (outcome,) = proof.outcomes + assert not outcome.finished + + +#: A flow whose bound is its config, for proving the config reaches it. +CONFIGURED = ''' +"""A loop held to whatever budget it is set up with.""" + +from hmz.flows import Agent, flow +from pydantic import BaseModel, Field + + +class Config(BaseModel): + model_config = {"extra": "forbid"} + + budget: float = Field(default=1.0, ge=0, description="millions of output tokens") + + +@flow +def run(agents: tuple[Agent], task: str, config: Config | None = None) -> None: + (agent,) = agents + held = config or Config() + while True: + agent(task, suppress=True) + if agent.spent().output >= held.budget * 1_000_000: + return +''' + + +def test_a_config_is_read_back_through_the_flows_own_model(tmp_path: Path) -> None: + at = written(tmp_path, "one", textwrap.dedent(CONFIGURED)) + # 0.3 million at 100k a turn is three turns: the setting reached the loop. + proof = proved(at, config={"budget": 0.3}, scenarios=(NEVER_DONE,)) + assert proof.findings == () + assert proof.outcomes[0].finished + assert proof.outcomes[0].turns == 3 + # And one the model refuses is refused before anything runs. + refused = proved(at, config={"budget": "a lot"}, scenarios=(NEVER_DONE,)) + assert [one.code for one in refused.findings] == ["refused-load"] + assert not refused.outcomes[0].finished + + +def test_an_empty_proof_only_loads_and_reads_the_live_config(tmp_path: Path) -> None: + at = written(tmp_path, "one", textwrap.dedent(CONFIGURED)) + proof = proved(at, scenarios=()) + assert proof == ((), ()) + # A config the static reading cannot see is still read here, off the model itself. + loose = written( + tmp_path, + "loose", + textwrap.dedent( + ''' + """A flow with a config that says nothing about itself.""" + + from hmz.flows import Agent, flow + from pydantic import BaseModel + + + class Config(BaseModel): + budget: float = 1.0 + + + @flow + def run(agents: tuple[Agent], task: str, config: Config | None = None) -> None: + agents[0](task) + ''' + ), + ) + told = proved(loose, scenarios=()) + assert [one.code for one in told.findings] == ["loose-config", "unsaid-field"] + assert {one.severity for one in told.findings} == {"warning"} + + +#: A flow reading a shaped answer: one guarded, one not, for the silent world to tell apart. +GUARDED = ''' +"""A flow that guards what a turn answered.""" + +from hmz.flows import Agent, flow +from pydantic import BaseModel, Field + + +class Review(BaseModel): + model_config = {"extra": "forbid"} + + done: bool = Field(description="whether it is over") + + +@flow +def run(agents: tuple[Agent], task: str) -> None: + (agent,) = agents + while True: + review = agent(task, suppress=True, schema=Review) + if review is not None and review.done: + return + if agent.spent().output >= 200_000: + return +''' + +UNGUARDED = ''' +"""A flow that reads a field off whatever came back.""" + +from hmz.flows import Agent, flow +from pydantic import BaseModel, Field + + +class Review(BaseModel): + model_config = {"extra": "forbid"} + + done: bool = Field(description="whether it is over") + + +@flow +def run(agents: tuple[Agent], task: str) -> None: + review = agents[0](task, suppress=True, schema=Review) + print(review.done) +''' + + +def test_the_silent_world_is_every_guard_tried_at_once(tmp_path: Path) -> None: + guarded = written(tmp_path, "guarded", textwrap.dedent(GUARDED)) + proof = proved(guarded, scenarios=(SILENT,)) + assert proof.outcomes[0].finished # None answers taken again, until the budget + unguarded = written(tmp_path, "unguarded", textwrap.dedent(UNGUARDED)) + told = proved(unguarded, scenarios=(SILENT,)) + assert not told.outcomes[0].finished + assert "AttributeError" in told.outcomes[0].said + + +def test_the_person_answers_what_the_scenario_says(tmp_path: Path) -> None: + at = written( + tmp_path, + "one", + textwrap.dedent( + ''' + """A conversation, over when the person says nothing.""" + + from typing import NamedTuple + + from hmz.flows import Agent, Person, flow + + + class Chat(NamedTuple): + assistant: Agent + human: Person + + + @flow + def run(agents: Chat, task: str) -> None: + conversation = agents.assistant.new() + said = task + while said: + answered = conversation(said, suppress=True) + said = agents.human(answered) + ''' + ), + ) + # Nobody at the prompt: the flow does the one thing it was given and stops. + silent = proved(at, scenarios=(SILENT,)) + assert silent.outcomes[0].finished + assert silent.outcomes[0].turns == 2 # the assistant's turn, and the person's "" + # A person who never stops talking is a conversation that never ends: the cap's. + chatty = proved(at, scenarios=(Scenario("chatty", True, "go on", turns=9),)) + assert not chatty.outcomes[0].finished + + +def test_an_async_flow_is_awaited(tmp_path: Path) -> None: + at = written( + tmp_path, + "one", + textwrap.dedent( + ''' + """A flow written as a coroutine.""" + + from hmz.flows import Agent, flow + + + @flow + async def run(agents: tuple[Agent], task: str) -> None: + await agents[0].aturn(task, suppress=True) + ''' + ), + ) + proof = proved(at, scenarios=(ALWAYS_DONE,)) + (outcome,) = proof.outcomes + assert outcome.finished, outcome + assert outcome.turns == 1 + + +class Nested(BaseModel): + said: str + fine: bool + + +class Shaped(BaseModel): + done: bool + notes: str + stage: Literal["draft", "final"] + rounds: int = 4 + weight: float + parts: list[str] + inner: Nested + extra: str | None = None + + +def test_a_shaped_answer_is_fabricated_field_by_field() -> None: + made = _made(Shaped, NEVER_DONE) + held = Shaped.model_validate(made) + assert held.done is False # the verdict, whatever the field is called + assert held.notes == NEVER_DONE.answer + assert held.stage == "draft" # the first of a literal's few + assert held.rounds == 4 # the default, where the field has one + assert held.weight == 0 + assert held.parts == [] + assert held.inner == Nested(said=NEVER_DONE.answer, fine=False) + assert held.extra == NEVER_DONE.answer # a string, even behind a union + + +def test_a_shape_nothing_can_be_fabricated_for_answers_nothing() -> None: + class Impossible(BaseModel): + count: int = Field(ge=5) # fabricated as 0, which the model then refuses + + assert _said(Impossible, ALWAYS_DONE) == "" + # And the silent world answers nothing whatever the shape. + assert _said(Shaped, SILENT) == "" + assert _said(None, SILENT) == "" + assert _said(None, ALWAYS_DONE) == ALWAYS_DONE.answer From bee485effc722c0602cd2ad34940d2ddffdb76ef Mon Sep 17 00:00:00 2001 From: dongyun Date: Fri, 28 Aug 2026 21:57:19 +0000 Subject: [PATCH 3/8] feat(flows): add the capability catalogue the compiler steers by catalogue() reads what this installed humanize serves at call time: the primitives every backend has, each moment outside EVERYWHERE with the backends whose drivers declare it, the shape a turn can be held to, the tools a flow may offer, and the goal feature -- all off the live enum, the DRIVEN table and the interfaces surface() reads, so a flow written against the catalogue is written against this installation rather than against a snapshot that drifts. briefed() renders it as the one page a compiler -- or a person choosing what to build on -- steers by, split into what every backend serves and what has to be declared on the place. Co-Authored-By: Claude Fable 5 --- src/hmz/flows/__init__.py | 5 +- src/hmz/flows/checking.py | 240 +++++++++++++++++++++++++++++++++++++- tests/test_catalogue.py | 90 ++++++++++++++ tests/test_proving.py | 4 +- 4 files changed, 336 insertions(+), 3 deletions(-) create mode 100644 tests/test_catalogue.py diff --git a/src/hmz/flows/__init__.py b/src/hmz/flows/__init__.py index cb3c25a..ecf8627 100644 --- a/src/hmz/flows/__init__.py +++ b/src/hmz/flows/__init__.py @@ -52,7 +52,7 @@ def run(agents: tuple[Agent, Agent], task: str) -> None: from typing import TYPE_CHECKING, Any, NamedTuple, overload from .agent import Agent, Driven, Person, Session -from .checking import Finding, checked +from .checking import Capability, Finding, briefed, catalogue, checked from .driving import ( NotAFlow, Place, @@ -129,6 +129,7 @@ def run(agents: tuple[Agent, Agent], task: str) -> None: "Agent", "AgentConfig", "AgentDefaults", + "Capability", "Driven", "Event", "Failed", @@ -164,7 +165,9 @@ def run(agents: tuple[Agent, Agent], task: str) -> None: "about", "at", "backends", + "briefed", "carries", + "catalogue", "checked", "configures", "container", diff --git a/src/hmz/flows/checking.py b/src/hmz/flows/checking.py index edcbf91..85491e9 100644 --- a/src/hmz/flows/checking.py +++ b/src/hmz/flows/checking.py @@ -22,6 +22,7 @@ from __future__ import annotations import ast +import contextlib from dataclasses import dataclass, field from pathlib import Path from typing import TYPE_CHECKING, Literal, NamedTuple, Protocol @@ -32,7 +33,15 @@ import os from collections.abc import Iterator -__all__ = ["Finding", "checked", "offered", "surface"] +__all__ = [ + "Capability", + "Finding", + "briefed", + "catalogue", + "checked", + "offered", + "surface", +] class Finding(NamedTuple): @@ -1557,3 +1566,232 @@ def _sleeps(body: list[ast.stmt]) -> bool: continue return False return True + + +# --------------------------------------------------------------------------------------- +# The capability catalogue: what this installed humanize serves, read off it live. +# --------------------------------------------------------------------------------------- + + +class Capability(NamedTuple): + """One thing a flow may build on, and which backends serve it. + + Attributes: + name: What it is called: a primitive by one word, and a moment only some backends + reach as `moment:`. + backends: The backends that serve it, or empty for one every backend serves. + said: What the ask looks like, with the code spelled out: this is what a compiler + or a person choosing what to build on is shown. + """ + + name: str + backends: frozenset[str] + said: str + + +def catalogue() -> tuple[Capability, ...]: + """Everything a flow may build on here, read off the installed interface at call time. + + At call time rather than written down, because this is what keeps a generated flow + honest across versions: the moments come off the live enum, the backend sets off the + driver classes' own declarations, and the asks off the same interfaces `surface` reads + -- so what the catalogue promises is what this installation serves, not what some + edition of it once did. + + Returns: + One capability apiece: the primitives every backend serves, then what only some do + -- each moment outside `EVERYWHERE`, the shape a turn can be held to, the tools a + flow may offer, and the goal feature. + """ + import inspect + import sys as running + + from hmz.agents import DRIVEN, EVERYWHERE, Moment + + agents = {name: held[0] for name, held in DRIVEN.items()} + sessions: dict[str, type] = {} + for name, cls in agents.items(): + # The session class, read off what `new` says it answers with: the class itself + # is what carries `shapes` and `takes_tools`. By the name in the driver's own + # module rather than through `get_type_hints`, which would ask every annotation + # in the signature to resolve -- and a driver is free to keep `os` under + # TYPE_CHECKING. + told: object = None + with contextlib.suppress(Exception): + told = inspect.signature(cls.new).return_annotation + if isinstance(told, str): + told = vars(running.modules[cls.__module__]).get(told) + if isinstance(told, type): + sessions[name] = told + held: list[Capability] = [ + Capability( + "turns", + frozenset(), + "one turn in a session of its own -- agent(prompt, suppress=True) -- or many " + "at once with agent.batch(prompts); suppress=True makes a failed turn answer " + "'' (or None, for a shaped one) instead of raising", + ), + Capability( + "sessions", + frozenset(), + "one conversation held across turns -- session = agent.new(cwd=...) then " + "session(prompt) -- and dropping the session is how a flow forgets", + ), + Capability( + "schema", + frozenset(), + "a turn read back as an object -- session(prompt, suppress=True, " + "schema=Model) answers the model or None, and the answer is guarded before a " + "field is read off it", + ), + Capability( + "budgets", + frozenset(), + "what a loop's bound reads -- agent.spent().output climbs as the run spends, " + "and session.rate() and session.juice() say how fast", + ), + Capability( + "hooks", + frozenset(), + "a word in at the moments of a turn -- agent.hooks.on(Moment.STOP, hook) " + "hangs a callable, and a Stop hook that refuses sends the agent on with what " + "it said", + ), + Capability( + "subflows", + frozenset(), + "one flow runs another -- load('official/rlar')(agents, task) -- found by " + "the same name -f takes, and refused where it is asked for if nothing " + "answers to it", + ), + Capability( + "person", + frozenset(), + "the person at the prompt is a place like any other -- agents that include a " + "Person field -- and person(said) asks them; run where nobody is at a " + "prompt, they answer nothing, and a flow written to stop on nothing stops", + ), + Capability( + "board", + frozenset(), + "named lines the flow and the person both write on and neither waits at -- " + "person.board.put('todo', said) and person.board.get('todo')", + ), + Capability( + "state", + frozenset(), + "a flow marked @flow(resumable=True) is handed a dict as its last argument, " + "holding what it wrote there last time -- and clears it when the run is over", + ), + Capability( + "config", + frozenset(), + "what a flow can be set up with is a pydantic model third argument -- " + "config: Config | None = None -- whose model_config refuses extras and whose " + "every field carries Field(description=...)", + ), + Capability( + "skills", + frozenset(), + "a flow's own skills live in its skills/ directory, one directory per skill " + "with a SKILL.md in it, and a session says which it carries with " + "session.loads([...])", + ), + Capability( + "clone", + frozenset(), + "an agent set up differently is another agent -- " + "agent.clone(config=replace(agent.config, effort='high')) -- having opened " + "nothing and spent nothing", + ), + Capability( + "moments", + frozenset(), + "the moments every backend reaches, for a hook to hang on: " + + ", ".join(f"Moment.{one.name}" for one in Moment if one in EVERYWHERE), + ), + ] + for moment in Moment: + if moment in EVERYWHERE: + continue + held.append( + Capability( + f"moment:{moment.value}", + frozenset( + name for name, cls in agents.items() if moment in cls.moments + ), + f"Moment.{moment.name} is reached only where the backend says so -- " + f"declare it on the place, Annotated[Agent, Moment.{moment.name}], and " + "the run is refused an agent that cannot run it before its first turn", + ) + ) + held.append( + Capability( + "shapes", + frozenset( + name for name, one in sessions.items() if getattr(one, "shapes", False) + ), + "a turn held to the shape rather than asked to keep to it -- every backend " + "takes schema=, and these are the ones the answer is certain on", + ) + ) + held.append( + Capability( + "tools", + frozenset( + name + for name, one in sessions.items() + if getattr(one, "takes_tools", False) + ), + "a flow's own callbacks put in front of the agent -- " + "session.offers([Tool(...)]) -- which a backend not among these refuses; " + "type(session).takes_tools says so beforehand", + ) + ) + pursuing = frozenset(name for name, cls in agents.items() if cls.pursues) + held.append( + Capability( + "pursue", + pursuing, + "the backend's own goal feature -- session.pursue(objective) keeps the " + "agent going until it decides for itself the objective is met", + ) + ) + held.append( + Capability( + "goal", + pursuing, + "a place run under that feature declares it -- Annotated[Agent, Goal] -- " + "and is refused an agent whose backend has none before the first turn", + ) + ) + return tuple(held) + + +def briefed() -> str: + """The catalogue rendered as the one page a compiler steers by. + + Returns: + What every backend serves, then what only some do -- each with the backends that + do, so that a flow built on one can say where it runs. + """ + held = catalogue() + lines = [ + "What a flow may build on here, read off this installed humanize.", + "", + "Every backend:", + ] + lines.extend(f"- {one.name}: {one.said}" for one in held if not one.backends) + lines += [ + "", + ( + "Only some backends -- a flow built on one of these says so where it " + "declares the place, and is refused an unfit agent before the first turn:" + ), + ] + lines.extend( + f"- {one.name} ({', '.join(sorted(one.backends))}): {one.said}" + for one in held + if one.backends + ) + return "\n".join(lines) diff --git a/tests/test_catalogue.py b/tests/test_catalogue.py new file mode 100644 index 0000000..5967ae0 --- /dev/null +++ b/tests/test_catalogue.py @@ -0,0 +1,90 @@ +"""The capability catalogue, held to saying only what this installation actually serves. + +Honesty tests: every name the catalogue uses is a real moment, a real backend or a real +member of the interfaces a flow is written against, and every backend set is exactly what +the live driver classes declare. The catalogue is what a compiler steers by, and a +capability it invented -- or one that drifted from the drivers -- is a generated flow that +asks for what nothing serves. +""" + +from __future__ import annotations + +from hmz.agents import DRIVEN, EVERYWHERE, Moment +from hmz.flows import Agent, Person, Session +from hmz.flows.checking import briefed, catalogue, offered, surface + + +def test_every_conditional_moment_is_real_and_exactly_whose_drivers_say() -> None: + told = {one.name: one for one in catalogue() if one.name.startswith("moment:")} + outside = {one for one in Moment if one not in EVERYWHERE} + assert set(told) == {f"moment:{one.value}" for one in outside} + for moment in outside: + assert told[f"moment:{moment.value}"].backends == frozenset( + name for name, (cls, _) in DRIVEN.items() if moment in cls.moments + ) + + +def test_the_backend_facts_are_the_drivers_own() -> None: + told = {one.name: one.backends for one in catalogue()} + assert told["pursue"] == frozenset( + name for name, (cls, _) in DRIVEN.items() if cls.pursues + ) + assert told["goal"] == told["pursue"] + # The two facts a session carries, checked against the backends known to carry them: + # the sets themselves are read off the session classes, so what is pinned here is that + # the reading reaches them at all. + assert {"claude", "codex"} <= told["shapes"] + assert "claude" in told["tools"] + for one in catalogue(): + assert one.backends <= set(DRIVEN), one.name + + +def test_every_ask_the_catalogue_spells_is_on_the_interfaces() -> None: + """The primitives are described in code, and the code has to be the real interface.""" + asks = surface(Agent) | surface(Session) | surface(Person) + anchored = { + "turns": "batch", + "sessions": "new", + "budgets": "spent", + "hooks": "hooks", + "board": "board", + "clone": "clone", + "skills": "loads", + "pursue": "pursue", + "tools": "offers", + } + said = {one.name: one.said for one in catalogue()} + for name, member in anchored.items(): + assert member in asks + assert member in said[name], name + # And the ones whose anchor is the vocabulary hmz.flows hands through. + offers = offered() + for name, word in { + "subflows": "load", + "person": "Person", + "state": "flow", + "hooks": "Moment", + "goal": "Goal", + }.items(): + assert word in offers + assert word in said[name], name + + +def test_the_moments_every_backend_reaches_are_everywhere() -> None: + (moments,) = (one for one in catalogue() if one.name == "moments") + assert moments.backends == frozenset() + for one in EVERYWHERE: + assert f"Moment.{one.name}" in moments.said + + +def test_the_briefing_mentions_every_capability_and_its_backends() -> None: + page = briefed() + for one in catalogue(): + assert f"- {one.name}" in page + for backend in one.backends: + assert backend in page + # The split the compiler steers by: what needs declaring is under the second heading. + assert "Every backend:" in page + assert "Only some backends" in page + assert page.index("- turns:") < page.index("Only some backends") + assert page.index("Only some backends") < page.index("- pursue") diff --git a/tests/test_proving.py b/tests/test_proving.py index 7c1178a..19e9e4e 100644 --- a/tests/test_proving.py +++ b/tests/test_proving.py @@ -30,7 +30,9 @@ #: A world that says no quickly: few turns allowed, and a short clock, so a flow that #: cannot end is caught in test time rather than in a minute apiece. -QUICKLY = Scenario("never-done", verdict=False, answer="did some of it", turns=6, seconds=20.0) +QUICKLY = Scenario( + "never-done", verdict=False, answer="did some of it", turns=6, seconds=20.0 +) #: A budget loop in miniature: each stub turn climbs 100k output tokens, so three turns #: spend the 250k this flow holds itself to. From 8bc541047fb650cff42b4c2f19ac15f238367506 Mon Sep 17 00:00:00 2001 From: dongyun Date: Fri, 28 Aug 2026 22:12:16 +0000 Subject: [PATCH 4/8] feat(cli): add hmz check The two readings from a command line: the static one over every file the flow holds, which executes nothing, then the flow loaded in a subprocess held to a clock so its live config model is read too. One finding a line with a count under them, --json for a script, --static to keep the flow unloaded, --strict to hold warnings to the bar. Exit 0 with nothing blocking, 1 with any error -- or any warning under --strict -- and 2 for a line to correct or a name no flow answers to. The readings are reached through Hmz().flows.check, so anything else that checks a flow makes the same call and is refused the same way. Co-Authored-By: Claude Fable 5 --- docs/reference/cli.md | 35 +++++++- docs/reference/sdk.md | 1 + src/hmz/SPEC.md | 22 +++++ src/hmz/cli/__init__.py | 16 ++++ src/hmz/cli/check.py | 119 +++++++++++++++++++++++++++ src/hmz/sdk/SPEC.md | 6 ++ src/hmz/sdk/flows.py | 35 +++++++- tests/test_check_command.py | 156 ++++++++++++++++++++++++++++++++++++ tests/test_cli.py | 1 + 9 files changed, 389 insertions(+), 2 deletions(-) create mode 100644 src/hmz/cli/check.py create mode 100644 tests/test_check_command.py diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 7cc3fc8..5186895 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -277,6 +277,39 @@ to a shell on that machine — read [Security](/guide/security). hmz anchor serve --listen 0.0.0.0:7777 --export /srv/project --token "$SECRET" ``` +## `hmz check` + +``` +hmz check [--static] [--strict] [--json] FLOW [FLOW...] +``` + +Reads a flow for what will not run — before anything runs it. Two readings, in their order: a +static one over every file the flow holds, which executes nothing and is safe to point at a +flow nobody has read, and then the flow loaded — in a subprocess held to a clock — so its live +config model is read too. A flow is named the way `-f` names one: `chat`, `official/rlar`, a +path of your own. + +| Argument | | +| --- | --- | +| `--static` | Only the reading that executes nothing: do not load the flow at all. | +| `--strict` | Exit non-zero on warnings too. | +| `--json` | One JSON object per finding, one a line, for a script to read. | + +Each finding prints as `file:line: severity: code: what is wrong`, with a count under them. +An error is a flow that cannot run, cannot be answered or cannot end — a loop nothing inside +can end, a name the interfaces do not answer to, an import of humanize's own internals — and +a warning is a flow that runs and may be regretted: a loop whose only way out is an agent's +verdict, a shaped answer read without a guard, a config that takes anything. + +It exits `0` for flows with nothing blocking (warnings print and pass), `1` where any error +was found — or any warning, under `--strict` — and `2` for a line to correct or a name no +flow answers to. + +```sh +hmz check official/rlar # one warning: a loop only its reviewer ends +hmz check --strict local/mine # hold a flow of your own to the whole bar +``` + ## `hmz flowverses` Where flows come from: a git repository with a `flows/` directory apiece, cloned under @@ -579,7 +612,7 @@ into them. | | | | --- | --- | | `0` | It did what it was asked. | -| `1` | It could not: the target could not be reached, the listener could not be started, there is no such provider, a turn could not be supervised. | +| `1` | It could not: the target could not be reached, the listener could not be started, there is no such provider, a turn could not be supervised, `hmz check` found something blocking. | | `2` | The command line was wrong — argparse's own rejections, a flow that is not there or takes other agents, a malformed listen address, a non-loopback listener with no token. | | `130` | Interrupted. | | *the agent's own* | `hmz anchor` exits with the status of the program it ran, and `hmz providers add` with that of the login it ran. | diff --git a/docs/reference/sdk.md b/docs/reference/sdk.md index f83a0a4..a3b649b 100644 --- a/docs/reference/sdk.md +++ b/docs/reference/sdk.md @@ -86,6 +86,7 @@ run.wait(timeout=60) | `find(named)` | The file one flow is written in. Raises `NotAFlow` for a name nothing answers to. | | `about(named)` | The line a flow says about itself. | | `places(named)` | Every agent it needs chosen for it, in the order it takes them. | +| `check(named, static=False)` | Reads it for what will not run, before anything runs it: one finding per thing found, errors and warnings. `static=True` keeps to the reading that executes nothing. | | `configures(named)` | What it can be [set up with](/reference/flows#settings-of-the-flow-s-own), or `None`. | | `resumes(named)` | Whether it says it can be [picked up](/guide/resuming). | | `fork(named, into=None)` | Copies it into this project's own flows, whole. | diff --git a/src/hmz/SPEC.md b/src/hmz/SPEC.md index 9f6f7f7..8d52f4c 100644 --- a/src/hmz/SPEC.md +++ b/src/hmz/SPEC.md @@ -670,6 +670,28 @@ that can happen to a flowverse -- added, fetched again, taken away. that will not go -- MUST say so where it can be read and exit non-zero, and MUST leave the list as it was. None of those MUST reach whoever typed the line as a traceback. +## `hmz check` + +```shell +hmz check [--static] [--strict] [--json] [...] +``` + +Reads a flow for what will not run, before anything runs it. + +- The two readings MUST run in their order: the static one over every file the flow holds, + which MUST NOT import or execute anything of it -- the flow most worth checking is one + nobody has read -- and then the flow loaded and its live config model read. The second MUST + run only in a subprocess with a clock held over it, MUST NOT run where the first found an + error, and `--static` MUST leave it out altogether: a flow that cannot run is not one to + run to find out more about. +- Every finding MUST print one a line -- the file, the line, the severity, the code and what + is wrong -- with a count under them, and `--json` MUST say the same as one JSON object a + line for a script to read. Everything wrong MUST be said at once rather than first-failure + first: a checker is asked so that one reading answers for the whole flow. +- It MUST exit 0 for flows with nothing blocking -- warnings print and pass -- 1 where any + error was found, or any warning under `--strict`, and 2 for a line to correct or a name no + flow answers to, refused as argparse refuses one. + ## `hmz agents` ```shell diff --git a/src/hmz/cli/__init__.py b/src/hmz/cli/__init__.py index 65d758f..2442419 100644 --- a/src/hmz/cli/__init__.py +++ b/src/hmz/cli/__init__.py @@ -160,6 +160,21 @@ def _flowverses(argv: list[str]) -> int: return flowverses(argv) +def _check(argv: list[str]) -> int: + """Reads a flow for what will not run, before anything runs it. + + Args: + argv: What followed the command name. + + Returns: + Zero for a flow with nothing blocking, one for one with something, or two for a + line to correct. + """ + from .check import check + + return check(argv) + + def _cred(argv: list[str]) -> int: """Runs a program whose credentials are kept somewhere other than where it looks. @@ -552,6 +567,7 @@ def _daemon(argv: list[str]) -> int: ), "anchor": (_anchor, "run an agent here that acts on another machine"), "flowverses": (_flowverses, "the places flows come from"), + "check": (_check, "check a flow before anything runs it"), "agents": (_agents, "the agents written down under a name"), "providers": (_providers, "the accounts an agent may be run as"), "fallback": ( diff --git a/src/hmz/cli/check.py b/src/hmz/cli/check.py new file mode 100644 index 0000000..48234c8 --- /dev/null +++ b/src/hmz/cli/check.py @@ -0,0 +1,119 @@ +"""``hmz check`` -- reads a flow for what will not run, before anything runs it. + +Two readings, in their order. The static one is pure `ast` over every file the flow holds: +it executes nothing, so it is safe to point at a flow nobody has read -- generated, fetched, +forked. Then the flow is loaded and its live config model read, in a subprocess held to a +clock, which is what catches what only running the file can show. Both answer with findings, +one a line, so everything wrong is said at once. + +It is here because the moment to check a flow is before something runs it: a CI job holding +a flowverse to its own bar, a flow just written by an agent, a fork about to be tried. The +readings themselves are :mod:`hmz.flows.checking` and :mod:`hmz.flows.proving`, reached +through :class:`hmz.sdk.Hmz` -- the same call anything else that checks a flow makes. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from hmz.flows import Finding + +__all__ = ["check"] + + +def check(argv: list[str]) -> int: + """Carries out one `hmz check` line. + + Args: + argv: What followed the command name. + + Returns: + Zero for flows with nothing blocking -- warnings print and pass, unless `--strict` + says otherwise -- one where any finding blocks, and two for a line to correct or a + name no flow answers to. + """ + import argparse + + parser = argparse.ArgumentParser( + prog="hmz check", + description="Read a flow for what will not run -- before anything runs it: a " + "static reading that executes nothing, then the flow loaded in a subprocess held " + "to a clock. Errors are what no run survives; warnings are runs that may be " + "regretted.", + ) + parser.add_argument( + "flow", + nargs="+", + metavar="FLOW", + help="a flow, by the name `-f` takes -- `chat`, `official/rlar` -- or by a path", + ) + parser.add_argument( + "--static", + action="store_true", + help="only the reading that executes nothing: do not load the flow at all", + ) + parser.add_argument( + "--strict", + action="store_true", + help="exit non-zero on warnings too", + ) + parser.add_argument( + "--json", + action="store_true", + dest="as_json", + help="one JSON object per finding, one a line, for a script to read", + ) + args = parser.parse_args(argv) + + import os + + from hmz.sdk import Hmz + + flows = Hmz().flows + found: list[Finding] = [] + for named in args.flow: + # A name nothing answers to is a line to correct, refused the way argparse refuses + # one -- before anything is read, and for every name on the line at once. + if not os.path.isfile(flows.find(named)): + parser.error(f"no flow called {named!r}") + found.extend(flows.check(named, static=args.static)) + _said(found, as_json=args.as_json, flows=len(args.flow)) + errors = sum(one.severity == "error" for one in found) + warned = len(found) - errors + return 1 if errors or (args.strict and warned) else 0 + + +def _said(found: list[Finding], *, as_json: bool, flows: int) -> None: + """Prints what the readings found, for a person or for a script. + + Args: + found: The findings, in the order they were found. + as_json: Whether to say each as one JSON object a line, and no count under them. + flows: How many flows the line named, for the count. + """ + if as_json: + import json + + for one in found: + print( + json.dumps( + { + "code": one.code, + "severity": one.severity, + "where": str(one.where), + "line": one.line, + "said": one.said, + } + ) + ) + return + from . import many + + for one in found: + print(f"{one.where}:{one.line}: {one.severity}: {one.code}: {one.said}") + errors = sum(one.severity == "error" for one in found) + if found: + print(f"hmz check: {many(errors, 'error')}, {many(len(found) - errors, 'warning')}") + else: + print(f"hmz check: nothing to say about {many(flows, 'flow')}") diff --git a/src/hmz/sdk/SPEC.md b/src/hmz/sdk/SPEC.md index 7b88147..3a7dade 100644 --- a/src/hmz/sdk/SPEC.md +++ b/src/hmz/sdk/SPEC.md @@ -134,6 +134,9 @@ class Flows: def find(self, named: str) -> str: ... def about(self, named: str) -> str: ... def places(self, named: str | os.PathLike[str]) -> tuple[Place, ...]: ... + def check( + self, named: str | os.PathLike[str], *, static: bool = False + ) -> tuple[Finding, ...]: ... def configures(self, named: str | os.PathLike[str]) -> type[BaseModel] | None: ... def resumes(self, named: str | os.PathLike[str]) -> bool: ... def fork(self, named: str, into: str | os.PathLike[str] | None = None) -> str: ... @@ -143,6 +146,9 @@ class Flows: The flows there are, and the places they come from. +- `check` MUST be the two readings of `hmz.flows` in their order -- the static one, then the + flow loaded in a subprocess -- and the second MUST NOT run where the first found an error, + nor say again what the first already said: one call is one answer, whichever way in asked. - Where a flowverse came from MUST be answered here, and MUST be answered from which flowverse it is rather than from whether its URL is empty. Whatever was signed into a URL MUST be taken out of it in one place, which every way of showing one asks. diff --git a/src/hmz/sdk/flows.py b/src/hmz/sdk/flows.py index b4a749e..0fd97c0 100644 --- a/src/hmz/sdk/flows.py +++ b/src/hmz/sdk/flows.py @@ -18,7 +18,7 @@ from pydantic import BaseModel - from hmz.flows import Flowverse, Offer, Place, Running + from hmz.flows import Finding, Flowverse, Offer, Place, Running __all__ = ["Flows", "Flowverses"] @@ -212,6 +212,39 @@ def places(self, named: str | os.PathLike[str]) -> tuple[Place, ...]: return wanted(named) + def check( + self, named: str | os.PathLike[str], *, static: bool = False + ) -> tuple[Finding, ...]: + """Reads a flow for what will not run, before anything runs it. + + Two readings, in their order. The static one is pure `ast` over every file the + flow holds and executes nothing, which is the whole of what `static` keeps. The + second loads the flow and reads its live config model, in a subprocess held to a + clock -- and is left out where the first found an error: a flow that cannot run is + not one to run to find out more about. + + Args: + named: The flow, by the name `-f` takes or by a path. + static: Only the reading that executes nothing. + + Returns: + Every finding, the static reading's first and nothing said twice: a finding the + static reading already made is not repeated off the live model. + """ + from pathlib import Path + + from hmz.flows import ENTRY, checked, find, inside, proved + + at = Path(find(str(named))) + whole = at.parent if at.name == ENTRY else at + found = list(checked(whole)) + if static or any(one.severity == "error" for one in found): + return tuple(found) + proof = proved(whole, name=inside(str(named)), scenarios=()) + said = {one.code for one in found} + found.extend(one for one in proof.findings if one.code not in said) + return tuple(found) + def configures(self, named: str | os.PathLike[str]) -> type[BaseModel] | None: """What a flow can be set up with, or None for one that takes no setting up.""" from hmz.flows import configures diff --git a/tests/test_check_command.py b/tests/test_check_command.py new file mode 100644 index 0000000..c6fbf5a --- /dev/null +++ b/tests/test_check_command.py @@ -0,0 +1,156 @@ +"""``hmz check``, held to its three exit statuses and to printing what a script can read. + +What each reading finds is `test_checking.py`'s and `test_proving.py`'s business; what is +checked here is the command around them -- that a clean flow passes, a blocked one blocks, +`--strict` raises the bar, `--static` keeps the flow unloaded, a name nothing answers to is +a usage error, and `--json` says the same findings a script can parse back. +""" + +from __future__ import annotations + +import json +import textwrap +from typing import TYPE_CHECKING + +import pytest + +from hmz.cli.check import check +from tests.stubs import written + +if TYPE_CHECKING: + from pathlib import Path + +CLEAN = ''' +"""A flow with nothing to say about it.""" + +from hmz.flows import Agent, flow + + +@flow +def run(agents: tuple[Agent], task: str) -> None: + agents[0](task, suppress=True) +''' + +DEAD = ''' +"""A flow whose loop nothing can end.""" + +from hmz.flows import Agent, flow + + +@flow +def run(agents: tuple[Agent], task: str) -> None: + while True: + agents[0](task, suppress=True) +''' + +#: rlar's shape: legal, and one warning -- the loop only its reviewer ends. +WARNED = ''' +"""A flow whose loop waits on its reviewer.""" + +from hmz.flows import Agent, flow +from pydantic import BaseModel, Field + + +class Review(BaseModel): + model_config = {"extra": "forbid"} + + done: bool = Field(description="whether it is over") + + +@flow +def run(agents: tuple[Agent], task: str) -> None: + while True: + review = agents[0](task, suppress=True, schema=Review) + if review is not None and review.done: + return +''' + +#: Clean to the static reading, and refused the moment it is loaded. +UNLOADABLE = ''' +"""A flow whose file will not run.""" + +from hmz.flows import Agent, flow + + +@flow +def run(agents: tuple[Agent], task: str) -> None: + agents[0](task, suppress=True) + + +raise RuntimeError("read no further") +''' + + +def test_a_clean_flow_passes_and_says_so( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + at = written(tmp_path, "one", textwrap.dedent(CLEAN)) + assert check([str(at)]) == 0 + assert "nothing to say about 1 flow" in capsys.readouterr().out + + +def test_a_blocking_finding_is_exit_one( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + at = written(tmp_path, "one", textwrap.dedent(DEAD)) + assert check([str(at)]) == 1 + out = capsys.readouterr().out + assert f"{at / '__init__.py'}:" in out + assert "error: dead-loop:" in out + assert "1 error, 0 warnings" in out + + +def test_a_warning_passes_until_strict_raises_the_bar( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + at = written(tmp_path, "one", textwrap.dedent(WARNED)) + assert check([str(at)]) == 0 + assert "warning: unbounded-loop:" in capsys.readouterr().out + assert check(["--strict", str(at)]) == 1 + + +def test_static_keeps_the_flow_unloaded(tmp_path: Path) -> None: + at = written(tmp_path, "one", textwrap.dedent(UNLOADABLE)) + # The static reading has nothing to say about it; loading it is what refuses it. + assert check(["--static", str(at)]) == 0 + assert check([str(at)]) == 1 + + +def test_a_name_nothing_answers_to_is_a_usage_error( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + with pytest.raises(SystemExit) as stopped: + check([str(tmp_path / "nowhere")]) + assert stopped.value.code == 2 + assert "no flow called" in capsys.readouterr().err + + +def test_a_flag_it_does_not_take_is_a_usage_error(tmp_path: Path) -> None: + with pytest.raises(SystemExit) as stopped: + check(["--everything", str(tmp_path)]) + assert stopped.value.code == 2 + + +def test_json_says_the_same_findings_a_script_can_read( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + at = written(tmp_path, "one", textwrap.dedent(DEAD)) + assert check(["--json", "--static", str(at)]) == 1 + lines = capsys.readouterr().out.strip().splitlines() + held = [json.loads(one) for one in lines] # every line parses: no count under them + assert [one["code"] for one in held] == ["dead-loop"] + assert held[0]["severity"] == "error" + assert held[0]["where"].endswith("__init__.py") + assert held[0]["line"] > 0 + assert "cannot end" in held[0]["said"] + + +def test_several_flows_are_one_answer( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + good = written(tmp_path, "good", textwrap.dedent(CLEAN)) + bad = written(tmp_path, "bad", textwrap.dedent(DEAD)) + assert check(["--static", str(good), str(bad)]) == 1 + out = capsys.readouterr().out + assert "dead-loop" in out + assert "1 error, 0 warnings" in out diff --git a/tests/test_cli.py b/tests/test_cli.py index 8bf74b9..4f491dd 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -34,6 +34,7 @@ ("trace", set[str]()), ("anchor", {"coganchor"}), ("flowverses", set[str]()), + ("check", set[str]()), ("agents", set[str]()), ] From 779b8c4c3e972682a6fd0da1737665a0aaffe655 Mon Sep 17 00:00:00 2001 From: dongyun Date: Fri, 28 Aug 2026 22:25:24 +0000 Subject: [PATCH 5/8] docs(flows): say how a flow is checked The reference grows a Checking-a-flow section beside Testing-a-flow: the two readings, the rule table with what each code found, the scenarios a proof drives against, and the catalogue. A guide walks the command -- one line, what an error is against a warning, --static for a flow nobody has read, the never-done proof as a library call, --json in a script -- and Writing-a-flow points at it from Check-your-work. Co-Authored-By: Claude Fable 5 --- docs/.vitepress/config.mts | 1 + docs/guide/checking-flows.md | 81 +++++++++++++++++++++++++++++++++++ docs/guide/testing-flows.md | 1 + docs/guide/writing-a-flow.md | 10 ++++- docs/reference/flows.md | 82 ++++++++++++++++++++++++++++++++++++ src/hmz/cli/check.py | 8 ++-- 6 files changed, 179 insertions(+), 4 deletions(-) create mode 100644 docs/guide/checking-flows.md diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 5e9d024..5793ea7 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -223,6 +223,7 @@ export default defineConfig({ { text: 'Settings of its own', link: '/guide/flow-settings' }, { text: 'Many turns at once', link: '/guide/async-flows' }, { text: 'A flow that calls a flow', link: '/guide/calling-flows' }, + { text: 'Checking a flow', link: '/guide/checking-flows' }, { text: 'Testing a flow', link: '/guide/testing-flows' }, { text: 'Flowverses', link: '/guide/flowverses' }, ], diff --git a/docs/guide/checking-flows.md b/docs/guide/checking-flows.md new file mode 100644 index 0000000..484aedc --- /dev/null +++ b/docs/guide/checking-flows.md @@ -0,0 +1,81 @@ +# Checking a flow + +Check a flow before anything runs it: a static reading that executes nothing, then the flow +driven by stubs against a clock. Together they catch the mistakes that otherwise surface +hours into a run — the loop nothing can end, the field read off an answer that failed, the +name the interface does not answer to. + +## One line + +```sh +hmz check official/rlar +``` + +``` +…/rlar/__init__.py:125: warning: unbounded-loop: every way out of this loop waits for an +agent to say so, and an agent may never say it -- give the loop a bound of its own: a budget +read off spent(), a cap on the rounds, a range +hmz check: 0 errors, 1 warning +``` + +A flow is named the way `-f` names one — `chat`, `official/rlar`, a path of your own — and +everything wrong is said at once, one finding a line. The full table of codes is in the +[reference](/reference/flows#checking-a-flow). + +## What an error is, and what a warning is + +An **error** is a flow no run survives: it cannot run, cannot be answered, or cannot end. +`hmz check` exits `1` on any of those. A **warning** is a flow that runs, and a run of it +that may be regretted — rlar's warning above is real and documented: its loop is ended by its +reviewer alone, which is the flow's own shape. Warnings print and pass; `--strict` holds a +flow to the whole bar, which is the right setting for a flowverse's CI: + +```sh +hmz check --strict local/mine +``` + +## The reading that runs nothing + +The static reading is pure `ast` over every file the flow holds. Nothing is imported and +nothing is executed, so it is safe to point at a flow nobody has read — one an agent just +wrote, one fetched off the internet, one about to be forked: + +```sh +hmz check --static somebody-elses/flow +``` + +## The reading that runs it against the worst day + +Without `--static`, the flow is also loaded — in a subprocess held to a clock, never in your +process — and its live config model is read. The same machinery is a library, and the +scenarios are the questions worth asking of a loop: + +```python +from hmz.flows import NEVER_DONE, SILENT, proved + +proof = proved(".humanize/flows/mine", scenarios=(NEVER_DONE, SILENT)) +assert all(one.finished for one in proof.outcomes), proof.outcomes +``` + +`NEVER_DONE` is the reviewer that never says the work is done. The stubs answer every turn at +once — every boolean verdict `False`, every turn adding 100k output tokens to `spent()` — so +a loop held to a budget walks to the end of it in milliseconds, and one whose only exit is +the verdict is caught by the turn cap. That is the executable proof that a run of your flow +can end. `SILENT` answers every turn with nothing, which is what a failed turn answers: a +flow that reads a field off an unguarded answer falls over here rather than at hour three. + +## In a script + +`--json` says the same findings one JSON object a line, and the exit status is the answer: +`0` with nothing blocking, `1` with something, `2` for a line to correct. + +```sh +hmz check --json local/mine | jq -r .code +``` + +## See also + +- [`hmz check`](/reference/cli#hmz-check) — the command and its flags +- [Checking a flow](/reference/flows#checking-a-flow) — the library API and the rule table +- [Testing a flow](/guide/testing-flows) — driving a flow with stand-ins of your own +- [Writing a flow](/guide/writing-a-flow) diff --git a/docs/guide/testing-flows.md b/docs/guide/testing-flows.md index ba1a8c3..20f72b3 100644 --- a/docs/guide/testing-flows.md +++ b/docs/guide/testing-flows.md @@ -174,6 +174,7 @@ uv run pytest --run-agents # also drives the real coding agent CLIs ## See also +- [Checking a flow](/guide/checking-flows) - [Answers in a shape](/guide/shapes) - [config model](/guide/flow-settings) - [hook](/guide/hooks) diff --git a/docs/guide/writing-a-flow.md b/docs/guide/writing-a-flow.md index 404463d..89974f1 100644 --- a/docs/guide/writing-a-flow.md +++ b/docs/guide/writing-a-flow.md @@ -176,7 +176,15 @@ by taking its name. A file whose name starts with `_` is not a flow. ## Check your work -Check which agents the flow declares. +Check the flow itself, before anything runs it: a static reading that executes nothing, then +the flow driven by stubs against a clock — including the world where the reviewer never says +the work is done. See [Checking a flow](/guide/checking-flows). + +```sh +hmz check local/twice +``` + +And check which agents the flow declares. ```python from hmz.flows import drives diff --git a/docs/reference/flows.md b/docs/reference/flows.md index 8ecde32..23e9d39 100644 --- a/docs/reference/flows.md +++ b/docs/reference/flows.md @@ -1130,6 +1130,88 @@ finished. What the turn was doing is left where it got to. A stop that waited for a turn would not read as a stop — a model can think for minutes. +## Checking a flow + +Two readings, before anything runs it — what [`hmz check`](/reference/cli#hmz-check) runs +from a command line, reachable as a library for a test or a loop that writes flows. + +`checked` is the static one: pure `ast` over every file the flow's directory holds (what is +under its `skills/` excepted), executing nothing, so it is safe to point at a flow nobody has +read. It answers with findings rather than raising, one per thing found: + +```python +from hmz.flows import checked + +for one in checked(".humanize/flows/mine"): + print(f"{one.where}:{one.line}: {one.severity}: {one.code}: {one.said}") +``` + +A `Finding` carries `code`, `severity`, `where`, `line` and `said`. An **error** is a flow +that cannot run, cannot be answered, or cannot end — something no run of it survives. A +**warning** is a flow that runs, and a run of it that may be regretted. + +| Code | Severity | What it found | +| --- | --- | --- | +| `unread` | error | A file that will not parse. | +| `not-a-flow` | error | No `__init__.py`, or nothing marked `@flow()`. | +| `unsized-agents` | error | An `agents` annotation that does not state a fixed count. | +| `unread-annotation` | error | The annotation's names live under `TYPE_CHECKING`, where a run cannot read them. | +| `foreign-import` | error | An import of humanize's own modules other than `hmz.flows`. | +| `unknown-name` | error | `from hmz.flows import` a name it does not offer. | +| `unknown-ask` | error | An attribute asked of an agent, session or person that is not on the interfaces. | +| `dead-loop` | error | A constant-true loop with no `break`, `return` or `raise` inside it. | +| `sleeping-loop` | error | A constant-true loop that only sleeps: alive from outside, doing nothing. | +| `stateless-resume` | error | `@flow(resumable=True)` with nowhere to be handed its state. | +| `unbounded-loop` | warning | Every way out of a loop waits on an agent's verdict, and the function holds no bound of its own. | +| `unguarded-answer` | warning | A field read off a suppressed, shaped answer nothing tested against `None`. | +| `unsaid-moment` | warning | A hook hung on a moment only some backends run that no place declares. | +| `loose-config` | warning | A config whose `model_config` neither forbids extras nor freezes. | +| `unsaid-field` | warning | A config field without a `Field(description=...)`. | +| `unsaid-flow` | warning | An entry file without a docstring, so lists of flows show nothing for it. | +| `state-kept` | warning | Kept state something writes and nothing ever clears. | +| `twice-named` | warning | Two flows in one file under one name; the first wins. | + +Every rule is the proof of an absence, worked out one function at a time — no exit in this +loop, no bound in this function, no guard on this name. Nothing claims an exit reachable or +follows a value through a call: a flow that keeps its loop in one function and its bound in +another is a flow the reading trusts. + +`proved` is the second reading: the flow loaded and driven for real, in a subprocess per +scenario, by stubs that claim every capability over the real driver base classes — so the +hooks it hangs fire as they would, every turn lands at once, and each adds what the scenario +says to `spent()`. The parent holds the clock, sleeps are free, and the flow works in a +scratch directory taken away with the process. + +```python +from hmz.flows import ALWAYS_DONE, NEVER_DONE, SILENT, proved + +proof = proved(".humanize/flows/mine", scenarios=(NEVER_DONE, ALWAYS_DONE, SILENT)) +assert proof.findings == () +assert all(one.finished for one in proof.outcomes), proof.outcomes +``` + +A `Scenario` says how the world answers: every boolean field of a shaped answer says its +`verdict`, every string field its `answer`, each turn climbs `climb` output tokens, and the +proof ends at `turns` turns or `seconds` seconds, whichever the flow forces. Three are named: +`NEVER_DONE` is the reviewer that never says the work is done — a loop with a bound of its +own still ends here, which is the executable proof that a run can end; `ALWAYS_DONE` is the +shortest road through; `SILENT` answers every turn with nothing, which is what a failed turn +answers, so it is every guard tried at once. A flow the loading refuses comes back as a +`refused-load` finding, and the flow's live config model is read against the config rules +whether or not the static reading could see it. + +And the catalogue, for whoever — or whatever — is writing a flow against this installation: + +```python +from hmz.flows import briefed, catalogue + +catalogue() # one Capability per thing a flow may build on, with the backends that serve it +briefed() # the same, rendered as one page to steer by +``` + +Read off the live interface at call time — the moments off the enum, the backend sets off +the driver classes' own declarations — so what it promises is what this installation serves. + ## Testing a flow A flow is a function, so drive it with something that is not a coding agent: diff --git a/src/hmz/cli/check.py b/src/hmz/cli/check.py index 48234c8..029fa4e 100644 --- a/src/hmz/cli/check.py +++ b/src/hmz/cli/check.py @@ -66,7 +66,7 @@ def check(argv: list[str]) -> int: ) args = parser.parse_args(argv) - import os + from pathlib import Path from hmz.sdk import Hmz @@ -75,7 +75,7 @@ def check(argv: list[str]) -> int: for named in args.flow: # A name nothing answers to is a line to correct, refused the way argparse refuses # one -- before anything is read, and for every name on the line at once. - if not os.path.isfile(flows.find(named)): + if not Path(flows.find(named)).is_file(): parser.error(f"no flow called {named!r}") found.extend(flows.check(named, static=args.static)) _said(found, as_json=args.as_json, flows=len(args.flow)) @@ -114,6 +114,8 @@ def _said(found: list[Finding], *, as_json: bool, flows: int) -> None: print(f"{one.where}:{one.line}: {one.severity}: {one.code}: {one.said}") errors = sum(one.severity == "error" for one in found) if found: - print(f"hmz check: {many(errors, 'error')}, {many(len(found) - errors, 'warning')}") + print( + f"hmz check: {many(errors, 'error')}, {many(len(found) - errors, 'warning')}" + ) else: print(f"hmz check: nothing to say about {many(flows, 'flow')}") From a08617730802cf83981cb8781fd4e0e179e04147 Mon Sep 17 00:00:00 2001 From: dongyun Date: Fri, 28 Aug 2026 22:28:23 +0000 Subject: [PATCH 6/8] feat(flows): offer MINE, so a flow that writes flows knows where yours live The aot flow lands what it compiles in the flows of your own, and the two places those are is a fact written in verses.py -- handed through rather than spelled again where it would drift. Co-Authored-By: Claude Fable 5 --- src/hmz/flows/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hmz/flows/__init__.py b/src/hmz/flows/__init__.py index ecf8627..66fd277 100644 --- a/src/hmz/flows/__init__.py +++ b/src/hmz/flows/__init__.py @@ -119,6 +119,7 @@ def run(agents: tuple[Agent, Agent], task: str) -> None: "EVERYWHERE", "FLOWS", "LOCAL", + "MINE", "NEVER_DONE", "OFFICIAL", "PERMISSIONS", From 73e4c0a6959ecca2e4ac24058724e6bed3b51b4d Mon Sep 17 00:00:00 2001 From: dongyun Date: Sat, 29 Aug 2026 00:47:30 +0000 Subject: [PATCH 7/8] fix(flows): fabricate a shaped answer to a list's own lower bound A field that takes at least three answers was answered with none: the fabrication gave every list [], the model refused it, and a flow that bounds its own retries read as one that could not be driven at all. Now a list is answered with as many fabricated elements as its bound asks, and an Annotated element is answered as the type it annotates. Found scanning the official flows: parallel_flame_chase's coordinator takes a plan of at least three lanes. Its own cross-field validator is still past what deterministic fabrication can satisfy -- which the proof reports honestly, as a flow that fails closed after three tries. Co-Authored-By: Claude Fable 5 --- src/hmz/flows/proving.py | 13 ++++++++++++- tests/test_proving.py | 16 ++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/hmz/flows/proving.py b/src/hmz/flows/proving.py index b2cc5d5..e297a09 100644 --- a/src/hmz/flows/proving.py +++ b/src/hmz/flows/proving.py @@ -593,8 +593,13 @@ def _filled(kind: Any, field: Any, scenario: Scenario) -> Any: Returns: The value. """ + from typing import Annotated + from pydantic import BaseModel + if get_origin(kind) is Annotated: + # The constraints ride along in the field itself; what is answered is the type. + return _filled(get_args(kind)[0], field, scenario) for said in _unioned(kind): if said is bool: return scenario.verdict @@ -608,7 +613,13 @@ def _filled(kind: Any, field: Any, scenario: Scenario) -> Any: if said in (int, float): return 0 if get_origin(said) in (list, tuple, set, frozenset): - return [] + # As many as the field says it takes at the least, each made the same way: + # a shape that requires three lanes is answered with three, not refused. + fewest = 0 + for bound in getattr(field, "metadata", None) or (): + fewest = max(fewest, getattr(bound, "min_length", 0) or 0) + inner = next(iter(get_args(said)), None) + return [_filled(inner, None, scenario) for _ in range(fewest)] if get_origin(said) is dict: return {} if isinstance(said, type) and issubclass(said, BaseModel): diff --git a/tests/test_proving.py b/tests/test_proving.py index 19e9e4e..3e9bf66 100644 --- a/tests/test_proving.py +++ b/tests/test_proving.py @@ -382,3 +382,19 @@ class Impossible(BaseModel): assert _said(Shaped, SILENT) == "" assert _said(None, SILENT) == "" assert _said(None, ALWAYS_DONE) == ALWAYS_DONE.answer + + +def test_a_list_that_takes_at_least_some_is_answered_with_that_many() -> None: + from typing import Annotated + + class Planned(BaseModel): + lanes: list[Nested] = Field(min_length=3) + tags: list[Annotated[str, Field(min_length=1)]] = Field(min_length=1) + loose: list[str] + + made = _made(Planned, NEVER_DONE) + held = Planned.model_validate(made) + assert len(held.lanes) == 3 + assert held.lanes[0] == Nested(said=NEVER_DONE.answer, fine=False) + assert held.tags == [NEVER_DONE.answer] + assert held.loose == [] From 2ffd55c05c9d293c8b959b72dc702b64c301c248 Mon Sep 17 00:00:00 2001 From: dongyun Date: Sat, 29 Aug 2026 01:17:51 +0000 Subject: [PATCH 8/8] feat(flows): add unknown-verdict -- a comparison no answer can satisfy A shaped answer's Literal field compared against a value the shape does not offer -- review.verdict == "DONE" over Literal["done", "redo"] -- is a guard that never opens, or one that never shuts. Both readings pass, both provings pass, and the flow silently steers by a value no answer will ever hold; found where the shape is declared in the same file and left be where it is not. The rule oh-my-humanize's freeze checker runs on its edge conditions (references undeclared verdict), carried over to answers held to a shape. Co-Authored-By: Claude Fable 5 --- docs/reference/flows.md | 1 + src/hmz/flows/checking.py | 164 ++++++++++++++++++++++++++++++++++++++ tests/test_checking.py | 103 ++++++++++++++++++++++++ 3 files changed, 268 insertions(+) diff --git a/docs/reference/flows.md b/docs/reference/flows.md index 23e9d39..9dc1e9f 100644 --- a/docs/reference/flows.md +++ b/docs/reference/flows.md @@ -1164,6 +1164,7 @@ that cannot run, cannot be answered, or cannot end — something no run of it su | `stateless-resume` | error | `@flow(resumable=True)` with nowhere to be handed its state. | | `unbounded-loop` | warning | Every way out of a loop waits on an agent's verdict, and the function holds no bound of its own. | | `unguarded-answer` | warning | A field read off a suppressed, shaped answer nothing tested against `None`. | +| `unknown-verdict` | warning | An answer's field compared against a value its shape does not offer, so the comparison can never be what it reads as. | | `unsaid-moment` | warning | A hook hung on a moment only some backends run that no place declares. | | `loose-config` | warning | A config whose `model_config` neither forbids extras nor freezes. | | `unsaid-field` | warning | A config field without a `Field(description=...)`. | diff --git a/src/hmz/flows/checking.py b/src/hmz/flows/checking.py index 85491e9..27431bb 100644 --- a/src/hmz/flows/checking.py +++ b/src/hmz/flows/checking.py @@ -381,6 +381,13 @@ def _root(node: ast.expr) -> str: return node.id if isinstance(node, ast.Name) else "" +def _tip(node: ast.expr) -> str: + """The name at the tip of one dotted expression -- `Literal` of `typing.Literal`.""" + if isinstance(node, ast.Attribute): + return node.attr + return node.id if isinstance(node, ast.Name) else "" + + def _moments_in(annotation: ast.expr, read: _Read) -> set[str]: """Every moment one annotation names, which is a place saying what it needs.""" return { @@ -797,6 +804,9 @@ class _Answer(NamedTuple): shaped: bool suppressed: bool line: int + #: The name of the shape the turn was held to, where it was named plainly -- "" for a + #: turn held to no shape, or to one written some way this reading does not follow. + model: str = "" @dataclass @@ -1211,6 +1221,7 @@ def _called(value: ast.Call, scope: _Scope) -> frozenset[str] | _Crew | _Answer for one in value.keywords ), line=value.lineno, + model=_shaped_as(value), ) if isinstance(func, ast.Attribute) and func.attr in {"aturn", "pursue", "apursue"}: asked = _valued(func.value, scope) @@ -1224,10 +1235,19 @@ def _called(value: ast.Call, scope: _Scope) -> frozenset[str] | _Crew | _Answer for one in value.keywords ), line=value.lineno, + model=_shaped_as(value), ) return None +def _shaped_as(value: ast.Call) -> str: + """The name of the shape one turn is held to, where the call names it plainly.""" + for keyword in value.keywords: + if keyword.arg == "schema" and isinstance(keyword.value, ast.Name): + return keyword.value.id + return "" + + #: The comparisons that read as a bound: a count against something, ordered. _ORDERED = (ast.Lt, ast.LtE, ast.Gt, ast.GtE) @@ -1260,6 +1280,7 @@ def _expression(node: ast.expr, scope: _Scope) -> None: for side in [node.left, *node.comparators] ): scope.bounded = True + _never(node, scope) _guards(node, scope) for one in [node.left, *node.comparators]: _expression(one, scope) @@ -1389,6 +1410,149 @@ def _asked(node: ast.Attribute, scope: _Scope) -> None: ) +#: The comparisons that ask whether a value is exactly one of some others. +_BY_VALUE = (ast.Eq, ast.NotEq, ast.In, ast.NotIn) + + +def _never(node: ast.Compare, scope: _Scope) -> None: + """One comparison read against the shape behind it, for a value no answer holds. + + `review.verdict == "DONE"` over `Literal["done", "redo"]` is a guard that never opens, + and `!= "DONE"` one that never shuts: either way the flow steers by a value the shape + cannot answer with. Said only where everything is certain -- the answer's shape is a + model this same file declares, the field spells its values out, and the other side is + constants -- so a shape read from elsewhere is let be rather than guessed at. + """ + if len(node.ops) != 1 or not isinstance(node.ops[0], _BY_VALUE): + return + membership = isinstance(node.ops[0], (ast.In, ast.NotIn)) + pairs = [(node.left, node.comparators[0])] + if not membership: + # `"done" == review.verdict` reads the same either way round; `"d" in x` does not. + pairs.append((node.comparators[0], node.left)) + for asked, against in pairs: + if not (isinstance(asked, ast.Attribute) and isinstance(asked.value, ast.Name)): + continue + answer = scope.answers.get(asked.value.id) + if answer is None or not answer.model: + continue + held = _offered(answer.model, scope.read, asked.attr, set()) + values = _values_of(against, membership=membership) + if held is None or values is None: + return + offers = ", ".join(repr(one) for one in sorted(held, key=repr)) + for value in values: + if value not in held: + scope.findings.append( + Finding( + "unknown-verdict", + "warning", + scope.read.where, + node.lineno, + f"no answer holds {value!r} at {asked.attr!r} -- the shape " + f"offers {offers}, and a comparison against a value it cannot " + "hold reads as a guard and guards nothing", + ) + ) + return + + +def _offered( + model: str, read: _Read, field_name: str, seen: set[str] +) -> frozenset[object] | None: + """Every value one field of a model may hold, read off the model's own words. + + Follows local bases the way the strictness rule does -- a field a model inherits is as + much its shape as one it declares. + + Args: + model: The model's name. + read: The file, whose models are the only ones read. + field_name: The field asked about. + seen: The models already walked, which stops a circular inheritance. + + Returns: + The values, or None where the model or the field is not here, or the field's + annotation does not spell its values out. + """ + if model in seen or model not in read.models: + return None + seen.add(model) + declared = read.models[model] + for node in declared.body: + if ( + isinstance(node, ast.AnnAssign) + and isinstance(node.target, ast.Name) + and node.target.id == field_name + ): + return _options_in(node.annotation) + for base in declared.bases: + if isinstance(base, ast.Name): + held = _offered(base.id, read, field_name, seen) + if held is not None: + return held + return None + + +def _options_in(annotation: ast.expr | None) -> frozenset[object] | None: + """Every value one annotation admits, where it spells them all out. + + `Literal` through and through -- unions, `Optional` and `Annotated` read through -- + or None for anything open: a field that may hold a plain `str` is a field any + comparison against is an honest one. + """ + said = _unquoted(annotation) + if said is None: + return None + if isinstance(said, ast.Constant) and said.value is None: + return frozenset({None}) + if isinstance(said, ast.BinOp) and isinstance(said.op, ast.BitOr): + left = _options_in(said.left) + right = _options_in(said.right) + if left is None or right is None: + return None + return left | right + if not isinstance(said, ast.Subscript): + return None + head = _tip(said.value) + parts = _elements(said.slice) + if head == "Literal": + options: set[object] = set() + for part in parts: + if not isinstance(part, ast.Constant): + return None + options.add(part.value) + return frozenset(options) + if head == "Annotated" and parts: + return _options_in(parts[0]) + if head == "Optional" and parts: + inner = _options_in(parts[0]) + return None if inner is None else inner | {None} + if head == "Union": + gathered: frozenset[object] = frozenset() + for part in parts: + inner = _options_in(part) + if inner is None: + return None + gathered |= inner + return gathered + return None + + +def _values_of(node: ast.expr, *, membership: bool) -> list[object] | None: + """The constant values one side of a comparison holds, or None where it is not sure.""" + if not membership: + return [node.value] if isinstance(node, ast.Constant) else None + if isinstance(node, (ast.Tuple, ast.List, ast.Set)): + values: list[object] = [] + for elt in node.elts: + if not isinstance(elt, ast.Constant): + return None + values.append(elt.value) + return values + return None + + # --------------------------------------------------------------------------------------- # The loop rules: a loop is legal when something inside it can end it. # --------------------------------------------------------------------------------------- diff --git a/tests/test_checking.py b/tests/test_checking.py index 7fe6da7..02b4f3d 100644 --- a/tests/test_checking.py +++ b/tests/test_checking.py @@ -36,6 +36,7 @@ "stateless-resume": "error", "unbounded-loop": "warning", "unguarded-answer": "warning", + "unknown-verdict": "warning", "unsaid-moment": "warning", "loose-config": "warning", "unsaid-field": "warning", @@ -488,6 +489,108 @@ def run(agents: tuple[Agent], task: str) -> None: pytest.param( DOC + """ +from typing import Literal + +from hmz.flows import Agent, flow +from pydantic import BaseModel, Field + +class Review(BaseModel): + model_config = {"extra": "forbid"} + + verdict: Literal["done", "redo"] = Field(description="how the round went") + +@flow +def run(agents: tuple[Agent], task: str) -> None: + review = agents[0](task, suppress=True, schema=Review) + if review and review.verdict == "DONE": + return +""", + {"unknown-verdict"}, + id="unknown-verdict", + ), + pytest.param( + DOC + + """ +from typing import Literal + +from hmz.flows import Agent, flow +from pydantic import BaseModel, Field + +class Review(BaseModel): + model_config = {"extra": "forbid"} + + verdict: Literal["done", "redo"] = Field(description="how the round went") + +@flow +def run(agents: tuple[Agent], task: str) -> None: + review = agents[0](task, suppress=True, schema=Review) + if review and review.verdict in ("done", "OOPS"): + return +""", + {"unknown-verdict"}, + id="unknown-verdict-a-dead-member", + ), + pytest.param( + DOC + + """ +from typing import Literal + +from hmz.flows import Agent, flow +from pydantic import BaseModel, Field + +class Review(BaseModel): + model_config = {"extra": "forbid"} + + verdict: Literal["done", "redo"] = Field(description="how the round went") + +@flow +def run(agents: tuple[Agent], task: str) -> None: + review = agents[0](task, suppress=True, schema=Review) + if review and review.verdict == "done": + return +""", + CLEAN, + id="unknown-verdict-edge-a-value-the-shape-offers", + ), + pytest.param( + DOC + + """ +from hmz.flows import Agent, flow +from pydantic import BaseModel, Field + +class Review(BaseModel): + model_config = {"extra": "forbid"} + + verdict: str = Field(description="how the round went, in its own words") + +@flow +def run(agents: tuple[Agent], task: str) -> None: + review = agents[0](task, suppress=True, schema=Review) + if review and review.verdict == "DONE": + return +""", + CLEAN, + id="unknown-verdict-edge-a-field-left-open", + ), + pytest.param( + DOC + + """ +from _shapes import Review + +from hmz.flows import Agent, flow + +@flow +def run(agents: tuple[Agent], task: str) -> None: + review = agents[0](task, suppress=True, schema=Review) + if review and review.verdict == "DONE": + return +""", + CLEAN, + id="unknown-verdict-edge-a-shape-declared-elsewhere", + ), + pytest.param( + DOC + + """ from hmz.flows import Agent, Moment, flow @flow