diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index fc03fdc8..0f9775f6 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -32,5 +32,8 @@ jobs: git config --global user.name "github-actions[bot]" git config --global user.email "github-actions[bot]@users.noreply.github.com" + - name: Validate docs in strict mode + run: hatch run docs:build --strict + - name: Deploy docs to GitHub Pages run: hatch run docs:deploy diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 182584e9..6f9acdfe 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -5,6 +5,9 @@ on: tags: - "v[0-9].[0-9]+.[0-9]+*" +env: + HATCH_VERSION: "1.16.5" + jobs: release-on-pypi: runs-on: ubuntu-latest @@ -17,7 +20,7 @@ jobs: uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - name: Install Hatch - run: pip install hatch + run: pip install hatch==${{ env.HATCH_VERSION }} - name: Build run: hatch build diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9a1e6c86..b00d5e59 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -40,6 +40,18 @@ jobs: tests-haystack-v3: runs-on: ubuntu-latest + services: + redis: + image: redis:6.2-alpine + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 2s + --health-timeout 2s + --health-retries 20 + env: + HAYHOOKS_TEST_REDIS_URL: redis://127.0.0.1:6379/15 strategy: matrix: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] @@ -55,15 +67,21 @@ jobs: - name: Install Haystack v3 into the test env run: | - hatch env create test - hatch -e test env run -- uv pip install --upgrade "haystack-ai>=3" - hatch -e test env run -- python -c "import haystack, sys; v = haystack.__version__; print('haystack', v); sys.exit(0 if int(v.split('.')[0]) >= 3 else 'expected Haystack v3+, got ' + v)" + hatch env create test-v3 + hatch -e test-v3 env run -- uv pip install --upgrade "haystack-ai>=3" + hatch -e test-v3 env run -- python -c "import haystack, sys; v = haystack.__version__; print('haystack', v); sys.exit(0 if int(v.split('.')[0]) >= 3 else 'expected Haystack v3+, got ' + v)" - - name: Run unit tests - run: hatch run test:unit + - name: Run tests + run: hatch run test-v3:all + + - name: Run durable process-recovery smoke test + if: matrix.python-version == '3.12' + env: + HAYHOOKS_TEST_PROCESS_RECOVERY: "1" + run: hatch run test-v3:all tests/test_durable_process_recovery.py - name: Ty - check types (Haystack v3) - run: hatch run test:types + run: hatch run test-v3:types linting: runs-on: ubuntu-slim @@ -83,6 +101,9 @@ jobs: - name: Ty - check types run: hatch run test:types + - name: Build documentation in strict mode + run: hatch run docs:build --strict + dashboard-tests: runs-on: ubuntu-latest steps: diff --git a/.gitignore b/.gitignore index 048a3b8d..1345bd25 100644 --- a/.gitignore +++ b/.gitignore @@ -192,6 +192,11 @@ chainlit.md .env.local .env.*.local +# Local A2A long-running demo artifacts +.a2a-long-running-demo-session.json +/a2a-req +/a2a-res + # Temporary files *.tmp *.temp diff --git a/README.md b/README.md index ae14176f..c5de60da 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ With Hayhooks, you can: - πŸ“¦ **Deploy your Haystack pipelines and agents as REST APIs** with maximum flexibility and minimal boilerplate code. - πŸ› οΈ **Expose your Haystack pipelines and agents over the MCP protocol**, making them available as tools in AI dev environments like [Cursor](https://cursor.com) or [Claude Desktop](https://claude.ai/download). Under the hood, Hayhooks runs as an [MCP Server](https://modelcontextprotocol.io/docs/concepts/architecture), exposing each pipeline and agent as an [MCP Tool](https://modelcontextprotocol.io/docs/concepts/tools). - 🀝 **Expose your Haystack pipelines and agents over the [A2A protocol](https://a2a-protocol.org)** (`pip install "hayhooks[a2a]"`), so other agents can discover them through auto-generated agent cards and delegate tasks to them via `hayhooks a2a run`. +- ♻️ **Run Haystack 3 pipelines and agents as durable background work** (`pip install "hayhooks[durable]"`) with Redis-backed checkpoints, retries, cancellation, wait/resume, progress, and process recovery. - πŸ’¬ **Integrate your Haystack pipelines and agents with [Open WebUI](https://openwebui.com)** as OpenAI-compatible chat completion backends with streaming support. - πŸ–₯️ **Embed a [Chainlit](https://chainlit.io/) chat UI** directly in Hayhooks with `pip install "hayhooks[chainlit]"` and `hayhooks run --with-chainlit` -- zero-configuration frontend with streaming, pipeline selection, and custom UI widgets. - πŸ•ΉοΈ **Control Hayhooks core API endpoints through chat** - deploy, undeploy, list, or run Haystack pipelines and agents by chatting with [Claude Desktop](https://claude.ai/download), [Cursor](https://cursor.com), or any other MCP client. @@ -53,6 +54,7 @@ from hayhooks import BasePipelineWrapper, async_streaming_generator def weather_function(location): return f"The weather in {location} is sunny." + weather_tool = Tool( name="weather_tool", description="Provides weather information for a given location.", @@ -64,6 +66,7 @@ weather_tool = Tool( function=weather_function, ) + class PipelineWrapper(BasePipelineWrapper): def setup(self) -> None: self.agent = Agent( @@ -73,7 +76,7 @@ class PipelineWrapper(BasePipelineWrapper): ) # This will create a POST /my_agent/run endpoint - #Β `question` will be the input argument and will be auto-validated by a Pydantic model + # `question` will be the input argument and will be auto-validated by a Pydantic model async def run_api_async(self, question: str) -> str: result = await self.agent.run_async(messages=[ChatMessage.from_user(question)]) return result["last_message"].text @@ -82,9 +85,7 @@ class PipelineWrapper(BasePipelineWrapper): async def run_chat_completion_async( self, model: str, messages: list[dict], body: dict ) -> AsyncGenerator[str, None]: - chat_messages = [ - ChatMessage.from_openai_dict_format(message) for message in messages - ] + chat_messages = [ChatMessage.from_openai_dict_format(message) for message in messages] return async_streaming_generator( pipeline=self.agent, @@ -154,11 +155,22 @@ Or chat with it in the [embedded Chainlit UI](docs/features/chainlit-integration - Built-in support for handling file uploads in pipelines - Perfect for RAG systems and document processing +### ♻️ Durable Execution + +- Run typed Pipeline and Agent work outside the request and recover it after a process restart +- Preserve checkpoints, bounded retries, progress, cancellation, wait/resume, idempotency, and owner isolation +- Project managed durable Agents over A2A and retain terminal results with Redis TTL + +Durable execution is intentionally an at-least-once engine for low-to-moderate workloads with one to three replicas. +It uses fixed polling and cooperative cancellation; it is not a DAG orchestrator, high-scale fair queue, live migration +system, or exactly-once boundary for external side effects. See the [supported scope and tradeoffs](docs/advanced/durable-engine.md#supported-scope-and-tradeoffs). + ## Next Steps - [Quick Start Guide](docs/getting-started/quick-start.md) - Get started with Hayhooks - [Installation](docs/getting-started/installation.md) - Install Hayhooks and dependencies - [Configuration](docs/getting-started/configuration.md) - Configure Hayhooks for your needs +- [Durable Engine](docs/advanced/durable-engine.md) - Understand restart-safe execution and its boundaries - [Tracing Dashboard Frontend](dashboard/README.md) - Local dashboard setup and frontend development commands - [Examples](docs/examples/overview.md) - Explore example implementations diff --git a/docs/advanced/durable-engine-vs-temporal.md b/docs/advanced/durable-engine-vs-temporal.md new file mode 100644 index 00000000..706a7266 --- /dev/null +++ b/docs/advanced/durable-engine-vs-temporal.md @@ -0,0 +1,67 @@ +# Hayhooks durable engine and Temporal + +Hayhooks provides focused durable execution for Haystack 3 Pipelines and Agents. Temporal is a general-purpose +durable workflow platform. + +## Core comparison + +| Requirement | Hayhooks durable engine | Temporal | +|---|---|---| +| Persistence and recovery | Redis records plus explicit Pipeline or Agent checkpoints | Event History plus deterministic [Workflow replay](https://docs.temporal.io/workflows) | +| Delivery safety | At-least-once with fenced leases and one active owner | Workflow logic is effectively once; [Activities](https://docs.temporal.io/activity-execution) may retry | +| Retries | Explicit, bounded retries from the latest checkpoint | Declarative, independently configurable [Activity retry policies](https://docs.temporal.io/encyclopedia/retry-policies) | +| Interaction | Typed inspect, wait/resume, progress, and result APIs | [Queries, Signals, and Updates](https://docs.temporal.io/encyclopedia/workflow-message-passing) plus durable Workflow state | +| Cancellation | Cooperative checks at safe boundaries | Cooperative cancellation with propagation policies | +| Orchestration | One Pipeline or Agent execution with delayed retries | Durable timers, Activities, [schedules](https://docs.temporal.io/schedule), and [Child Workflows](https://docs.temporal.io/child-workflows) | +| Versioning | Exact revision gate prevents incompatible recovery | Replay-safe patching and [Worker Versioning](https://docs.temporal.io/production-deployment/worker-deployments/worker-versioning) | +| Haystack example | Run a RAG Pipeline with `checkpoint_at=["generator"]`; after a crash, restore its `PipelineSnapshot` before generation | Invoke retrieval and generation as separate Activities wrapping Haystack components; completed Activity results are not repeated during Workflow replay | +| Best fit | Focused, moderate-scale durable Haystack workloads | Large-scale or cross-service orchestration around Haystack | + +## Hayhooks code map + +| Concern | Relevant implementation | +|---|---| +| Lifecycle and fencing | [`engine.py`](https://github.com/deepset-ai/hayhooks/blob/main/src/hayhooks/durable/engine.py) | +| Store contract and Redis persistence | [`store.py`](https://github.com/deepset-ai/hayhooks/blob/main/src/hayhooks/durable/store.py), [`redis.py`](https://github.com/deepset-ai/hayhooks/blob/main/src/hayhooks/durable/redis.py) | +| Pipeline and Agent checkpoints | [`adapters.py`](https://github.com/deepset-ai/hayhooks/blob/main/src/hayhooks/durable/adapters.py) | +| Retries and worker recovery | [`context.py`](https://github.com/deepset-ai/hayhooks/blob/main/src/hayhooks/durable/context.py), [`manager.py`](https://github.com/deepset-ai/hayhooks/blob/main/src/hayhooks/durable/manager.py) | +| Progress, wait/resume, and inspection | [`context.py`](https://github.com/deepset-ai/hayhooks/blob/main/src/hayhooks/durable/context.py), [`routes.py`](https://github.com/deepset-ai/hayhooks/blob/main/src/hayhooks/server/durable/routes.py) | +| Cooperative cancellation | [`context.py`](https://github.com/deepset-ai/hayhooks/blob/main/src/hayhooks/durable/context.py), [`engine.py`](https://github.com/deepset-ai/hayhooks/blob/main/src/hayhooks/durable/engine.py) | +| Revision and deployment safety | [`runtime.py`](https://github.com/deepset-ai/hayhooks/blob/main/src/hayhooks/durable/runtime.py), [`deploy_utils.py`](https://github.com/deepset-ai/hayhooks/blob/main/src/hayhooks/server/utils/deploy_utils.py) | +| Durable A2A projection and recovery | [`durable_executor.py`](https://github.com/deepset-ai/hayhooks/blob/main/src/hayhooks/server/a2a/durable_executor.py), [`redis_task_store.py`](https://github.com/deepset-ai/hayhooks/blob/main/src/hayhooks/server/a2a/redis_task_store.py) | + +Hayhooks makes an existing Haystack Pipeline or Agent durable with minimal restructuring. Temporal offers finer-grained +orchestration, but obtaining that granularity usually means deciding which Haystack operations should become separate +Activities; the whole Pipeline can still run as one Activity when independent step recovery is unnecessary. + +## What Temporal adds + +| Gain | Why it matters | +|---|---| +| Independent step policies | Retrieval, generation, payments, or notifications can have separate retries, timeouts, workers, and resource limits | +| Cross-service orchestration | A Workflow can coordinate Haystack with databases, external APIs, approval systems, and other services | +| Durable time and interaction | Native timers, schedules, callbacks, Signals, Queries, and Updates | +| Horizontal routing | Activities can run on different task queues, worker pools, languages, or infrastructure | +| Production operations | Execution history, search, UI, batch operations, metrics, and mature failure investigation | +| Long-running deployments | Worker versioning, pinning, gradual rollout, rollback, and Workflows that span releases | +| Reduced engine ownership | Temporal owns task delivery, persistence, recovery, and orchestration semantics instead of Hayhooks maintaining them in Redis | + +For restart-safe Haystack Pipelines and Agents with checkpoints, retries, cancellation, and resume, Temporal provides +little immediate functional gain and adds infrastructure plus integration work. It becomes valuable when an execution +grows into a long-lived, multi-step workflow spanning Haystack and other systems. + +Temporal does not remove the need for idempotent external side effects: an Activity may execute more than once when its +result is lost and the task is retried. + +## References + +- [Hayhooks durable engine](durable-engine.md) +- [Temporal Workflows](https://docs.temporal.io/workflows) +- [Temporal Activities](https://docs.temporal.io/activities) +- [Temporal Activity execution](https://docs.temporal.io/activity-execution) +- [Temporal retry policies](https://docs.temporal.io/encyclopedia/retry-policies) +- [Temporal Workflow message passing](https://docs.temporal.io/encyclopedia/workflow-message-passing) +- [Temporal Child Workflows](https://docs.temporal.io/child-workflows) +- [Temporal Schedules](https://docs.temporal.io/schedule) +- [Temporal Visibility](https://docs.temporal.io/visibility) +- [Temporal Worker Versioning](https://docs.temporal.io/production-deployment/worker-deployments/worker-versioning) diff --git a/docs/advanced/durable-engine.md b/docs/advanced/durable-engine.md new file mode 100644 index 00000000..76d6df78 --- /dev/null +++ b/docs/advanced/durable-engine.md @@ -0,0 +1,318 @@ +# Durable engine + +Hayhooks durable execution provides detached, checkpointed work for **Haystack +3** Pipelines and Agents. + +It accepts validated work, runs it outside the request, resumes from Haystack +checkpoints, and recovers after a worker or process disappears. External +writes use application idempotency keys derived from the execution ID and +logical step, keeping replay safe when a process exits between an effect and +its next checkpoint. + +## Supported capabilities + +- Detached, typed Pipeline and Agent execution through durable REST endpoints. +- Pipeline snapshots and Agent state checkpoints with bounded retries, + progress, cancellation, and typed wait/resume. +- Redis-backed fenced claims and lease recovery across process restarts. +- Live SSE chunk streaming that clients can detach from and reattach to. +- Idempotent submission, optional owner-isolated REST access, and managed A2A + task projection. +- Native Redis TTL for terminal records, plus an equivalent volatile + in-memory backend for local development and tests. +- Explicit revision checks and safe deploy/undeploy handling. + +## Supported scope and tradeoffs + +- validated durable input before submission succeeds; +- at-least-once execution with one fenced worker owner at a time; +- checkpoints, progress, retries, cancellation, wait/resume, and terminal + results; and +- a public result that excludes private input, checkpoint, state, owner, and + fence details. + +The pure reducer in `hayhooks.durable.engine` is the only lifecycle +decision-maker. Storage atomically persists its plan and derives indexes from +the old and new control records. + +```text +queued ── claim ──> running ── complete/fail/cancel ──> terminal + β”‚ β”‚ + β”‚ β”œβ”€β”€ checkpoint / heartbeat + β”‚ β”œβ”€β”€ retry ──> queued (due later) + β”‚ └── suspend ──> waiting ── resume ──> queued + └── cancel ──> terminal +``` + +## Embedding the runtime + +Applications can import the runtime, providers, deployment contracts, public +exceptions, and FastAPI adapter directly from `hayhooks.durable`. A standalone +runtime starts only deployments attached to that runtime; it never inspects the +Hayhooks pipeline registry. + +This durable integration fragment assumes the host FastAPI application already +has authentication middleware that sets a stable principal on `request.state` +before the owner dependency runs. The authentication middleware itself is +application-specific and intentionally omitted: + +```python +from contextlib import asynccontextmanager + +from fastapi import FastAPI, Request +from haystack import Pipeline +from pydantic import BaseModel + +from hayhooks import BasePipelineWrapper, DurableContext +from hayhooks.durable import DurableRuntime, DurableSettings, create_durable_router + + +class JobRequest(BaseModel): + document_id: str + + +class JobResult(BaseModel): + indexed: bool + + +class JobWrapper(BasePipelineWrapper): + durable_revision = "job-v1" + + def setup(self) -> None: + self.pipeline = Pipeline() + + async def run_durable_async(self, context: DurableContext, request: JobRequest) -> JobResult: + # Replace this example body with checkpointed Pipeline work. + return JobResult(indexed=bool(request.document_id)) + + +runtime = DurableRuntime( + durable_settings=DurableSettings( + durable_store="memory", # Development only; use Redis in production. + durable_poll_interval=0.05, + ) +) + +wrapper = JobWrapper() +wrapper.setup() +deployment = runtime.deployment("jobs", wrapper) + + +def current_owner_id(request: Request) -> str: + principal = request.state.principal + return f"{principal.tenant_id}:{principal.subject_id}" + + +@asynccontextmanager +async def lifespan(_app: FastAPI): + try: + await runtime.start() + yield + finally: + await runtime.close() + + +app = FastAPI(lifespan=lifespan) +app.include_router( + create_durable_router(deployment, owner_id_dependency=current_owner_id), + prefix="/jobs", +) +``` + +The adapter exposes typed submit, inspect, stream, cancel, and resume routes. +It does not start workers or own the runtime. Host middleware and dependencies +retain control of authentication and authorization; the durable layer persists +only the stable owner ID returned by the dependency, which must be a non-empty +string of at most 512 characters (a misconfigured dependency fails closed with +500). Durable wrapper code can read that value through `context.owner_id` after +process recovery. The same deployment is also drivable without HTTP through +`deployment.submit(...)` and the store's resume/cancel operations; see +[Streaming chunks](#streaming-chunks) for the reattachable SSE endpoint. + +Passing `owner_id_dependency=None` is an explicit unscoped security choice: + +```python +app.include_router( + create_durable_router(deployment, owner_id_dependency=None), + prefix="/internal-jobs", +) +``` + +In this mode, possession of the unguessable execution ID grants access. Use it +only behind one application-wide authorization boundary or for local +development. + +For production Redis, give each application/environment an isolated prefix: + +```python +from hayhooks.durable import DurableRuntime, RedisExecutionStoreProvider + +provider = RedisExecutionStoreProvider( + redis_url="redis://localhost:6379/0", + key_prefix="myapp:production:durable", +) +runtime = DurableRuntime(provider) +``` + +When the host owns an existing binary Redis client, pass `redis=client` and +`close_redis=False`, then close the durable runtime before closing the client. +Do not use `decode_responses=True`. + +Every Uvicorn worker owns a runtime, Redis pool, and worker tasks. Redis leases +and fences coordinate them, so the setting is a per-process, per-deployment +ceiling and effective concurrency is `processes Γ— +durable_execution_concurrency`. Keep wrapper revisions identical across replicas +and start with one to three processes and conservative concurrency. + +Built-in providers snapshot `DurableSettings`, and the runtime adopts that +snapshot when a provider is supplied. Conflicting runtime and provider settings +are rejected before deployment creation. The selected provider is fixed for the +runtime's lifetime; create a new runtime to change storage backends. + +## Redis layout + +Each deployment has an isolated namespace with controls and opaque input, +checkpoint, result, error, wait, and progress payload keys, plus one `chunks` +stream per execution. It has exactly two sorted-set indexes: + +| Key | Purpose | +|---|---| +| `runnable` | All queued work, scored by its retry deadline or immediate transition time. | +| `lease-expiry` | Running fences, scored by their Redis-server lease deadline. | +| `exec::chunks` | Bounded append-only display chunks, read by the execution SSE stream. | + +The namespace also contains a `capacity` hash with only `nonterminal` and one +idempotency binding per execution. Terminal execution and idempotency keys use +native Redis TTL. The in-memory backend schedules equivalent cleanup. The store +assigns progress sequence numbers when it commits each transition, preserving +checkpoint and cancellation progress when they race. + +Workers poll `runnable` every configured poll interval, one second by default. +They use Redis `TIME` and read one due member without removing it. Multiple +replicas can observe it; the watched control and fence let exactly one +transition to `running`. Lease maintenance uses the same interval and recovers +at most 100 expired entries per pass. The default averages about 500 ms claim +latency and can add up to one second to claim or lease recovery. + +The controlled beta deployment profile uses one logical deployment with one to +three replicas and low-to-moderate load. Its two indexes, native TTL, and +single reducer keep the worker model observable during normal operation and +recovery. + +## Streaming chunks + +`GET /{pipeline}/executions/{execution_id}/stream` is a Server-Sent Events +stream of one execution's display chunks, followed by a terminal `completed`, +`failed`, or `canceled` event carrying the same public projection the inspect +route returns. The submit response advertises it as the `stream` link. + +Chunks are best-effort display data, deliberately outside the durable fence. A +chunk append uses one non-transactional Redis pipeline and never writes the +control record, so token-rate streaming cannot invalidate a heartbeat. Nothing +about a chunk can fail a run either: an invalid or oversized payload (chunks are +capped at 64 KB) or a backend blip drops that chunk and logs it, because +replaying a pipeline to recover a display token is never the right trade. +Progress events remain the coarse durable audit trail. + +```python +class StreamingWrapper(BasePipelineWrapper): + durable_revision = "streaming-v1" + + async def run_durable_async(self, context: DurableContext, request: Question) -> Answer: + result = await context.run_agent_async( + messages=[ChatMessage.from_user(request.query)], + streaming_callback=context.stream_chunk, + ) + return Answer(reply=result["messages"][-1].text) +``` + +Bind the callback to the component when the work is a Pipeline rather than an +Agent. `async_streaming_generator` passes it per run in `pipeline_run_args`, and +that does not survive here: `run_pipeline_async` data is serialized into the +`PipelineSnapshot`, Haystack drops the callable it cannot serialize, and +`Pipeline.run` rebuilds its `data` from the snapshot when resuming, so a callback +passed as run data disappears at the first checkpoint. + +Hayhooks provides the synchronous callback that a Pipeline component needs. +Binding it to a shared component is safe for the same reason +`async_streaming_generator` can hand the same module-level +`_async_streaming_callback` to every concurrent run: the callback carries no +per-run state and resolves its destination on each call from a `ContextVar`. +Hayhooks routes on `_ASYNC_STREAMING_QUEUE`; the durable path routes on the +execution context, which the engine sets per execution task and `asyncio.to_thread` +copies into the Pipeline's worker thread: + +```python +from hayhooks import durable_streaming_callback + + +class StreamingPipelineWrapper(BasePipelineWrapper): + durable_revision = "streaming-pipeline-v1" + + def setup(self) -> None: + self.pipeline = Pipeline.loads(...) + self.pipeline.get_component("llm").streaming_callback = durable_streaming_callback +``` + +`run_pipeline_async` drives the Pipeline on a worker thread, which is why that +callback uses the synchronous bridge. The Agent path is the other way round: +`run_agent_async` awaits `Agent.run_async` on the server loop, where the bridge +cannot work, so an Agent takes `context.stream_chunk` as shown above. What +`pipeline_run_args` injection actually buys `async_streaming_generator` is +control over *which* components stream without permanently mutating a shared +Pipeline; here the callback simply does nothing outside a durable execution. A +run-time `streaming_callback` still takes precedence over a bound one, so an +ordinary streaming endpoint on the same wrapper is unaffected. See +`examples/durable_chat_with_website` for the whole wrapper. Each SSE event +carries the entry ID as `id:` and the producing `attempt` in its payload; a +client resets its buffer when `attempt` increases, because a retried attempt +re-streams from its checkpoint. The server ignores chunks from an older attempt +once a newer one is known. Reconnecting clients resend `Last-Event-ID` +automatically and resume from that cursor. If the bounded log no longer contains +that cursor, the stream emits a `gap` event, replays the retained tail, and then +continues; reset or mark the client buffer as partial when that happens. A +browser `EventSource` reconnects whenever the server closes the connection, +including after the terminal event, so call `close()` once that event arrives. + +The stream follows one execution for its whole life, not just one run of it: an +execution that suspends into `waiting` keeps its stream open and heartbeating +until it is resumed and reaches a terminal state, which for an approval-gated +workflow can be a long time. Detach and reattach with `Last-Event-ID` if a +connection parked that long is not what you want. + +`durable_max_stream_chunks` bounds the log per execution (10 000 by default); +`0` disables chunk production entirely while leaving the endpoint working. +`durable_max_stream_chunk_bytes` caps a single chunk at 64 KB by default; an +oversized chunk is dropped, never failed, and it also sets how many entries one +read returns, so that a single read stays under 4 MB whatever the cap is. The +two bounds multiply: size Redis for +`durable_max_stream_chunks * durable_max_stream_chunk_bytes` per streaming +execution, times the executions running at once. The log expires with its +execution under `durable_terminal_ttl_seconds`. A new viewer that starts after +the log has overflowed receives only the retained tail; the terminal result +remains authoritative. + +A stream that breaks after its headers were sent has no status code left to +report with, so it ends in an `error` event instead. Treat it the way a client +treats a dropped connection: reattach with `Last-Event-ID`, or read the +execution's terminal state from the inspect route. + +## Revisions and rollout + +Every durable wrapper, including a managed A2A Agent, declares a non-empty +`durable_revision`. Use an image digest or Git SHA in production and update it +with checkpoint-relevant code, prompts, configuration, or dependencies. Claims +and resumes verify that persisted work matches the active revision. + +## Operations + +`DurableExecutionManager.health_snapshot()` reports `nonterminal`, `runnable`, +`lease_expiry`, and the current worker store-error streak. Repeated claim or +transition failures make the deployment health snapshot unhealthy until a +worker completes a store operation successfully; `/status/{pipeline_name}` +then returns `503`. Alert on sustained runnable growth, repeated lease +recovery, worker/store health failures, and runs that exceed their expected +duration. + +See [Durable execution operations](durable-execution-operations.md) for +deployment, retention, and incident guidance. diff --git a/docs/advanced/durable-execution-operations.md b/docs/advanced/durable-execution-operations.md new file mode 100644 index 00000000..dd816cf0 --- /dev/null +++ b/docs/advanced/durable-execution-operations.md @@ -0,0 +1,96 @@ +# Durable execution operations + +Hayhooks durable execution provides fenced, at-least-once recovery. Use an +idempotency key derived from the execution ID and logical step for every +external write so recovered work remains safe to replay. + +## Controlled beta deployment profile + +- Use authenticated Redis 6.2+ with TLS, persistence, backups, and + `maxmemory-policy noeviction`. +- Run one logical Hayhooks deployment, normally one replica and at most two or + three, against one isolated namespace. +- Set a non-empty `durable_revision` on every durable Pipeline wrapper and + managed A2A Agent. An image digest or Git SHA is the recommended value. +- Treat `HAYHOOKS_DURABLE_EXECUTION_CONCURRENCY` as the per-deployment ceiling. + Keep its default of one until every Pipeline/Agent component and tool is proven + concurrency-safe. +- Put REST and A2A behind authentication, request-size, rate, and tenant + controls. `HAYHOOKS_DURABLE_MAX_NONTERMINAL_EXECUTIONS` is an optional + deployment-wide secondary admission cap. + +## Execution and recovery + +The namespace holds a control record, opaque payloads, one bounded `chunks` +stream per execution, one `runnable` ZSET, one `lease-expiry` ZSET, a +`nonterminal` capacity field, and idempotency bindings. The control is +authoritative; the indexes are derived atomically with it. + +Workers poll due runnable work at the configured interval using Redis `TIME`. +Candidate reads are non-destructive. A watched control hash and monotonically +increasing fence make concurrent replica claims safe. Lease maintenance uses +the same interval and processes up to 100 expired fences. Delayed retries +remain in `runnable` with their Redis-server due timestamp and are invisible +until due. + +## Retention and rollout + +Terminal control and payload keys and their idempotency binding receive the +configured Redis TTL when a run first becomes terminal. +`HAYHOOKS_DURABLE_MAX_STREAM_CHUNKS` bounds each execution's SSE chunk log and +`0` disables it, which is the kill switch if streaming misbehaves. +`HAYHOOKS_DURABLE_MAX_STREAM_CHUNK_BYTES` caps one chunk (64 KB by default); +oversized chunks are dropped, never failed. Every Redis chunk append refreshes +the configured TTL, so stale writers cannot create permanent keys; a nonterminal +execution quiet for that entire retention window may lose old display history +and reports a cursor gap on reattachment. Memory refuses appends after its +cleanup has run. Do not delete records manually while they are nonterminal. + +Begin a new controlled-beta deployment with an empty durable namespace, then +retain its terminal records through the configured Redis TTL. + +## Streaming load + +A stream reads its chunk log without blocking, so an attached viewer holds a +Redis connection only for the microseconds of each read rather than for its +whole lifetime. That matters because streams share the engine's connection pool: +a blocking read pins one connection per viewer, which caps concurrent streams at +the pool size and starves the workers, whose heartbeats and terminal transitions +need that same pool. The cost of polling instead is up to 100 ms of extra +live-display latency. + +Polling is two-speed: 100 ms while chunks are moving, backing off to one second +after a second of silence, since an execution can sit in `waiting` for hours with +a viewer attached. A partial page waits for the live interval; only a full +backlog page is drained immediately. An idle stream rereads its execution record +every second, so a terminal event arrives within about two seconds of the last +chunk; a run that keeps generating after a cancellation request reports terminal +only after it stops producing chunks. An idle stream costs about one chunk read, +three record reads, and one heartbeat per second per viewer, and an execution parked in +`waiting` keeps paying that until it is resumed. One read returns at most +`4 MB / durable_max_stream_chunk_bytes` chunks β€” 62 at the default cap β€” so a +client reattaching to a full log catches up over several reads rather than +materializing the whole log at once. Chunks are display data and are never a +reason to replay a run. A non-empty page also performs one control lookup before +delivery, which prevents a lease-lost worker from leaking stale chunks before +its replacement emits. + +Each producer also waits for one two-command Redis pipeline per chunk (`XADD` +and rolling `EXPIRE`). +That keeps ordering and shutdown behavior simple, but caps token throughput at +roughly one chunk per Redis round trip. Batch only if production measurements +show that remote Redis latency is slowing generation. + +## Health and incidents + +Health exposes `nonterminal`, `runnable`, `lease_expiry`, and +`worker_store_error_streak`. A claim or transition store failure marks the +deployment health snapshot unhealthy until that worker completes a store +operation successfully; `/status/{pipeline_name}` then returns `503`. Investigate +a growing runnable count, repeated lease recovery, store failures, or executions +that remain running/waiting longer than expected. Pause submissions, preserve +the Redis namespace, and inspect controls and fences before changing code or +restarting workers. + +Use Redis 6.2 or later. Monitor the durable counts alongside Redis availability +and latency to keep execution recovery healthy. diff --git a/docs/advanced/running-pipelines.md b/docs/advanced/running-pipelines.md index 6566ca1a..86af7f5a 100644 --- a/docs/advanced/running-pipelines.md +++ b/docs/advanced/running-pipelines.md @@ -23,10 +23,7 @@ Execute deployed pipelines via CLI, HTTP API, or programmatically. ```python import requests - resp = requests.post( - "http://localhost:1416/my_pipeline/run", - json={"query": "What is Haystack?"} - ) + resp = requests.post("http://localhost:1416/my_pipeline/run", json={"query": "What is Haystack?"}) print(resp.json()) ``` @@ -36,14 +33,13 @@ Execute deployed pipelines via CLI, HTTP API, or programmatically. import httpx import asyncio + async def main(): async with httpx.AsyncClient() as client: - r = await client.post( - "http://localhost:1416/my_pipeline/run", - json={"query": "What is Haystack?"} - ) + r = await client.post("http://localhost:1416/my_pipeline/run", json={"query": "What is Haystack?"}) print(r.json()) + asyncio.run(main()) ``` @@ -95,10 +91,7 @@ See [File Upload Support](../features/file-upload-support.md) for implementation ```python import requests -resp = requests.post( - "http://localhost:1416/my_pipeline/run", - json={"query": "What is Haystack?"} -) +resp = requests.post("http://localhost:1416/my_pipeline/run", json={"query": "What is Haystack?"}) print(resp.json()) ``` @@ -110,14 +103,13 @@ print(resp.json()) import httpx import asyncio + async def main(): async with httpx.AsyncClient() as client: - r = await client.post( - "http://localhost:1416/my_pipeline/run", - json={"query": "What is Haystack?"} - ) + r = await client.post("http://localhost:1416/my_pipeline/run", json={"query": "What is Haystack?"}) print(r.json()) + asyncio.run(main()) ``` @@ -132,6 +124,7 @@ import requests from requests.exceptions import RequestException import time + def run_with_retry(pipeline_name, params, max_retries=3): url = f"http://localhost:1416/{pipeline_name}/run" @@ -143,7 +136,7 @@ def run_with_retry(pipeline_name, params, max_retries=3): except RequestException as e: if attempt == max_retries - 1: raise - time.sleep(2 ** attempt) # Exponential backoff + time.sleep(2**attempt) # Exponential backoff ``` ## Logging @@ -153,6 +146,7 @@ Add logging to your pipeline wrappers: ```python from hayhooks import log + class PipelineWrapper(BasePipelineWrapper): def run_api(self, query: str) -> str: log.info("Processing query: {}", query) diff --git a/docs/concepts/pipeline-wrapper.md b/docs/concepts/pipeline-wrapper.md index cf2d4b88..91bb042b 100644 --- a/docs/concepts/pipeline-wrapper.md +++ b/docs/concepts/pipeline-wrapper.md @@ -20,6 +20,7 @@ from haystack import Pipeline from hayhooks import BasePipelineWrapper, get_last_user_message, async_streaming_generator, streaming_generator + class PipelineWrapper(BasePipelineWrapper): def setup(self) -> None: pipeline_yaml = (Path(__file__).parent / "pipeline.yml").read_text() @@ -67,7 +68,7 @@ def setup(self) -> None: {% endfor %} Answer the given question: {{query}} {% endmessage %}""", - required_variables="*" + required_variables="*", ) llm = OpenAIChatGenerator(model="gpt-4o-mini") @@ -165,6 +166,7 @@ Hayhooks can stream results from `run_api()` or `run_api_async()` when you retur from collections.abc import Generator from hayhooks import streaming_generator + def run_api(self, query: str) -> Generator: return streaming_generator( pipeline=self.pipeline, @@ -178,6 +180,7 @@ For async pipelines: from collections.abc import AsyncGenerator from hayhooks import async_streaming_generator + async def run_api_async(self, query: str) -> AsyncGenerator: return async_streaming_generator( pipeline=self.pipeline, @@ -217,6 +220,7 @@ If you need SSE (for browsers, [EventSource](https://developer.mozilla.org/en-US ```python from hayhooks import SSEStream, streaming_generator + def run_api(self, query: str): return SSEStream( streaming_generator( @@ -231,6 +235,7 @@ For async pipelines: ```python from hayhooks import SSEStream, async_streaming_generator + async def run_api_async(self, query: str): return SSEStream( async_streaming_generator( @@ -248,6 +253,7 @@ Hayhooks can return binary files (images, PDFs, audio, etc.) directly from `run_ import tempfile from fastapi.responses import FileResponse + def run_api(self, prompt: str) -> FileResponse: image = self.generate_image(prompt) @@ -269,6 +275,56 @@ For a full working example, see the [Image Generation example](https://github.co ## Optional Methods +### Durable execution + +Implement `run_durable()` or `run_durable_async()` on an ordinary `BasePipelineWrapper` to submit restart-safe work. +The method receives a `DurableContext` and one Pydantic request model; it returns a Pydantic result model. Hayhooks +owns records, worker lifecycle, Redis, cancellation, and resume. + +Ordinary exceptions from a durable wrapper are terminal failures. For a +transient dependency failure (for example an LLM timeout or rate limit), call +`await context.retry(...)` so Hayhooks persists the checkpoint and schedules a +bounded retry; do not simply re-raise the transient error. + +```python +from haystack import Pipeline +from pydantic import BaseModel + +from hayhooks import BasePipelineWrapper, DurableContext + + +class JobRequest(BaseModel): + source: str + + +class JobResult(BaseModel): + processed: int + + +class PipelineWrapper(BasePipelineWrapper): + durable_revision = "my-image-digest-or-git-sha" + + def setup(self) -> None: + self.pipeline = Pipeline() + # Add and connect components. + + async def run_durable_async(self, context: DurableContext, request: JobRequest) -> JobResult: + outputs = await context.run_pipeline_async({"source": {"value": request.source}}, checkpoint_at=["process"]) + return JobResult(processed=outputs["process"]["count"]) +``` + +Hayhooks exposes `POST /{pipeline}/run-durable`, `GET /{pipeline}/executions/{execution_id}`, and cancel/resume +endpoints. The submit response and status endpoint expose only a safe result view; validated inputs and checkpoint +state remain server-side. The built-in Redis store is the default. Set `HAYHOOKS_DURABLE_STORE=memory` only for +volatile local development. `run_pipeline_async()` uses a worker thread around +Haystack's synchronous snapshot API; it does not call `Pipeline.run_async()`. +See the [durable engine](../advanced/durable-engine.md) for checkpoint, +recovery, and side-effect boundaries. +Every durable wrapper must set a non-empty `durable_revision` class attribute +from the immutable build identifier; Hayhooks does not fingerprint source or +configuration automatically. See also the +[complete durable example](https://github.com/deepset-ai/hayhooks/tree/main/examples/durable_execution). + ### run_api_async() The asynchronous version of `run_api()` for better performance under high load. @@ -367,7 +423,7 @@ async def run_chat_completion_async(self, model: str, messages: list[dict], body return async_streaming_generator( pipeline=self.pipeline, pipeline_run_args={"prompt": {"query": question}}, - allow_sync_streaming_callbacks=True # βœ… Auto-detect and enable hybrid mode + allow_sync_streaming_callbacks=True, # βœ… Auto-detect and enable hybrid mode ) ``` @@ -390,12 +446,12 @@ When you set `allow_sync_streaming_callbacks=True`, the system enables **intelli ```python # Option 1: Strict mode (Default - Recommended) -allow_sync_streaming_callbacks=False +allow_sync_streaming_callbacks = False # β†’ Raises error if sync-only components found # β†’ Best for: New code, ensuring proper async components, best performance # Option 2: Auto-detection (Compatibility mode) -allow_sync_streaming_callbacks=True +allow_sync_streaming_callbacks = True # β†’ Automatically detects and enables hybrid mode only when needed # β†’ Best for: Legacy pipelines, components without async support, gradual migration ``` @@ -431,16 +487,14 @@ class SyncOnlyWrapper(BasePipelineWrapper): self.pipeline = Pipeline() self.pipeline.add_component("llm", SyncOnlyGenerator()) - async def run_chat_completion_async( - self, model: str, messages: list[dict], body: dict - ) -> AsyncGenerator: + async def run_chat_completion_async(self, model: str, messages: list[dict], body: dict) -> AsyncGenerator: question = get_last_user_message(messages) # Enable hybrid mode so the sync-only component can stream in an async pipeline return async_streaming_generator( pipeline=self.pipeline, pipeline_run_args={"llm": {"prompt": question}}, - allow_sync_streaming_callbacks=True # βœ… Handles sync component + allow_sync_streaming_callbacks=True, # βœ… Handles sync component ) ``` @@ -497,11 +551,8 @@ class MultiLLMWrapper(BasePipelineWrapper): self.pipeline.add_component( "prompt_1", ChatPromptBuilder( - template=[ - ChatMessage.from_system("You are a helpful assistant."), - ChatMessage.from_user("{{query}}") - ] - ) + template=[ChatMessage.from_system("You are a helpful assistant."), ChatMessage.from_user("{{query}}")] + ), ) self.pipeline.add_component("llm_1", OpenAIChatGenerator(model="gpt-4o-mini")) @@ -511,11 +562,9 @@ class MultiLLMWrapper(BasePipelineWrapper): ChatPromptBuilder( template=[ ChatMessage.from_system("You are a helpful assistant that refines responses."), - ChatMessage.from_user( - "Previous response: {{previous_response[0].text}}\n\nRefine this." - ) + ChatMessage.from_user("Previous response: {{previous_response[0].text}}\n\nRefine this."), ] - ) + ), ) self.pipeline.add_component("llm_2", OpenAIChatGenerator(model="gpt-4o-mini")) @@ -528,10 +577,7 @@ class MultiLLMWrapper(BasePipelineWrapper): question = get_last_user_message(messages) # By default, only llm_2 (the last streaming component) will stream - return streaming_generator( - pipeline=self.pipeline, - pipeline_run_args={"prompt_1": {"query": question}} - ) + return streaming_generator(pipeline=self.pipeline, pipeline_run_args={"prompt_1": {"query": question}}) ``` **What happens:** Only `llm_2` (the last streaming-capable component) streams its responses token by token. The first LLM (`llm_1`) executes normally without streaming, and only the final refined output streams to the user. @@ -548,7 +594,7 @@ def run_chat_completion(self, model: str, messages: list[dict], body: dict) -> G return streaming_generator( pipeline=self.pipeline, pipeline_run_args={"prompt_1": {"query": question}}, - streaming_components=["llm_1", "llm_2"] # Stream both components + streaming_components=["llm_1", "llm_2"], # Stream both components ) ``` @@ -558,16 +604,16 @@ You can also selectively enable streaming for specific components: ```python # Stream only the first LLM -streaming_components=["llm_1"] +streaming_components = ["llm_1"] # Stream only the second LLM (same as default) -streaming_components=["llm_2"] +streaming_components = ["llm_2"] # Stream ALL capable components (shorthand) -streaming_components="all" +streaming_components = "all" -#Β Stream ALL capable components (specific list) -streaming_components=["llm_1", "llm_2"] +# Stream ALL capable components (specific list) +streaming_components = ["llm_1", "llm_2"] ``` ### Using the "all" Keyword @@ -578,7 +624,7 @@ The `"all"` keyword is a convenient shorthand to enable streaming for all capabl return streaming_generator( pipeline=self.pipeline, pipeline_run_args={...}, - streaming_components="all" # Enable all streaming components + streaming_components="all", # Enable all streaming components ) ``` @@ -686,27 +732,24 @@ See the [Multi-LLM Streaming Example](https://github.com/deepset-ai/hayhooks/tre For streaming responses, pass `include_outputs_from` to `streaming_generator()` or `async_streaming_generator()`, and use the `on_pipeline_end` callback to access intermediate outputs. For example: ```python - def run_chat_completion(self, model: str, messages: List[dict], body: dict) -> Generator: - question = get_last_user_message(messages) +def run_chat_completion(self, model: str, messages: List[dict], body: dict) -> Generator: + question = get_last_user_message(messages) - # Store retrieved documents for citations - self.retrieved_docs = [] + # Store retrieved documents for citations + self.retrieved_docs = [] - def on_pipeline_end(result: dict[str, Any]) -> None: - # Access intermediate outputs here - if "retriever" in result: - self.retrieved_docs = result["retriever"]["documents"] - # Use for citations, logging, analytics, etc. + def on_pipeline_end(result: dict[str, Any]) -> None: + # Access intermediate outputs here + if "retriever" in result: + self.retrieved_docs = result["retriever"]["documents"] + # Use for citations, logging, analytics, etc. - return streaming_generator( - pipeline=self.pipeline, - pipeline_run_args={ - "retriever": {"query": question}, - "prompt_builder": {"query": question} - }, - include_outputs_from={"retriever"}, # Make retriever outputs available - on_pipeline_end=on_pipeline_end - ) + return streaming_generator( + pipeline=self.pipeline, + pipeline_run_args={"retriever": {"query": question}, "prompt_builder": {"query": question}}, + include_outputs_from={"retriever"}, # Make retriever outputs available + on_pipeline_end=on_pipeline_end, + ) ``` **What happens:** The `on_pipeline_end` callback receives both `llm` and `retriever` outputs in the `result` dict, allowing you to access retrieved documents alongside the generated response. @@ -723,12 +766,9 @@ async def run_chat_completion_async(self, model: str, messages: List[dict], body return async_streaming_generator( pipeline=self.async_pipeline, - pipeline_run_args={ - "retriever": {"query": question}, - "prompt_builder": {"query": question} - }, + pipeline_run_args={"retriever": {"query": question}, "prompt_builder": {"query": question}}, include_outputs_from={"retriever"}, - on_pipeline_end=on_pipeline_end + on_pipeline_end=on_pipeline_end, ) ``` @@ -738,10 +778,7 @@ For non-streaming `run_api` or `run_api_async` endpoints, pass `include_outputs_ ```python def run_api(self, query: str) -> dict: - result = self.pipeline.run( - data={"retriever": {"query": query}}, - include_outputs_from={"retriever"} - ) + result = self.pipeline.run(data={"retriever": {"query": query}}, include_outputs_from={"retriever"}) # Build custom response with both answer and sources return {"answer": result["llm"]["replies"][0], "sources": result["retriever"]["documents"]} ``` @@ -751,8 +788,7 @@ Same pattern for async: ```python async def run_api_async(self, query: str) -> dict: result = await self.async_pipeline.run_async( - data={"retriever": {"query": query}}, - include_outputs_from={"retriever"} + data={"retriever": {"query": query}}, include_outputs_from={"retriever"} ) return {"answer": result["llm"]["replies"][0], "sources": result["retriever"]["documents"]} ``` @@ -787,6 +823,7 @@ def on_reasoning( """ return text + def run_chat_completion(self, model: str, messages: list[dict], body: dict) -> Generator: return streaming_generator( pipeline=self.pipeline, @@ -812,6 +849,7 @@ Hayhooks can handle file uploads by adding a `files` parameter: ```python from fastapi import UploadFile + def run_api(self, files: list[UploadFile] | None = None, query: str = "") -> str: if files: # Process uploaded files @@ -860,6 +898,7 @@ Your pipeline wrapper may require additional dependencies: # pipeline_wrapper.py import trafilatura # Additional dependency + def run_api(self, urls: list[str], question: str) -> str: # Use additional library content = trafilatura.fetch(urls[0]) @@ -886,6 +925,7 @@ Implement proper error handling in production: from hayhooks import log from fastapi import HTTPException + class PipelineWrapper(BasePipelineWrapper): def setup(self) -> None: try: diff --git a/docs/examples/overview.md b/docs/examples/overview.md index c8456f68..d7d2ba41 100644 --- a/docs/examples/overview.md +++ b/docs/examples/overview.md @@ -23,6 +23,8 @@ This page lists all maintained Hayhooks examples with detailed descriptions and | Example | Docs | Code | Description | |---|---|---|---| +| Durable Pipeline | [Pipeline wrapper](../concepts/pipeline-wrapper.md#durable-execution) | [GitHub](https://github.com/deepset-ai/hayhooks/tree/main/examples/durable_execution) | Typed REST wrapper with Redis-backed restart recovery, inspection, cancellation, and resume | +| Durable A2A Agent | [A2A Support](../features/a2a-support.md) | [GitHub](https://github.com/deepset-ai/hayhooks/tree/main/examples/a2a_long_running) | Haystack 3 Agent checkpoints, detached A2A tasks, progress, and restart recovery | | RAG: Indexing and Query with Elasticsearch | [rag-system.md](rag-system.md) | [GitHub](https://github.com/deepset-ai/hayhooks/tree/main/examples/rag_indexing_query) | Full indexing/query pipelines with Elasticsearch | | API Key Authentication | [advanced-configuration.md](../advanced/advanced-configuration.md) | [GitHub](https://github.com/deepset-ai/hayhooks/tree/main/examples/programmatic/api_key_auth) | Middleware-based API key auth with multi-key support and Swagger Authorize | diff --git a/docs/features/a2a-support.md b/docs/features/a2a-support.md index e0da2f17..9c809986 100644 --- a/docs/features/a2a-support.md +++ b/docs/features/a2a-support.md @@ -8,14 +8,16 @@ A2A complements [MCP support](mcp-support.md): MCP exposes pipelines as **tools* The Hayhooks A2A Server: -- Exposes every deployed pipeline that implements `run_chat_completion` or `run_chat_completion_async` as an A2A agent +- Exposes deployed chat and durable Agent wrappers as A2A agents - Serves a per-agent [Agent Card](#agent-cards) for discovery, auto-generated from the pipeline and customizable from the wrapper - Implements the JSON-RPC protocol binding of the [A2A specification](https://a2a-protocol.org/latest/specification/) (v1.0), including SSE streaming - Streams pipeline output incrementally as task artifact updates +- Supports detached long-running task execution with polling, subscription, and cooperative async cancellation ## Requirements -- Install with `pip install hayhooks[a2a]` (uses the official [a2a-sdk](https://github.com/a2aproject/a2a-python)) +- Install with `pip install hayhooks[a2a]` (uses the official [a2a-sdk](https://github.com/a2aproject/a2a-python) + and [redis-py](https://redis.io/docs/latest/develop/clients/redis-py/) clients) ## Getting Started @@ -44,11 +46,22 @@ HAYHOOKS_A2A_EXTERNAL_URL= # Base URL advertised in agent cards # (set when behind a reverse proxy) HAYHOOKS_A2A_V0_3_COMPAT=true # Also accept A2A spec 0.3 requests # (used by older clients and tools) +HAYHOOKS_A2A_TASK_STORE=auto # auto, memory, or redis +HAYHOOKS_A2A_REDIS_URL=redis://localhost:6379/0 +HAYHOOKS_A2A_REDIS_KEY_PREFIX=hayhooks:a2a +HAYHOOKS_DURABLE_STORE=redis # Redis by default; memory is volatile +HAYHOOKS_DURABLE_REDIS_URL=redis://localhost:6379/0 +HAYHOOKS_DURABLE_REDIS_KEY_PREFIX=hayhooks:durable +HAYHOOKS_DURABLE_EXECUTION_CONCURRENCY=1 + # Operator ceiling per durable Agent ``` ## Which pipelines are exposed -A deployed pipeline is exposed as an A2A agent when it implements `run_chat_completion` or `run_chat_completion_async` β€” the same methods used by the [OpenAI-compatible chat endpoints](openai-compatibility.md). No extra method is needed. +A deployed pipeline is exposed as an A2A agent when it uses either authoring mode: + +- **Chat compatibility**: implement `run_chat_completion` or `run_chat_completion_async`, the same methods used by the [OpenAI-compatible chat endpoints](openai-compatibility.md). +- **Durable Agent**: inherit from `hayhooks.a2a.A2APipelineWrapper` and assign a Haystack 3 `Agent` to `self.pipeline`. Hayhooks supplies the detached executor, checkpoints, progress projection, and durable store. To exclude a chat-capable pipeline from A2A, set `skip_a2a` on the wrapper: @@ -65,7 +78,7 @@ Each exposed pipeline is mounted under its own path prefix: |----------|-------------| | `GET /{pipeline_name}/.well-known/agent-card.json` | The pipeline's agent card | | `POST /{pipeline_name}/` | JSON-RPC binding (`SendMessage`, `SendStreamingMessage`, `GetTask`, ...) | -| `GET /status` | Server status and the list of exposed agents | +| `GET /status` | Operational readiness and the list of exposed agents; not an A2A protocol method | For example, with a deployed `weather_agent` pipeline: @@ -73,6 +86,13 @@ For example, with a deployed `weather_agent` pipeline: curl http://localhost:1418/weather_agent/.well-known/agent-card.json ``` +The operational status endpoint returns `200` only when the configured task +store, executor lifecycle, maintenance loop, and (for managed +durable Agents) durable execution runtime are healthy. It returns `503` with +`status: unavailable` otherwise. The A2A specification does not define a +health method; Agent Cards and their advertised interfaces remain the +standards-compliant discovery and protocol surface. + ## Agent Cards Agent cards are generated automatically: the card name is the pipeline name, the description comes from the pipeline's registry metadata, and a single default skill is created. Override any of it with the `a2a_card` class attribute: @@ -136,14 +156,146 @@ curl -s http://localhost:1418/weather_agent/ \ "parts": [{"text": "Weather in Berlin?"}]}}}' ``` -## Task lifecycle and streaming +## Chat-compatibility task lifecycle and streaming -Each request is handled as an A2A task: +In chat-compatibility mode, each request is handled as an A2A task: 1. A `Task` is created from the incoming message. 2. The task transitions to `working` and the pipeline's chat completion method runs. 3. Pipeline output is emitted as a single `response` artifact. Streaming results (generators returned by `streaming_generator` / `async_streaming_generator`) are emitted incrementally as artifact chunk updates, so `SendStreamingMessage` clients receive text as it is produced. -4. The task ends in `completed` (or `failed`, with the error in the status message β€” enable `HAYHOOKS_SHOW_TRACEBACKS` to include tracebacks). +4. The task ends in `completed`, `failed`, or `canceled`. Enable `HAYHOOKS_SHOW_TRACEBACKS` to include tracebacks in failure messages. + +Native executors own their protocol lifecycle. Durable Agents instead project +their authoritative durable execution into an A2A task as described under +[Durable Haystack Agents](#durable-haystack-agents). + +By default, non-streaming `SendMessage` remains blocking for backward compatibility: the response is returned after the task reaches a terminal or interrupted state. + +For detached execution, set `configuration.returnImmediately`: + +```bash +curl -s http://localhost:1418/weather_agent/ \ + -H "Content-Type: application/json" -H "A2A-Version: 1.0" \ + -d '{"jsonrpc": "2.0", "id": "1", "method": "SendMessage", + "params": {"configuration": {"returnImmediately": true}, + "message": {"messageId": "m1", "role": "ROLE_USER", + "parts": [{"text": "Start the long task"}]}}}' +``` + +The response contains a non-terminal task. Poll it with `GetTask`: + +```bash +curl -s http://localhost:1418/weather_agent/ \ + -H "Content-Type: application/json" -H "A2A-Version: 1.0" \ + -d '{"jsonrpc": "2.0", "id": "2", "method": "GetTask", + "params": {"id": ""}}' +``` + +Or subscribe to an active task with `SubscribeToTask` to receive the latest task snapshot followed by task updates over SSE: + +```bash +curl -N http://localhost:1418/weather_agent/ \ + -H "Content-Type: application/json" -H "A2A-Version: 1.0" \ + -d '{"jsonrpc": "2.0", "id": "3", "method": "SubscribeToTask", + "params": {"id": ""}}' +``` + +When `HAYHOOKS_A2A_V0_3_COMPAT=true`, A2A 0.3 clients can request the same detached behavior with `configuration.blocking=false`. + +## Task storage + +With `HAYHOOKS_A2A_TASK_STORE=auto`, Hayhooks uses Redis for Redis-backed durable Agents and otherwise gives each exposed agent its own A2A SDK `InMemoryTaskStore`. An explicit `memory` choice is never overridden. + +Task storage is server infrastructure rather than pipeline configuration. Hayhooks includes independent in-memory and Redis-backed providers. Select the built-in Redis provider with `HAYHOOKS_A2A_TASK_STORE=redis` or `hayhooks a2a run --task-store redis`; configure its URL and key prefix with `HAYHOOKS_A2A_REDIS_URL` and `HAYHOOKS_A2A_REDIS_KEY_PREFIX`. + +The SDK in-memory store is for one Hayhooks process only. It loses tasks on +restart, and a request routed to another replica cannot see tasks created by +the first. Multi-replica deployments must set the task store to `redis` and +give every replica the same Redis URL and key prefix. `auto` does this only for +Redis-backed durable Agents, so scaled chat-compatible wrappers must select `redis` explicitly. + +The A2A extra includes the official Redis client. Redis task records are protobuf payloads scoped by agent and resolved owner. The configured owner resolver is the sole owner decision point; the default distinguishes unauthenticated requests from an authenticated user named `anonymous`. Durable execution IDs are internal fixed-size hashes of that owner and the opaque A2A task ID, so clients must keep using the original A2A task ID. + +Redis stores each task and its version in one hash, with derived owner, active, and expiry indexes. Writes use bounded `WATCH`/`MULTI` transactions and Redis server time; they do not require Lua or `EVAL`. All keys for one agent share a Redis Cluster hash slot. Restart recovery relies on the durable engine's idempotent submission and persists each complete task projection with one compare-and-set write, so a stale replica cannot overwrite a newer task state. Persistent task records do not replay historical live event queues. + +A durable Agent accepts another user message only while its execution is +waiting for input. A follow-up sent while it is running is rejected rather than +being treated as an idempotent redelivery; clients should wait for +`INPUT_REQUIRED` before continuing a task. + +Terminal tasks use `HAYHOOKS_A2A_TERMINAL_TASK_TTL_SECONDS`. Runtime maintenance performs cleanup even when no later A2A request arrives and removes the protobuf payload and task indexes. Execution-record retention remains independent. If a task projection expires first, `GetTask` with the original task ID reconstructs its current state from the retained execution; the expired history and list entry remain gone. + +Applications constructing the server directly can use a configured provider: + +```python +from hayhooks.a2a import RedisTaskStoreProvider +from hayhooks.server.a2a.app import create_a2a_app +from hayhooks.server.a2a.runtime import A2ARuntime + +runtime = A2ARuntime( + task_store_provider=RedisTaskStoreProvider( + redis_url="redis://localhost:6379/0", + key_prefix="my-app:a2a", + ) +) +app = create_a2a_app(runtime=runtime) +``` + +Persisting task records alone does not make execution recoverable. For durable Agents, use Redis durable execution +(the default) and configure the A2A task store for the task history retention your clients need. Chat-compatible +execution remains process-local. + +### Durable Haystack Agents + +For the managed mode, use an `A2APipelineWrapper` with a Haystack 3 `Agent`. Hayhooks creates the +execution record using the A2A task ID, captures public Agent state at model/tool boundaries, and projects safe +progress, waiting, completion, failure, and cancellation states back to the A2A task. + +The execution record is authoritative; the A2A Task is its persisted client-facing projection. This is why the two +records have separate retention settings, and why a persistent A2A task store alone cannot recover interrupted work. + +```mermaid +flowchart LR + Client["A2A client"] --> Server["A2A server\nmanaged durable executor"] + Server --> Task["A2A Task store\nclient-facing task"] + Server --> Execution["Durable execution record\nsource of truth"] + Execution --> Manager["DurableExecutionManager\nclaim + fenced lease"] + Manager --> Agent["Haystack Agent\ntools + checkpoints"] + Agent --> Execution + Execution --> Projection["A2A task projection\nprogress / waiting / terminal state"] + Projection --> Task + Task --> Client +``` + +The projection updates the task; it does not run the Agent. After a restart, +the durable worker recovers execution. Startup then refreshes any persisted +active task snapshot once; later `GetTask` and list requests project the latest +durable state directly. + +```python +from haystack.components.agents import Agent +from haystack.components.generators.chat import OpenAIChatGenerator + +from hayhooks import A2APipelineWrapper + + +class PipelineWrapper(A2APipelineWrapper): + def setup(self) -> None: + self.pipeline = Agent(chat_generator=OpenAIChatGenerator(), tools=[]) +``` + +The wrapper does not create an executor, worker, record, queue, or Redis client. Durable execution uses Redis by +default; set `HAYHOOKS_DURABLE_STORE=memory` only for non-recoverable local development. +`HAYHOOKS_DURABLE_EXECUTION_CONCURRENCY` is the per-process ceiling. Increasing it requires the Agent, tools, and +their shared dependencies to be concurrency-safe. + +Snapshots, validated messages, and internal tool state remain server-side. A restarted Hayhooks process reclaims +incomplete Redis work from its last safe checkpoint. Tool effects before a checkpoint may be replayed, so tools should +be idempotent. Before exposing a durable Agent, apply the +[controlled beta deployment profile](../advanced/durable-execution-operations.md#controlled-beta-deployment-profile); +in particular, authenticate the A2A endpoint and enforce request and admission +limits at the gateway. See the +[durable A2A example](https://github.com/deepset-ai/hayhooks/tree/main/examples/a2a_long_running). ## Inspecting agents with a2a-inspector @@ -155,8 +307,9 @@ See [examples/a2a_multi_agent](https://github.com/deepset-ai/hayhooks/tree/main/ ## Current limitations -- **Request-bound task execution**: Hayhooks currently treats A2A as a chat-shaped bridge. Each task runs inside the request handler by calling `run_chat_completion` / `run_chat_completion_async`, so non-streaming `SendMessage` returns after the task has completed or failed. This means detached task execution via [`returnImmediately`](https://a2a-protocol.org/latest/specification/#322-sendmessageconfiguration), [`input-required`](https://a2a-protocol.org/latest/specification/#63-multi-turn-interaction) pauses, and [push notification delivery](https://a2a-protocol.org/latest/specification/#353-push-notification-delivery) are not supported yet. +- **Process-owned execution**: chat-compatible execution pauses while the A2A server is offline; durable Agents persist checkpoints and reclaim incomplete work after Hayhooks starts again. +- **Automatic task-store selection**: `auto` selects Redis only for Redis-backed durable Agents. Explicit `memory` stays process-local. A persistent task store preserves the protocol projection, but does not recover interrupted execution by itself. +- **Push notifications**: push notification delivery is not enabled yet, and agent cards do not advertise it. - **Static agents list**: A2A routes are built from the registry at startup. Pipelines deployed or undeployed at runtime require restarting `hayhooks a2a run`. -- **In-memory task store**: task state is kept in memory and lost on restart. - **Path-prefixed agent cards**: one server hosts many agents, so cards live under `/{pipeline_name}/.well-known/agent-card.json` instead of the domain root. If a consumer requires strict root-level discovery, run one A2A server instance per agent (separate `--pipelines-dir` and `--port`). -- **Cancellation is best-effort**: cancelling a task marks it canceled but does not interrupt a running pipeline. +- **Cancellation is cooperative**: async chat wrappers and durable Agents can observe cancellation. A durable A2A task reports cancellation requested first and becomes A2A canceled only after the execution record is terminal canceled. Synchronous work cannot be forcibly interrupted and retains its fenced claim until it returns. diff --git a/docs/features/cli-commands.md b/docs/features/cli-commands.md index 3e9b0c9b..694def12 100644 --- a/docs/features/cli-commands.md +++ b/docs/features/cli-commands.md @@ -143,6 +143,14 @@ hayhooks a2a run --host 0.0.0.0 --port 1418 | `--pipelines-dir` | | Directory for pipeline definitions | `./pipelines` | | `--additional-python-path` | | Additional Python path | `None` | | `--external-url` | | Base URL advertised in agent cards | `None` | +| `--task-store` | | Built-in A2A task-store backend: `auto`, `memory`, or `redis` | `auto` | +| `--a2a-redis-url` | | Redis URL for the built-in A2A task store | `redis://localhost:6379/0` | +| `--a2a-redis-key-prefix` | | Redis key prefix for the built-in A2A task store | `hayhooks:a2a` | +| `--execution-store` | | Built-in durable execution backend: `memory` or `redis` | `redis` | +| `--execution-redis-url` | | Redis URL for durable execution storage | `redis://localhost:6379/0` | +| `--execution-redis-key-prefix` | | Redis key prefix for durable execution storage | `hayhooks:durable` | +| `--durable-execution-concurrency` | | Operator concurrency ceiling per deployed durable Agent | `1` | +| `--debug` | | Include tracebacks in errors | `false` | ## Pipeline Management Commands diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md index d68539c5..fac962e3 100644 --- a/docs/getting-started/installation.md +++ b/docs/getting-started/installation.md @@ -38,7 +38,17 @@ This guide covers how to install Hayhooks and its dependencies. Includes all standard features plus [A2A Server](../features/a2a-support.md) support, exposing deployed pipelines and agents over the [A2A protocol](https://a2a-protocol.org) so other agents can discover - and delegate tasks to them. + and delegate tasks to them. This extra also includes Redis support for durable execution. + +=== "With Durable Execution" + + ```bash + pip install "hayhooks[durable]" + ``` + + Includes Redis support and Haystack 3, required for restart-safe Pipeline and Agent execution. See the + [durable Pipeline example](https://github.com/deepset-ai/hayhooks/tree/main/examples/durable_execution). + Durable execution requires Redis server 6.2 or newer. === "With Tracing Support" diff --git a/docs/guides/production-best-practices.md b/docs/guides/production-best-practices.md index 289fd0f7..a30e7a4a 100644 --- a/docs/guides/production-best-practices.md +++ b/docs/guides/production-best-practices.md @@ -100,6 +100,7 @@ Pipelines that spend most of their time waiting on external services -- LLM API ```python from hayhooks import BasePipelineWrapper, Pipeline + class PipelineWrapper(BasePipelineWrapper): def setup(self) -> None: self.pipeline = Pipeline() @@ -157,7 +158,7 @@ healthcheck: start_period: 40s ``` -For Kubernetes, use a liveness probe on the same endpoint: +For Kubernetes, use the same endpoint as a liveness probe: ```yaml livenessProbe: @@ -168,6 +169,23 @@ livenessProbe: periodSeconds: 30 ``` +For a durable pipeline, use `/status/{pipeline_name}` as its readiness probe. +It returns `503` when that deployment has no healthy durable worker slots. + +## Treat Durable Execution as a Controlled Beta + +Durable Pipelines and managed durable A2A Agents have stricter requirements +than ordinary request/response pipelines. Before using them with production +traffic, follow the authoritative +[controlled beta deployment profile](../advanced/durable-execution-operations.md#controlled-beta-deployment-profile). +It covers the supported Redis topology, immutable revisions, drained upgrades, +concurrency, ingress limits, operation timeouts, idempotent effects, +observability, and incident recovery. + +Do not use rolling mixed-version upgrades for the durable engine, and do not +assume Pipeline or Agent checkpoints remain compatible across Haystack, +Hayhooks, or application upgrades. + ## Docker and Container Tips Follow these practices when running Hayhooks in containers: diff --git a/docs/reference/api-reference.md b/docs/reference/api-reference.md index eef7674b..c29dcae1 100644 --- a/docs/reference/api-reference.md +++ b/docs/reference/api-reference.md @@ -99,7 +99,11 @@ Get status of all deployed pipelines. "pipeline1", "pipeline2" ], - "status": "Up!" + "status": "Up!", + "durable": { + "healthy": true, + "deployments": {} + } } ``` @@ -129,6 +133,33 @@ Execute a deployed pipeline. } ``` +#### Durable execution + +Wrappers that implement exactly one of `run_durable()` or +`run_durable_async()` expose these typed resources: + +| Endpoint | Description | +|---|---| +| `POST /{pipeline_name}/run-durable` | Validate and persist a detached execution; accepts an optional `Idempotency-Key` header | +| `GET /{pipeline_name}/executions/{execution_id}` | Inspect safe status, progress, waiting state, error, or result | +| `GET /{pipeline_name}/executions/{execution_id}/stream` | Stream display chunks over SSE, ending in the terminal event; resumable with `Last-Event-ID` | +| `POST /{pipeline_name}/executions/{execution_id}/cancel` | Request cooperative cancellation | +| `POST /{pipeline_name}/executions/{execution_id}/resume` | Resume an execution waiting for input | + +Submission normally returns `202 Accepted` and a `Location` header. The +`Idempotency-Key` is never used as the execution ID directly: the server derives +an unguessable ID from it, so follow the `Location` header or the returned +`execution_id`. An +idempotent replay of a retained terminal execution returns `200 OK`. Validated +input, checkpoints, application state, ownership, and fence details remain +server-side. See [Pipeline wrapper durable execution](../concepts/pipeline-wrapper.md#durable-execution) +and the [durable engine contract](../advanced/durable-engine.md). + +Existing FastAPI applications can expose the same routes with the public +`hayhooks.durable.create_durable_router()` factory. The application owns a +`DurableRuntime` in its lifespan and may provide a normal FastAPI dependency +that returns a stable owner ID. See [Embedding the runtime](../advanced/durable-engine.md#embedding-the-runtime). + ### OpenAI Compatibility #### Chat Completion @@ -252,10 +283,7 @@ Currently, Hayhooks does not include built-in rate limiting. Consider implementi ```python import requests - response = requests.post( - "http://localhost:1416/chat_pipeline/run", - json={"query": "Hello!"} - ) + response = requests.post("http://localhost:1416/chat_pipeline/run", json={"query": "Hello!"}) print(response.json()) ``` @@ -287,12 +315,7 @@ Currently, Hayhooks does not include built-in rate limiting. Consider implementi response = requests.post( "http://localhost:1416/v1/chat/completions", - json={ - "model": "chat_pipeline", - "messages": [ - {"role": "user", "content": "Hello!"} - ] - } + json={"model": "chat_pipeline", "messages": [{"role": "user", "content": "Hello!"}]}, ) print(response.json()) ``` @@ -304,15 +327,10 @@ Currently, Hayhooks does not include built-in rate limiting. Consider implementi client = OpenAI( base_url="http://localhost:1416/v1", - api_key="not-needed" # Hayhooks doesn't require auth by default + api_key="not-needed", # Hayhooks doesn't require auth by default ) - response = client.chat.completions.create( - model="chat_pipeline", - messages=[ - {"role": "user", "content": "Hello!"} - ] - ) + response = client.chat.completions.create(model="chat_pipeline", messages=[{"role": "user", "content": "Hello!"}]) print(response.choices[0].message.content) ``` diff --git a/docs/reference/environment-variables.md b/docs/reference/environment-variables.md index 5e0cfd0b..d8bd0055 100644 --- a/docs/reference/environment-variables.md +++ b/docs/reference/environment-variables.md @@ -141,6 +141,146 @@ export HAYHOOKS_DEPLOY_CONCURRENCY=parallel - Default: `true` - Description: Accept A2A spec 0.3 requests on the same endpoints. Many clients and tools (e.g. the a2a-inspector) still speak 0.3 during the 1.0 transition +### HAYHOOKS_A2A_TASK_STORE + +- Default: `auto` +- Description: Built-in A2A task-store backend +- Options: + - `auto`: Select Redis for Redis-backed durable A2A Agents, otherwise memory + - `memory`: Process-local task records; use only one Hayhooks process, not load-balanced replicas or restart-safe tasks + - `redis`: Persistent task records using the configured Redis URL and key prefix + +### HAYHOOKS_A2A_REDIS_URL + +- Default: `redis://localhost:6379/0` +- Description: Redis URL used by the built-in A2A task store + +### HAYHOOKS_A2A_REDIS_KEY_PREFIX + +- Default: `hayhooks:a2a` +- Description: Prefix applied to built-in Redis A2A task-store keys. Use a distinct prefix when multiple environments share Redis. + +### HAYHOOKS_A2A_REDIS_SOCKET_TIMEOUT / HAYHOOKS_A2A_REDIS_SOCKET_CONNECT_TIMEOUT + +- Default: `5.0` seconds each +- Description: Bound established-socket operations and new Redis connections for the built-in A2A task store. + +### HAYHOOKS_A2A_REDIS_HEALTH_CHECK_INTERVAL + +- Default: `30` seconds +- Description: Redis-py connection health-check interval for the built-in A2A task store. Set `0` to disable proactive checks. + +### HAYHOOKS_A2A_TERMINAL_TASK_TTL_SECONDS + +- Default: `604800` (seven days) +- Description: Retention window for terminal A2A tasks. Cleanup also removes their owner-update and recovery indexes. + +### HAYHOOKS_A2A_TASK_SNAPSHOT_CACHE_SIZE + +- Default: `1024` +- Description: Maximum loaded protobuf task snapshots retained per A2A Redis task-store instance for optimistic version checks. The cache is a global LRU across task IDs. + +### HAYHOOKS_A2A_LIST_SCAN_BATCH_SIZE + +- Default: `500` +- Description: Maximum task IDs and payloads loaded in one Redis batch while applying filtered A2A task-list queries. Exact filtered counts still require scanning the owner's update index. + +## Durable execution + +### HAYHOOKS_DURABLE_STORE + +- Default: `redis` +- Description: Built-in durable execution-store backend. Both choices use the same state reducer; Redis is required for restart recovery. +- Options: + - `memory`: Request-detached execution whose records are lost on process exit + - `redis`: Redis control records, runnable/lease indexes, checkpoints, cancellation, and restart recovery + +### HAYHOOKS_DURABLE_REDIS_URL + +- Default: `redis://localhost:6379/0` +- Description: Redis URL used by durable REST and A2A execution. + +### HAYHOOKS_DURABLE_REDIS_KEY_PREFIX + +- Default: `hayhooks:durable` +- Description: Key prefix used by durable execution records and queues. + +### HAYHOOKS_DURABLE_REDIS_SOCKET_TIMEOUT / HAYHOOKS_DURABLE_REDIS_SOCKET_CONNECT_TIMEOUT + +- Default: `5.0` seconds each +- Description: Bound established-socket operations and new Redis connections for durable execution. A timeout is reported as a store failure, causing worker backoff and marking the affected durable deployment unhealthy rather than waiting indefinitely. + +### HAYHOOKS_DURABLE_REDIS_HEALTH_CHECK_INTERVAL + +- Default: `30` seconds +- Description: Redis-py connection health-check interval for durable execution. Set `0` to disable proactive checks. + +### HAYHOOKS_DURABLE_LEASE_DURATION_MS + +- Default: `30000` +- Description: Milliseconds for the renewable execution lease. Workers renew it at one-third of this duration. + +### HAYHOOKS_DURABLE_LEASE_COMMIT_SAFETY_MS + +- Default: `1500` +- Description: Server-clock margin before a lease deadline within which an owned transition is rejected. This prevents a transition that is near expiry from committing after another worker may recover the lease. + +### HAYHOOKS_DURABLE_TERMINAL_TTL_SECONDS + +- Default: `604800` (seven days) +- Description: Retention period for terminal execution records. + +### HAYHOOKS_DURABLE_MAX_PROGRESS_EVENTS + +- Default: `100` +- Description: Maximum retained client-visible progress events per execution. + +### HAYHOOKS_DURABLE_MAX_RECORD_BYTES + +- Default: `1000000` +- Description: Maximum JSON payload size for validated input, a checkpoint, wait data, a result, or an error. The engine reserves space for each payload independently, so retained state remains bounded even while a run has input, a checkpoint, and progress history. + +### HAYHOOKS_DURABLE_MAX_NONTERMINAL_EXECUTIONS + +- Default: `0` (disabled) +- Description: Maximum queued, running, or waiting executions for one logical deployment. New work is rejected atomically with `503` and `Retry-After` when the bound is full; idempotent replay remains available. + +### HAYHOOKS_DURABLE_SHUTDOWN_GRACE_PERIOD + +- Default: `5.0` +- Description: Seconds to wait for workers during shutdown. Synchronous work that exceeds the window retains its claim and heartbeat until the underlying thread exits. + +### HAYHOOKS_DURABLE_MAX_ATTEMPTS + +- Default: `3` +- Description: Maximum application attempts, including the first attempt and recovered abandoned claims, before retry exhaustion becomes terminal failure. + +### HAYHOOKS_DURABLE_RETRY_BASE_DELAY + +- Default: `1.0` +- Description: Base seconds for bounded exponential retry delay when application code does not provide a delay. + +### HAYHOOKS_DURABLE_RETRY_MAX_DELAY + +- Default: `60.0` +- Description: Maximum seconds for the next retry delay, including explicit application overrides. + +### HAYHOOKS_DURABLE_TRUSTED_OWNER_HEADER + +- Default: `""` (bearer execution-ID mode) +- Description: Trusted reverse-proxy header containing the authenticated owner. When set, submit, inspect, cancel, and resume enforce owner equality. +- Security: The proxy must remove client-supplied copies and inject this header over a trusted hop. + +### HAYHOOKS_DURABLE_EXECUTION_CONCURRENCY + +- Default: `1` +- Description: Per-process ceiling for concurrent durable executions of each deployment. Active work scales from zero to this limit as executions become runnable. + +### HAYHOOKS_DURABLE_POLL_INTERVAL + +- Default: `1.0` +- Description: Shared worker and lease-maintenance polling interval in seconds. At concurrency 1, `0.25` uses about 16 idle Redis commands per second with 125 ms average pickup latency, `0.5` uses about 8 with 250 ms latency, and `1.0` uses about 4 with 500 ms latency. Counts are per durable deployment and replica. + ## Chainlit UI ### HAYHOOKS_CHAINLIT_ENABLED diff --git a/examples/README.md b/examples/README.md index 9889595e..fa5efd70 100644 --- a/examples/README.md +++ b/examples/README.md @@ -22,6 +22,9 @@ This directory contains various examples demonstrating different use cases and f | [responses_with_file_upload](./pipeline_wrappers/responses_with_file_upload/) | Agent-based Responses API with file reading | β€’ Haystack Agent with `read_file` tool
β€’ `run_response_async` with streaming
β€’ `run_file_upload` with in-memory store
β€’ `_strip_tool_calls` for agentic clients
β€’ Codex CLI compatible | Building an agent that reads local files and uploaded files via the Responses API, compatible with Codex CLI and the OpenAI Python client | | [chat_completion_with_file_upload](./pipeline_wrappers/chat_completion_with_file_upload/) | Chat Completions API with `/v1/files` upload | β€’ `run_chat_completion_async` with streaming
β€’ `run_file_upload` with in-memory store
β€’ Resolves `{"type": "file"}` content parts
β€’ OpenAI file input format | Using the Chat Completions API with files uploaded via `/v1/files` and referenced using OpenAI's multi-part content format | | [a2a_multi_agent](./a2a_multi_agent/) | Two agents with their own MCP tools, communicating over A2A | β€’ `hayhooks a2a run` hosting two agents
β€’ Per-agent A2A agent cards
β€’ Agent-to-agent delegation via A2A client tool
β€’ One MCP tool server per agent (FastMCP)
β€’ Streaming A2A client | Building multi-agent systems where Haystack Agents expose themselves over A2A and delegate tasks to each other while using MCP for their own tools | +| [a2a_long_running](./a2a_long_running/) | Recoverable OpenAI agent over A2A | β€’ `OpenAIChatGenerator`-based Haystack Agent
β€’ Redis-backed fenced execution checkpoints
β€’ Hayhooks and client restart recovery
β€’ `input-required` continuation
β€’ Polling and durable cancellation | Building tool-using A2A agents whose accepted work resumes after process restarts | +| [durable_execution](./durable_execution/) | First-class durable Pipeline | β€’ Typed `/run-durable` REST endpoint
β€’ Built-in Redis store and fenced claims
β€’ Restart recovery, inspection, cancellation, and resume | Running recoverable background jobs through an ordinary wrapper without A2A | +| [durable_chat_with_website](./durable_chat_with_website/) | Durable Pipeline that streams its answer | β€’ SSE token streaming over `/executions/{id}/stream`
β€’ Detach and reattach with `Last-Event-ID`
β€’ Checkpointed page fetch, streamed generation
β€’ Bounded chunk log outside the durable fence | Giving a recoverable job a live UI without making display chunks part of durable state | | [rag_indexing_query](./rag_indexing_query/) | Complete RAG system with Elasticsearch | β€’ Document indexing pipeline
β€’ Query pipeline
β€’ Elasticsearch integration
β€’ Multiple file format support (PDF, Markdown, Text)
β€’ Sentence transformers embeddings | Implementing production-ready RAG systems for document search and knowledge retrieval | | [shared_code_between_wrappers](./shared_code_between_wrappers/) | Code sharing between pipeline wrappers | β€’ Shared library imports
β€’ HAYHOOKS_ADDITIONAL_PYTHON_PATH
β€’ Multiple deployment strategies
β€’ Code reusability | Organizing complex projects with multiple pipelines that share common functionality | @@ -33,12 +36,30 @@ This directory contains various examples demonstrating different use cases and f ## Getting Started -Each example includes: +Examples intentionally stay lightweight. Every runnable example includes its +source files; examples that need non-default setup or dependencies also include +a dedicated README and/or `requirements.txt`. - **Pipeline wrapper implementation** (`pipeline_wrapper.py`) - **Pipeline configuration** (`.yml` files where applicable) -- **Dependencies** (`requirements.txt` where applicable) -- **Documentation** (individual README files with setup instructions) +- **Dependencies** (`requirements.txt` where a demo needs them) +- **Documentation** (individual README files where a demo needs dedicated setup instructions) + +For the durable examples, use this presentation order: + +1. [`durable_execution`](./durable_execution/) β€” the deterministic reference + for typed submission, retry, approval, checkpoints, crash recovery, and + cancellation. +2. [`durable_chat_with_website`](./durable_chat_with_website/) β€” the same + engine with a live SSE token stream, showing where display chunks sit + relative to the durable fence. +3. [`a2a_long_running`](./a2a_long_running/) β€” durable Agent execution exposed + through standard A2A task lifecycle and continuation messages. + +Each durable example's Compose file publishes Redis on `localhost:6379`. +Run these examples one at a time, or change the host port and corresponding +`HAYHOOKS_DURABLE_REDIS_URL`. Each stack has its own named volume; `compose +down` retains it and `compose down -v` resets it. ## Common Prerequisites @@ -54,7 +75,7 @@ Most examples require: 1. Navigate to the `/examples` directory 2. Create and activate a virtual environment (recommended) 3. Install dependencies: `pip install -r requirements.txt` (if present) -4. Follow the specific example's README for deployment and testing +4. Follow the example-specific README when present; otherwise deploy its wrapper using the standard Hayhooks command ## Support diff --git a/examples/a2a_long_running/README.md b/examples/a2a_long_running/README.md new file mode 100644 index 00000000..bf68b56d --- /dev/null +++ b/examples/a2a_long_running/README.md @@ -0,0 +1,204 @@ +# Durable long-running A2A Agent + +This example runs a Haystack Agent as a durable A2A task. Redis persists both +the execution checkpoints and the A2A task projection, so accepted work can +survive a Hayhooks restart. + +This is a local reliability demonstration, not a production Redis or ingress +configuration. Before a production evaluation, apply the +[controlled beta deployment profile](../../docs/advanced/durable-execution-operations.md#controlled-beta-deployment-profile). + +Run the setup commands from the repository root. The example requires `curl` +and `jq`. All durable examples use the same Compose service and Redis volume. + +```bash +docker compose -f examples/durable-compose.yaml up -d +python -m pip install -e ".[durable,a2a]" + +export OPENAI_API_KEY=... +export HAYHOOKS_DURABLE_REDIS_URL=redis://localhost:6379/0 +export HAYHOOKS_A2A_TASK_STORE=auto +export HAYHOOKS_EXAMPLE_TOOL_DELAY_SECONDS=15 +export HAYHOOKS_EXAMPLE_RECEIPT_DELAY_SECONDS=0 +export HAYHOOKS_DURABLE_LEASE_DURATION_MS=5000 +``` + +The five-second execution lease keeps the restart demonstration short. Active +workers heartbeat their claims; production deployments should tune this value +for their own workload and failure environment. + +Open a first terminal and start Hayhooks in the foreground. It prints the PID +used for the forced-crash demonstration: + +```bash +sh -c 'echo "Hayhooks PID: $$"; exec hayhooks a2a run --pipelines-dir examples/a2a_long_running/pipelines' +``` + +For a quick happy-path rehearsal, run the Rich demo client in a second +terminal. It creates unique message IDs, submits work, waits for approval, +approves it, and follows the task to completion: + +```bash +python examples/a2a_long_running/demo.py +``` + +Use the manual requests below when presenting the protocol or demonstrating +crash recovery and cancellation. + +In a second terminal, define a helper that sends the current request file and +prints the response: + +```bash +send_a2a_request() { + local request_file="$1" + local response_file="${2:-a2a-res}" + + curl -fsS http://localhost:1418/long_running_agent/ \ + -H 'content-type: application/json' \ + -H 'A2A-Version: 1.0' \ + --data-binary @"$request_file" \ + --output "$response_file" + + jq . "$response_file" +} +``` + +Submit detached work and save the returned task ID: + +```bash +jq -n '{ + "jsonrpc":"2.0", + "id":"submit", + "method":"SendMessage", + "params":{ + "message":{ + "messageId":"prepare-demo", + "role":"ROLE_USER", + "parts":[{ + "text":"Prepare this document for indexing. document_id: hayhooks-guide. content: Hayhooks durable A2A work survives restarts." + }] + }, + "configuration":{"returnImmediately":true} + } +}' > a2a-req + +send_a2a_request a2a-req a2a-res + +TASK_ID=$(jq -er '.result.task.id // .result.id' a2a-res) +``` + +Inspect the task with `GetTask`. Repeat these commands until its state is +`TASK_STATE_INPUT_REQUIRED`: + +```bash +jq -n --arg id "$TASK_ID" \ + '{"jsonrpc":"2.0","id":"poll","method":"GetTask","params":{"id":$id}}' \ + > a2a-req + +send_a2a_request a2a-req a2a-res +``` + +Approve the task with a follow-up A2A message. The persisted Agent checkpoint +already contains the original request: + +```bash +jq -n --arg task_id "$TASK_ID" '{ + "jsonrpc":"2.0", + "id":"resume", + "method":"SendMessage", + "params":{ + "message":{ + "messageId":"approval-demo", + "taskId":$task_id, + "role":"ROLE_USER", + "parts":[{"text":"Approved; proceed."}] + }, + "configuration":{"returnImmediately":true} + } +}' > a2a-req + +send_a2a_request a2a-req a2a-res +``` + +## Crash and replay + +Run the `GetTask` commands again until progress contains β€œIndexing effect +committed.” The tool has written its SQLite row and remains open for 15 seconds. + +In the second terminal, replace `` with the PID printed in the first +terminal to kill Hayhooks without graceful shutdown: + +```bash +kill -9 +``` + +Return to the first terminal and run the same foreground command again against +the same Redis: + +```bash +sh -c 'echo "Hayhooks PID: $$"; exec hayhooks a2a run --pipelines-dir examples/a2a_long_running/pipelines' +``` + +After about five seconds, run the same `GetTask` commands again. Redis reclaims +the interrupted execution and the Agent replays the tool from its previous +checkpoint. Continue inspecting until the task is terminal. + +The indexing tool uses the execution ID and document ID as a SQLite primary +key. The first attempt reports `side_effect_applied: true`; replay reports +`false` instead of inserting the same effect twice. This demonstrates the +at-least-once contract: Hayhooks protects execution state, while external +effects still require application-level idempotency. + +## Checkpoint efficiency: resume after indexing + +The Agent follows a realistic ingestion workflow: it first cleans and splits +the document, then makes a small catalog receipt update. The durable +`after_tool` hook checkpoints the completed indexing result before the Agent +asks the model to call the receipt tool. That means a crash while the receipt +is running resumes from the indexed document instead of repeating the expensive +cleaning and splitting work. + +For this demonstration, start a fresh task, approve it, and use no indexing +delay but a long receipt delay: + +```bash +export HAYHOOKS_EXAMPLE_TOOL_DELAY_SECONDS=0 +export HAYHOOKS_EXAMPLE_RECEIPT_DELAY_SECONDS=15 +``` + +Wait until task progress contains β€œIndexing is checkpointed; holding the +lightweight receipt update,” then kill and restart Hayhooks as in the crash +replay section. The recovered Agent receives the saved indexing result and +continues with `publish_indexing_receipt`; it must not call +`prepare_document_for_indexing` again. The indexing table retains the example's +idempotency proof; the receipt tool has no irreversible effect, so replaying it +is safe. + +## Cancellation + +For a cancellation demonstration, submit and approve a fresh task, then send: + +```bash +jq -n --arg id "$TASK_ID" \ + '{"jsonrpc":"2.0","id":"cancel","method":"CancelTask","params":{"id":$id}}' \ + > a2a-req + +send_a2a_request a2a-req a2a-res +``` + +Cancellation is cooperative. A synchronous tool may finish its current work +before the Agent reaches the next cancellation checkpoint. + +The SQLite database is a local effect-store demonstration. It defaults to the +operating system's temporary directory and survives a process restart on the +same machine. Set `HAYHOOKS_EXAMPLE_INDEX_DB` to a mounted path if the effect +must survive container replacement. + +Press Ctrl-C in the first terminal to stop Hayhooks, then stop Redis. The static message IDs are intended +for a clean rehearsal; remove the volume before starting again: + +```bash +docker compose -f examples/durable-compose.yaml down +# docker compose -f examples/durable-compose.yaml down -v +rm -f a2a-req a2a-res +``` diff --git a/examples/a2a_long_running/demo.py b/examples/a2a_long_running/demo.py new file mode 100644 index 00000000..f9413a47 --- /dev/null +++ b/examples/a2a_long_running/demo.py @@ -0,0 +1,149 @@ +"""Run the durable A2A example's submit, approval, and completion flow.""" + +from __future__ import annotations + +import argparse +import asyncio +import sys +import time + +import httpx +from a2a.client import A2AClientError, Client, ClientConfig, create_client +from a2a.helpers import new_text_message +from a2a.types import GetTaskRequest, Role, SendMessageConfiguration, SendMessageRequest, Task, TaskState +from rich.console import Console +from rich.panel import Panel +from rich.table import Table + +TERMINAL_STATES = { + TaskState.TASK_STATE_COMPLETED, + TaskState.TASK_STATE_FAILED, + TaskState.TASK_STATE_CANCELED, + TaskState.TASK_STATE_REJECTED, +} + + +class A2ADemoError(RuntimeError): + """A readable failure raised by the demo client.""" + + +async def send_message(client: Client, text: str, *, task_id: str = "") -> Task: + request = SendMessageRequest( + message=new_text_message(text, task_id=task_id or None, role=Role.ROLE_USER), + configuration=SendMessageConfiguration(return_immediately=True), + ) + async for response in client.send_message(request): + if response.HasField("task"): + return response.task + msg = "A2A response did not contain a task" + raise A2ADemoError(msg) + + +async def wait_for_state( # noqa: PLR0913 - polling controls are explicit CLI inputs + client: Client, + task_id: str, + expected: set[int], + *, + deadline: float, + poll_interval: float, + console: Console, +) -> Task: + previous_state = None + while time.monotonic() < deadline: + task = await client.get_task(GetTaskRequest(id=task_id)) + state = task.status.state + if state != previous_state: + console.print(f" [cyan]A2A state[/cyan] β†’ [bold]{TaskState.Name(state)}[/bold]") + previous_state = state + if state in expected: + return task + if state in TERMINAL_STATES: + names = ", ".join(TaskState.Name(item) for item in sorted(expected)) + msg = f"Task became {TaskState.Name(state)} before reaching {names}" + raise A2ADemoError(msg) + await asyncio.sleep(poll_interval) + names = ", ".join(TaskState.Name(item) for item in sorted(expected)) + msg = f"Timed out waiting for {names}" + raise A2ADemoError(msg) + + +def print_summary(task: Task, console: Console) -> None: + table = Table(title="Durable A2A result", show_header=False) + table.add_column("Field", style="cyan") + table.add_column("Value") + table.add_row("Task", task.id) + table.add_row("State", TaskState.Name(task.status.state)) + table.add_row("History messages", str(len(task.history))) + table.add_row("Artifacts", ", ".join(artifact.name or "unnamed" for artifact in task.artifacts)) + console.print(table) + + result = next((artifact for artifact in task.artifacts if artifact.name == "durable-result"), None) + if result: + text = "\n".join(part.text for part in result.parts if part.WhichOneof("content") == "text") + if text: + console.print(Panel(text, title="Agent result", border_style="green")) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--url", default="http://localhost:1418/long_running_agent/", help="A2A agent base URL") + parser.add_argument("--document-id", default="hayhooks-guide") + parser.add_argument("--content", default="Hayhooks durable A2A work survives restarts.") + parser.add_argument("--timeout", type=float, default=120, help="Overall timeout in seconds") + parser.add_argument("--poll-interval", type=float, default=0.5) + return parser.parse_args() + + +async def run(args: argparse.Namespace, console: Console) -> None: + if args.timeout <= 0 or args.poll_interval <= 0: + msg = "timeout and poll interval must be positive" + raise A2ADemoError(msg) + + deadline = time.monotonic() + args.timeout + async with httpx.AsyncClient(timeout=15) as http: + client = await create_client( + args.url, + ClientConfig(streaming=False, polling=True, httpx_client=http), + ) + console.print(Panel.fit("Submit β†’ input required β†’ approve β†’ complete", title="Durable A2A demo")) + task = await send_message( + client, + f"Prepare this document for indexing. document_id: {args.document_id}. content: {args.content}", + ) + console.print(f"[green]βœ“[/green] Submitted task [bold]{task.id}[/bold]") + await wait_for_state( + client, + task.id, + {TaskState.TASK_STATE_INPUT_REQUIRED}, + deadline=deadline, + poll_interval=args.poll_interval, + console=console, + ) + console.print("[green]βœ“[/green] Approval requested") + await send_message(client, "Approved; proceed.", task_id=task.id) + console.print("[green]βœ“[/green] Approval sent") + print_summary( + await wait_for_state( + client, + task.id, + {TaskState.TASK_STATE_COMPLETED}, + deadline=deadline, + poll_interval=args.poll_interval, + console=console, + ), + console, + ) + + +def main() -> int: + console = Console() + try: + asyncio.run(run(parse_args(), console)) + except (A2ADemoError, A2AClientError, httpx.HTTPError, ValueError) as error: + console.print(f"[bold red]Demo failed:[/bold red] {error}") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/a2a_long_running/pipelines/long_running_agent/pipeline_wrapper.py b/examples/a2a_long_running/pipelines/long_running_agent/pipeline_wrapper.py new file mode 100644 index 00000000..dc516a33 --- /dev/null +++ b/examples/a2a_long_running/pipelines/long_running_agent/pipeline_wrapper.py @@ -0,0 +1,176 @@ +"""A durable A2A Agent that calls a real Haystack document-preparation Pipeline.""" + +import json +import os +import sqlite3 +import tempfile +import time +from pathlib import Path +from typing import Annotated + +from haystack import Document, Pipeline +from haystack.components.agents import Agent +from haystack.components.agents.state import State +from haystack.components.generators.chat import OpenAIChatGenerator +from haystack.components.preprocessors import DocumentCleaner, DocumentSplitter +from haystack.hooks.from_function import FunctionHook +from haystack.tools import tool + +from hayhooks import A2APipelineWrapper, current_durable_context, current_execution_id + +_MAX_DEMO_TOOL_DELAY_SECONDS = 300.0 + + +def _demo_delay_seconds(name: str, default: str = "0") -> float: + raw_delay = os.getenv(name, default) + try: + delay = float(raw_delay) + except ValueError as error: + msg = f"{name} must be a number" + raise ValueError(msg) from error + if not 0 <= delay <= _MAX_DEMO_TOOL_DELAY_SECONDS: + msg = f"{name} must be between 0 and 300" + raise ValueError(msg) + return delay + + +def require_approval(state: State) -> None: # noqa: ARG001 + """Suspend before the first model call so A2A exposes input-required.""" + context = current_durable_context() + if context is None or context.state.get("approval_requested"): + return + context.state["approval_requested"] = True + context.suspend_sync( + { + "kind": "approval", + "message": "Approve the indexing side effect", + "expected_input_schema": { + "type": "object", + "properties": {"message": {"type": "string"}}, + "required": ["message"], + }, + } + ) + + +async def require_approval_async(state: State) -> None: # noqa: ARG001 + context = current_durable_context() + if context is None or context.state.get("approval_requested"): + return + context.state["approval_requested"] = True + await context.suspend( + { + "kind": "approval", + "message": "Approve the indexing side effect", + "expected_input_schema": { + "type": "object", + "properties": {"message": {"type": "string"}}, + "required": ["message"], + }, + } + ) + + +@tool +def prepare_document_for_indexing( + document_id: Annotated[str, "A stable identifier for the source document"], + content: Annotated[str, "Raw document text to clean and split into chunks"], +) -> str: + """Clean, chunk, and idempotently record an indexing side effect.""" + context = current_durable_context() + execution_id = current_execution_id() + if context is None or execution_id is None: + msg = "This example tool must run inside a durable execution" + raise RuntimeError(msg) + effect_key = f"{execution_id}:index:{document_id}" + preparation_pipeline = Pipeline() + preparation_pipeline.add_component("clean", DocumentCleaner(remove_empty_lines=True)) + preparation_pipeline.add_component( + "split", + DocumentSplitter(split_by="word", split_length=80, split_overlap=10), + ) + preparation_pipeline.connect("clean.documents", "split.documents") + outputs = preparation_pipeline.run( + {"clean": {"documents": [Document(id=document_id, content=content, meta={"document_id": document_id})]}} + ) + chunks = outputs["split"]["documents"] + default_database = Path(tempfile.gettempdir()) / "hayhooks-durable-a2a.sqlite3" + database = os.getenv("HAYHOOKS_EXAMPLE_INDEX_DB", str(default_database)) + with sqlite3.connect(database) as connection: + connection.execute( + "CREATE TABLE IF NOT EXISTS indexing_effects " + "(idempotency_key TEXT PRIMARY KEY, document_id TEXT NOT NULL, chunk_count INTEGER NOT NULL)" + ) + cursor = connection.execute( + "INSERT OR IGNORE INTO indexing_effects (idempotency_key, document_id, chunk_count) VALUES (?, ?, ?)", + (effect_key, document_id, len(chunks)), + ) + applied = cursor.rowcount == 1 + + # Hold the tool open *after* its external effect. Killing the server in + # this window replays the tool from the previous Agent checkpoint, while + # the SQLite primary key proves that the effect is still applied once. + delay = _demo_delay_seconds("HAYHOOKS_EXAMPLE_TOOL_DELAY_SECONDS", "3") + context.report_progress_sync( + f"Indexing effect committed; holding the tool open for {delay:g} seconds", + kind="side_effect_committed", + metadata={ + "idempotency_key": effect_key, + "side_effect_applied": applied, + }, + ) + if delay: + time.sleep(delay) + + return json.dumps( + { + "document_id": document_id, + "chunk_count": len(chunks), + "idempotency_key": effect_key, + "side_effect_applied": applied, + "chunks": [{"chunk_id": str(chunk.id), "preview": (chunk.content or "")[:160]} for chunk in chunks], + } + ) + + +@tool +def publish_indexing_receipt( + document_id: Annotated[str, "The stable identifier returned by document preparation"], + chunk_count: Annotated[int, "The prepared chunk count returned by document preparation"], +) -> str: + """Perform the inexpensive follow-up step after document preparation succeeds.""" + context = current_durable_context() + if context is None: + msg = "This example tool must run inside a durable execution" + raise RuntimeError(msg) + + delay = _demo_delay_seconds("HAYHOOKS_EXAMPLE_RECEIPT_DELAY_SECONDS") + context.report_progress_sync( + f"Indexing is checkpointed; holding the lightweight receipt update for {delay:g} seconds", + kind="receipt_started", + ) + if delay: + time.sleep(delay) + + return json.dumps({"document_id": document_id, "chunk_count": chunk_count, "receipt": "published"}) + + +class PipelineWrapper(A2APipelineWrapper): + """Let Hayhooks map this real tool-using Agent to durable A2A executions.""" + + durable_revision = "a2a-long-running-agent" + + def setup(self) -> None: + self.pipeline = Agent( + chat_generator=OpenAIChatGenerator(model="gpt-4o-mini"), + tools=[prepare_document_for_indexing, publish_indexing_receipt], + system_prompt=( + "You prepare documents for retrieval and publish their catalog status. When a user supplies a " + "document identifier and content, first call only prepare_document_for_indexing. After its result, " + "in a later tool turn call only publish_indexing_receipt with its document_id and chunk_count. " + "Do not prepare a document again once its successful result is present. Then report the number " + "of chunks and a concise readiness summary. Treat a follow-up approval message as authorization " + "to proceed." + ), + hooks={"before_llm": [FunctionHook(function=require_approval, async_function=require_approval_async)]}, + ) diff --git a/examples/durable-compose.yaml b/examples/durable-compose.yaml new file mode 100644 index 00000000..67cc5304 --- /dev/null +++ b/examples/durable-compose.yaml @@ -0,0 +1,18 @@ +services: + redis: + image: redis:7.4-alpine + command: + - redis-server + - --appendonly + - "yes" + - --appendfsync + - everysec + - --maxmemory-policy + - noeviction + ports: + - "6379:6379" + volumes: + - durable-redis:/data + +volumes: + durable-redis: diff --git a/examples/durable_chat_with_website/README.md b/examples/durable_chat_with_website/README.md new file mode 100644 index 00000000..1cf6abbb --- /dev/null +++ b/examples/durable_chat_with_website/README.md @@ -0,0 +1,169 @@ +# Durable chat-with-website with token streaming + +A durable Pipeline that fetches live web pages, answers a question about them, +and streams the answer token by token over Server-Sent Events. It is the +streaming counterpart to `examples/durable_execution`: the same engine owns +records, the Redis runnable queue, fenced workers, checkpoints, and retention, +while the SSE stream carries display chunks alongside it. + +The two halves are deliberately separate: + +- **Durable state** is fenced and checkpointed. A `PipelineSnapshot` is + persisted once the pages are fetched and converted, so a restart resumes into + generation instead of hitting the network again. +- **Chunks are display data.** They live in a bounded append-only log outside + the fence, so a token cannot contend with the lease heartbeat, and a dropped + token can never fail or replay the execution. + +The streaming callback is bound to the `llm` component in `setup()` rather than +passed per run the way `async_streaming_generator` passes it. Per-run injection +cannot work under checkpointing: run data is serialized into the +`PipelineSnapshot`, Haystack drops the callable it cannot serialize, and +`Pipeline.run` rebuilds its `data` from the snapshot when resuming, so the +callback disappears at the first checkpoint. + +The built-in `durable_streaming_callback` resolves the execution from a +`ContextVar` on each call, so one callback bound to a shared component still +keeps concurrent streams isolated. Per-run injection buys the ordinary helper +control over *which* components stream, not isolation between runs. A run-time +`streaming_callback` still takes precedence, so ordinary streaming endpoints on +the same wrapper behave normally. + +The included `httpx` client submits a question, streams the answer, then +deliberately drops the connection mid-answer and reattaches with +`Last-Event-ID` to show that nothing in between is lost. + +Run each command from the repository root. This is a local demonstration, not a +production Redis configuration. + +1. Start Redis and install the example dependencies. + +```bash +docker compose -f examples/durable-compose.yaml up -d && python -m pip install -e ".[durable]" httpx rich +``` + +2. Point Hayhooks at Redis and supply an OpenAI key. The five-second lease keeps + the restart demonstration in step 5 short; the 30-second default would leave a + killed execution waiting that long before another worker may reclaim it. + +```bash +export HAYHOOKS_DURABLE_REDIS_URL=redis://localhost:6379/0 OPENAI_API_KEY=sk-... HAYHOOKS_DURABLE_LEASE_DURATION_MS=5000 +``` + +3. In a first terminal, start Hayhooks. + +```bash +hayhooks run --pipelines-dir examples/durable_chat_with_website/pipelines +``` + +4. In a second terminal, run the client. + +```bash +python examples/durable_chat_with_website/demo.py +``` + +The answer prints token by token. After eight tokens the client drops the +connection on purpose, prints the `Last-Event-ID` it reached, reattaches, and +finishes the answer without a gap. The stream ends with a `completed` event +carrying the same projection the inspect route returns. + +### Run the three-pane recovery show + +For the full demo, set the durable concurrency ceiling to two: + +```bash +HAYHOOKS_DURABLE_EXECUTION_CONCURRENCY=2 hayhooks run \ + --pipelines-dir examples/durable_chat_with_website/pipelines +``` + +Then launch the show from another terminal: + +```bash +python examples/durable_chat_with_website/showcase.py +``` + +Two long answers stream concurrently in the top panes. **ATLAS** cuts its SSE +connection after 12 seconds and **COMET** after 16 seconds. Each execution keeps +running while its client is away, an inspect request proves that from the +control plane, and the client reattaches three seconds later with its saved +`Last-Event-ID`. A successful pane finishes with an exact-replay proof: the +chunks seen across both connections reconstruct the durable result with no gap +or duplicates. + +The bottom HTTP flight recorder makes the sequence visible across both clients: + +```text +POST β†’ 202 submit execution +GET β†’ 200 open SSE stream +CLOSE βœ‚ drop only the client connection +GET β†’ 200 execution is still running +GET β†’ 200 reattach with Last-Event-ID +SSE completed +``` + +The live dashboard uses Rich, which Hayhooks already installs. Enlarge the +terminal to at least 100 columns by 28 rows so both streams have room to breathe. + +### Manual curl handoff + +To show the resumable stream in two side-by-side terminals, run the start script +in the first terminal. It requests a deliberately long answer, saves the stream +URL and raw SSE transcript in the system temporary directory, and uses +`jq --unbuffered` to print only the generated text: + +```bash +./examples/durable_chat_with_website/start_stream.sh +``` + +After at least ten seconds, press Ctrl-C in the first terminal. Then run the +resume script in the second terminal. It reads the saved cursor and asks +Hayhooks for only the events after it: + +```bash +./examples/durable_chat_with_website/resume_stream.sh +``` + +The generated text continues in the second terminal without repeating completed +chunks, while the printed `Last-Event-ID` makes the resume cursor visible. If +the bounded chunk log has discarded that cursor, Hayhooks prints the `gap` +detail before replaying the retained tail. + +5. To watch the durable half, restart Hayhooks while a request is in flight. + +The client reports the broken stream and keeps reattaching. Once the lease +expires, a worker in the restarted process reclaims the execution, and the +progress log records what the second attempt actually did: + +``` +fetch Fetching 2 page(s) +checkpoint Checkpoint saved before pipeline component 'prompt' +resume Resuming from the fetch checkpoint +completed Answer complete +``` + +The checkpoint lands about a second after submission, so kill the server after +that to see the `resume` line. Kill it sooner and the second attempt honestly +reports a second `fetch`, because there was no snapshot to resume from yet. +The retried attempt replays from its checkpoint, so the client prints an +attempt marker when tokens repeat. A server-side `error` event is treated the +same way as a dropped connection: reattach from `Last-Event-ID`. + +Expect recovery to take roughly the lease duration plus one poll interval, so +about six seconds with the settings above. The same bounded retry also covers a +transient generator failure, which is the everyday reason that checkpoint earns +its keep. + +Streaming is bounded and can be switched off without touching the wrapper: + +```bash +# 10 000 chunks per execution by default; 0 disables the log and leaves the endpoint working +export HAYHOOKS_DURABLE_MAX_STREAM_CHUNKS=0 +``` + +Stop Hayhooks with Ctrl-C, then stop Redis. Add `-v` to remove the retained +volume before a clean rehearsal: + +```bash +docker compose -f examples/durable-compose.yaml down +# docker compose -f examples/durable-compose.yaml down -v +``` diff --git a/examples/durable_chat_with_website/demo.py b/examples/durable_chat_with_website/demo.py new file mode 100644 index 00000000..43720fee --- /dev/null +++ b/examples/durable_chat_with_website/demo.py @@ -0,0 +1,136 @@ +"""Submit a durable chat-with-website question and follow its token stream.""" + +from __future__ import annotations + +import argparse +import json +import time +import uuid +from collections.abc import Iterator +from typing import Any +from urllib.parse import urljoin + +import httpx +from rich.console import Console +from rich.json import JSON +from rich.panel import Panel + +_PAUSE_SECONDS = 2 +# Detaching mid-answer is the point of the demo: the execution keeps running and +# the chunk log keeps every token until this client reattaches. +_DETACH_AFTER_CHUNKS = 8 + + +def sse_events(client: httpx.Client, url: str, cursor: str | None) -> Iterator[dict[str, str]]: + """Yield one dict of SSE fields per event, resuming from *cursor* when given.""" + headers = {"Last-Event-ID": cursor} if cursor else {} + with client.stream("GET", url, headers=headers, timeout=None) as response: + response.raise_for_status() + fields: dict[str, str] = {} + for line in response.iter_lines(): + if line.startswith(":"): + continue # a heartbeat comment, sent while the execution is quiet + if line: + name, _, value = line.partition(": ") + fields[name] = value + elif fields: + yield fields + fields = {} + + +class StreamDroppedError(Exception): + """The server ended the stream with an error event instead of a terminal one.""" + + +class StreamGapError(Exception): + """The bounded chunk log no longer contains the requested cursor.""" + + +def follow( + console: Console, client: httpx.Client, url: str, cursor: str | None, *, detach_after: int | None +) -> tuple[str | None, dict[str, Any] | None]: + """Print chunks from one connection; return the cursor and the terminal body.""" + attempt: int | None = None + for printed, event in enumerate(sse_events(client, url, cursor), start=1): + cursor = event.get("id", cursor) + if event["event"] == "error": + raise StreamDroppedError(json.loads(event["data"]).get("detail", "execution stream interrupted")) + if event["event"] == "gap": + raise StreamGapError(json.loads(event["data"])["detail"]) + if event["event"] != "chunk": + console.print() + return cursor, json.loads(event["data"]) + chunk = json.loads(event["data"]) + if attempt is None: + attempt = chunk["attempt"] + elif chunk["attempt"] < attempt: + continue + elif chunk["attempt"] > attempt: + # A retried attempt replays from its checkpoint, so tokens printed + # before the crash arrive again. Printing cannot retract them; a + # client with a rewritable buffer would reset it here instead. + attempt = chunk["attempt"] + console.print() + console.print(f"[dim]Attempt {attempt}: the execution resumed from its checkpoint, so tokens repeat.[/dim]") + console.print(chunk["payload"]["content"] or "", end="") + if printed == detach_after: + return cursor, None + return cursor, None + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", default="http://localhost:1416") + parser.add_argument("--question", default="What does Haystack do, and how does Redis fit in?") + args = parser.parse_args() + base_url = args.base_url.rstrip("/") + console = Console() + + with httpx.Client(timeout=10) as client: + submitted = client.post( + f"{base_url}/chat_with_website/run-durable", + headers={"Idempotency-Key": f"ask-{uuid.uuid4().hex[:12]}"}, + json={"question": args.question}, + ) + console.print(Panel(JSON.from_data(submitted.json()), title=f"{submitted.status_code} submitted")) + if submitted.is_error: + return 1 + stream_url = urljoin(f"{base_url}/", submitted.json()["links"]["stream"]) + + console.print(Panel.fit(f"[bold cyan]GET[/] {stream_url}", title="Streaming", border_style="cyan")) + cursor, terminal, detached = None, None, False + while terminal is None: + gap = False + try: + cursor, terminal = follow( + console, client, stream_url, cursor, detach_after=None if detached else _DETACH_AFTER_CHUNKS + ) + except (httpx.HTTPError, StreamDroppedError) as error: + console.print() + console.print(Panel(str(error), title="Stream interrupted", border_style="red")) + except StreamGapError as error: + gap = True + cursor = None + console.print() + console.print( + Panel(f"{error}. Replaying the retained tail.", title="Partial stream", border_style="red") + ) + if terminal is None: + detached = True + if not gap: + console.print() + console.print( + Panel( + f"Reattaching from Last-Event-ID {cursor}. Nothing between here and there is lost.", + title="Detached", + border_style="yellow", + ) + ) + time.sleep(_PAUSE_SECONDS) + + console.print(Panel(JSON.from_data(terminal), title=f"Terminal event: {terminal['status']}")) + return 0 if terminal["status"] == "completed" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/durable_chat_with_website/pipelines/chat_with_website/chat_with_website.yml b/examples/durable_chat_with_website/pipelines/chat_with_website/chat_with_website.yml new file mode 100644 index 00000000..86c58ba6 --- /dev/null +++ b/examples/durable_chat_with_website/pipelines/chat_with_website/chat_with_website.yml @@ -0,0 +1,48 @@ +components: + converter: + type: haystack.components.converters.html.HTMLToDocument + init_parameters: + extraction_kwargs: null + + fetcher: + init_parameters: + raise_on_failure: true + retry_attempts: 2 + timeout: 3 + user_agents: + - haystack/LinkContentFetcher/2.0.0b8 + type: haystack.components.fetchers.link_content.LinkContentFetcher + + llm: + init_parameters: + api_key: + env_vars: + - OPENAI_API_KEY + strict: true + type: env_var + generation_kwargs: {} + model: gpt-4o-mini + type: haystack.components.generators.chat.openai.OpenAIChatGenerator + + prompt: + init_parameters: + template: | + {% message role="user" %} + According to the contents of this website: + {% for document in documents %} + {{document.content}} + {% endfor %} + Answer the given question: {{query}} + {% endmessage %} + required_variables: "*" + type: haystack.components.builders.chat_prompt_builder.ChatPromptBuilder + +connections: + - receiver: converter.sources + sender: fetcher.streams + - receiver: prompt.documents + sender: converter.documents + - receiver: llm.messages + sender: prompt.prompt + +metadata: {} \ No newline at end of file diff --git a/examples/durable_chat_with_website/pipelines/chat_with_website/pipeline_wrapper.py b/examples/durable_chat_with_website/pipelines/chat_with_website/pipeline_wrapper.py new file mode 100644 index 00000000..610df284 --- /dev/null +++ b/examples/durable_chat_with_website/pipelines/chat_with_website/pipeline_wrapper.py @@ -0,0 +1,66 @@ +"""A durable chat-with-website Pipeline that streams its answer while it runs.""" + +from pathlib import Path + +from haystack import Pipeline +from haystack.core.errors import PipelineRuntimeError +from pydantic import BaseModel, Field + +from hayhooks import BasePipelineWrapper, DurableContext, durable_streaming_callback + +DEFAULT_URLS = ["https://haystack.deepset.ai", "https://www.redis.io"] + + +class ChatRequest(BaseModel): + """A question to answer from the live contents of a few web pages.""" + + question: str = Field(min_length=1, max_length=2_000) + urls: list[str] = Field(default=DEFAULT_URLS, min_length=1, max_length=5) + + +class ChatAnswer(BaseModel): + """The finished answer, already delivered token by token over the stream.""" + + reply: str + urls: list[str] + + +class PipelineWrapper(BasePipelineWrapper): + """Fetch pages durably, then stream the generated answer to the SSE endpoint.""" + + durable_revision = "durable-chat-with-website-v1" + def setup(self) -> None: + self.pipeline = Pipeline.loads((Path(__file__).parent / "chat_with_website.yml").read_text()) + # `async_streaming_generator` passes its callback per run in `pipeline_run_args`. + # That cannot work under checkpointing: run data is serialized into the + # PipelineSnapshot, Haystack drops the callable it cannot serialize, and + # `Pipeline.run` rebuilds `data` from the snapshot on resume, so the callback + # is gone from the first checkpoint on. Binding it to the component survives. + # + # The helper resolves its destination per call from a ContextVar, so one bound + # callback keeps concurrent durable executions isolated. A run-time callback + # still wins, leaving ordinary streaming endpoints unaffected. + self.pipeline.get_component("llm").streaming_callback = durable_streaming_callback + + async def run_durable_async(self, context: DurableContext, request: ChatRequest) -> ChatAnswer: + # This body re-runs from the top on every attempt, so the message has to say + # what the attempt will actually do rather than what the first one did. + resumed = context.record.checkpoint is not None + await context.report_progress( + "Resuming from the fetch checkpoint" if resumed else f"Fetching {len(request.urls)} page(s)", + kind="resume" if resumed else "fetch", + ) + try: + outputs = await context.run_pipeline_async( + {"fetcher": {"urls": request.urls}, "prompt": {"query": request.question}}, + # Fetching is the slow, flaky step. Checkpointing once it is done means a + # later attempt resumes into generation instead of hitting the network again. + checkpoint_at=["prompt"], + ) + except PipelineRuntimeError as error: + # Without this, a transient generator failure ends the execution and the + # checkpoint above never pays for itself. The retry is bounded by + # `durable_max_attempts`. + await context.retry(f"Pipeline attempt failed: {error}") + await context.report_progress("Answer complete", kind="completed") + return ChatAnswer(reply=outputs["llm"]["replies"][0].text, urls=request.urls) diff --git a/examples/durable_chat_with_website/resume_stream.sh b/examples/durable_chat_with_website/resume_stream.sh new file mode 100755 index 00000000..b156e4f7 --- /dev/null +++ b/examples/durable_chat_with_website/resume_stream.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +set -euo pipefail + +STATE="${TMPDIR:-/tmp}/hayhooks-resumable-stream-demo" +command -v curl >/dev/null +command -v jq >/dev/null + +if [[ ! -s "$STATE.url" || ! -s "$STATE.events" ]]; then + printf 'Run start_stream.sh first, then stop it with Ctrl-C.\n' >&2 + exit 1 +fi + +STREAM_URL="$(<"$STATE.url")" +LAST_EVENT_ID="$( + awk ' + {sub(/\r$/, "")} + /^id: / {candidate = substr($0, 5)} + /^$/ && candidate != "" {last = candidate; candidate = ""} + END {print last} + ' "$STATE.events" +)" + +if [[ -z "$LAST_EVENT_ID" ]]; then + printf 'No complete streamed event is available to resume yet.\n' >&2 + exit 1 +fi + +printf 'Resuming after Last-Event-ID: %s\n\n' "$LAST_EVENT_ID" +curl -NfsS -H "Last-Event-ID: $LAST_EVENT_ID" "$STREAM_URL" | + awk '/^data: / {print substr($0, 7); fflush()}' | + jq --unbuffered -jr ' + .payload.content? // + if .status? then "\n\n[\(.status)]\n" + elif .detail? then "\n\n[\(.detail)]\n" + else empty + end + ' diff --git a/examples/durable_chat_with_website/showcase.py b/examples/durable_chat_with_website/showcase.py new file mode 100644 index 00000000..a683cbb9 --- /dev/null +++ b/examples/durable_chat_with_website/showcase.py @@ -0,0 +1,452 @@ +"""Run two concurrent resumable streams in a three-pane terminal dashboard.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import time +import uuid +from dataclasses import dataclass +from typing import Any +from urllib.parse import urljoin, urlparse + +import httpx +from rich.console import Console, Group +from rich.layout import Layout +from rich.live import Live +from rich.panel import Panel +from rich.table import Table +from rich.text import Text + +_PIPELINE = "chat_with_website" +_REATTACH_DELAY_SECONDS = 3.0 +_MIN_TERMINAL_WIDTH = 100 +_MIN_TERMINAL_HEIGHT = 28 + + +@dataclass(frozen=True) +class Lane: + name: str + color: str + disconnect_after: float + question: str + + +@dataclass +class LaneState: + lane: Lane + phase: str = "READY" + detail: str = "waiting for launch" + answer: str = "" + cursor: str | None = None + connected_at: float | None = None + attempt: int | None = None + terminal: dict[str, Any] | None = None + done: bool = False + error: str | None = None + + +@dataclass(frozen=True) +class Activity: + timestamp: float + lane: str + method: str + status: str + target: str + note: str + + +_LANES = ( + Lane( + name="ATLAS", + color="bright_cyan", + disconnect_after=12.0, + question=( + "Write a lively 1,800 to 2,200 word field guide titled THE ATLAS TRANSMISSIONS. " + "Use exactly 14 numbered dispatches to explain what Haystack is, what people build with it, " + "how its pipelines and components fit together, and where Redis is useful. Begin with the exact " + "marker ATLAS-LIVE. Base factual claims on the supplied websites, use vivid but accurate analogies, " + "and keep going until all 14 dispatches and a short field checklist are complete." + ), + ), + Lane( + name="COMET", + color="bright_magenta", + disconnect_after=16.0, + question=( + "Write a lively 1,800 to 2,200 word incident chronicle titled THE COMET LOG. Use exactly 12 " + "timestamped scenes to explore what Haystack can build, how pipelines organize work, what Redis " + "provides, and how those ideas can support reliable AI systems. Begin with the exact marker " + "COMET-LIVE. Base factual claims on the supplied websites, label imaginative examples clearly, " + "and finish with a six-item launch checklist." + ), + ), +) + + +class StreamEventError(RuntimeError): + """The SSE endpoint reported an error event.""" + + +def _path(url: str) -> str: + parsed = urlparse(url) + parts = (parsed.path or "/").split("/") + if len(parts) > 3 and parts[2] == "executions": # noqa: PLR2004 - path segment indexes are the format + parts[3] = f"{parts[3][:8]}…" + return "/".join(parts) + + +def _activity( # noqa: PLR0913 - one compact call records one HTTP timeline row + events: list[Activity], started: float, lane: str, method: str, status: str, target: str, note: str = "" +) -> None: + events.append(Activity(time.monotonic() - started, lane, method, status, target, note)) + + +def _apply_sse_event(state: LaneState, fields: dict[str, str]) -> dict[str, Any] | None: + event = fields.get("event", "message") + state.cursor = fields.get("id", state.cursor) + data = json.loads(fields.get("data", "{}")) + if event == "chunk": + state.attempt = data["attempt"] + state.answer += data["payload"].get("content") or "" + return None + if event == "gap": + state.detail = data["detail"] + return None + if event == "error": + raise StreamEventError(data.get("detail", "execution stream interrupted")) + return data + + +async def _read_sse(response: httpx.Response, state: LaneState) -> dict[str, Any] | None: + fields: dict[str, str] = {} + async for line in response.aiter_lines(): + if line.startswith(":"): + continue + if line: + name, separator, value = line.partition(":") + if separator: + fields[name] = value.lstrip() + continue + if fields: + terminal = _apply_sse_event(state, fields) + fields = {} + if terminal is not None: + return terminal + return None + + +async def _open_stream( # noqa: PLR0913 - stream context is deliberately explicit at the call site + client: httpx.AsyncClient, + state: LaneState, + stream_url: str, + events: list[Activity], + started: float, + *, + disconnect_after: float | None, +) -> tuple[dict[str, Any] | None, bool]: + headers = {"Last-Event-ID": state.cursor} if state.cursor else {} + note = f"Last-Event-ID: {state.cursor}" if state.cursor else "new event stream" + _activity(events, started, state.lane.name, "GET", "β†’", _path(stream_url), note) + async with client.stream("GET", stream_url, headers=headers) as response: + _activity( + events, started, state.lane.name, "GET", str(response.status_code), _path(stream_url), "SSE connected" + ) + response.raise_for_status() + state.connected_at = time.monotonic() + try: + if disconnect_after is None: + return await _read_sse(response, state), False + return await asyncio.wait_for(_read_sse(response, state), timeout=disconnect_after), False + except asyncio.TimeoutError: + return None, True + + +async def _run_lane( # noqa: PLR0915 - the full visible lifecycle reads linearly in one coroutine + client: httpx.AsyncClient, state: LaneState, base_url: str, events: list[Activity], started: float +) -> None: + lane = state.lane + submit_url = f"{base_url}/{_PIPELINE}/run-durable" + state.phase = "SUBMITTING" + state.detail = "creating durable execution" + _activity(events, started, lane.name, "POST", "β†’", _path(submit_url), "submit durable execution") + try: + submitted = await client.post( + submit_url, + headers={"Idempotency-Key": f"show-{lane.name.lower()}-{uuid.uuid4().hex[:10]}"}, + json={"question": lane.question}, + ) + body = submitted.json() + execution_id = body.get("execution_id", "unknown") + _activity( + events, + started, + lane.name, + "POST", + str(submitted.status_code), + _path(submit_url), + f"accepted {execution_id[:12]}", + ) + submitted.raise_for_status() + stream_url = urljoin(f"{base_url}/", body["links"]["stream"]) + inspect_url = urljoin(f"{base_url}/", body["links"]["self"]) + + state.phase = "LIVE" + state.detail = f"client link will cut at {lane.disconnect_after:.0f}s" + terminal, interrupted = await _open_stream( + client, + state, + stream_url, + events, + started, + disconnect_after=lane.disconnect_after, + ) + if not interrupted: + msg = f"answer completed before the planned {lane.disconnect_after:.0f}s network cut" + raise RuntimeError(msg) + + state.phase = "LINK CUT" + state.detail = f"socket closed on purpose; cursor {state.cursor or 'start'} saved" + _activity( + events, + started, + lane.name, + "CLOSE", + "βœ‚", + _path(stream_url), + f"client cut at {lane.disconnect_after:.0f}s; cursor {state.cursor or 'start'}", + ) + await asyncio.sleep(_REATTACH_DELAY_SECONDS / 2) + _activity(events, started, lane.name, "GET", "β†’", _path(inspect_url), "check execution without a viewer") + inspected = await client.get(inspect_url) + inspected.raise_for_status() + control = inspected.json() + _activity( + events, + started, + lane.name, + "GET", + str(inspected.status_code), + _path(inspect_url), + f"status={control['status']}; attempt={control['attempt']}", + ) + state.detail = f"control plane: {control['status']}; reattaching from {state.cursor or 'start'}" + await asyncio.sleep(_REATTACH_DELAY_SECONDS / 2) + + state.phase = "RESUMED" + state.detail = f"Last-Event-ID: {state.cursor or 'start'}" + terminal, _ = await _open_stream( + client, + state, + stream_url, + events, + started, + disconnect_after=None, + ) + if terminal is None: + msg = "stream ended without a terminal event" + raise RuntimeError(msg) + state.terminal = terminal + exact = terminal.get("result", {}).get("reply") == state.answer + if terminal.get("status") != "completed" or not exact: + msg = "terminal result did not exactly match the replayed stream" + raise RuntimeError(msg) + + state.phase = "EXACT REPLAY" + state.detail = f"{len(state.answer):,} characters; no gaps; no duplicates" + state.done = True + _activity(events, started, lane.name, "SSE", "βœ“", _path(stream_url), state.detail) + except (httpx.HTTPError, json.JSONDecodeError, KeyError, RuntimeError, StreamEventError) as error: + state.phase = "FAILED" + state.detail = str(error) + state.error = str(error) + state.done = True + _activity(events, started, lane.name, "SSE", "!", _path(submit_url), str(error)) + + +def _tail_text(value: str, console: Console, *, width: int, height: int) -> Text: + if not value: + return Text("Waiting for the first streamed token…") + lines = console.render_lines(Text(value), console.options.update(width=width, height=None), pad=False) + plain_lines = ["".join(segment.text for segment in line).rstrip() for line in lines] + if len(plain_lines) <= height: + return Text("\n".join(plain_lines)) + visible_lines = plain_lines[-height:] + if height == 1: + return Text.assemble(("… ", "dim"), visible_lines[0]) + truncated = "\n".join(visible_lines[1:]) + return Text.assemble(("… replay transcript continues …\n", "dim"), visible_lines[0], "\n", truncated) + + +def _lane_panel(state: LaneState, console: Console, *, width: int, height: int) -> Panel: + color = "red" if state.error else state.lane.color + elapsed = time.monotonic() - state.connected_at if state.connected_at and state.phase == "LIVE" else None + progress = "" + if elapsed is not None: + width = max(10, min(24, console.width // 6)) + filled = min(width, int(width * elapsed / state.lane.disconnect_after)) + progress = f" {'━' * filled}{'Β·' * (width - filled)} {elapsed:4.1f}/{state.lane.disconnect_after:.0f}s" + + status = Text() + status.append(state.phase, style=f"bold {color}") + status.append(progress, style=color) + status.append(f"\n{state.detail}", style="dim") + if state.attempt is not None: + status.append(f" attempt {state.attempt}", style="dim") + + content_width = max(1, width - 4) # panel border and horizontal padding + header = Group(status, Text("─" * min(28, max(10, console.width // 4)), style=color)) + header_height = len( + console.render_lines(header, console.options.update(width=content_width, height=None), pad=False) + ) + answer = _tail_text(state.answer, console, width=content_width, height=max(1, height - header_height - 2)) + return Panel( + Group(header, answer), + title=f" {state.lane.name} Β· CUT AT {state.lane.disconnect_after:.0f}s ", + border_style=color, + padding=(0, 1), + ) + + +def _activity_panel(events: list[Activity], console: Console) -> Panel: + table = Table.grid(expand=True, padding=(0, 1)) + table.add_column(width=7, style="dim", no_wrap=True) + table.add_column(width=8, no_wrap=True) + table.add_column(width=7, no_wrap=True) + table.add_column(width=7, no_wrap=True) + table.add_column(ratio=1, overflow="ellipsis") + rows = max(4, int(console.height * 0.3) - 4) + for event in events[-rows:]: + lane_color = "bright_cyan" if event.lane == "ATLAS" else "bright_magenta" + status_color = "green" if event.status.startswith("2") or event.status == "βœ“" else "red" + if event.status == "β†’": + status_color = "bright_yellow" + table.add_row( + f"{event.timestamp:5.1f}s", + Text(event.lane, style=f"bold {lane_color}"), + Text(event.method, style="bold"), + Text(event.status, style=f"bold {status_color}"), + f"{event.target} [dim]{event.note}[/]", + ) + if not events: + table.add_row("0.0s", "SHOW", "READY", "Β·", "waiting for both POST requests") + return Panel(table, title=" HTTP FLIGHT RECORDER Β· POST β†’ CUT β†’ INSPECT β†’ RESUME ", border_style="bright_yellow") + + +def _dashboard(states: list[LaneState], events: list[Activity], console: Console) -> Layout: + layout = Layout() + layout.split_column( + Layout(name="streams", ratio=7, minimum_size=12), Layout(name="activity", ratio=3, minimum_size=8) + ) + layout["streams"].split_row(Layout(name="atlas"), Layout(name="comet")) + regions = layout.render(console, console.options) + for state, name in zip(states, ("atlas", "comet"), strict=True): + region = regions[layout[name]].region + layout[name].update(_lane_panel(state, console, width=region.width, height=region.height)) + layout["activity"].update(_activity_panel(events, console)) + return layout + + +def _preflight(base_url: str, console: Console) -> bool: + try: + response = httpx.get(f"{base_url}/status", timeout=3) + response.raise_for_status() + status = response.json() + except (httpx.HTTPError, json.JSONDecodeError) as error: + console.print( + Panel(f"Hayhooks is not ready at {base_url}: {error}", title=" SERVER MISSING ", border_style="red") + ) + return False + deployment = status.get("durable", {}).get("deployments", {}).get(_PIPELINE) + if not deployment or not deployment.get("healthy"): + console.print( + Panel(f"The durable {_PIPELINE} pipeline is not healthy.", title=" PREFLIGHT FAILED ", border_style="red") + ) + return False + if deployment.get("configured_slots", 0) < len(_LANES): + console.print( + Panel( + "The server's durable execution concurrency ceiling is below two, " + "so this two-lane demo would run serially.\n\n" + "Set [bold]HAYHOOKS_DURABLE_EXECUTION_CONCURRENCY=2[/] or higher and restart Hayhooks.", + title=" CONCURRENCY CEILING TOO LOW ", + border_style="bright_yellow", + ) + ) + return False + if console.width < _MIN_TERMINAL_WIDTH or console.height < _MIN_TERMINAL_HEIGHT: + console.print("[yellow]Tip: enlarge this terminal to at least 100x28 for the best show.[/]") + return True + + +async def _show(base_url: str, console: Console) -> int: + states = [LaneState(lane) for lane in _LANES] + events: list[Activity] = [] + started = time.monotonic() + timeout = httpx.Timeout(10, read=None) + async with httpx.AsyncClient(timeout=timeout) as client: + tasks = [asyncio.create_task(_run_lane(client, state, base_url, events, started)) for state in states] + with Live(_dashboard(states, events, console), console=console, refresh_per_second=12) as live: + while not all(task.done() for task in tasks): + live.update(_dashboard(states, events, console)) + await asyncio.sleep(0.08) + results = await asyncio.gather(*tasks, return_exceptions=True) + for state, result in zip(states, results, strict=True): + if isinstance(result, BaseException): + state.phase = "FAILED" + state.detail = str(result) + state.error = str(result) + state.done = True + _activity(events, started, state.lane.name, "SSE", "!", "/", str(result)) + live.update(_dashboard(states, events, console), refresh=True) + console.print( + "[bold green]Recovery race complete.[/] Both client connections were cut and both answers replayed exactly." + if not any(state.error for state in states) + else "[bold red]Recovery race failed.[/] Read the pane marked FAILED above." + ) + return 1 if any(state.error for state in states) else 0 + + +def _self_test() -> int: + state = LaneState(_LANES[0]) + assert ( + _apply_sse_event( + state, + {"id": "1-0", "event": "chunk", "data": '{"attempt":1,"payload":{"content":"hello"}}'}, + ) + is None + ) + assert state.answer == "hello" + assert state.cursor == "1-0" + terminal = _apply_sse_event(state, {"event": "completed", "data": '{"status":"completed"}'}) + assert terminal == {"status": "completed"} + assert _path("http://localhost:1416/a?b=1") == "/a" + assert _path("http://localhost/chat_with_website/executions/123456789/stream").endswith("/12345678…/stream") + tail = _tail_text("zero one two three four", Console(width=8), width=8, height=3) + assert tail.plain.endswith("three\nfour") and "zero" not in tail.plain + test_console = Console(width=100, height=28) + states = [LaneState(lane, answer=("old text " * 500) + f"LATEST-{lane.name}") for lane in _LANES] + lines = test_console.render_lines(_dashboard(states, [], test_console), test_console.options) + rendered = "\n".join("".join(segment.text for segment in line) for line in lines) + assert all(f"LATEST-{lane.name}" in rendered for lane in _LANES) + Console().print("showcase self-test passed") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", default="http://localhost:1416") + parser.add_argument("--self-test", action="store_true", help=argparse.SUPPRESS) + args = parser.parse_args() + if args.self_test: + return _self_test() + base_url = args.base_url.rstrip("/") + console = Console() + if not _preflight(base_url, console): + return 1 + return asyncio.run(_show(base_url, console)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/durable_chat_with_website/start_stream.sh b/examples/durable_chat_with_website/start_stream.sh new file mode 100755 index 00000000..cca55225 --- /dev/null +++ b/examples/durable_chat_with_website/start_stream.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +set -euo pipefail + +BASE_URL="${HAYHOOKS_BASE_URL:-http://localhost:1416}" +BASE_URL="${BASE_URL%/}" +STATE="${TMPDIR:-/tmp}/hayhooks-resumable-stream-demo" +QUESTION="${QUESTION:-Write a detailed briefing of at least 2,500 words in 16 titled sections. Explain what Haystack does, how its pipelines and agents work, what Redis does, and how Redis can support durable execution. Include a comparison, an architecture walkthrough, a failure-and-recovery scenario, trade-offs, and a conclusion. Do not abbreviate.}" + +command -v curl >/dev/null +command -v jq >/dev/null + +STREAM_PATH="$( + curl -fsS -X POST "$BASE_URL/chat_with_website/run-durable" \ + -H 'Content-Type: application/json' \ + -H "Idempotency-Key: curl-demo-$(date +%s)-$$" \ + -d "$(jq -nc --arg question "$QUESTION" '{question: $question}')" | + jq -er '.links.stream' +)" +STREAM_URL="$BASE_URL$STREAM_PATH" +printf '%s\n' "$STREAM_URL" > "$STATE.url" +printf 'Streaming %s\nStop with Ctrl-C after at least 10 seconds.\n\n' "$STREAM_URL" + +trap 'printf "\n"' EXIT +curl -NfsS "$STREAM_URL" | + tee "$STATE.events" | + awk '/^data: / {print substr($0, 7); fflush()}' | + jq --unbuffered -jr ' + .payload.content? // + if .status? then "\n\n[\(.status)]\n" + elif .detail? then "\n\n[\(.detail)]\n" + else empty + end + ' diff --git a/examples/durable_execution/README.md b/examples/durable_execution/README.md new file mode 100644 index 00000000..83d808a3 --- /dev/null +++ b/examples/durable_execution/README.md @@ -0,0 +1,70 @@ +# Durable document-preparation Pipeline + +This is the canonical REST durable-execution example. The wrapper owns typed +application behavior; Hayhooks owns records, the Redis runnable queue, fenced workers, +checkpoints, retry delay, waiting/resume, cancellation, and retention. +The wrapper declares the stable `durable_revision` required by the engine; use +an image digest or Git SHA instead for production releases. + +The real Haystack Pipeline cleans and splits a document. Hayhooks persists a +`PipelineSnapshot` after `clean`, so a crash during the later delay resumes +from that checkpoint without repeating the completed cleaning step. + +The included `httpx` client pauses five seconds between requests, prints every +URL and response with Rich, automatically handles the retry and approval, and +stays alive while you restart Hayhooks. + +Run each command from the repository root. This is a local reliability +demonstration, not a production Redis configuration. + +1. Start Redis and install the example dependencies. + +```bash +docker compose -f examples/durable-compose.yaml up -d && python -m pip install -e ".[durable]" httpx +``` + +2. Set the durable settings. The five-second lease keeps recovery short. + +```bash +export HAYHOOKS_DURABLE_REDIS_URL=redis://localhost:6379/0 HAYHOOKS_DURABLE_LEASE_DURATION_MS=5000 HAYHOOKS_DURABLE_MAX_ATTEMPTS=4 +``` + +3. Open a first terminal and start Hayhooks. It prints the PID for the forced + crash. + +```bash +sh -c 'echo "Hayhooks PID: $$"; exec hayhooks run --pipelines-dir examples/durable_execution/pipelines' +``` + +4. Open a second terminal and run the client. It submits the document, shows + the intentional retry, waits for approval, approves it, and polls every five + seconds. + +```bash +python examples/durable_execution/demo.py +``` + +5. When the client prints β€œThe clean checkpoint is persisted,” return to the + first terminal and press `Ctrl-C` to stop Hayhooks. The next client request + reports the expected connection failure and waits five seconds before trying + again. + +6. In the first terminal, start Hayhooks again with the same command. The + client detects recovery and prints the completed response. + +```bash +sh -c 'echo "Hayhooks PID: $$"; exec hayhooks run --pipelines-dir examples/durable_execution/pipelines' +``` + +The recovered Pipeline skips `clean`, repeats the interrupted `demo_delay`, +then runs `split`. Durable execution is at least once: if a Pipeline has an +external side effect, use the execution ID and logical step as its idempotency +key. + +Press Ctrl-C in the first terminal to stop Hayhooks, then stop Redis. Add `-v` +to remove the retained Redis volume before a clean rehearsal: + +```bash +docker compose -f examples/durable-compose.yaml down +# docker compose -f examples/durable-compose.yaml down -v +``` diff --git a/examples/durable_execution/demo.py b/examples/durable_execution/demo.py new file mode 100644 index 00000000..603cb820 --- /dev/null +++ b/examples/durable_execution/demo.py @@ -0,0 +1,129 @@ +"""Submit and follow the durable Pipeline recovery demonstration.""" + +from __future__ import annotations + +import argparse +import json +import time +import uuid +from collections.abc import Callable +from typing import Any +from urllib.parse import urljoin + +import httpx +from rich.console import Console +from rich.json import JSON +from rich.panel import Panel + +_PAUSE_SECONDS = 5 +_TERMINAL_STATUSES = {"completed", "failed", "canceled"} + + +def request(console: Console, client: httpx.Client, method: str, url: str, **kwargs: Any) -> httpx.Response | None: + console.print(Panel.fit(f"[bold cyan]{method}[/] {url}", title="Request", border_style="cyan")) + if payload := kwargs.get("json"): + console.print(JSON.from_data(payload)) + try: + response = client.request(method, url, **kwargs) + except httpx.HTTPError as error: + console.print(Panel(str(error), title="Connection failed", border_style="red")) + return None + try: + body = JSON.from_data(response.json()) + except json.JSONDecodeError: + body = response.text + console.print(Panel(body, title=f"{response.status_code} {response.url}", border_style="green")) + return response + + +def pause(console: Console) -> None: + console.print(f"[dim]Waiting {_PAUSE_SECONDS} seconds before the next request...[/]") + time.sleep(_PAUSE_SECONDS) + + +def poll_until( + console: Console, + client: httpx.Client, + execution_url: str, + matches: Callable[[dict[str, Any]], bool], +) -> dict[str, Any] | None: + while True: + response = request(console, client, "GET", execution_url) + if response is not None: + body = response.json() + if matches(body): + return body + if body["status"] in _TERMINAL_STATUSES: + return None + pause(console) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", default="http://localhost:1416") + args = parser.parse_args() + base_url = args.base_url.rstrip("/") + execution_id = f"prepare-hayhooks-guide-{uuid.uuid4().hex[:12]}" + console = Console() + + with httpx.Client(timeout=2) as client: + while ( + submitted := request( + console, + client, + "POST", + f"{base_url}/durable_job/run-durable", + headers={"Idempotency-Key": execution_id}, + json={ + "documents": [ + { + "document_id": "hayhooks-guide", + "content": "Hayhooks durable Pipelines survive restarts.", + } + ], + "fail_first_attempt": True, + "require_approval": True, + "demo_delay_seconds": 30, + }, + ) + ) is None: + pause(console) + if submitted.is_error: + return 1 + links = submitted.json()["links"] + execution_url = urljoin(f"{base_url}/", links["self"]) + resume_url = urljoin(f"{base_url}/", links["resume"]) + + pause(console) + if poll_until(console, client, execution_url, lambda body: body["status"] == "waiting") is None: + return 1 + + pause(console) + resumed = request(console, client, "POST", resume_url, json={"approved": True}) + if resumed is None or resumed.is_error: + return 1 + + pause(console) + checkpoint = poll_until( + console, + client, + execution_url, + lambda body: any(event["kind"] == "demo_delay" for event in body["progress"]), + ) + if checkpoint is None: + return 1 + console.print( + Panel( + "Kill Hayhooks now, then restart it. This client will keep polling.", + title="The clean checkpoint is persisted", + border_style="yellow", + ) + ) + + pause(console) + completed = poll_until(console, client, execution_url, lambda body: body["status"] in _TERMINAL_STATUSES) + return 0 if completed is not None and completed["status"] == "completed" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/durable_execution/pipelines/durable_job/pipeline_wrapper.py b/examples/durable_execution/pipelines/durable_job/pipeline_wrapper.py new file mode 100644 index 00000000..998cee12 --- /dev/null +++ b/examples/durable_execution/pipelines/durable_job/pipeline_wrapper.py @@ -0,0 +1,132 @@ +"""A durable document-preparation Pipeline using real Haystack components.""" + +import time + +from haystack import Document, Pipeline, component +from haystack.components.preprocessors import DocumentCleaner, DocumentSplitter +from pydantic import BaseModel, Field + +from hayhooks import BasePipelineWrapper, DurableContext, current_durable_context + + +class SourceDocument(BaseModel): + """One raw document accepted by the durable indexing-preparation job.""" + + document_id: str = Field(min_length=1, max_length=128) + content: str = Field(min_length=1, max_length=10_000) + + +class DocumentPreparationRequest(BaseModel): + """Documents to clean and split into embedding-ready chunks.""" + + documents: list[SourceDocument] = Field(min_length=1, max_length=25) + fail_first_attempt: bool = False + require_approval: bool = False + demo_delay_seconds: float = Field(default=0, ge=0, le=300) + + +class ApprovalInput(BaseModel): + """Typed input accepted by the generated resume endpoint.""" + + approved: bool + + +class PreparedChunk(BaseModel): + """A compact, client-safe projection of a Haystack Document chunk.""" + + document_id: str + chunk_id: str + content: str + + +class DocumentPreparationResult(BaseModel): + """The chunks produced by the real Haystack preprocessing Pipeline.""" + + document_count: int + chunk_count: int + chunks: list[PreparedChunk] + + +@component +class DemoDelay: + """Optional pause after a persisted checkpoint, used only for restart demonstrations.""" + + @component.output_types(documents=list[Document]) + def run(self, documents: list[Document], seconds: float) -> dict[str, list[Document]]: + if seconds: + context = current_durable_context() + if context is not None: + context.report_progress_sync( + f"Checkpointed demo delay started for {seconds:g} seconds", + kind="demo_delay", + ) + time.sleep(seconds) + return {"documents": documents} + + +class PipelineWrapper(BasePipelineWrapper): + """Clean and chunk documents before a later embedding/indexing stage.""" + + durable_revision = "durable-document-preparation" + durable_resume_model = ApprovalInput + + def setup(self) -> None: + self.pipeline = Pipeline() + self.pipeline.add_component("clean", DocumentCleaner(remove_empty_lines=True)) + self.pipeline.add_component("demo_delay", DemoDelay()) + self.pipeline.add_component( + "split", + DocumentSplitter(split_by="word", split_length=80, split_overlap=10), + ) + self.pipeline.connect("clean.documents", "demo_delay.documents") + self.pipeline.connect("demo_delay.documents", "split.documents") + + async def run_durable_async( + self, context: DurableContext, request: DocumentPreparationRequest + ) -> DocumentPreparationResult: + await context.report_progress("Document preparation accepted", kind="accepted") + if request.fail_first_attempt and context.attempt == 1: + await context.report_progress("Demonstrating one bounded retry", kind="retry_demo") + await context.retry("Intentional first-attempt failure", delay=1) + + if request.require_approval and context.resume_input is None: + await context.suspend( + { + "kind": "approval", + "message": "Approve document preparation", + "expected_input_schema": ApprovalInput.model_json_schema(), + } + ) + if request.require_approval: + approval = ApprovalInput.model_validate(context.take_resume_input()) + if not approval.approved: + msg = "Document preparation was not approved" + raise ValueError(msg) + + documents = [ + Document(id=source.document_id, content=source.content, meta={"document_id": source.document_id}) + for source in request.documents + ] + outputs = await context.run_pipeline_async( + { + "clean": {"documents": documents}, + "demo_delay": {"seconds": request.demo_delay_seconds}, + }, + # The delay begins after clean has completed and the snapshot before + # demo_delay has been persisted, creating a reliable crash window. + checkpoint_at=["clean", "demo_delay", "split"], + ) + chunks = outputs["split"]["documents"] + await context.report_progress("Document preparation completed", kind="completed") + return DocumentPreparationResult( + document_count=len(documents), + chunk_count=len(chunks), + chunks=[ + PreparedChunk( + document_id=str(chunk.meta["document_id"]), + chunk_id=str(chunk.id), + content=chunk.content or "", + ) + for chunk in chunks + ], + ) diff --git a/mkdocs.yml b/mkdocs.yml index 7e217187..7c450d51 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -30,6 +30,9 @@ nav: - Advanced Usage: - Running Pipelines: advanced/running-pipelines.md - Advanced Configuration: advanced/advanced-configuration.md + - Durable Engine: advanced/durable-engine.md + - Durable Engine vs Temporal: advanced/durable-engine-vs-temporal.md + - Durable Execution Operations: advanced/durable-execution-operations.md - Code Sharing: advanced/code-sharing.md - Guides: - Development Best Practices: guides/development-best-practices.md @@ -118,6 +121,9 @@ plugins: Advanced Usage: - advanced/running-pipelines.md: Execute Haystack Pipelines and manage runs programmatically. - advanced/advanced-configuration.md: Fine-tune advanced configuration settings. + - advanced/durable-engine.md: Current contract, state model, and architectural boundaries for durable Pipeline and Agent execution. + - advanced/durable-engine-vs-temporal.md: Compare Hayhooks durable execution with Temporal and understand when each fits. + - advanced/durable-execution-operations.md: Operate durable execution with explicit Redis, recovery, security, and scaling contracts. - advanced/code-sharing.md: Share Python code securely across deployments. Guides: - guides/development-best-practices.md: Development workflow tips for Hayhooks pipelines. diff --git a/pyproject.toml b/pyproject.toml index 7a8e0dad..174d5fd7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,11 @@ mcp = [ ] a2a = [ "a2a-sdk[http-server]>=1.1.0,<2.0", + "redis>=5,<9", +] +durable = [ + "haystack-ai>=3,<4", + "redis>=5,<9", ] chainlit = [ "chainlit>=2.0.0", @@ -138,6 +143,24 @@ all = "pytest -vv {args:tests}" all-cov = "all --cov=hayhooks" types = "ty check {args:src/hayhooks}" +[tool.hatch.envs.test-v3] +features = ["durable", "a2a", "mcp"] +extra-dependencies = [ + "qdrant-haystack", + "trafilatura", + "pytest", + "pytest-asyncio", + "pytest-cov", + "pytest-mock", + "ty", +] + +[tool.hatch.envs.test-v3.scripts] +unit = "pytest -vv -m 'not integration' {args:tests}" +integration = "pytest -vv -m integration {args:tests}" +all = "pytest -vv {args:tests}" +types = "ty check {args:src/hayhooks}" + [tool.ty.environment] python-version = "3.10" diff --git a/src/hayhooks/__init__.py b/src/hayhooks/__init__.py index fcc93a3a..5dc6d5e4 100644 --- a/src/hayhooks/__init__.py +++ b/src/hayhooks/__init__.py @@ -1,25 +1,41 @@ +"""Public Hayhooks authoring API.""" + +from hayhooks.a2a import A2APipelineWrapper from hayhooks.callbacks import default_on_pipeline_end, default_on_tool_call_end, default_on_tool_call_start +from hayhooks.durable import ( + ExecutionProgress, + ExecutionResult, + current_durable_context, + current_execution_id, + durable_streaming_callback, +) +from hayhooks.durable.context import DurableContext +from hayhooks.durable.models import ExecutionStatus from hayhooks.events import PipelineEvent from hayhooks.server.app import create_app, run_app from hayhooks.server.logger import log from hayhooks.server.pipelines.sse import SSEStream +from hayhooks.server.pipelines.streaming import async_streaming_generator, streaming_generator from hayhooks.server.pipelines.utils import ( - async_streaming_generator, chat_messages_from_openai_response, coerce_pipeline_inputs, get_input_files, get_last_user_input_text, get_last_user_message, is_user_message, - streaming_generator, ) from hayhooks.server.utils.base_pipeline_wrapper import BasePipelineWrapper from hayhooks.server.utils.haystack_compat import AsyncPipeline, Pipeline from hayhooks.server.utils.yaml_pipeline_wrapper import YAMLPipelineWrapper __all__ = [ + "A2APipelineWrapper", "AsyncPipeline", "BasePipelineWrapper", + "DurableContext", + "ExecutionProgress", + "ExecutionResult", + "ExecutionStatus", "Pipeline", "PipelineEvent", "SSEStream", @@ -28,9 +44,12 @@ "chat_messages_from_openai_response", "coerce_pipeline_inputs", "create_app", + "current_durable_context", + "current_execution_id", "default_on_pipeline_end", "default_on_tool_call_end", "default_on_tool_call_start", + "durable_streaming_callback", "get_input_files", "get_last_user_input_text", "get_last_user_message", diff --git a/src/hayhooks/a2a.py b/src/hayhooks/a2a.py new file mode 100644 index 00000000..1a21e856 --- /dev/null +++ b/src/hayhooks/a2a.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable + +from hayhooks.server.utils.base_pipeline_wrapper import BasePipelineWrapper + +if TYPE_CHECKING: + from a2a.server.context import ServerCallContext + from a2a.server.tasks import TaskStore + from a2a.types import Task + + from hayhooks.server.a2a.redis_task_store import RedisTaskStore, RedisTaskStoreProvider + + +class A2APipelineWrapper(BasePipelineWrapper): + """Base class for wrappers that expose a managed durable A2A Agent.""" + + durable: bool = True + + +def default_a2a_owner(context: ServerCallContext) -> str: + """Return the built-in stable owner for an A2A request.""" + user = context.user + if not user.is_authenticated: + return "anonymous" + if not user.user_name: + msg = "Authenticated A2A users must have a non-empty user name" + raise ValueError(msg) + return f"user:{user.user_name}" + + +def validate_a2a_owner(owner_id: str) -> str: + """Reject owner resolvers that cannot isolate persisted tasks.""" + if not isinstance(owner_id, str) or not owner_id: + msg = "A2A owner resolvers must return a non-empty string" + raise ValueError(msg) + return owner_id + + +class TaskStoreProvider(ABC): + """Create A2A SDK task stores for the agents mounted by the server.""" + + @abstractmethod + def create_task_store(self, agent_name: str) -> TaskStore: + """Return the task store for an exposed agent.""" + raise NotImplementedError + + async def initialize(self) -> None: + """Validate provider resources during A2A application startup.""" + return None + + async def health(self) -> dict[str, Any]: + """Return a payload-safe readiness projection for provider resources.""" + return {"healthy": True, "provider": type(self).__name__} + + async def close(self) -> None: + """Release resources owned by the provider when the A2A server stops.""" + return None + + +@runtime_checkable +class RecoverableTaskStore(Protocol): + """Optional Redis operations used to recover durable A2A projections.""" + + def owner_id_for_context(self, context: ServerCallContext) -> str: ... + + async def recoverable_task_batch( + self, cursor: int, limit: int + ) -> tuple[list[tuple[Task, str, int]], int | None]: ... + + async def save_projection(self, task: Task, owner: str, expected_version: int) -> bool: + """Persist a projected task through its optimistic version fence.""" + ... + + +def __getattr__(name: str) -> Any: + """Lazily expose optional Redis task-store types without importing A2A runtime.""" + if name in {"RedisTaskStore", "RedisTaskStoreProvider"}: + from hayhooks.server.a2a.redis_task_store import RedisTaskStore, RedisTaskStoreProvider + + return {"RedisTaskStore": RedisTaskStore, "RedisTaskStoreProvider": RedisTaskStoreProvider}[name] + msg = f"module {__name__!r} has no attribute {name!r}" + raise AttributeError(msg) + + +__all__ = [ + "A2APipelineWrapper", + "RecoverableTaskStore", + "RedisTaskStore", + "RedisTaskStoreProvider", + "TaskStoreProvider", + "default_a2a_owner", + "validate_a2a_owner", +] diff --git a/src/hayhooks/cli/a2a.py b/src/hayhooks/cli/a2a.py index 2c0e0cbd..ec9cedf9 100644 --- a/src/hayhooks/cli/a2a.py +++ b/src/hayhooks/cli/a2a.py @@ -1,5 +1,5 @@ import sys -from typing import Annotated +from typing import Annotated, Literal import typer @@ -20,6 +20,38 @@ def run( # noqa: PLR0913 str | None, typer.Option("--external-url", help="Base URL advertised in agent cards (e.g. behind a reverse proxy)"), ] = None, + task_store: Annotated[ + Literal["auto", "memory", "redis"] | None, + typer.Option("--task-store", help="Built-in A2A task-store backend"), + ] = None, + a2a_redis_url: Annotated[ + str | None, + typer.Option("--a2a-redis-url", help="Redis URL for the built-in A2A task store"), + ] = None, + a2a_redis_key_prefix: Annotated[ + str | None, + typer.Option("--a2a-redis-key-prefix", help="Redis key prefix for the built-in A2A task store"), + ] = None, + execution_store: Annotated[ + Literal["memory", "redis"] | None, + typer.Option("--execution-store", help="Built-in durable execution-store backend"), + ] = None, + execution_redis_url: Annotated[ + str | None, + typer.Option("--execution-redis-url", help="Redis URL for durable execution storage"), + ] = None, + execution_redis_key_prefix: Annotated[ + str | None, + typer.Option("--execution-redis-key-prefix", help="Redis key prefix for durable execution storage"), + ] = None, + durable_execution_concurrency: Annotated[ + int | None, + typer.Option( + "--durable-execution-concurrency", + min=1, + help="Operator concurrency ceiling per deployed durable Agent", + ), + ] = None, debug: Annotated[bool, typer.Option("--debug", help="If true, tracebacks should be returned on errors")] = False, ) -> None: """ @@ -28,13 +60,12 @@ def run( # noqa: PLR0913 # Lazy imports of settings, logger and uvicorn import uvicorn + from hayhooks.durable.runtime import DurableRuntime + from hayhooks.server.a2a.app import create_a2a_app from hayhooks.server.logger import intercept_stdlib_logging, log - from hayhooks.server.utils.a2a_utils import a2a_import, create_a2a_app from hayhooks.server.utils.deploy_utils import deploy_pipelines from hayhooks.settings import settings - a2a_import.check() - # Fill defaults from settings when command executes host = host or settings.a2a_host port = port or settings.a2a_port @@ -48,24 +79,51 @@ def run( # noqa: PLR0913 if external_url: settings.a2a_external_url = external_url + if task_store is not None: + settings.a2a_task_store = task_store + + if a2a_redis_url is not None: + settings.a2a_redis_url = a2a_redis_url + + if a2a_redis_key_prefix is not None: + settings.a2a_redis_key_prefix = a2a_redis_key_prefix + + if execution_store is not None: + settings.durable_store = execution_store + + if execution_redis_url is not None: + settings.durable_redis_url = execution_redis_url + + if execution_redis_key_prefix is not None: + settings.durable_redis_key_prefix = execution_redis_key_prefix + + if durable_execution_concurrency is not None: + settings.durable_execution_concurrency = durable_execution_concurrency + if additional_python_path: settings.additional_python_path = additional_python_path sys.path.append(additional_python_path) log.trace("Added '{}' to sys.path", additional_python_path) + durable_runtime = DurableRuntime(app_settings=settings) + # Deploy the pipelines - deploy_pipelines() + deploy_pipelines(durable_runtime=durable_runtime) # Setup the Starlette app exposing pipelines as A2A agents log.debug( - "Starting A2A server with host={}, port={}, pipelines_dir={}, external_url={}, v0.3_compat={}", + "Starting A2A server with host={}, port={}, pipelines_dir={}, external_url={}, " + "v0.3_compat={}, task_store={}, durable_store={}, durable_execution_concurrency={}", host, port, pipelines_dir, settings.a2a_external_url or "", settings.a2a_v0_3_compat, + settings.a2a_task_store, + settings.durable_store, + settings.durable_execution_concurrency, ) - app = create_a2a_app(debug=debug) + app = create_a2a_app(debug=debug, durable_runtime=durable_runtime) # Run the A2A server # NOTE: reload and workers options are not supported in this context diff --git a/src/hayhooks/cli/mcp.py b/src/hayhooks/cli/mcp.py index 87008365..8003c188 100644 --- a/src/hayhooks/cli/mcp.py +++ b/src/hayhooks/cli/mcp.py @@ -33,6 +33,7 @@ def run( # noqa: PLR0913 # Lazy imports of settings, logger and uvicorn import uvicorn + from hayhooks.durable.runtime import DurableRuntime from hayhooks.server.logger import intercept_stdlib_logging, log from hayhooks.server.utils.deploy_utils import deploy_pipelines from hayhooks.server.utils.mcp_utils import create_mcp_server, create_starlette_app @@ -54,14 +55,21 @@ def run( # noqa: PLR0913 sys.path.append(additional_python_path) log.trace("Added '{}' to sys.path", additional_python_path) + durable_runtime = DurableRuntime(app_settings=settings) + # Deploy the pipelines - deploy_pipelines() + deploy_pipelines(durable_runtime=durable_runtime) # Setup the MCP server - server: Server = create_mcp_server() + server: Server = create_mcp_server(durable_runtime=durable_runtime) # Setup the Starlette app - app = create_starlette_app(server, debug=debug, json_response=json_response) + app = create_starlette_app( + server, + debug=debug, + json_response=json_response, + durable_runtime=durable_runtime, + ) # Run the MCP server # NOTE: reload and workers options are not supported in this context diff --git a/src/hayhooks/durable/__init__.py b/src/hayhooks/durable/__init__.py new file mode 100644 index 00000000..7fd6e37a --- /dev/null +++ b/src/hayhooks/durable/__init__.py @@ -0,0 +1,66 @@ +"""Public durable execution API.""" + +from __future__ import annotations + +from hayhooks.durable.context import DurableContext, durable_streaming_callback, get_current_durable_context +from hayhooks.durable.fastapi import create_durable_router +from hayhooks.durable.mode import DurableAuthoringMode, durable_authoring_mode +from hayhooks.durable.models import ( + ExecutionAdmissionError, + ExecutionCanceledError, + ExecutionProgress, + ExecutionRecordSizeError, + ExecutionResult, + ExecutionStatus, + ExecutionStoreError, + ExecutionSuspendedError, + RetryableExecutionError, +) +from hayhooks.durable.runtime import ( + DefinitionRevisionConflictError, + DurableDeployment, + DurableRuntime, + ExecutionStoreProvider, + IdempotencyConflictError, + durable_runtime, +) +from hayhooks.durable.settings import DurableSettings +from hayhooks.durable.store import ExecutionStore, InMemoryExecutionStoreProvider, RedisExecutionStoreProvider + + +def current_execution_id() -> str | None: + """Return the active durable execution ID for hooks and idempotent tools.""" + context = get_current_durable_context() + return context.execution_id if context is not None else None + + +current_durable_context = get_current_durable_context + +__all__ = [ + "DefinitionRevisionConflictError", + "DurableAuthoringMode", + "DurableContext", + "DurableDeployment", + "DurableRuntime", + "DurableSettings", + "ExecutionAdmissionError", + "ExecutionCanceledError", + "ExecutionProgress", + "ExecutionRecordSizeError", + "ExecutionResult", + "ExecutionStatus", + "ExecutionStore", + "ExecutionStoreError", + "ExecutionStoreProvider", + "ExecutionSuspendedError", + "IdempotencyConflictError", + "InMemoryExecutionStoreProvider", + "RedisExecutionStoreProvider", + "RetryableExecutionError", + "create_durable_router", + "current_durable_context", + "current_execution_id", + "durable_authoring_mode", + "durable_runtime", + "durable_streaming_callback", +] diff --git a/src/hayhooks/durable/adapters.py b/src/hayhooks/durable/adapters.py new file mode 100644 index 00000000..e8fffc39 --- /dev/null +++ b/src/hayhooks/durable/adapters.py @@ -0,0 +1,391 @@ +""" +Haystack 3 adapters used by :class:`hayhooks.durable.context.DurableContext`. + +The adapters use only Haystack's public PipelineSnapshot, Agent, State, and +hook APIs. The imports intentionally stay lazy so the base Hayhooks install +continues to support Haystack 2 for non-durable deployments. +""" + +from __future__ import annotations + +import asyncio +import importlib +from collections.abc import Mapping +from typing import Any, cast + +from haystack.lazy_imports import LazyImport + +from hayhooks.durable.context import DurableContext +from hayhooks.durable.models import ExecutionCheckpoint, ExecutionKind, RetryableExecutionError, validate_json + +_HAYSTACK_V3_ERROR = ( + "Durable execution requires Haystack 3. Install `hayhooks[durable]` in the durable server environment." +) +_AGENT_CHECKPOINT_PHASE = "_hayhooks_agent_checkpoint_phase" +_AGENT_FINAL_PHASE = "after_run" +_AGENT_INTERNAL_STATE_KEYS = frozenset(("continue_run", "tools", "hook_context")) + + +async def _run_fenced_thread(function: Any, /, *args: Any, **kwargs: Any) -> Any: + """Keep the caller's durable claim alive until non-cancellable thread work exits.""" + task = asyncio.create_task(asyncio.to_thread(function, *args, **kwargs)) + try: + return await asyncio.shield(task) + except asyncio.CancelledError: + return await task + + +# Keep every Haystack 3-only symbol behind Haystack's supported optional-import +# boundary. This module is imported by the base package in Haystack 2 +# environments, where Agent, hooks, and snapshots intentionally do not exist. +with LazyImport(_HAYSTACK_V3_ERROR) as haystack_v3_import: + from haystack import Pipeline + from haystack.components.agents import Agent + from haystack.components.agents.state import State + from haystack.core.errors import BreakpointException, PipelineRuntimeError + from haystack.dataclasses import ChatMessage + from haystack.dataclasses.breakpoints import Breakpoint, PipelineSnapshot + + FunctionHook = importlib.import_module("haystack.hooks.from_function").FunctionHook + + +def require_haystack_v3() -> None: + """Fail durable deployment explicitly when the optional v3 extra is missing.""" + try: + haystack_v3_import.check() + import haystack + except ImportError as error: # pragma: no cover - dependency failure + raise RuntimeError(_HAYSTACK_V3_ERROR) from error + major = str(getattr(haystack, "__version__", "0")).split(".", maxsplit=1)[0] + if major != "3": + raise RuntimeError(_HAYSTACK_V3_ERROR) + + +class HaystackDurableAdapter: + """Bind a validated Haystack 3 Pipeline or Agent to execution contexts.""" + + def __init__(self, pipeline: Any, kind: ExecutionKind) -> None: + require_haystack_v3() + self.pipeline = pipeline + self.kind = kind + if kind is ExecutionKind.PIPELINE: + self._validate_pipeline() + else: + self._validate_agent() + self._install_agent_checkpoint_hooks() + + def _validate_pipeline(self) -> None: + haystack_v3_import.check() + if not isinstance(self.pipeline, Pipeline): + msg = "run_durable Pipeline wrappers must set self.pipeline to a Haystack 3 Pipeline" + raise TypeError(msg) + + def _validate_agent(self) -> None: + haystack_v3_import.check() + if not isinstance(self.pipeline, Agent): + msg = "durable Agent wrappers must set self.pipeline to a Haystack 3 Agent" + raise TypeError(msg) + + async def run_pipeline_async( + self, context: DurableContext, data: dict[str, Any], *, checkpoint_at: list[str] + ) -> dict[str, Any]: + return cast( + dict[str, Any], + await _run_fenced_thread(self.run_pipeline, context, data, checkpoint_at=checkpoint_at), + ) + + def run_pipeline( + self, context: DurableContext, data: dict[str, Any], *, checkpoint_at: list[str] + ) -> dict[str, Any]: + if self.kind is not ExecutionKind.PIPELINE: + msg = "run_pipeline is available only when self.pipeline is a Haystack Pipeline" + raise TypeError(msg) + snapshot = None + if context.record.checkpoint is not None: + checkpoint = context.record.checkpoint + if checkpoint.kind is not ExecutionKind.PIPELINE: + msg = "The persisted checkpoint is not a PipelineSnapshot" + raise TypeError(msg) + snapshot = PipelineSnapshot.from_dict(cast(dict[str, Any], checkpoint.data["snapshot"])) + boundaries = list(checkpoint_at) + if snapshot is not None: + break_point: Any = snapshot.break_point + completed_visits = snapshot.pipeline_state.component_visits + boundaries = [ + name + for name in boundaries + if completed_visits.get(name, 0) == 0 + and not (name == break_point.component_name and break_point.visit_count == 0) + ] + + next_data = data if snapshot is None else {} + try: + while boundaries: + component_name = boundaries.pop(0) + break_point = Breakpoint(component_name=component_name) + try: + return cast( + dict[str, Any], + self.pipeline.run(data=next_data, pipeline_snapshot=snapshot, break_point=break_point), + ) + except BreakpointException as error: + if error.pipeline_snapshot is None: + msg = "Haystack breakpoint did not expose a PipelineSnapshot" + raise RetryableExecutionError(msg) from error + snapshot = error.pipeline_snapshot + context.record.append_progress( + f"Checkpoint saved before pipeline component '{component_name}'", + kind="checkpoint", + ) + context._sync_await(context.checkpoint(_pipeline_checkpoint(context, snapshot))) + next_data = {} + return cast(dict[str, Any], self.pipeline.run(data=next_data, pipeline_snapshot=snapshot)) + except PipelineRuntimeError as error: + if error.pipeline_snapshot is not None: + context._sync_await(context.checkpoint(_pipeline_checkpoint(context, error.pipeline_snapshot))) + raise + + async def run_agent_async(self, context: DurableContext, *, messages: list[Any], **kwargs: Any) -> dict[str, Any]: + if self.kind is not ExecutionKind.AGENT: + msg = "run_agent_async is available only when self.pipeline is a Haystack Agent" + raise TypeError(msg) + final_result = _final_agent_result(context) + if final_result is not None: + return final_result + method = getattr(self.pipeline, "run_async", None) + if callable(method): + return cast(dict[str, Any], await method(messages=messages, **kwargs)) + return cast( + dict[str, Any], + await _run_fenced_thread(self.run_agent, context, messages=messages, **kwargs), + ) + + def run_agent(self, context: DurableContext, *, messages: list[Any], **kwargs: Any) -> dict[str, Any]: + if self.kind is not ExecutionKind.AGENT: + msg = "run_agent is available only when self.pipeline is a Haystack Agent" + raise TypeError(msg) + final_result = _final_agent_result(context) + if final_result is not None: + return final_result + return cast(dict[str, Any], self.pipeline.run(messages=messages, **kwargs)) + + def _install_agent_checkpoint_hooks(self) -> None: # noqa: C901, PLR0915 + """Install once; hooks select the active execution through ContextVar.""" + if getattr(self.pipeline, "_hayhooks_durable_hooks_installed", False): + return + + def restore_before_run(state: State) -> None: + if context := _current_durable_context(): + _restore_agent_state(context, state) + + async def restore_before_run_async(state: State) -> None: + if context := _current_durable_context(): + _restore_agent_state(context, state) + + def check_cancelled_before_llm(state: State) -> None: + del state + if context := _current_durable_context(): + context.check_cancelled_sync() + + async def check_cancelled_before_llm_async(state: State) -> None: + del state + if context := _current_durable_context(): + await context.check_cancelled() + + def checkpoint_after_tool(state: State) -> None: + context = _current_durable_context() + if context is None: + return + if not _agent_exits_after_tools(state, self.pipeline.exit_conditions): + context._sync_await(_checkpoint_agent_state(context, state)) + context.check_cancelled_sync() + + async def checkpoint_after_tool_async(state: State) -> None: + context = _current_durable_context() + if context is None: + return + if not _agent_exits_after_tools(state, self.pipeline.exit_conditions): + await _checkpoint_agent_state(context, state) + await context.check_cancelled() + + def checkpoint_on_exit(state: State) -> None: + context = _current_durable_context() + if context is not None and state.data["continue_run"]: + context._sync_await(_checkpoint_agent_state(context, state)) + + async def checkpoint_on_exit_async(state: State) -> None: + context = _current_durable_context() + if context is not None and state.data["continue_run"]: + await _checkpoint_agent_state(context, state) + + def checkpoint_after_run(state: State) -> None: + if context := _current_durable_context(): + context._sync_await(_checkpoint_agent_state(context, state, final=True)) + + async def checkpoint_after_run_async(state: State) -> None: + if context := _current_durable_context(): + await _checkpoint_agent_state(context, state, final=True) + + # The module uses postponed annotations, while Haystack validates hook + # signatures with ``inspect.signature`` rather than resolving hints. + for function in ( + restore_before_run, + restore_before_run_async, + check_cancelled_before_llm, + check_cancelled_before_llm_async, + checkpoint_after_tool, + checkpoint_after_tool_async, + checkpoint_on_exit, + checkpoint_on_exit_async, + checkpoint_after_run, + checkpoint_after_run_async, + ): + function.__annotations__["state"] = State + + hooks = dict(getattr(self.pipeline, "hooks", {}) or {}) + hooks["before_run"] = [ + FunctionHook(function=restore_before_run, async_function=restore_before_run_async), + *hooks.get("before_run", []), + ] + hooks["before_llm"] = [ + FunctionHook(function=check_cancelled_before_llm, async_function=check_cancelled_before_llm_async), + *hooks.get("before_llm", []), + ] + hooks["after_tool"] = [ + *hooks.get("after_tool", []), + FunctionHook(function=checkpoint_after_tool, async_function=checkpoint_after_tool_async), + ] + hooks["on_exit"] = [ + *hooks.get("on_exit", []), + FunctionHook(function=checkpoint_on_exit, async_function=checkpoint_on_exit_async), + ] + hooks["after_run"] = [ + *hooks.get("after_run", []), + FunctionHook(function=checkpoint_after_run, async_function=checkpoint_after_run_async), + ] + self.pipeline.hooks = hooks + self.pipeline._hayhooks_durable_hooks_installed = True + + +def chat_messages(values: Any) -> list[Any]: + """Decode a persisted list of serialized chat messages.""" + if not isinstance(values, list): + return [] + return [ChatMessage.from_dict(value) for value in values if isinstance(value, dict)] + + +def _current_durable_context() -> DurableContext | None: + from hayhooks.durable.context import get_current_durable_context + + return get_current_durable_context() + + +def _pipeline_checkpoint(context: DurableContext, snapshot: Any) -> ExecutionCheckpoint: + return ExecutionCheckpoint( + ExecutionKind.PIPELINE, + {"snapshot": validate_json(snapshot.to_dict(), limit=context.record.max_record_bytes, label="snapshot")}, + ) + + +def _checkpoint_data(state: Any, context: DurableContext, *, final: bool = False) -> dict[str, Any]: + """Exclude live resources from State's otherwise public serialization.""" + payload = _without_live_agent_resources(state.to_dict()) + if final: + payload[_AGENT_CHECKPOINT_PHASE] = _AGENT_FINAL_PHASE + return cast( + dict[str, Any], + validate_json(payload, limit=context.record.max_record_bytes, label="Agent state"), + ) + + +def _without_live_agent_resources(payload: Mapping[str, Any]) -> dict[str, Any]: + """Remove per-run Agent resources from a serialized state checkpoint.""" + cleaned = dict(payload) + data = dict(cast(Mapping[str, Any], cleaned.get("data", {}))) + schema = dict(cast(Mapping[str, Any], cleaned.get("schema", {}))) + serialization_schema = dict(data.get("serialization_schema", {})) + properties = dict(serialization_schema.get("properties", {})) + serialized_data = dict(data.get("serialized_data", {})) + for key in ("tools", "hook_context"): + schema.pop(key, None) + properties.pop(key, None) + serialized_data.pop(key, None) + serialization_schema["properties"] = properties + data["serialization_schema"] = serialization_schema + data["serialized_data"] = serialized_data + cleaned["schema"] = schema + cleaned["data"] = data + return cleaned + + +async def _checkpoint_agent_state(context: DurableContext, state: Any, *, final: bool = False) -> None: + context.record.append_progress( + "Agent final checkpoint saved" if final else "Agent step checkpoint saved", kind="checkpoint" + ) + await context.checkpoint(ExecutionCheckpoint(ExecutionKind.AGENT, _checkpoint_data(state, context, final=final))) + + +def _agent_exits_after_tools(state: State, exit_conditions: list[str]) -> bool: + """Return whether Haystack will stop after the current tool-result messages.""" + if exit_conditions == ["text"]: + return False + matched = False + for message in reversed(state.data.get("messages", [])): + result = message.tool_call_result + if result is None: + break + if result.origin.tool_name not in exit_conditions: + continue + if result.error: + return False + matched = True + return matched + + +def _final_agent_result(context: DurableContext) -> dict[str, Any] | None: + """Return a checkpointed terminal Agent result without re-entering the Agent loop.""" + checkpoint = context.record.checkpoint + if ( + checkpoint is None + or checkpoint.kind is not ExecutionKind.AGENT + or checkpoint.data.get(_AGENT_CHECKPOINT_PHASE) != _AGENT_FINAL_PHASE + ): + return None + state = State.from_dict(_without_live_agent_resources(checkpoint.data)) + result = {key: value for key, value in state.data.items() if key not in _AGENT_INTERNAL_STATE_KEYS} + if messages := result.get("messages"): + result["last_message"] = messages[-1] + return result + + +def _restore_agent_state(context: DurableContext, state: Any) -> None: + """Restore a recovered State, retaining fresh per-run live resources.""" + checkpoint = context.record.checkpoint + if checkpoint is None or checkpoint.kind is not ExecutionKind.AGENT: + return + restored = State.from_dict(_without_live_agent_resources(checkpoint.data)) + live_tools = state.data.get("tools") + live_hook_context = state.data.get("hook_context") + state.data.clear() + state.data.update(restored.data) + if live_tools is not None: + state.data["tools"] = live_tools + if live_hook_context is not None: + state.data["hook_context"] = live_hook_context + resume = context.take_resume_input() + if isinstance(resume, dict): + state.data.setdefault("messages", []).extend(chat_messages(resume.get("messages"))) + + +def execution_kind(pipeline: Any) -> ExecutionKind: + """Classify a real Haystack 3 Pipeline or Agent behind the lazy boundary.""" + require_haystack_v3() + if isinstance(pipeline, Pipeline): + return ExecutionKind.PIPELINE + if isinstance(pipeline, Agent): + return ExecutionKind.AGENT + msg = "Durable wrappers must set self.pipeline to a real Haystack 3 Pipeline or Agent" + raise TypeError(msg) + + +__all__ = ["HaystackDurableAdapter", "chat_messages", "execution_kind", "require_haystack_v3"] diff --git a/src/hayhooks/durable/backend.py b/src/hayhooks/durable/backend.py new file mode 100644 index 00000000..cf955bce --- /dev/null +++ b/src/hayhooks/durable/backend.py @@ -0,0 +1,227 @@ +"""Backend-neutral durable-store policy and contract.""" +# ruff: noqa: EM101, EM102 + +from __future__ import annotations + +import json +import re +from collections.abc import Callable, Mapping +from dataclasses import dataclass, replace +from typing import Any, Protocol + +from hayhooks.durable.engine import ( + ExecutionCommand, + ExecutionControl, + ExecutionPayloadSizeError, + PayloadKind, + TransitionPlan, + validate_run_id, +) +from hayhooks.durable.models import ExecutionAdmissionError, ExecutionStoreError + +MAINTENANCE_BATCH_SIZE = 100 +CHUNK_CURSOR_START = "0-0" +# One read caps its own fan-in: a client reattaching from the start of a full log +# would otherwise materialize max_stream_chunks * max_stream_chunk_bytes at once. +# The cap is on bytes, not entries, because that is the resource being protected. +CHUNK_READ_MAX_BYTES = 4_000_000 +_CHUNK_CURSOR = re.compile(r"^\d{1,20}-\d{1,20}$") +_MAX_STREAM_ID_PART = 2**64 - 1 +DEFAULT_TRANSACTION_MAX_RETRIES = 8 +DEFAULT_TRANSACTION_BACKOFF_MAX_MS = 25 + + +class ExecutionStoreCorruptionError(ExecutionStoreError): + """Persisted backend state cannot be safely decoded as durable control data.""" + + +class ExecutionContentionError(ExecutionStoreError): + """A bounded optimistic transaction could not obtain a stable snapshot.""" + + +class ExecutionIdempotencyConflictError(RuntimeError): + """A logical idempotency key was reused for a different request binding.""" + + +class ChunkCursorExpiredError(RuntimeError): + """A chunk cursor is no longer present in the bounded stream log.""" + + +@dataclass(frozen=True, slots=True) +class SubmissionResult: + created: bool + control: ExecutionControl + + +@dataclass(frozen=True, slots=True) +class ExecutionStoreConfig: + key_prefix: str = "hayhooks:durable" + transaction_max_retries: int = DEFAULT_TRANSACTION_MAX_RETRIES + transaction_backoff_max_ms: int = DEFAULT_TRANSACTION_BACKOFF_MAX_MS + lease_commit_safety_ms: int = 50 + terminal_ttl_seconds: int = 604_800 + max_nonterminal_executions: int = 0 + max_input_bytes: int = 256_000 + max_checkpoint_bytes: int = 512_000 + max_result_bytes: int = 512_000 + max_error_bytes: int = 64_000 + max_wait_bytes: int = 64_000 + max_progress_events: int = 100 + max_progress_event_bytes: int = 8_192 + # Zero disables the append-only display chunk log; see append_chunk. + max_stream_chunks: int = 10_000 + max_stream_chunk_bytes: int = 64_000 + + def __post_init__(self) -> None: + for name in ( + "transaction_max_retries", + "terminal_ttl_seconds", + "max_input_bytes", + "max_checkpoint_bytes", + "max_result_bytes", + "max_error_bytes", + "max_wait_bytes", + "max_progress_events", + "max_progress_event_bytes", + "max_stream_chunk_bytes", + ): + if getattr(self, name) < 1: + raise ValueError(f"{name} must be positive") + if ( + min( + self.transaction_backoff_max_ms, + self.lease_commit_safety_ms, + self.max_nonterminal_executions, + self.max_stream_chunks, + ) + < 0 + ): + raise ValueError("durable limits cannot be negative") + + +class ExecutionBackend(Protocol): + """Internal operations needed by the durable adapter.""" + + config: ExecutionStoreConfig + deployment: str + + async def initialize(self) -> None: ... + + async def submit( + self, control: ExecutionControl, input_payload: bytes, *, binding_digest: str + ) -> SubmissionResult: ... + + async def get(self, run_id: str) -> ExecutionControl | None: ... + + async def read_payloads(self, run_id: str, kinds: tuple[PayloadKind, ...]) -> dict[PayloadKind, bytes | None]: ... + + async def read_progress(self, run_id: str) -> list[bytes]: ... + + async def append_chunk(self, run_id: str, attempt: int, chunk: bytes) -> None: ... + + async def read_chunks(self, run_id: str, after: str) -> list[tuple[str, int, bytes]]: ... + + async def transition( + self, run_id: str, command: ExecutionCommand, *, candidate: bool = False + ) -> TransitionPlan: ... + + async def read_candidate(self) -> str | None: ... + + async def maintain(self, command_factory: Callable[[int, int], ExecutionCommand]) -> int: ... + + async def operational_counts(self) -> dict[str, int]: ... + + +def chunk_read_count(config: ExecutionStoreConfig) -> int: + """Return the entry cap that keeps one chunk read under ``CHUNK_READ_MAX_BYTES``.""" + return max(1, CHUNK_READ_MAX_BYTES // config.max_stream_chunk_bytes) + + +def parse_chunk_cursor(value: str) -> tuple[int, int]: + """Decode a chunk-log entry ID; the cursor arrives from an untrusted SSE client.""" + if not _CHUNK_CURSOR.match(value): + raise ValueError("chunk cursor must be a '