diff --git a/e2e/bub/harbor-tasks/locomo-support-group/environment/Dockerfile b/e2e/bub/harbor-tasks/locomo-support-group/environment/Dockerfile new file mode 100644 index 000000000..b00911daf --- /dev/null +++ b/e2e/bub/harbor-tasks/locomo-support-group/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM python:3.12-slim-bookworm + +WORKDIR /workspace diff --git a/e2e/bub/harbor-tasks/locomo-support-group/steps/capture/instruction.md b/e2e/bub/harbor-tasks/locomo-support-group/steps/capture/instruction.md new file mode 100644 index 000000000..0f2a4d82f --- /dev/null +++ b/e2e/bub/harbor-tasks/locomo-support-group/steps/capture/instruction.md @@ -0,0 +1 @@ +,powercontext.remember text='Caroline attended an LGBTQ support group on 7 May 2023. The group made her feel accepted and gave her courage to embrace herself. She plans to continue her education and explore counseling or mental-health work.' kind='conversation-fact' reason='Pinned LoCoMo-derived e2e sample' diff --git a/e2e/bub/harbor-tasks/locomo-support-group/steps/capture/tests/test.sh b/e2e/bub/harbor-tasks/locomo-support-group/steps/capture/tests/test.sh new file mode 100644 index 000000000..77ea7cd40 --- /dev/null +++ b/e2e/bub/harbor-tasks/locomo-support-group/steps/capture/tests/test.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu + +echo 1 > /logs/verifier/reward.txt diff --git a/e2e/bub/harbor-tasks/locomo-support-group/steps/recall/instruction.md b/e2e/bub/harbor-tasks/locomo-support-group/steps/recall/instruction.md new file mode 100644 index 000000000..31814a1bb --- /dev/null +++ b/e2e/bub/harbor-tasks/locomo-support-group/steps/recall/instruction.md @@ -0,0 +1 @@ +,powercontext.context query='When did Caroline go to the LGBTQ support group?' diff --git a/e2e/bub/harbor-tasks/locomo-support-group/steps/recall/tests/test.sh b/e2e/bub/harbor-tasks/locomo-support-group/steps/recall/tests/test.sh new file mode 100644 index 000000000..77ea7cd40 --- /dev/null +++ b/e2e/bub/harbor-tasks/locomo-support-group/steps/recall/tests/test.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu + +echo 1 > /logs/verifier/reward.txt diff --git a/e2e/bub/harbor-tasks/locomo-support-group/task.toml b/e2e/bub/harbor-tasks/locomo-support-group/task.toml new file mode 100644 index 000000000..c3e4009a6 --- /dev/null +++ b/e2e/bub/harbor-tasks/locomo-support-group/task.toml @@ -0,0 +1,20 @@ +version = "1.3" +multi_step_reward_strategy = "final" + +[metadata] +sample = "locomo" + +[agent] +timeout_sec = 300.0 + +[verifier] +timeout_sec = 60.0 + +[environment] +build_timeout_sec = 300.0 + +[[steps]] +name = "capture" + +[[steps]] +name = "recall" diff --git a/e2e/bub/harbor-tasks/project-database-decision/environment/Dockerfile b/e2e/bub/harbor-tasks/project-database-decision/environment/Dockerfile new file mode 100644 index 000000000..b00911daf --- /dev/null +++ b/e2e/bub/harbor-tasks/project-database-decision/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM python:3.12-slim-bookworm + +WORKDIR /workspace diff --git a/e2e/bub/harbor-tasks/project-database-decision/steps/capture/instruction.md b/e2e/bub/harbor-tasks/project-database-decision/steps/capture/instruction.md new file mode 100644 index 000000000..1e1fd3f22 --- /dev/null +++ b/e2e/bub/harbor-tasks/project-database-decision/steps/capture/instruction.md @@ -0,0 +1 @@ +,powercontext.remember text='The project selected OceanBase because it needs MySQL-compatible, multi-node persistent storage for shared agent context.' kind='project-decision' reason='Durable architecture decision' diff --git a/e2e/bub/harbor-tasks/project-database-decision/steps/capture/tests/test.sh b/e2e/bub/harbor-tasks/project-database-decision/steps/capture/tests/test.sh new file mode 100644 index 000000000..77ea7cd40 --- /dev/null +++ b/e2e/bub/harbor-tasks/project-database-decision/steps/capture/tests/test.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu + +echo 1 > /logs/verifier/reward.txt diff --git a/e2e/bub/harbor-tasks/project-database-decision/steps/recall/instruction.md b/e2e/bub/harbor-tasks/project-database-decision/steps/recall/instruction.md new file mode 100644 index 000000000..c0dfb7bc4 --- /dev/null +++ b/e2e/bub/harbor-tasks/project-database-decision/steps/recall/instruction.md @@ -0,0 +1 @@ +,powercontext.context query='Which project decision selected multi-node persistent storage?' diff --git a/e2e/bub/harbor-tasks/project-database-decision/steps/recall/tests/test.sh b/e2e/bub/harbor-tasks/project-database-decision/steps/recall/tests/test.sh new file mode 100644 index 000000000..77ea7cd40 --- /dev/null +++ b/e2e/bub/harbor-tasks/project-database-decision/steps/recall/tests/test.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu + +echo 1 > /logs/verifier/reward.txt diff --git a/e2e/bub/harbor-tasks/project-database-decision/task.toml b/e2e/bub/harbor-tasks/project-database-decision/task.toml new file mode 100644 index 000000000..ea5f588e9 --- /dev/null +++ b/e2e/bub/harbor-tasks/project-database-decision/task.toml @@ -0,0 +1,17 @@ +version = "1.3" +multi_step_reward_strategy = "final" + +[agent] +timeout_sec = 300.0 + +[verifier] +timeout_sec = 60.0 + +[environment] +build_timeout_sec = 300.0 + +[[steps]] +name = "capture" + +[[steps]] +name = "recall" diff --git a/e2e/bub/src/powercontext_e2e/catalog.py b/e2e/bub/src/powercontext_e2e/catalog.py new file mode 100644 index 000000000..533a5ec1b --- /dev/null +++ b/e2e/bub/src/powercontext_e2e/catalog.py @@ -0,0 +1,134 @@ +"""Validated catalog contracts for end-to-end workloads.""" + +from __future__ import annotations + +import hashlib +from pathlib import Path +from typing import Literal + +import yaml +from pydantic import BaseModel, ConfigDict, Field, model_validator + + +class CatalogModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class Provenance(CatalogModel): + source: str + revision: str + selection: str + case_ids: tuple[str, ...] = Field(min_length=1) + + +class HarborDatasetSpec(CatalogModel): + path: Path | None = None + name: str | None = None + version: str | None = None + task_id: str = Field(min_length=1) + checksum: str = Field(pattern=r"^[0-9a-f]{64}$") + + @model_validator(mode="after") + def require_one_source(self) -> HarborDatasetSpec: + if (self.path is None) == (self.name is None): + raise ValueError("A Harbor dataset requires exactly one of path or name") # noqa: TRY003 + if self.path is not None and self.version is not None: + raise ValueError("A local Harbor dataset cannot declare a version") # noqa: TRY003 + return self + + +class BubExecutionSpec(CatalogModel): + type: Literal["bub"] = "bub" + model: bool = False + max_steps: int = Field(default=50, ge=1, le=200) + max_tokens: int = Field(default=16384, ge=256) + + +class CaptureThresholds(CatalogModel): + capture_coverage: float = Field(default=0, ge=0, le=1) + groundedness: float = Field(default=0, ge=0, le=1) + probe_coverage: float = Field(default=1, ge=0, le=1) + minimum_in_run_contexts: int = Field(default=0, ge=0) + + +class RecallProbeSpec(CatalogModel): + id: str = Field(pattern=r"^[a-z0-9][a-z0-9_-]*$") + query: str = Field(min_length=1, max_length=8192) + expected_context: tuple[str, ...] = () + + +class MemoryEvaluationSpec(CatalogModel): + capture_events: bool = False + checkpoint_every_events: int = Field(default=5, ge=1, le=100) + max_event_bytes: int = Field(default=8192, ge=512, le=32768) + require_checkpoint: bool = False + expected_memory: tuple[str, ...] = () + probes: tuple[RecallProbeSpec, ...] = Field(min_length=1) + thresholds: CaptureThresholds = Field(default_factory=CaptureThresholds) + + @model_validator(mode="after") + def require_unique_probe_ids(self) -> MemoryEvaluationSpec: + probe_ids = [probe.id for probe in self.probes] + if len(probe_ids) != len(set(probe_ids)): + raise ValueError("Recall probe IDs must be unique") # noqa: TRY003 + return self + + +class E2ETask(CatalogModel): + schema_: Literal["powercontext.e2e-task/v1"] = Field(alias="schema") + id: str = Field(pattern=r"^[a-z0-9][a-z0-9_-]*$") + categories: tuple[str, ...] = Field(min_length=1) + provenance: Provenance | None = None + dataset: HarborDatasetSpec + execution: BubExecutionSpec + evaluation: MemoryEvaluationSpec + + +class TaskSelectionError(ValueError): + """Report unknown IDs or categories at the catalog boundary.""" + + def __init__(self, selector: str, values: set[str]) -> None: + super().__init__(f"Unknown e2e workload {selector}: {sorted(values)!r}") + + +def load_tasks(path: Path) -> tuple[E2ETask, ...]: + task_paths = sorted(path.glob("*.yaml")) if path.is_dir() else [path] + tasks = tuple(E2ETask.model_validate(yaml.safe_load(item.read_text(encoding="utf-8"))) for item in task_paths) + ids = [task.id for task in tasks] + if not tasks: + raise ValueError(f"No e2e workload manifests found at {path}") # noqa: TRY003 + if len(ids) != len(set(ids)): + raise ValueError("E2E workload IDs must be unique") # noqa: TRY003 + for task in tasks: + _validate_provenance(task) + return tasks + + +def select_tasks( + tasks: tuple[E2ETask, ...], + *, + ids: tuple[str, ...] = (), + categories: tuple[str, ...] = (), +) -> tuple[E2ETask, ...]: + requested_ids = set(ids) + requested_categories = set(categories) + available_ids = {task.id for task in tasks} + available_categories = {category for task in tasks for category in task.categories} + if missing_ids := requested_ids - available_ids: + raise TaskSelectionError("IDs", missing_ids) + if missing_categories := requested_categories - available_categories: + raise TaskSelectionError("categories", missing_categories) + if not requested_ids and not requested_categories: + requested_categories = {"acceptance"} + return tuple( + task for task in tasks if task.id in requested_ids or requested_categories.intersection(task.categories) + ) + + +def _validate_provenance(task: E2ETask) -> None: + if task.provenance is None: + return + source = Path(task.provenance.source) + digest = hashlib.sha256(source.read_bytes()).hexdigest() + if digest != task.provenance.revision: + raise ValueError(f"Task source fingerprint changed: {source}") # noqa: TRY003 diff --git a/e2e/bub/tasks/locomo-support-group.yaml b/e2e/bub/tasks/locomo-support-group.yaml new file mode 100644 index 000000000..4c52e9f8f --- /dev/null +++ b/e2e/bub/tasks/locomo-support-group.yaml @@ -0,0 +1,34 @@ +schema: powercontext.e2e-task/v1 +id: locomo-support-group +categories: + - acceptance + - sample +provenance: + source: benchmark/locomo/dataset/locomo10.json + revision: 4448275ea2c5cd0af5774d80aea7b05b5a16e1b996caf8554ca3d762a301ae84 + selection: first-conversation-first-question/v1 + case_ids: + - conv-26 + - conv-26:q001 + - D1 +dataset: + path: e2e/bub/harbor-tasks + task_id: locomo-support-group + checksum: 13b88f406933d47c3a826d30143a2d5404f1990a0166b3d98ec25a24bcc0069c +execution: + type: bub + model: false + max_steps: 10 + max_tokens: 4096 +evaluation: + expected_memory: + - LGBTQ support group + - 7 May 2023 + probes: + - id: support-group-date + query: When did Caroline go to the LGBTQ support group? + expected_context: + - LGBTQ support group + - 7 May 2023 + thresholds: + probe_coverage: 1 diff --git a/e2e/bub/tasks/project-database-decision.yaml b/e2e/bub/tasks/project-database-decision.yaml new file mode 100644 index 000000000..e1efc028d --- /dev/null +++ b/e2e/bub/tasks/project-database-decision.yaml @@ -0,0 +1,29 @@ +schema: powercontext.e2e-task/v1 +id: project-database-decision +categories: + - acceptance + - sample + - smoke +dataset: + path: e2e/bub/harbor-tasks + task_id: project-database-decision + checksum: a892da6ad2cdddc9f3815a66c9543d615db4aa2616777aa4c7f5e5c279ff6bb5 +execution: + type: bub + model: false + max_steps: 10 + max_tokens: 4096 +evaluation: + expected_memory: + - OceanBase + - MySQL-compatible + - multi-node persistent storage + probes: + - id: database-decision + query: Which project decision selected multi-node persistent storage? + expected_context: + - OceanBase + - MySQL-compatible + - multi-node persistent storage + thresholds: + probe_coverage: 1 diff --git a/e2e/bub/tasks/terminal-bench-db-wal-recovery.yaml b/e2e/bub/tasks/terminal-bench-db-wal-recovery.yaml new file mode 100644 index 000000000..4b958857e --- /dev/null +++ b/e2e/bub/tasks/terminal-bench-db-wal-recovery.yaml @@ -0,0 +1,32 @@ +schema: powercontext.e2e-task/v1 +id: terminal-bench-db-wal-recovery +categories: + - long-horizon + - terminal-bench +dataset: + name: terminal-bench + version: "2.0" + task_id: db-wal-recovery + checksum: 01f470c86f3a1f7ce4d91bdf0aaaa89fa96a4124ac48617990e2735b5291913a +execution: + type: bub + model: true + max_steps: 200 + max_tokens: 16384 +evaluation: + capture_events: true + checkpoint_every_events: 5 + max_event_bytes: 8192 + require_checkpoint: true + probes: + - id: investigation + query: What records and SQLite table were found in /app/main.db? + - id: decisions + query: What happened to the WAL file for /app/main.db? + - id: current-state + query: What is the current recovery status and what is in /app/recovered.json? + thresholds: + capture_coverage: 0.9 + groundedness: 0.8 + probe_coverage: 0.67 + minimum_in_run_contexts: 0 diff --git a/e2e/bub/tests/test_workload_catalog.py b/e2e/bub/tests/test_workload_catalog.py new file mode 100644 index 000000000..296dff7e3 --- /dev/null +++ b/e2e/bub/tests/test_workload_catalog.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from pathlib import Path + +from powercontext_e2e.catalog import load_tasks, select_tasks + + +def test_workloads_can_be_selected_by_multiple_ids_or_category() -> None: + repository = Path(__file__).resolve().parents[3] + tasks = load_tasks(repository / "e2e" / "bub" / "tasks") + + assert [task.id for task in tasks] == [ + "locomo-support-group", + "project-database-decision", + "terminal-bench-db-wal-recovery", + ] + assert {task.execution.type for task in tasks} == {"bub"} + assert {task.id for task in tasks if task.execution.model} == {"terminal-bench-db-wal-recovery"} + + selected_ids = select_tasks( + tasks, + ids=("locomo-support-group", "terminal-bench-db-wal-recovery"), + ) + acceptance = select_tasks(tasks, categories=("acceptance",)) + + assert [task.id for task in selected_ids] == [ + "locomo-support-group", + "terminal-bench-db-wal-recovery", + ] + assert [task.id for task in acceptance] == [ + "locomo-support-group", + "project-database-decision", + ]