feat(e2e): add workload catalog

e2e/bub/harbor-tasks/locomo-support-group/environment/Dockerfile

@@ -0,0 +1,3 @@
+FROM python:3.12-slim-bookworm
+
+WORKDIR /workspace

e2e/bub/harbor-tasks/locomo-support-group/steps/capture/instruction.md

@@ -0,0 +1 @@
+,powercontext.remember text='Caroline attended an LGBTQ support group on 7 May 2023. The group made her feel accepted and gave her courage to embrace herself. She plans to continue her education and explore counseling or mental-health work.' kind='conversation-fact' reason='Pinned LoCoMo-derived e2e sample'

e2e/bub/harbor-tasks/locomo-support-group/steps/capture/tests/test.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+set -eu
+
+echo 1 > /logs/verifier/reward.txt

e2e/bub/harbor-tasks/locomo-support-group/steps/recall/instruction.md

@@ -0,0 +1 @@
+,powercontext.context query='When did Caroline go to the LGBTQ support group?'

e2e/bub/harbor-tasks/locomo-support-group/steps/recall/tests/test.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+set -eu
+
+echo 1 > /logs/verifier/reward.txt

e2e/bub/harbor-tasks/locomo-support-group/task.toml

@@ -0,0 +1,20 @@
+version = "1.3"
+multi_step_reward_strategy = "final"
+
+[metadata]
+sample = "locomo"
+
+[agent]
+timeout_sec = 300.0
+
+[verifier]
+timeout_sec = 60.0
+
+[environment]
+build_timeout_sec = 300.0
+
+[[steps]]
+name = "capture"
+
+[[steps]]
+name = "recall"

e2e/bub/harbor-tasks/project-database-decision/environment/Dockerfile

@@ -0,0 +1,3 @@
+FROM python:3.12-slim-bookworm
+
+WORKDIR /workspace

e2e/bub/harbor-tasks/project-database-decision/steps/capture/instruction.md

@@ -0,0 +1 @@
+,powercontext.remember text='The project selected OceanBase because it needs MySQL-compatible, multi-node persistent storage for shared agent context.' kind='project-decision' reason='Durable architecture decision'

e2e/bub/harbor-tasks/project-database-decision/steps/capture/tests/test.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+set -eu
+
+echo 1 > /logs/verifier/reward.txt

e2e/bub/harbor-tasks/project-database-decision/steps/recall/instruction.md

@@ -0,0 +1 @@
+,powercontext.context query='Which project decision selected multi-node persistent storage?'

e2e/bub/harbor-tasks/project-database-decision/steps/recall/tests/test.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+set -eu
+
+echo 1 > /logs/verifier/reward.txt

e2e/bub/harbor-tasks/project-database-decision/task.toml

@@ -0,0 +1,17 @@
+version = "1.3"
+multi_step_reward_strategy = "final"
+
+[agent]
+timeout_sec = 300.0
+
+[verifier]
+timeout_sec = 60.0
+
+[environment]
+build_timeout_sec = 300.0
+
+[[steps]]
+name = "capture"
+
+[[steps]]
+name = "recall"

e2e/bub/src/powercontext_e2e/catalog.py

@@ -0,0 +1,134 @@
+"""Validated catalog contracts for end-to-end workloads."""
+
+from __future__ import annotations
+
+import hashlib
+from pathlib import Path
+from typing import Literal
+
+import yaml
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+
+class CatalogModel(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+
+class Provenance(CatalogModel):
+    source: str
+    revision: str
+    selection: str
+    case_ids: tuple[str, ...] = Field(min_length=1)
+
+
+class HarborDatasetSpec(CatalogModel):
+    path: Path | None = None
+    name: str | None = None
+    version: str | None = None
+    task_id: str = Field(min_length=1)
+    checksum: str = Field(pattern=r"^[0-9a-f]{64}$")
+
+    @model_validator(mode="after")
+    def require_one_source(self) -> HarborDatasetSpec:
+        if (self.path is None) == (self.name is None):
+            raise ValueError("A Harbor dataset requires exactly one of path or name")  # noqa: TRY003
+        if self.path is not None and self.version is not None:
+            raise ValueError("A local Harbor dataset cannot declare a version")  # noqa: TRY003
+        return self
+
+
+class BubExecutionSpec(CatalogModel):
+    type: Literal["bub"] = "bub"
+    model: bool = False
+    max_steps: int = Field(default=50, ge=1, le=200)
+    max_tokens: int = Field(default=16384, ge=256)
+
+
+class CaptureThresholds(CatalogModel):
+    capture_coverage: float = Field(default=0, ge=0, le=1)
+    groundedness: float = Field(default=0, ge=0, le=1)
+    probe_coverage: float = Field(default=1, ge=0, le=1)
+    minimum_in_run_contexts: int = Field(default=0, ge=0)
+
+
+class RecallProbeSpec(CatalogModel):
+    id: str = Field(pattern=r"^[a-z0-9][a-z0-9_-]*$")
+    query: str = Field(min_length=1, max_length=8192)
+    expected_context: tuple[str, ...] = ()
+
+
+class MemoryEvaluationSpec(CatalogModel):
+    capture_events: bool = False
+    checkpoint_every_events: int = Field(default=5, ge=1, le=100)
+    max_event_bytes: int = Field(default=8192, ge=512, le=32768)
+    require_checkpoint: bool = False
+    expected_memory: tuple[str, ...] = ()
+    probes: tuple[RecallProbeSpec, ...] = Field(min_length=1)
+    thresholds: CaptureThresholds = Field(default_factory=CaptureThresholds)
+
+    @model_validator(mode="after")
+    def require_unique_probe_ids(self) -> MemoryEvaluationSpec:
+        probe_ids = [probe.id for probe in self.probes]
+        if len(probe_ids) != len(set(probe_ids)):
+            raise ValueError("Recall probe IDs must be unique")  # noqa: TRY003
+        return self
+
+
+class E2ETask(CatalogModel):
+    schema_: Literal["powercontext.e2e-task/v1"] = Field(alias="schema")
+    id: str = Field(pattern=r"^[a-z0-9][a-z0-9_-]*$")
+    categories: tuple[str, ...] = Field(min_length=1)
+    provenance: Provenance | None = None
+    dataset: HarborDatasetSpec
+    execution: BubExecutionSpec
+    evaluation: MemoryEvaluationSpec
+
+
+class TaskSelectionError(ValueError):
+    """Report unknown IDs or categories at the catalog boundary."""
+
+    def __init__(self, selector: str, values: set[str]) -> None:
+        super().__init__(f"Unknown e2e workload {selector}: {sorted(values)!r}")
+
+
+def load_tasks(path: Path) -> tuple[E2ETask, ...]:
+    task_paths = sorted(path.glob("*.yaml")) if path.is_dir() else [path]
+    tasks = tuple(E2ETask.model_validate(yaml.safe_load(item.read_text(encoding="utf-8"))) for item in task_paths)
+    ids = [task.id for task in tasks]
+    if not tasks:
+        raise ValueError(f"No e2e workload manifests found at {path}")  # noqa: TRY003
+    if len(ids) != len(set(ids)):
+        raise ValueError("E2E workload IDs must be unique")  # noqa: TRY003
+    for task in tasks:
+        _validate_provenance(task)
+    return tasks
+
+
+def select_tasks(
+    tasks: tuple[E2ETask, ...],
+    *,
+    ids: tuple[str, ...] = (),
+    categories: tuple[str, ...] = (),
+) -> tuple[E2ETask, ...]:
+    requested_ids = set(ids)
+    requested_categories = set(categories)
+    available_ids = {task.id for task in tasks}
+    available_categories = {category for task in tasks for category in task.categories}
+    if missing_ids := requested_ids - available_ids:
+        raise TaskSelectionError("IDs", missing_ids)
+    if missing_categories := requested_categories - available_categories:
+        raise TaskSelectionError("categories", missing_categories)
+    if not requested_ids and not requested_categories:
+        requested_categories = {"acceptance"}
+    return tuple(
+        task for task in tasks if task.id in requested_ids or requested_categories.intersection(task.categories)
+    )
+
+
+def _validate_provenance(task: E2ETask) -> None:
+    if task.provenance is None:
+        return
+    source = Path(task.provenance.source)
+    digest = hashlib.sha256(source.read_bytes()).hexdigest()
+    if digest != task.provenance.revision:
+        raise ValueError(f"Task source fingerprint changed: {source}")  # noqa: TRY003

e2e/bub/tasks/locomo-support-group.yaml

@@ -0,0 +1,34 @@
+schema: powercontext.e2e-task/v1
+id: locomo-support-group
+categories:
+  - acceptance
+  - sample
+provenance:
+  source: benchmark/locomo/dataset/locomo10.json
+  revision: 4448275ea2c5cd0af5774d80aea7b05b5a16e1b996caf8554ca3d762a301ae84
+  selection: first-conversation-first-question/v1
+  case_ids:
+    - conv-26
+    - conv-26:q001
+    - D1
+dataset:
+  path: e2e/bub/harbor-tasks
+  task_id: locomo-support-group
+  checksum: 13b88f406933d47c3a826d30143a2d5404f1990a0166b3d98ec25a24bcc0069c
+execution:
+  type: bub
+  model: false
+  max_steps: 10
+  max_tokens: 4096
+evaluation:
+  expected_memory:
+    - LGBTQ support group
+    - 7 May 2023
+  probes:
+    - id: support-group-date
+      query: When did Caroline go to the LGBTQ support group?
+      expected_context:
+        - LGBTQ support group
+        - 7 May 2023
+  thresholds:
+    probe_coverage: 1

e2e/bub/tasks/project-database-decision.yaml

@@ -0,0 +1,29 @@
+schema: powercontext.e2e-task/v1
+id: project-database-decision
+categories:
+  - acceptance
+  - sample
+  - smoke
+dataset:
+  path: e2e/bub/harbor-tasks
+  task_id: project-database-decision
+  checksum: a892da6ad2cdddc9f3815a66c9543d615db4aa2616777aa4c7f5e5c279ff6bb5
+execution:
+  type: bub
+  model: false
+  max_steps: 10
+  max_tokens: 4096
+evaluation:
+  expected_memory:
+    - OceanBase
+    - MySQL-compatible
+    - multi-node persistent storage
+  probes:
+    - id: database-decision
+      query: Which project decision selected multi-node persistent storage?
+      expected_context:
+        - OceanBase
+        - MySQL-compatible
+        - multi-node persistent storage
+  thresholds:
+    probe_coverage: 1

e2e/bub/tasks/terminal-bench-db-wal-recovery.yaml

@@ -0,0 +1,32 @@
+schema: powercontext.e2e-task/v1
+id: terminal-bench-db-wal-recovery
+categories:
+  - long-horizon
+  - terminal-bench
+dataset:
+  name: terminal-bench
+  version: "2.0"
+  task_id: db-wal-recovery
+  checksum: 01f470c86f3a1f7ce4d91bdf0aaaa89fa96a4124ac48617990e2735b5291913a
+execution:
+  type: bub
+  model: true
+  max_steps: 200
+  max_tokens: 16384
+evaluation:
+  capture_events: true
+  checkpoint_every_events: 5
+  max_event_bytes: 8192
+  require_checkpoint: true
+  probes:
+    - id: investigation
+      query: What records and SQLite table were found in /app/main.db?
+    - id: decisions
+      query: What happened to the WAL file for /app/main.db?
+    - id: current-state
+      query: What is the current recovery status and what is in /app/recovered.json?
+  thresholds:
+    capture_coverage: 0.9
+    groundedness: 0.8
+    probe_coverage: 0.67
+    minimum_in_run_contexts: 0

e2e/bub/tests/test_workload_catalog.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from powercontext_e2e.catalog import load_tasks, select_tasks
+
+
+def test_workloads_can_be_selected_by_multiple_ids_or_category() -> None:
+    repository = Path(__file__).resolve().parents[3]
+    tasks = load_tasks(repository / "e2e" / "bub" / "tasks")
+
+    assert [task.id for task in tasks] == [
+        "locomo-support-group",
+        "project-database-decision",
+        "terminal-bench-db-wal-recovery",
+    ]
+    assert {task.execution.type for task in tasks} == {"bub"}
+    assert {task.id for task in tasks if task.execution.model} == {"terminal-bench-db-wal-recovery"}
+
+    selected_ids = select_tasks(
+        tasks,
+        ids=("locomo-support-group", "terminal-bench-db-wal-recovery"),
+    )
+    acceptance = select_tasks(tasks, categories=("acceptance",))
+
+    assert [task.id for task in selected_ids] == [
+        "locomo-support-group",
+        "terminal-bench-db-wal-recovery",
+    ]
+    assert [task.id for task in acceptance] == [
+        "locomo-support-group",
+        "project-database-decision",
+    ]