Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ RUN microdnf install -y git && microdnf clean all
# Research Lambda invocation). Treat `Dockerfile` + `Dockerfile.alerts`
# + `requirements.txt` as one tri-state pin that must move in lockstep.
COPY requirements.txt ${LAMBDA_TASK_ROOT}/
RUN pip install --no-cache-dir "alpha-engine-lib[arcticdb,flow_doctor,rag] @ git+https://github.com/cipher813/alpha-engine-lib@v0.34.0" && \
RUN pip install --no-cache-dir "alpha-engine-lib[arcticdb,flow_doctor,rag] @ git+https://github.com/cipher813/alpha-engine-lib@v0.41.0" && \
grep -vE "^#|^$|^pytest|^python-dotenv|^boto3|^botocore|^s3transfer|^alpha-engine-lib" requirements.txt > /tmp/req-lambda.txt && \
pip install --no-cache-dir -r /tmp/req-lambda.txt && \
rm -rf /root/.cache/pip /tmp/req-lambda.txt
Expand Down
2 changes: 1 addition & 1 deletion Dockerfile.alerts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ FROM --platform=linux/amd64 public.ecr.aws/lambda/python:3.12
# v0.1.0 — but moving the alerts pin in lockstep avoids two-track lib
# state across this repo's Lambda images.
COPY requirements-alerts.txt ${LAMBDA_TASK_ROOT}/
RUN pip install --no-cache-dir "alpha-engine-lib[flow_doctor] @ git+https://github.com/cipher813/alpha-engine-lib@v0.34.0" && \
RUN pip install --no-cache-dir "alpha-engine-lib[flow_doctor] @ git+https://github.com/cipher813/alpha-engine-lib@v0.41.0" && \
pip install --no-cache-dir -r requirements-alerts.txt && \
rm -rf /root/.cache/pip

Expand Down
42 changes: 22 additions & 20 deletions evals/judge.py
Original file line number Diff line number Diff line change
Expand Up @@ -472,13 +472,23 @@ def build_batch_request(
``with_structured_output`` call shape — same structured-output
semantics, just transported via the Batches API).

Delegates payload construction to
``alpha_engine_lib.anthropic_payload.build_batches_request_params``
(L334 chokepoint — second consumer of the lib's anthropic_payload
substrate after morning-signal). The chokepoint enforces the
server-tool ⊥ assistant-prefill invariant on the embedded
``params`` dict so future RubricEval extensions that add a server
tool (web_search etc.) can't reach Anthropic's HTTP 400.

Raises ``ValueError`` if ``artifact.agent_id`` has no rubric
mapped — callers must pre-filter via ``resolve_rubric_for_agent``.

Empty-input short-circuit is handled by the orchestrator BEFORE
this function is invoked (the skip artifact is persisted
client-side without spending a batch slot).
"""
from alpha_engine_lib.anthropic_payload import build_batches_request_params

rubric_name = resolve_rubric_for_agent(artifact.agent_id)
if rubric_name is None:
raise ValueError(
Expand All @@ -491,26 +501,18 @@ def build_batch_request(
rendered = _render_rubric(artifact, loaded_prompt)
tool_spec = _build_rubric_tool_spec()

return {
"custom_id": custom_id,
"params": {
"model": judge_model,
"max_tokens": max_tokens,
"tools": [tool_spec],
# Force the model to call the rubric tool — equivalent to
# ``with_structured_output(...)``'s ``tool_choice`` posture.
# Without this the model can decide to emit prose, which
# would fall through every parser in this module.
"tool_choice": {"type": "tool", "name": _RUBRIC_TOOL_NAME},
"messages": [
{"role": "user", "content": rendered},
],
# ``metadata.user_id`` is reserved for end-user identification
# in Anthropic's contract; we pass the rubric+version pair
# via ``metadata`` for batch-side observability without
# putting it on a schema-validated field.
},
}
# Force the model to call the rubric tool — equivalent to
# ``with_structured_output(...)``'s ``tool_choice`` posture. Without
# this the model can decide to emit prose, which would fall through
# every parser in this module.
return build_batches_request_params(
custom_id=custom_id,
model=judge_model,
max_tokens=max_tokens,
user_content=rendered,
tools=[tool_spec],
tool_choice={"type": "tool", "name": _RUBRIC_TOOL_NAME},
)


def parse_batch_message(
Expand Down
2 changes: 1 addition & 1 deletion requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ pydantic~=2.7
# graph/research_graph rag.is_available probe)
# Lib is now public — no PAT required. Lambda Dockerfile installs via the
# git URL; CI pulls directly.
alpha-engine-lib[arcticdb,flow_doctor,rag] @ git+https://github.com/cipher813/alpha-engine-lib@v0.34.0
alpha-engine-lib[arcticdb,flow_doctor,rag] @ git+https://github.com/cipher813/alpha-engine-lib@v0.41.0

# Scheduling / calendar
exchange-calendars~=4.5
Expand Down
Loading