Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@ them and feeds results back until it has an answer. The **data plane** —
the dashboard, widgets, scheduled jobs, database migrations — does not
depend on the model at all. If your LLM endpoint is down, the dashboard
still renders, because widgets are plain `async def` functions that read
the database directly; nothing about them calls the model.
the database directly; nothing about them calls the model, and the framework
enforces it — `app.llm()` raises inside a widget. To show model output, have a
job or tool compute it, store it, and let the widget read the stored result.

Apps are code, not configuration:

Expand Down
21 changes: 20 additions & 1 deletion src/dudamel/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,13 @@
from dudamel.contract.renderers import ACTION_LABEL_MAX, RENDERERS, clean_action_label
from dudamel.contract.schema import ToolSchema
from dudamel.contract.types import TOOL_NAME_RE, Job, Tool, Widget
from dudamel.exceptions import AppSettingsError, RegistryError, RuntimeNotBoundError
from dudamel.exceptions import (
AppSettingsError,
LLMNotAllowedInWidgetError,
RegistryError,
RuntimeNotBoundError,
)
from dudamel.widgets import IN_WIDGET

if TYPE_CHECKING:
from zoneinfo import ZoneInfo
Expand Down Expand Up @@ -199,6 +205,11 @@ def widget(
actions: Sequence[str] = (),
timeout: float = 15.0,
) -> Callable:
"""Register a dashboard widget: an `async def` with no required
arguments that returns the payload `renderer` expects. Widgets read
stored data; they cannot call `app.llm()` (it raises
`LLMNotAllowedInWidgetError`), so the dashboard renders with every
model tier down."""
if renderer not in RENDERERS:
raise RegistryError(f"unknown renderer {renderer!r}; choose one of {sorted(RENDERERS)}")

Expand Down Expand Up @@ -269,6 +280,14 @@ def wrap(fn: Callable[[], Awaitable[None]]) -> Callable:

# --- runtime capabilities ------------------------------------------------
async def llm(self, *args: Any, **kwargs: Any) -> Any:
"""Ask the model. Available to tools and jobs; refused inside a widget,
which must render with every model tier down."""
if IN_WIDGET.get():
raise LLMNotAllowedInWidgetError(
"widgets cannot call app.llm(): the dashboard must render when the model "
"is down. Compute the result in a scheduled job or tool, store it, and "
"have the widget read the stored result."
)
if self._llm is None:
raise RuntimeNotBoundError(
"app.llm is bound at run time by Runtime; not in tests/import"
Expand Down
11 changes: 11 additions & 0 deletions src/dudamel/exceptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,17 @@ class RuntimeNotBoundError(DudamelError):
"""A runtime capability (db/llm/notify) used before the run phase bound it."""


class LLMNotAllowedInWidgetError(DudamelError):
"""`app.llm()` was called from inside a widget.

The dashboard is the part of the system that keeps working when every
model tier is down, so a widget that could reach the model would make that
claim depend on every app author's discipline. Enforced instead of
documented. The message is shown to the operator on the widget's error
card, so it names the rule and the way out.
"""


class AppSettingsError(DudamelError):
"""An app's [apps.<name>] config block has an unknown key or fails the
app's own settings model."""
Expand Down
4 changes: 3 additions & 1 deletion src/dudamel/models_core.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,9 @@ class JobRun(CoreBase):
__tablename__ = "job_runs"
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
job_id: Mapped[str] = mapped_column(String(255), index=True)
status: Mapped[str] = mapped_column(String(32)) # ok|error|timeout|skipped|misfired|cancelled
status: Mapped[str] = mapped_column(
String(32)
) # running|ok|error|timeout|skipped|misfired|cancelled|interrupted
started_at: Mapped[datetime] = mapped_column(DateTime, default=_utcnow)
finished_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
detail: Mapped[str | None] = mapped_column(Text, nullable=True)
Expand Down
184 changes: 165 additions & 19 deletions src/dudamel/scheduler.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
"""APScheduler wiring: turns every registered Job into a running APScheduler
job and records EVERY outcome (ok/error/timeout/skipped/misfired/cancelled)
as a `job_runs` row.
as a `job_runs` row. A run that starts leaves a `running` row immediately,
updated in place when it ends, so a process that dies mid-run still leaves
evidence; `start()` turns what a dead process left into `interrupted` and
accounts for fires missed while nothing was running as one `misfired` row.

Thin wrapper — the scheduler calls only `job.fn()`. Job functions are
command-plane (unlike widgets): they may use `app.llm`/`app.notify`, but
command-plane (unlike widgets, which `App.llm` refuses): they may use `app.llm`/`app.notify`, but
those bindings are Runtime's concern (already wired up before this ever
fires); the scheduler itself never touches the LLM directly.

Expand All @@ -18,7 +21,7 @@
import asyncio
import logging
import traceback
from datetime import UTC, datetime
from datetime import UTC, datetime, timedelta
from typing import Any
from zoneinfo import ZoneInfo

Expand All @@ -27,6 +30,7 @@
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from apscheduler.triggers.cron import CronTrigger
from apscheduler.triggers.interval import IntervalTrigger
from sqlalchemy import func, select, update
from sqlalchemy.exc import SQLAlchemyError

from dudamel.contract.types import Job
Expand All @@ -46,6 +50,11 @@
# the whole shutdown sequence.
_DRAIN_TIMEOUT = 2.0

# Counting missed fires is a loop over the trigger; an install that was off for
# a year with a per-minute job must not spin through half a million iterations
# at startup just to print a number nobody needs to the digit.
_MISSED_COUNT_CAP = 1000


def _utcnow() -> datetime:
return datetime.now(UTC).replace(tzinfo=None)
Expand Down Expand Up @@ -106,6 +115,10 @@ async def _run_job(self, job: Job) -> None:
self._inflight.add(task)
task.add_done_callback(self._inflight.discard)
started = _utcnow()
# Written before the job runs, not after: a row that only appears when
# the run finishes is exactly the row a crash, SIGKILL or power loss
# never gets to write.
run_id = await self._begin(job.id, started)
try:
# `asyncio.timeout` lets us tell OUR deadline apart from a
# TimeoutError the job raised itself. Since Python 3.11
Expand All @@ -121,11 +134,11 @@ async def _run_job(self, job: Job) -> None:
if cm.expired():
detail = f"job {job.id} timed out after {job.timeout}s"
logger.warning(detail)
await self._record(job.id, "timeout", started, detail)
await self._record(job.id, "timeout", started, detail, run_id=run_id)
else:
# The job raised its own TimeoutError -- a job error, recorded
# with the traceback like any other raised exception.
await self._record_error(job.id, started, e)
await self._record_error(job.id, started, e, run_id)
except asyncio.CancelledError:
# The executor cancels in-flight jobs at shutdown. Record the
# outcome, then let the cancellation continue -- a run that was
Expand All @@ -136,22 +149,24 @@ async def _run_job(self, job: Job) -> None:
detail = f"job {job.id} was cancelled before it finished"
logger.info(detail)
try:
await self._record(job.id, "cancelled", started, detail)
await self._record(job.id, "cancelled", started, detail, run_id=run_id)
except Exception as e: # noqa: BLE001 — recording is best-effort
logger.warning("failed to record cancelled run for job %s: %s", job.id, e)
raise
except Exception as e: # job bugs must not kill the scheduler
await self._record_error(job.id, started, e)
await self._record_error(job.id, started, e, run_id)
else:
await self._record(job.id, "ok", started, None)
await self._record(job.id, "ok", started, None, run_id=run_id)

async def _record_error(self, job_id: str, started: datetime, e: BaseException) -> None:
async def _record_error(
self, job_id: str, started: datetime, e: BaseException, run_id: int | None = None
) -> None:
"""Record a job that raised as an "error" row, capturing the active
traceback. Must be called from inside the handling `except` block so
`traceback.format_exc()` refers to `e`."""
detail = f"{type(e).__name__}: {e}\n{traceback.format_exc()}"[:_DETAIL_CAP]
logger.warning("job %s raised: %s", job_id, e)
await self._record(job_id, "error", started, detail)
await self._record(job_id, "error", started, detail, run_id=run_id)

def _on_missed(self, event: JobExecutionEvent) -> None:
"""APScheduler listener (sync callback, invoked from the running
Expand All @@ -176,20 +191,53 @@ def _on_max_instances(self, event: JobExecutionEvent) -> None:
self._background_tasks.add(task)
task.add_done_callback(self._background_tasks.discard)

async def _begin(self, job_id: str, started_at: datetime) -> int | None:
"""Insert the `running` row for a run that is starting; returns its id.

None when the write failed -- the run still goes ahead (a job must not
be skipped because its bookkeeping could not be written) and `_record`
then inserts a complete row at the end instead, so the outcome is
still recorded exactly once.
"""
try:
async with self._db.session() as s:
row = JobRun(job_id=job_id, status="running", started_at=started_at)
s.add(row)
await s.flush()
return row.id
except SQLAlchemyError as e:
logger.warning("failed to record running row for job %s: %s", job_id, e)
return None

async def _record(
self, job_id: str, status: str, started_at: datetime, detail: str | None
self,
job_id: str,
status: str,
started_at: datetime,
detail: str | None,
*,
run_id: int | None = None,
) -> None:
"""Record an outcome: finish the `running` row `run_id` names, or
insert a complete row when there is none (skipped/misfired never
started a run; a failed `_begin` left no row to finish)."""
try:
async with self._db.session() as s:
s.add(
JobRun(
job_id=job_id,
status=status,
started_at=started_at,
finished_at=_utcnow(),
detail=detail,
row = await s.get(JobRun, run_id) if run_id is not None else None
if row is None:
s.add(
JobRun(
job_id=job_id,
status=status,
started_at=started_at,
finished_at=_utcnow(),
detail=detail,
)
)
)
else:
row.status = status
row.finished_at = _utcnow()
row.detail = detail
except SQLAlchemyError as e:
# The job itself already ran to whatever conclusion it reached; a
# DB failure recording that outcome must not raise into the
Expand All @@ -201,6 +249,101 @@ async def _record(
# same reason the cancelled path already catches broadly.
logger.warning("failed to record job_runs row for job %s: %s", job_id, e)

async def _reconcile_history(self) -> None:
"""Make `job_runs` honest about the time the process was not running.
Called by `start()` before anything is scheduled. Best-effort: a
database failure here is logged and must not stop the scheduler."""
try:
await self._mark_interrupted()
await self._record_downtime_misfires()
except SQLAlchemyError as e:
logger.warning("failed to reconcile job_runs at startup: %s", e)

async def _mark_interrupted(self) -> None:
"""A row still `running` at startup belongs to a process that died
mid-run. Nothing else can own it: `serve()` holds the exclusive
`data_dir/.dudamel.lock` for the life of the process, so no second
instance is executing jobs against this database, and this scheduler
has not started anything yet."""
async with self._db.session() as s:
result = await s.execute(
update(JobRun)
.where(JobRun.status == "running")
.values(
status="interrupted",
finished_at=_utcnow(),
detail="the process ended while this run was in progress "
"(crash, kill or power loss); the outcome is unknown",
)
)
if result.rowcount:
logger.warning(
"%d job run(s) were interrupted by the previous shutdown", result.rowcount
)

async def _record_downtime_misfires(self) -> None:
"""One `misfired` row per job that should have fired while the process
was down, saying how many fires were missed.

Needed because the in-memory job store recomputes every job's next fire
from "now" at each start: APScheduler never learns a fire was due while
we were away, so its own EVENT_JOB_MISSED (which `_on_missed` records)
cannot fire for downtime. That listener covers the other case -- a fire
that came due while the process WAS up but ran more than
`misfire_grace_time` late. The two windows do not overlap: this one
ends at the moment of startup, before the scheduler can run anything.

The window starts at the job's latest recorded run (any status, so a
previous startup's `misfired` row moves it forward and the same
downtime is never reported twice). A job with no rows is new or the
database is fresh: nothing was missed, nothing is recorded.
"""
now = _utcnow()
async with self._db.session() as s:
last_by_job = dict(
(
await s.execute(
select(JobRun.job_id, func.max(JobRun.started_at)).group_by(JobRun.job_id)
)
).all()
)
for job_id, aps_job in self._aps_jobs.items():
last = last_by_job.get(job_id)
if last is None:
continue
missed = self._count_missed(aps_job, last.replace(tzinfo=UTC), now.replace(tzinfo=UTC))
if missed == 0:
continue
count = f"{_MISSED_COUNT_CAP}+" if missed > _MISSED_COUNT_CAP else str(missed)
await self._record(
job_id,
"misfired",
now,
f"{count} scheduled run(s) were missed while dudamel was not running "
f"(last run started {last:%Y-%m-%d %H:%M:%S} UTC)",
)

@staticmethod
def _count_missed(aps_job: APSJob, last: datetime, now: datetime) -> int:
"""Fires of this job's trigger in (last, now], capped at one past
`_MISSED_COUNT_CAP`. Uses the trigger the job was registered with, so
the framework zone it carries decides what "every day at 20:00" means."""
trigger = aps_job.trigger
if isinstance(trigger, IntervalTrigger):
# An interval trigger is anchored at its own construction time --
# which is this startup -- so it cannot say when it would have
# fired last week. Intervals count from the last run instead.
step = trigger.interval.total_seconds()
if step <= 0:
return 0
return min(int((now - last).total_seconds() // step), _MISSED_COUNT_CAP + 1)
count = 0
fire = trigger.get_next_fire_time(None, last + timedelta(microseconds=1))
while fire is not None and fire <= now and count <= _MISSED_COUNT_CAP:
count += 1
fire = trigger.get_next_fire_time(fire, fire + timedelta(microseconds=1))
return count

def list_jobs(self) -> list[dict[str, Any]]:
"""Registered jobs with a best-effort next-fire time, for the
dashboard's /jobs page. Computed from each job's trigger directly
Expand All @@ -220,6 +363,9 @@ def list_jobs(self) -> list[dict[str, Any]]:
return jobs

async def start(self) -> None:
# Before the scheduler can run anything, so nothing it starts is
# mistaken for a leftover `running` row.
await self._reconcile_history()
self._scheduler.start()

async def shutdown(self) -> None:
Expand Down
Loading
Loading