diff --git a/README.md b/README.md index dcd23ef..667b36c 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,9 @@ them and feeds results back until it has an answer. The **data plane** — the dashboard, widgets, scheduled jobs, database migrations — does not depend on the model at all. If your LLM endpoint is down, the dashboard still renders, because widgets are plain `async def` functions that read -the database directly; nothing about them calls the model. +the database directly; nothing about them calls the model, and the framework +enforces it — `app.llm()` raises inside a widget. To show model output, have a +job or tool compute it, store it, and let the widget read the stored result. Apps are code, not configuration: diff --git a/src/dudamel/app.py b/src/dudamel/app.py index 9c06039..5069606 100644 --- a/src/dudamel/app.py +++ b/src/dudamel/app.py @@ -13,7 +13,13 @@ from dudamel.contract.renderers import ACTION_LABEL_MAX, RENDERERS, clean_action_label from dudamel.contract.schema import ToolSchema from dudamel.contract.types import TOOL_NAME_RE, Job, Tool, Widget -from dudamel.exceptions import AppSettingsError, RegistryError, RuntimeNotBoundError +from dudamel.exceptions import ( + AppSettingsError, + LLMNotAllowedInWidgetError, + RegistryError, + RuntimeNotBoundError, +) +from dudamel.widgets import IN_WIDGET if TYPE_CHECKING: from zoneinfo import ZoneInfo @@ -199,6 +205,11 @@ def widget( actions: Sequence[str] = (), timeout: float = 15.0, ) -> Callable: + """Register a dashboard widget: an `async def` with no required + arguments that returns the payload `renderer` expects. Widgets read + stored data; they cannot call `app.llm()` (it raises + `LLMNotAllowedInWidgetError`), so the dashboard renders with every + model tier down.""" if renderer not in RENDERERS: raise RegistryError(f"unknown renderer {renderer!r}; choose one of {sorted(RENDERERS)}") @@ -269,6 +280,14 @@ def wrap(fn: Callable[[], Awaitable[None]]) -> Callable: # --- runtime capabilities ------------------------------------------------ async def llm(self, *args: Any, **kwargs: Any) -> Any: + """Ask the model. Available to tools and jobs; refused inside a widget, + which must render with every model tier down.""" + if IN_WIDGET.get(): + raise LLMNotAllowedInWidgetError( + "widgets cannot call app.llm(): the dashboard must render when the model " + "is down. Compute the result in a scheduled job or tool, store it, and " + "have the widget read the stored result." + ) if self._llm is None: raise RuntimeNotBoundError( "app.llm is bound at run time by Runtime; not in tests/import" diff --git a/src/dudamel/exceptions.py b/src/dudamel/exceptions.py index 0e7eb9a..8ca18fd 100644 --- a/src/dudamel/exceptions.py +++ b/src/dudamel/exceptions.py @@ -55,6 +55,17 @@ class RuntimeNotBoundError(DudamelError): """A runtime capability (db/llm/notify) used before the run phase bound it.""" +class LLMNotAllowedInWidgetError(DudamelError): + """`app.llm()` was called from inside a widget. + + The dashboard is the part of the system that keeps working when every + model tier is down, so a widget that could reach the model would make that + claim depend on every app author's discipline. Enforced instead of + documented. The message is shown to the operator on the widget's error + card, so it names the rule and the way out. + """ + + class AppSettingsError(DudamelError): """An app's [apps.] config block has an unknown key or fails the app's own settings model.""" diff --git a/src/dudamel/models_core.py b/src/dudamel/models_core.py index 3f59e29..b69e18e 100644 --- a/src/dudamel/models_core.py +++ b/src/dudamel/models_core.py @@ -66,7 +66,9 @@ class JobRun(CoreBase): __tablename__ = "job_runs" id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) job_id: Mapped[str] = mapped_column(String(255), index=True) - status: Mapped[str] = mapped_column(String(32)) # ok|error|timeout|skipped|misfired|cancelled + status: Mapped[str] = mapped_column( + String(32) + ) # running|ok|error|timeout|skipped|misfired|cancelled|interrupted started_at: Mapped[datetime] = mapped_column(DateTime, default=_utcnow) finished_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) detail: Mapped[str | None] = mapped_column(Text, nullable=True) diff --git a/src/dudamel/scheduler.py b/src/dudamel/scheduler.py index e833395..f257033 100644 --- a/src/dudamel/scheduler.py +++ b/src/dudamel/scheduler.py @@ -1,9 +1,12 @@ """APScheduler wiring: turns every registered Job into a running APScheduler job and records EVERY outcome (ok/error/timeout/skipped/misfired/cancelled) -as a `job_runs` row. +as a `job_runs` row. A run that starts leaves a `running` row immediately, +updated in place when it ends, so a process that dies mid-run still leaves +evidence; `start()` turns what a dead process left into `interrupted` and +accounts for fires missed while nothing was running as one `misfired` row. Thin wrapper — the scheduler calls only `job.fn()`. Job functions are -command-plane (unlike widgets): they may use `app.llm`/`app.notify`, but +command-plane (unlike widgets, which `App.llm` refuses): they may use `app.llm`/`app.notify`, but those bindings are Runtime's concern (already wired up before this ever fires); the scheduler itself never touches the LLM directly. @@ -18,7 +21,7 @@ import asyncio import logging import traceback -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from typing import Any from zoneinfo import ZoneInfo @@ -27,6 +30,7 @@ from apscheduler.schedulers.asyncio import AsyncIOScheduler from apscheduler.triggers.cron import CronTrigger from apscheduler.triggers.interval import IntervalTrigger +from sqlalchemy import func, select, update from sqlalchemy.exc import SQLAlchemyError from dudamel.contract.types import Job @@ -46,6 +50,11 @@ # the whole shutdown sequence. _DRAIN_TIMEOUT = 2.0 +# Counting missed fires is a loop over the trigger; an install that was off for +# a year with a per-minute job must not spin through half a million iterations +# at startup just to print a number nobody needs to the digit. +_MISSED_COUNT_CAP = 1000 + def _utcnow() -> datetime: return datetime.now(UTC).replace(tzinfo=None) @@ -106,6 +115,10 @@ async def _run_job(self, job: Job) -> None: self._inflight.add(task) task.add_done_callback(self._inflight.discard) started = _utcnow() + # Written before the job runs, not after: a row that only appears when + # the run finishes is exactly the row a crash, SIGKILL or power loss + # never gets to write. + run_id = await self._begin(job.id, started) try: # `asyncio.timeout` lets us tell OUR deadline apart from a # TimeoutError the job raised itself. Since Python 3.11 @@ -121,11 +134,11 @@ async def _run_job(self, job: Job) -> None: if cm.expired(): detail = f"job {job.id} timed out after {job.timeout}s" logger.warning(detail) - await self._record(job.id, "timeout", started, detail) + await self._record(job.id, "timeout", started, detail, run_id=run_id) else: # The job raised its own TimeoutError -- a job error, recorded # with the traceback like any other raised exception. - await self._record_error(job.id, started, e) + await self._record_error(job.id, started, e, run_id) except asyncio.CancelledError: # The executor cancels in-flight jobs at shutdown. Record the # outcome, then let the cancellation continue -- a run that was @@ -136,22 +149,24 @@ async def _run_job(self, job: Job) -> None: detail = f"job {job.id} was cancelled before it finished" logger.info(detail) try: - await self._record(job.id, "cancelled", started, detail) + await self._record(job.id, "cancelled", started, detail, run_id=run_id) except Exception as e: # noqa: BLE001 — recording is best-effort logger.warning("failed to record cancelled run for job %s: %s", job.id, e) raise except Exception as e: # job bugs must not kill the scheduler - await self._record_error(job.id, started, e) + await self._record_error(job.id, started, e, run_id) else: - await self._record(job.id, "ok", started, None) + await self._record(job.id, "ok", started, None, run_id=run_id) - async def _record_error(self, job_id: str, started: datetime, e: BaseException) -> None: + async def _record_error( + self, job_id: str, started: datetime, e: BaseException, run_id: int | None = None + ) -> None: """Record a job that raised as an "error" row, capturing the active traceback. Must be called from inside the handling `except` block so `traceback.format_exc()` refers to `e`.""" detail = f"{type(e).__name__}: {e}\n{traceback.format_exc()}"[:_DETAIL_CAP] logger.warning("job %s raised: %s", job_id, e) - await self._record(job_id, "error", started, detail) + await self._record(job_id, "error", started, detail, run_id=run_id) def _on_missed(self, event: JobExecutionEvent) -> None: """APScheduler listener (sync callback, invoked from the running @@ -176,20 +191,53 @@ def _on_max_instances(self, event: JobExecutionEvent) -> None: self._background_tasks.add(task) task.add_done_callback(self._background_tasks.discard) + async def _begin(self, job_id: str, started_at: datetime) -> int | None: + """Insert the `running` row for a run that is starting; returns its id. + + None when the write failed -- the run still goes ahead (a job must not + be skipped because its bookkeeping could not be written) and `_record` + then inserts a complete row at the end instead, so the outcome is + still recorded exactly once. + """ + try: + async with self._db.session() as s: + row = JobRun(job_id=job_id, status="running", started_at=started_at) + s.add(row) + await s.flush() + return row.id + except SQLAlchemyError as e: + logger.warning("failed to record running row for job %s: %s", job_id, e) + return None + async def _record( - self, job_id: str, status: str, started_at: datetime, detail: str | None + self, + job_id: str, + status: str, + started_at: datetime, + detail: str | None, + *, + run_id: int | None = None, ) -> None: + """Record an outcome: finish the `running` row `run_id` names, or + insert a complete row when there is none (skipped/misfired never + started a run; a failed `_begin` left no row to finish).""" try: async with self._db.session() as s: - s.add( - JobRun( - job_id=job_id, - status=status, - started_at=started_at, - finished_at=_utcnow(), - detail=detail, + row = await s.get(JobRun, run_id) if run_id is not None else None + if row is None: + s.add( + JobRun( + job_id=job_id, + status=status, + started_at=started_at, + finished_at=_utcnow(), + detail=detail, + ) ) - ) + else: + row.status = status + row.finished_at = _utcnow() + row.detail = detail except SQLAlchemyError as e: # The job itself already ran to whatever conclusion it reached; a # DB failure recording that outcome must not raise into the @@ -201,6 +249,101 @@ async def _record( # same reason the cancelled path already catches broadly. logger.warning("failed to record job_runs row for job %s: %s", job_id, e) + async def _reconcile_history(self) -> None: + """Make `job_runs` honest about the time the process was not running. + Called by `start()` before anything is scheduled. Best-effort: a + database failure here is logged and must not stop the scheduler.""" + try: + await self._mark_interrupted() + await self._record_downtime_misfires() + except SQLAlchemyError as e: + logger.warning("failed to reconcile job_runs at startup: %s", e) + + async def _mark_interrupted(self) -> None: + """A row still `running` at startup belongs to a process that died + mid-run. Nothing else can own it: `serve()` holds the exclusive + `data_dir/.dudamel.lock` for the life of the process, so no second + instance is executing jobs against this database, and this scheduler + has not started anything yet.""" + async with self._db.session() as s: + result = await s.execute( + update(JobRun) + .where(JobRun.status == "running") + .values( + status="interrupted", + finished_at=_utcnow(), + detail="the process ended while this run was in progress " + "(crash, kill or power loss); the outcome is unknown", + ) + ) + if result.rowcount: + logger.warning( + "%d job run(s) were interrupted by the previous shutdown", result.rowcount + ) + + async def _record_downtime_misfires(self) -> None: + """One `misfired` row per job that should have fired while the process + was down, saying how many fires were missed. + + Needed because the in-memory job store recomputes every job's next fire + from "now" at each start: APScheduler never learns a fire was due while + we were away, so its own EVENT_JOB_MISSED (which `_on_missed` records) + cannot fire for downtime. That listener covers the other case -- a fire + that came due while the process WAS up but ran more than + `misfire_grace_time` late. The two windows do not overlap: this one + ends at the moment of startup, before the scheduler can run anything. + + The window starts at the job's latest recorded run (any status, so a + previous startup's `misfired` row moves it forward and the same + downtime is never reported twice). A job with no rows is new or the + database is fresh: nothing was missed, nothing is recorded. + """ + now = _utcnow() + async with self._db.session() as s: + last_by_job = dict( + ( + await s.execute( + select(JobRun.job_id, func.max(JobRun.started_at)).group_by(JobRun.job_id) + ) + ).all() + ) + for job_id, aps_job in self._aps_jobs.items(): + last = last_by_job.get(job_id) + if last is None: + continue + missed = self._count_missed(aps_job, last.replace(tzinfo=UTC), now.replace(tzinfo=UTC)) + if missed == 0: + continue + count = f"{_MISSED_COUNT_CAP}+" if missed > _MISSED_COUNT_CAP else str(missed) + await self._record( + job_id, + "misfired", + now, + f"{count} scheduled run(s) were missed while dudamel was not running " + f"(last run started {last:%Y-%m-%d %H:%M:%S} UTC)", + ) + + @staticmethod + def _count_missed(aps_job: APSJob, last: datetime, now: datetime) -> int: + """Fires of this job's trigger in (last, now], capped at one past + `_MISSED_COUNT_CAP`. Uses the trigger the job was registered with, so + the framework zone it carries decides what "every day at 20:00" means.""" + trigger = aps_job.trigger + if isinstance(trigger, IntervalTrigger): + # An interval trigger is anchored at its own construction time -- + # which is this startup -- so it cannot say when it would have + # fired last week. Intervals count from the last run instead. + step = trigger.interval.total_seconds() + if step <= 0: + return 0 + return min(int((now - last).total_seconds() // step), _MISSED_COUNT_CAP + 1) + count = 0 + fire = trigger.get_next_fire_time(None, last + timedelta(microseconds=1)) + while fire is not None and fire <= now and count <= _MISSED_COUNT_CAP: + count += 1 + fire = trigger.get_next_fire_time(fire, fire + timedelta(microseconds=1)) + return count + def list_jobs(self) -> list[dict[str, Any]]: """Registered jobs with a best-effort next-fire time, for the dashboard's /jobs page. Computed from each job's trigger directly @@ -220,6 +363,9 @@ def list_jobs(self) -> list[dict[str, Any]]: return jobs async def start(self) -> None: + # Before the scheduler can run anything, so nothing it starts is + # mistaken for a leftover `running` row. + await self._reconcile_history() self._scheduler.start() async def shutdown(self) -> None: diff --git a/src/dudamel/widgets.py b/src/dudamel/widgets.py index ea09a55..d93f05a 100644 --- a/src/dudamel/widgets.py +++ b/src/dudamel/widgets.py @@ -1,6 +1,7 @@ """Widget execution: turns a registered Widget into dashboard-ready JSON. -Thin runner — zero business logic, zero LLM calls. A widget that raises, +Thin runner — zero business logic, zero LLM calls, and no way for a widget to +make one: `App.llm()` refuses while `IN_WIDGET` is set. A widget that raises, times out its own fn(), or returns a payload its renderer rejects must never take down the whole dashboard: it degrades to an error entry instead of propagating. @@ -11,6 +12,7 @@ import asyncio import logging from collections.abc import Mapping +from contextvars import ContextVar from typing import Any from pydantic import BaseModel, ValidationError @@ -26,6 +28,21 @@ logger = logging.getLogger("dudamel.widgets") +# True for exactly the duration of a widget's fn(). `App.llm()` reads it and +# refuses: the dashboard must render with every model tier dead, and "widgets +# don't call the model" is only a guarantee if the framework enforces it. Lives +# here (not in app.py) because widgets.py is the one place that knows when a +# widget is running; app.py imports it, never the reverse. +# +# A ContextVar rather than a plain flag so concurrent renders (the dashboard +# runs widgets in parallel) cannot see each other's state. Because a context is +# copied into any task created from it, a task the widget spawns INHERITS the +# flag -- intended: spawning a task must not be a way around the rule. A thread +# started via `app.to_thread` also copies the context (asyncio.to_thread does), +# but a thread the widget starts by other means does not; that is outside what +# a runtime guard can see. +IN_WIDGET: ContextVar[bool] = ContextVar("in_widget", default=False) + def _to_jsonable(value: Any) -> Any: """validate_widget_payload returns pydantic models (stat/table), a list of @@ -117,7 +134,13 @@ async def run_widget( # the generic handler below and reports its real message, not a # fabricated "widget timed out after Ns". async with asyncio.timeout(widget.timeout) as cm: - raw = await widget.fn() + # Reset through the token in `finally` so the flag cannot outlive + # the widget, on success, raise or timeout alike. + token = IN_WIDGET.set(True) + try: + raw = await widget.fn() + finally: + IN_WIDGET.reset(token) except TimeoutError as e: if not cm.expired(): logger.warning("widget %s (app %s) failed: %s", widget.id, widget.app_name, e) diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index b02e9ea..76ea5c7 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -31,13 +31,20 @@ async def make_db(tmp_path: Path) -> Database: async def poll_job_runs( db: Database, job_id: str, *, count: int = 1, timeout: float = 5.0 ) -> list[JobRun]: - """Poll job_runs for `job_id` until at least `count` rows exist or the - timeout elapses -- never a single sleep-and-hope.""" + """Poll job_runs for `job_id` until at least `count` FINISHED rows exist or + the timeout elapses -- never a single sleep-and-hope. A `running` row exists + from the instant a run starts, so it is not an outcome yet.""" deadline = time.monotonic() + timeout while True: async with db.session() as s: rows = list( - (await s.execute(select(JobRun).where(JobRun.job_id == job_id))).scalars().all() + ( + await s.execute( + select(JobRun).where(JobRun.job_id == job_id, JobRun.status != "running") + ) + ) + .scalars() + .all() ) if len(rows) >= count: return rows @@ -578,3 +585,173 @@ async def nightly() -> None: ) finally: await rt.stop() + + +# --- job_runs: a run that starts leaves a row, and downtime is accounted for -- + + +async def _rows(db: Database, job_id: str) -> list[JobRun]: + async with db.session() as s: + return list( + (await s.execute(select(JobRun).where(JobRun.job_id == job_id).order_by(JobRun.id))) + .scalars() + .all() + ) + + +async def test_a_row_exists_with_status_running_while_the_job_executes(tmp_path) -> None: + app = App("stats", description="d") + inside, release = asyncio.Event(), asyncio.Event() + + @app.job(interval_seconds=3600) + async def slow() -> None: + inside.set() + await release.wait() + + orc = Orchestrator(apps=[app]) + db = await make_db(tmp_path) + sched = JobScheduler(orc.registry, db, timezone=ZoneInfo("UTC")) + try: + run = asyncio.create_task(sched._run_job(orc.registry.jobs[0])) + await asyncio.wait_for(inside.wait(), 5) + mid = await _rows(db, "stats.slow") + assert [(r.status, r.finished_at) for r in mid] == [("running", None)] + release.set() + await run + # The same row is updated in place: one run, one row. + done = await _rows(db, "stats.slow") + assert [r.status for r in done] == ["ok"] + assert done[0].id == mid[0].id + assert done[0].finished_at is not None + finally: + await db.dispose() + + +async def test_a_running_row_left_by_a_dead_process_becomes_interrupted(tmp_path) -> None: + app = App("stats", description="d") + + @app.job(interval_seconds=3600) + async def poll() -> None: + return None + + orc = Orchestrator(apps=[app]) + db = await make_db(tmp_path) + async with db.session() as s: + s.add(JobRun(job_id="stats.poll", status="running", started_at=_naive(2026, 1, 1, 10))) + sched = JobScheduler(orc.registry, db, timezone=ZoneInfo("UTC")) + await sched.start() + try: + rows = await _rows(db, "stats.poll") + finally: + await sched.shutdown() + await db.dispose() + assert rows[0].status == "interrupted" + assert "process" in rows[0].detail + assert rows[0].finished_at is not None + + +def _naive(*args: int) -> datetime: + return datetime(*args) + + +async def _cron_hourly_sched(tmp_path: Path, monkeypatch, now: datetime): + app = App("stats", description="d") + + @app.job(cron="0 * * * *") + async def hourly() -> None: + return None + + orc = Orchestrator(apps=[app]) + db = await make_db(tmp_path) + monkeypatch.setattr("dudamel.scheduler._utcnow", lambda: now) + return JobScheduler(orc.registry, db, timezone=ZoneInfo("UTC")), db + + +async def test_fires_missed_during_downtime_record_one_misfired_row_with_the_count( + tmp_path, monkeypatch +) -> None: + # Last run 10:30; down until 13:30 -> the 11:00, 12:00 and 13:00 fires. + sched, db = await _cron_hourly_sched(tmp_path, monkeypatch, _naive(2026, 1, 1, 13, 30)) + async with db.session() as s: + s.add(JobRun(job_id="stats.hourly", status="ok", started_at=_naive(2026, 1, 1, 10, 30))) + try: + await sched._reconcile_history() + rows = await _rows(db, "stats.hourly") + finally: + await db.dispose() + assert [r.status for r in rows] == ["ok", "misfired"] + assert "3 " in rows[1].detail + + +async def test_no_fire_in_the_downtime_window_records_nothing(tmp_path, monkeypatch) -> None: + sched, db = await _cron_hourly_sched(tmp_path, monkeypatch, _naive(2026, 1, 1, 10, 50)) + async with db.session() as s: + s.add(JobRun(job_id="stats.hourly", status="ok", started_at=_naive(2026, 1, 1, 10, 30))) + try: + await sched._reconcile_history() + rows = await _rows(db, "stats.hourly") + finally: + await db.dispose() + assert [r.status for r in rows] == ["ok"] + + +async def test_a_job_with_no_history_records_no_misfire(tmp_path, monkeypatch) -> None: + sched, db = await _cron_hourly_sched(tmp_path, monkeypatch, _naive(2026, 1, 1, 13, 30)) + try: + await sched._reconcile_history() + assert await _rows(db, "stats.hourly") == [] + finally: + await db.dispose() + + +async def test_the_missed_count_is_bounded(tmp_path, monkeypatch) -> None: + sched, db = await _cron_hourly_sched(tmp_path, monkeypatch, _naive(2026, 12, 1)) + async with db.session() as s: + s.add(JobRun(job_id="stats.hourly", status="ok", started_at=_naive(2026, 1, 1))) + try: + await sched._reconcile_history() + rows = await _rows(db, "stats.hourly") + finally: + await db.dispose() + assert "1000+" in rows[1].detail + + +async def test_interval_downtime_counts_whole_intervals_since_the_last_run( + tmp_path, monkeypatch +) -> None: + app = App("stats", description="d") + + @app.job(interval_seconds=600) + async def poll() -> None: + return None + + orc = Orchestrator(apps=[app]) + db = await make_db(tmp_path) + monkeypatch.setattr("dudamel.scheduler._utcnow", lambda: _naive(2026, 1, 1, 11, 5)) + sched = JobScheduler(orc.registry, db, timezone=ZoneInfo("UTC")) + async with db.session() as s: + s.add(JobRun(job_id="stats.poll", status="ok", started_at=_naive(2026, 1, 1, 10, 0))) + try: + await sched._reconcile_history() + rows = await _rows(db, "stats.poll") + finally: + await db.dispose() + assert [r.status for r in rows] == ["ok", "misfired"] + assert "6 " in rows[1].detail + + +async def test_a_normal_run_through_the_scheduler_is_one_ok_row(tmp_path) -> None: + app = App("stats", description="d") + + @app.job(interval_seconds=3600) + async def poll() -> None: + return None + + orc = Orchestrator(apps=[app]) + db = await make_db(tmp_path) + sched = JobScheduler(orc.registry, db, timezone=ZoneInfo("UTC")) + try: + await sched._run_job(orc.registry.jobs[0]) + assert [r.status for r in await _rows(db, "stats.poll")] == ["ok"] + finally: + await db.dispose() diff --git a/tests/test_widgets_exec.py b/tests/test_widgets_exec.py index 2d5364c..33e23ea 100644 --- a/tests/test_widgets_exec.py +++ b/tests/test_widgets_exec.py @@ -383,3 +383,70 @@ async def today() -> list[dict[str, object]]: card = await run_widget(app.widgets["today"], _actions_of(app)) assert "error" in card assert "must not be empty" in card["error"] + + +# --- widgets cannot reach the model ------------------------------------------ +# The positioning claim is that the dashboard works with every tier dead. These +# bind a counting stand-in for the Runtime's `app._llm`, so "the provider is +# never called" is observed rather than inferred from the error text. + + +def _app_with_counting_llm() -> tuple[App, list[str]]: + app = App("gym", description="d") + calls: list[str] = [] + + async def fake_llm(prompt, **kw): + calls.append(prompt) + return "answer" + + app._llm = fake_llm + return app, calls + + +async def test_widget_calling_app_llm_gets_an_error_card_and_never_reaches_the_model() -> None: + app, calls = _app_with_counting_llm() + + async def fn() -> dict: + await app.llm("summarize") + return {"label": "x", "value": 1} + + out = await run_widget(make_widget(fn)) + assert calls == [] + assert "data" not in out + assert "widget" in out["error"] and "scheduled job" in out["error"] + + +async def test_llm_outside_a_widget_still_works() -> None: + app, calls = _app_with_counting_llm() + assert await app.llm("from a job or tool") == "answer" + assert calls == ["from a job or tool"] + + +async def test_widget_flag_does_not_leak_after_success_or_error() -> None: + from dudamel.widgets import IN_WIDGET + + async def ok() -> dict: + assert IN_WIDGET.get() is True + return {"label": "x", "value": 1} + + async def boom() -> dict: + raise RuntimeError("nope") + + assert IN_WIDGET.get() is False + await run_widget(make_widget(ok)) + assert IN_WIDGET.get() is False + await run_widget(make_widget(boom)) + assert IN_WIDGET.get() is False + + +async def test_a_task_a_widget_spawns_cannot_reach_the_model_either() -> None: + app, calls = _app_with_counting_llm() + + async def fn() -> dict: + # The task copies the context at creation, so it inherits the flag. + await asyncio.create_task(app.llm("sneaky")) + return {"label": "x", "value": 1} + + out = await run_widget(make_widget(fn)) + assert calls == [] + assert "error" in out