From 390f81f6f7f0f3aff2be52988b81286efd79b7d6 Mon Sep 17 00:00:00 2001 From: TheMightyTRexUK <196487809+TheMightyTRex@users.noreply.github.com> Date: Sat, 31 Oct 2026 10:18:10 +0000 Subject: [PATCH 1/2] Cancel abandoned page reads when the browser leaves Leaving a page while a slow request is still running kept its database query busy until it finished, and moving back and forth piled the queries up. Reads (GET and HEAD) on the requests, following and library pages are now cancelled when the browser disconnects before the response starts; the database driver then cancels the running query. Writes, reads that call outside services, and clean-up after a response always run to the end. Co-Authored-By: Claude Opus 5.5 --- services/app/disconnect.py | 64 ++++++++++++++++++++++++++++ services/app/main.py | 3 ++ tests/unit/test_disconnect.py | 79 +++++++++++++++++++++++++++++++++++ 3 files changed, 146 insertions(+) create mode 100644 services/app/disconnect.py create mode 100644 tests/unit/test_disconnect.py diff --git a/services/app/disconnect.py b/services/app/disconnect.py new file mode 100644 index 00000000..e6985205 --- /dev/null +++ b/services/app/disconnect.py @@ -0,0 +1,64 @@ +"""Stop a page's database work when the browser leaves before the answer arrives.""" + +import asyncio + +# Pages whose reads only query the database. Reads that call outside services (searches on MAM, +# Prowlarr or Hardcover, for example) are never cancelled: cancelling one part-way can leave that +# service marked busy for the next request. +DATABASE_READS = ("/api/requests", "/api/following", "/api/library") + + +class CancelOnDisconnect: + """ + For reads (GET and HEAD) on the pages in DATABASE_READS, cancel the request when the browser + disconnects, for example after moving to another page. Cancelling the task makes the + database driver cancel its running query, so an abandoned page no longer keeps a database + connection busy. Writes always run to the end, so a change is never left half done. Once the + answer has started, the request also runs to the end: servers report a disconnect as soon as + a response is complete, and the clean-up after it (closing the database session, background + tasks) must still happen. + """ + + def __init__(self, app): + self.app = app + + async def __call__(self, scope, receive, send): + if ( + scope["type"] != "http" + or scope["method"] not in {"GET", "HEAD"} + or not any( + scope["path"] == prefix or scope["path"].startswith(prefix + "/") + for prefix in DATABASE_READS + ) + ): + await self.app(scope, receive, send) + return + messages: asyncio.Queue = asyncio.Queue() + gone = asyncio.Event() + answered = asyncio.Event() + + async def listen(): + while True: + message = await receive() + await messages.put(message) + if message["type"] == "http.disconnect": + if not answered.is_set(): + gone.set() + handler.cancel() + return + + async def answer(message): + if message["type"] == "http.response.start": + answered.set() + await send(message) + + handler = asyncio.ensure_future(self.app(scope, messages.get, answer)) + listener = asyncio.ensure_future(listen()) + try: + await handler + except asyncio.CancelledError: + if not gone.is_set(): + raise + # The browser has gone, so there is no one to answer. + finally: + listener.cancel() diff --git a/services/app/main.py b/services/app/main.py index ec2179bf..512eb4eb 100644 --- a/services/app/main.py +++ b/services/app/main.py @@ -76,6 +76,7 @@ from app.config import get_settings from app.db.session import get_engine, session_factory from app.diagnostics import auth_configuration +from app.disconnect import CancelOnDisconnect from app.jobs.queue import get_queue from app.recovery import active_restore, restore_pending, runtime_lease @@ -234,6 +235,8 @@ async def frontend(path: str): return JSONResponse({"detail": "Not found"}, 404) return FileResponse(dist / "index.html") + # Added last so it is the outermost layer and sees the browser's disconnect first. + app.add_middleware(CancelOnDisconnect) return app diff --git a/tests/unit/test_disconnect.py b/tests/unit/test_disconnect.py new file mode 100644 index 00000000..779c3cf6 --- /dev/null +++ b/tests/unit/test_disconnect.py @@ -0,0 +1,79 @@ +import asyncio + +from app.disconnect import CancelOnDisconnect + + +def scope(method="GET", path="/api/requests"): + return {"type": "http", "method": method, "path": path} + + +async def run(method, path="/api/requests"): + state = {"finished": False, "cancelled": False} + + async def slow(scope, receive, send): + await receive() + try: + await asyncio.sleep(5) + state["finished"] = True + except asyncio.CancelledError: + state["cancelled"] = True + raise + + sent = [{"type": "http.request", "body": b"", "more_body": False}] + + async def receive(): + if sent: + return sent.pop() + await asyncio.sleep(0.05) # The browser leaves shortly after asking. + return {"type": "http.disconnect"} + + async def send(message): + pass + + await asyncio.wait_for(CancelOnDisconnect(slow)(scope(method, path), receive, send), 1) + return state + + +async def test_a_page_read_stops_when_the_browser_leaves(): + assert await run("GET") == {"finished": False, "cancelled": True} + + +async def test_writes_and_non_api_requests_are_left_alone(): + for method, path in ( + ("POST", "/api/requests"), + ("GET", "/assets/app.js"), + ("GET", "/api/books/1/sources"), # Searches outside services. + ("GET", "/api/requestsx"), + ): + try: + await asyncio.wait_for(run(method, path), 0.2) + except TimeoutError: + continue # Still running after the browser left: not cancelled. + raise AssertionError(f"{method} {path} was cancelled") + + +async def test_clean_up_after_the_answer_is_not_cancelled(): + state = {"cleaned_up": False} + + async def answers_then_cleans_up(scope, receive, send): + await receive() + await send({"type": "http.response.start", "status": 200, "headers": []}) + await send({"type": "http.response.body", "body": b"", "more_body": False}) + await asyncio.sleep(0.1) # Closing the database session after the answer. + state["cleaned_up"] = True + + sent = [{"type": "http.request", "body": b"", "more_body": False}] + complete = asyncio.Event() + + async def receive(): + if sent: + return sent.pop() + await complete.wait() + return {"type": "http.disconnect"} # Servers report this once the answer is complete. + + async def send(message): + if message["type"] == "http.response.body" and not message.get("more_body"): + complete.set() + + await asyncio.wait_for(CancelOnDisconnect(answers_then_cleans_up)(scope(), receive, send), 1) + assert state["cleaned_up"] From 2cdaff20e9e28cb8327a30523a7cc6277f6bd889 Mon Sep 17 00:00:00 2001 From: TheMightyTRexUK <196487809+TheMightyTRex@users.noreply.github.com> Date: Sat, 31 Oct 2026 11:31:56 +0000 Subject: [PATCH 2/2] Faster database lookups and a Background work setting Request counts and background jobs looked operations up by kind and status, and by the request id inside the command payload, with no index for either, so each lookup read the whole operations table. On a library with 52,000 operations the request counts took about 4 minutes; with the two indexes in migration 0081 they take under a second. JIT is turned off for the app's connections, because compiling the large status queries cost more than it saved (17 seconds on one count). Settings -> Background work chooses how often scheduled jobs do their work: Normal (unchanged), Relaxed (x5), Light (x20) or Minimal (x50). Each job keeps its schedule and skips the runs that are not due at the chosen pace, so the setting applies within a minute without a restart. Migration 0082 stores it. Co-Authored-By: Claude Opus 5.5 --- apps/web/src/api/schema.d.ts | 80 +++++++++++++ apps/web/src/pages/BackgroundWork.tsx | 105 ++++++++++++++++++ apps/web/src/pages/SettingsSections.tsx | 10 ++ apps/web/tests/ui/background-work.spec.ts | 47 ++++++++ docs/openapi.json | 77 +++++++++++++ services/app/api/background_work.py | 40 +++++++ .../versions/0081_operation_lookups.py | 33 ++++++ .../versions/0082_background_work.py | 28 +++++ services/app/db/models.py | 22 ++++ services/app/db/session.py | 3 +- services/app/jobs/pace.py | 43 +++++++ services/app/jobs/tasks.py | 25 +++++ services/app/main.py | 2 + services/app/state_bundle.py | 2 +- tests/integration/test_background_work.py | 95 ++++++++++++++++ tests/unit/test_background_pace.py | 45 ++++++++ 16 files changed, 655 insertions(+), 2 deletions(-) create mode 100644 apps/web/src/pages/BackgroundWork.tsx create mode 100644 apps/web/tests/ui/background-work.spec.ts create mode 100644 services/app/api/background_work.py create mode 100644 services/app/db/migrations/versions/0081_operation_lookups.py create mode 100644 services/app/db/migrations/versions/0082_background_work.py create mode 100644 services/app/jobs/pace.py create mode 100644 tests/integration/test_background_work.py create mode 100644 tests/unit/test_background_pace.py diff --git a/apps/web/src/api/schema.d.ts b/apps/web/src/api/schema.d.ts index 9976eab3..6439be70 100644 --- a/apps/web/src/api/schema.d.ts +++ b/apps/web/src/api/schema.d.ts @@ -141,6 +141,24 @@ export interface paths { patch?: never; trace?: never; }; + "/api/background-work": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Background Work */ + get: operations["background_work_api_background_work_get"]; + /** Save Background Work */ + put: operations["save_background_work_api_background_work_put"]; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/health/live": { parameters: { query?: never; @@ -6524,6 +6542,15 @@ export interface components { /** Parts Medium */ parts_medium?: string | null; }; + /** BackgroundWork */ + BackgroundWork: { + /** + * Pace + * @default normal + * @enum {string} + */ + pace: "normal" | "relaxed" | "light" | "minimal"; + }; /** BatchInput */ BatchInput: { /** Expected Content Revision */ @@ -15466,6 +15493,59 @@ export interface operations { }; }; }; + background_work_api_background_work_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["BackgroundWork"]; + }; + }; + }; + }; + save_background_work_api_background_work_put: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["BackgroundWork"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["BackgroundWork"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; liveness_api_health_live_get: { parameters: { query?: never; diff --git a/apps/web/src/pages/BackgroundWork.tsx b/apps/web/src/pages/BackgroundWork.tsx new file mode 100644 index 00000000..4e95c8d4 --- /dev/null +++ b/apps/web/src/pages/BackgroundWork.tsx @@ -0,0 +1,105 @@ +import { useState } from "react"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { api, result } from "../api/client"; +import type { components } from "../api/schema"; +import { Loading, Notice } from "../components"; + +type Pace = components["schemas"]["BackgroundWork"]["pace"]; + +const paces: { id: Pace; title: string; description: string }[] = [ + { + id: "normal", + title: "Normal", + description: "Every background job runs on its usual schedule.", + }, + { + id: "relaxed", + title: "Relaxed", + description: + "Background jobs run 5 times less often. Downloads, imports and list checks are noticed a little later.", + }, + { + id: "light", + title: "Light", + description: + "Background jobs run 20 times less often. Good for a small server that also does other work.", + }, + { + id: "minimal", + title: "Minimal", + description: + "Background jobs run 50 times less often. Finished downloads can take up to an hour to import.", + }, +]; + +export default function BackgroundWork() { + const cache = useQueryClient(); + const setting = useQuery({ + queryKey: ["background-work"], + queryFn: async () => result(await api.GET("/api/background-work")), + }); + const [draft, setDraft] = useState(null); + const pace = draft ?? setting.data?.pace ?? "normal"; + const save = useMutation({ + mutationFn: async (submitted: Pace) => + result( + await api.PUT("/api/background-work", { body: { pace: submitted } }), + ), + onSuccess: (saved, submitted) => { + setDraft((current) => (current === submitted ? null : current)); + cache.setQueryData(["background-work"], saved); + }, + }); + return ( +
+

Background work

+

+ Dewarr checks downloads, imports, lists and connections on a schedule. + On a small or shared server, a slower pace saves CPU and database work. + Searching and requesting books are not affected. +

+

+ New in this version. Normal keeps the schedule Dewarr has always used. +

+ {setting.isPending ? ( + + ) : setting.data ? ( +
{ + event.preventDefault(); + save.mutate(pace); + }} + > +
+ How often background jobs run + {paces.map((option) => ( + + ))} +
+ + {save.isSuccess && ( +

+ Background work saved. It applies within a minute. +

+ )} +
+ ) : null} + +
+ ); +} diff --git a/apps/web/src/pages/SettingsSections.tsx b/apps/web/src/pages/SettingsSections.tsx index 58a84b9f..cd51f69d 100644 --- a/apps/web/src/pages/SettingsSections.tsx +++ b/apps/web/src/pages/SettingsSections.tsx @@ -1,3 +1,4 @@ +import BackgroundWork from "./BackgroundWork"; import { lazy } from "react"; import SettingsGroup from "../components/SettingsGroup"; @@ -102,6 +103,15 @@ export function settingsSections(role: string, permissions: string[] = []) { title: "Notifications", content: , }, + ...(admin + ? [ + { + id: "background-work", + title: "Background work", + content: , + }, + ] + : []), { id: "logs", title: "Logs", content: }, ]; } diff --git a/apps/web/tests/ui/background-work.spec.ts b/apps/web/tests/ui/background-work.spec.ts new file mode 100644 index 00000000..838ac533 --- /dev/null +++ b/apps/web/tests/ui/background-work.spec.ts @@ -0,0 +1,47 @@ +import { expect, test } from "../fixtures"; + +test("admins choose how often background jobs run", async ({ page }) => { + const saved: unknown[] = []; + let pace = "normal"; + await page.route("**/api/**", async (route) => { + const request = route.request(); + const path = new URL(request.url()).pathname; + let data: unknown = { items: [], total: 0, offset: 0, limit: 40 }; + if (path === "/api/auth/me") + data = { + user: { + id: "admin", + role: "admin", + username: "admin", + display_name: "Admin", + }, + csrf_token: "test", + }; + else if (path === "/api/setup/onboarding") data = { status: "completed" }; + else if (path === "/api/library/libraries") data = []; + else if (path === "/api/background-work") { + if (request.method() === "PUT") { + saved.push(request.postDataJSON()); + pace = (request.postDataJSON() as { pace: string }).pace; + } + data = { pace }; + } + return route.fulfill({ json: data }); + }); + await page.goto("/settings#background-work"); + const section = page.getByRole("region", { name: "Background work" }); + await expect(section.getByText("New in this version.")).toBeVisible(); + await expect(section.getByRole("radio", { name: /^Normal/ })).toBeChecked(); + await section.getByRole("radio", { name: /^Light/ }).check(); + await section.getByRole("button", { name: "Save background work" }).click(); + await expect(section.getByRole("status")).toHaveText( + "Background work saved. It applies within a minute.", + ); + expect(saved).toEqual([{ pace: "light" }]); + await page.reload(); + await expect( + page + .getByRole("region", { name: "Background work" }) + .getByRole("radio", { name: /^Light/ }), + ).toBeChecked(); +}); diff --git a/docs/openapi.json b/docs/openapi.json index fca1ef4c..88a1a149 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -321,6 +321,66 @@ } } }, + "/api/background-work": { + "get": { + "tags": [ + "background work" + ], + "summary": "Background Work", + "operationId": "background_work_api_background_work_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BackgroundWork" + } + } + } + } + } + }, + "put": { + "tags": [ + "background work" + ], + "summary": "Save Background Work", + "operationId": "save_background_work_api_background_work_put", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BackgroundWork" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BackgroundWork" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/api/health/live": { "get": { "tags": [ @@ -20562,6 +20622,23 @@ "type": "object", "title": "Availability" }, + "BackgroundWork": { + "properties": { + "pace": { + "type": "string", + "enum": [ + "normal", + "relaxed", + "light", + "minimal" + ], + "title": "Pace", + "default": "normal" + } + }, + "type": "object", + "title": "BackgroundWork" + }, "BatchInput": { "properties": { "expected_content_revision": { diff --git a/services/app/api/background_work.py b/services/app/api/background_work.py new file mode 100644 index 00000000..0f910539 --- /dev/null +++ b/services/app/api/background_work.py @@ -0,0 +1,40 @@ +"""Settings -> System -> Background work: how often scheduled jobs do their work.""" + +from typing import Literal + +from fastapi import APIRouter +from pydantic import BaseModel +from sqlalchemy.dialects.postgresql import insert + +from app.api.dependencies import Admin, Database +from app.db.models import AuditEvent, BackgroundWorkSettings +from app.jobs import pace + +router = APIRouter(prefix="/background-work", tags=["background work"]) + +Pace = Literal["normal", "relaxed", "light", "minimal"] + + +class BackgroundWork(BaseModel): + pace: Pace = "normal" + + +@router.get("", response_model=BackgroundWork) +async def background_work(admin: Admin, db: Database): + row = await db.get(BackgroundWorkSettings, 1) + return BackgroundWork(pace=row.pace if row else "normal") + + +@router.put("", response_model=BackgroundWork) +async def save_background_work(body: BackgroundWork, admin: Admin, db: Database): + await db.execute( + insert(BackgroundWorkSettings) + .values(id=1, pace=body.pace) + .on_conflict_do_update(index_elements=[BackgroundWorkSettings.id], set_={"pace": body.pace}) + ) + db.add( + AuditEvent(actor_id=admin.id, action="background_work.changed", detail=body.model_dump()) + ) + await db.commit() + pace.remember(body.pace) + return body diff --git a/services/app/db/migrations/versions/0081_operation_lookups.py b/services/app/db/migrations/versions/0081_operation_lookups.py new file mode 100644 index 00000000..558bc570 --- /dev/null +++ b/services/app/db/migrations/versions/0081_operation_lookups.py @@ -0,0 +1,33 @@ +""" +Indexes for the most frequent operation lookups. Background jobs and request pages look operations +up by kind and status, and by the request id inside the command payload; without these indexes, +each lookup read the whole table. The request counts went from about 4 minutes to under a second. +""" + +import sqlalchemy as sa +from alembic import op + +revision = "0081_operation_lookups" +down_revision = "0080_member_setup" +branch_labels = depends_on = None + + +def upgrade(): + op.create_index( + "ix_operations_kind_status_created", + "operations", + ["kind", "status", "created_at"], + ) + op.create_index( + "ix_operations_command_intent", + "operations", + # Written with subscripts, exactly as the app's queries compile, or Postgres ignores it. Not + # a partial index: Postgres cannot tell that an equality on this value implies the key + # exists, so a partial index would never be used. + [sa.text("(payload['command'] ->> 'intent_id')"), "kind", "created_at"], + ) + + +def downgrade(): + op.drop_index("ix_operations_command_intent", table_name="operations") + op.drop_index("ix_operations_kind_status_created", table_name="operations") diff --git a/services/app/db/migrations/versions/0082_background_work.py b/services/app/db/migrations/versions/0082_background_work.py new file mode 100644 index 00000000..c2e03c2f --- /dev/null +++ b/services/app/db/migrations/versions/0082_background_work.py @@ -0,0 +1,28 @@ +""" +How often background jobs run. One install-wide row; "normal" keeps every job on its built-in +schedule, and the slower paces skip runs to save CPU and database work on small servers. +""" + +import sqlalchemy as sa +from alembic import op + +revision = "0082_background_work" +down_revision = "0081_operation_lookups" +branch_labels = depends_on = None + + +def upgrade(): + op.create_table( + "background_work_settings", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column("pace", sa.String(length=20), nullable=False, server_default="normal"), + sa.CheckConstraint("id = 1", name="background_work_settings_id_check"), + sa.CheckConstraint( + "pace IN ('normal', 'relaxed', 'light', 'minimal')", + name="background_work_settings_pace_check", + ), + ) + + +def downgrade(): + op.drop_table("background_work_settings") diff --git a/services/app/db/models.py b/services/app/db/models.py index e57cb4dc..8f273ba1 100644 --- a/services/app/db/models.py +++ b/services/app/db/models.py @@ -713,6 +713,15 @@ class Operation(Identity, Base): "created_at", postgresql_where=text("kind = 'lists.writeback'"), ), + # Background jobs look operations up by kind and status, and request pages by the + # request id inside the command payload. + Index("ix_operations_kind_status_created", "kind", "status", "created_at"), + Index( + "ix_operations_command_intent", + text("(payload['command'] ->> 'intent_id')"), + "kind", + "created_at", + ), ) owner_id: Mapped[UUID] = mapped_column(ForeignKey("users.id"), index=True) kind: Mapped[str] = mapped_column(String(60)) @@ -1213,6 +1222,19 @@ class OnboardingSettings(Base): ) +class BackgroundWorkSettings(Base): + __tablename__ = "background_work_settings" + __table_args__ = ( + CheckConstraint("id = 1", name="background_work_settings_id_check"), + CheckConstraint( + "pace IN ('normal', 'relaxed', 'light', 'minimal')", + name="background_work_settings_pace_check", + ), + ) + id: Mapped[int] = mapped_column(Integer, primary_key=True) + pace: Mapped[str] = mapped_column(String(20), default="normal", server_default="normal") + + class OrganizationSettings(Base): __tablename__ = "organization_settings" id: Mapped[int] = mapped_column(Integer, primary_key=True) diff --git a/services/app/db/session.py b/services/app/db/session.py index 11a8180c..e6ece7c0 100644 --- a/services/app/db/session.py +++ b/services/app/db/session.py @@ -20,7 +20,8 @@ def get_engine() -> AsyncEngine: pool_size=settings.db_pool_size, max_overflow=2, hide_parameters=True, - connect_args={"options": "-csearch_path=public,book_queue"}, + # JIT compiling the large status queries cost more than it saved (17 s on one count). + connect_args={"options": "-csearch_path=public,book_queue -cjit=off"}, ) diff --git a/services/app/jobs/pace.py b/services/app/jobs/pace.py new file mode 100644 index 00000000..498d50fe --- /dev/null +++ b/services/app/jobs/pace.py @@ -0,0 +1,43 @@ +""" +How often scheduled background jobs do their work, from Settings -> System -> Background work. + +Every scheduled job keeps its built-in schedule and receives the time it was scheduled for. At a +slower pace, a job skips the runs that are not due: at Light (x20), a job scheduled every minute +does its work every 20 minutes. The decision depends only on that time, so every worker agrees on +it without storing anything, and a new pace applies within a minute without a restart. +""" + +import time + +from app.db.models import BackgroundWorkSettings +from app.db.session import session_factory + +PACES = {"normal": 1, "relaxed": 5, "light": 20, "minimal": 50} +REFRESH_SECONDS = 60 +_cache: dict[str, float | str] = {"pace": "normal", "read_at": float("-inf")} + + +def remember(pace: str) -> None: + _cache.update(pace=pace, read_at=time.monotonic()) + + +async def current() -> str: + if time.monotonic() - float(_cache["read_at"]) >= REFRESH_SECONDS: + try: + async with session_factory()() as db: + row = await db.get(BackgroundWorkSettings, 1) + remember(row.pace if row and row.pace in PACES else "normal") + except Exception: + # Never stop scheduled work because the setting could not be read. + remember(str(_cache["pace"])) + return str(_cache["pace"]) + + +def is_due(timestamp: int, every_seconds: int, pace: str) -> bool: + factor = PACES.get(pace, 1) + return factor == 1 or (int(timestamp) // every_seconds) % factor == 0 + + +async def due(timestamp: int, every_seconds: int) -> bool: + """Whether a scheduled run at this time should do its work at the current pace.""" + return is_due(timestamp, every_seconds, await current()) diff --git a/services/app/jobs/tasks.py b/services/app/jobs/tasks.py index dff12a26..aa352a9b 100644 --- a/services/app/jobs/tasks.py +++ b/services/app/jobs/tasks.py @@ -8,6 +8,7 @@ from app.config import get_settings from app.db.models import AuditEvent, Integration, Operation, User from app.db.session import session_factory +from app.jobs import pace from app.jobs.queue import tasks from app.jobs.retry import ( CatalogRetryStrategy, @@ -58,6 +59,8 @@ async def confirm_batch(operation_ids: list[str]) -> None: @tasks.periodic(cron="* * * * *") @tasks.task(name="organization.confirm", queue="system", retry=3) async def schedule_import_confirmation(timestamp: int) -> None: + if not await pace.due(timestamp, 60): + return from sqlalchemy import text from app.db.models import ImportDestination, ImportEntry, Library @@ -279,6 +282,8 @@ async def evaluate_acquisition(operation_id: str) -> None: @tasks.periodic(cron="*/5 * * * *") @tasks.task(name="acquisition.reconcile", queue="acquisition", retry=3) async def reconcile_acquisition(timestamp: int) -> None: + if not await pace.due(timestamp, 300): + return from app.domain.acquisition import reconcile_requests await reconcile_requests() @@ -287,6 +292,8 @@ async def reconcile_acquisition(timestamp: int) -> None: @tasks.periodic(cron="*/5 * * * *") @tasks.task(name="library.schedule", queue="system", retry=3) async def schedule_inventory(timestamp: int) -> None: + if not await pace.due(timestamp, 300): + return if get_settings().recovery_mode: return from app.domain.operations import enqueue_sync @@ -342,6 +349,8 @@ async def reconcile_fulfillment(work_id: str) -> None: @tasks.periodic(cron="* * * * * */10") @tasks.task(name="acquisition.downloads.schedule", queue="acquisition", retry=3) async def schedule_downloads(timestamp: int) -> None: + if not await pace.due(timestamp, 10): + return from sqlalchemy import or_, text from app.db.models import AutomaticImport, AutomaticImportContinuation, DownloadAttempt @@ -448,6 +457,8 @@ async def compare_list_membership(operation_id: str) -> None: @tasks.periodic(cron="* * * * *") @tasks.task(name="lists.schedule", queue="lists", retry=3) async def schedule_shelves(timestamp: int) -> None: + if not await pace.due(timestamp, 60): + return from app.domain.list_subscriptions import schedule await schedule() @@ -481,6 +492,8 @@ async def select_best_release(operation_id: str) -> None: @tasks.periodic(cron="* * * * *") @tasks.task(name="lists.acquisition.schedule", queue="lists", retry=3) async def schedule_list_acquisition(timestamp: int) -> None: + if not await pace.due(timestamp, 60): + return from app.domain.list_automation import schedule await schedule() @@ -543,6 +556,8 @@ async def reuse_completed_transfer(continuation_id: str) -> None: @tasks.periodic(cron="*/10 * * * *") @tasks.task(name="releases.schedule", queue="metadata", retry=3) async def schedule_releases(timestamp: int) -> None: + if not await pace.due(timestamp, 600): + return from app.domain.libro_library import schedule as schedule_enrichment from app.domain.release_monitor import schedule as schedule_monitors @@ -553,6 +568,8 @@ async def schedule_releases(timestamp: int) -> None: @tasks.periodic(cron="*/10 * * * *") @tasks.task(name="discovery.schedule", queue="metadata", retry=3) async def schedule_discovery(timestamp: int) -> None: + if not await pace.due(timestamp, 600): + return from app.domain.discovery_catalog import schedule await schedule() @@ -568,6 +585,8 @@ async def refresh_discovery(user_id: str, collection_id: str, generation: int) - @tasks.periodic(cron="*/15 * * * *") @tasks.task(name="series.gap-schedule", queue="metadata", retry=3) async def schedule_series_gaps(timestamp: int) -> None: + if not await pace.due(timestamp, 900): + return from app.domain.series_gap_watch import schedule await schedule() @@ -590,6 +609,8 @@ async def quick_add_book(operation_id: str) -> None: @tasks.periodic(cron="* * * * *") @tasks.task(name="sources.mam.account", queue="system", retry=3) async def schedule_mam_account(timestamp: int) -> None: + if not await pace.due(timestamp, 60): + return if get_settings().recovery_mode: return from app.domain.account_automation import run @@ -600,6 +621,8 @@ async def schedule_mam_account(timestamp: int) -> None: @tasks.periodic(cron="* * * * *") @tasks.task(name="notifications.dispatch", queue="notifications", lock="notifications.dispatch") async def dispatch_notifications(timestamp: int = 0) -> None: + if not await pace.due(timestamp, 60): + return from app.notifications.delivery import tick await tick() @@ -622,6 +645,8 @@ async def reject_failed_import(attempt_id: str) -> None: @tasks.periodic(cron="*/5 * * * *") @tasks.task(name="connections.health", queue="system", lock="connections.health") async def check_connection_health(timestamp: int = 0) -> None: + if not await pace.due(timestamp, 300): + return from app.domain.connection_health import run await run() diff --git a/services/app/main.py b/services/app/main.py index 512eb4eb..f3ddcd7e 100644 --- a/services/app/main.py +++ b/services/app/main.py @@ -19,6 +19,7 @@ auth, automatic_imports, automatic_selection, + background_work, book_sources, capacity, catalog, @@ -113,6 +114,7 @@ def create_app() -> FastAPI: app = FastAPI(title="Dewarr", version="0.3.0", lifespan=lifespan) app.include_router(application_release.router, prefix="/api") app.include_router(notifications.router, prefix="/api") + app.include_router(background_work.router, prefix="/api") @app.exception_handler(RequestValidationError) async def invalid_request(request: Request, exception: RequestValidationError): diff --git a/services/app/state_bundle.py b/services/app/state_bundle.py index ead83b55..9d0b98aa 100644 --- a/services/app/state_bundle.py +++ b/services/app/state_bundle.py @@ -24,7 +24,7 @@ from app.db.models import Base from app.recovery import MAINTENANCE_LOCK -SCHEMA = "0080_member_setup" +SCHEMA = "0082_background_work" CONFIG_FIELDS = { "public_url", "cookie_secure", diff --git a/tests/integration/test_background_work.py b/tests/integration/test_background_work.py new file mode 100644 index 00000000..7a06674e --- /dev/null +++ b/tests/integration/test_background_work.py @@ -0,0 +1,95 @@ +from importlib import import_module + +import pytest +from alembic.migration import MigrationContext +from alembic.operations import Operations +from sqlalchemy import create_engine, select, text + +from app.config import get_settings +from app.db.models import AuditEvent +from app.jobs import pace + +pytestmark = pytest.mark.integration +INDEXES = {"ix_operations_kind_status_created", "ix_operations_command_intent"} + + +async def test_admins_choose_the_background_work_pace(client, admin, database): + assert (await client.get("/api/background-work")).json() == {"pace": "normal"} + saved = await client.put("/api/background-work", json={"pace": "light"}) + assert saved.status_code == 200, saved.text + assert saved.json() == {"pace": "light"} + assert (await client.get("/api/background-work")).json() == {"pace": "light"} + assert await pace.current() == "light" + assert (await client.put("/api/background-work", json={"pace": "turbo"})).status_code == 422 + async with database() as db: + actions = list(await db.scalars(select(AuditEvent.action))) + assert "background_work.changed" in actions + pace.remember("normal") + + +async def test_background_work_is_admin_only(client, admin): + created = await client.post( + "/api/auth/users", + json={ + "username": "pace-member", + "display_name": "Member", + "role": "member", + "password": "a separate long password", + }, + ) + assert created.status_code == 201 + login = await client.post( + "/api/auth/login", + json={"username": "pace-member", "password": "a separate long password"}, + ) + client.headers["X-CSRF-Token"] = login.json()["csrf_token"] + assert (await client.get("/api/background-work")).status_code == 403 + assert (await client.put("/api/background-work", json={"pace": "light"})).status_code == 403 + + +def indexes(connection): + return set( + connection.execute( + text("SELECT indexname FROM pg_indexes WHERE tablename = 'operations'") + ).scalars() + ) + + +def test_both_migrations_can_be_removed_and_restored(migrated_database): + lookups = import_module("app.db.migrations.versions.0081_operation_lookups") + background = import_module("app.db.migrations.versions.0082_background_work") + engine = create_engine(get_settings().database_url.get_secret_value()) + try: + with engine.connect() as connection, connection.begin(): + assert INDEXES <= indexes(connection) + with Operations.context(MigrationContext.configure(connection)): + background.downgrade() + assert ( + connection.execute( + text("SELECT to_regclass('public.background_work_settings')") + ).scalar() + is None + ) + lookups.downgrade() + assert not INDEXES & indexes(connection) + lookups.upgrade() + background.upgrade() + assert INDEXES <= indexes(connection) + assert connection.execute( + text("SELECT to_regclass('public.background_work_settings')") + ).scalar() + # The planner can use the request index for the expression the app's queries use. + connection.execute(text("SET LOCAL enable_seqscan = off")) + plan = "\n".join( + connection.execute( + text( + "EXPLAIN SELECT id FROM operations " + "WHERE payload['command'] ->> 'intent_id' = 'x' " + "AND kind = 'acquisition.auto-select'" + ) + ).scalars() + ) + assert "ix_operations_command_intent" in plan + connection.rollback() + finally: + engine.dispose() diff --git a/tests/unit/test_background_pace.py b/tests/unit/test_background_pace.py new file mode 100644 index 00000000..6cde563e --- /dev/null +++ b/tests/unit/test_background_pace.py @@ -0,0 +1,45 @@ +import pytest + +from app.jobs import pace, tasks + + +def test_normal_runs_every_scheduled_time(): + assert all(pace.is_due(minute * 60, 60, "normal") for minute in range(100)) + + +@pytest.mark.parametrize(("name", "factor"), [("relaxed", 5), ("light", 20), ("minimal", 50)]) +def test_slower_paces_run_one_scheduled_time_in_each_group(name, factor): + for every in (10, 60, 300): + runs = [step for step in range(factor * 4) if pace.is_due(step * every, every, name)] + assert runs == [0, factor, factor * 2, factor * 3] + + +def test_an_unknown_pace_falls_back_to_normal(): + assert pace.is_due(61, 60, "turbo") + + +async def test_a_job_that_is_not_due_skips_its_work(monkeypatch): + calls = [] + + async def reconcile_requests(): + calls.append("reconcile") + + monkeypatch.setattr("app.domain.acquisition.reconcile_requests", reconcile_requests) + pace.remember("light") + try: + await tasks.reconcile_acquisition.func(timestamp=300) # Run 1 of each 20: skipped. + assert calls == [] + await tasks.reconcile_acquisition.func(timestamp=0) # Run 0: due. + assert calls == ["reconcile"] + finally: + pace.remember("normal") + + +async def test_scheduled_work_continues_when_the_setting_cannot_be_read(monkeypatch): + def broken(): + raise RuntimeError("database unavailable") + + monkeypatch.setattr(pace, "session_factory", broken) + pace.remember("normal") + monkeypatch.setitem(pace._cache, "read_at", float("-inf")) + assert await pace.current() == "normal"