diff --git a/apps/web/src/api/schema.d.ts b/apps/web/src/api/schema.d.ts index 9976eab3..6439be70 100644 --- a/apps/web/src/api/schema.d.ts +++ b/apps/web/src/api/schema.d.ts @@ -141,6 +141,24 @@ export interface paths { patch?: never; trace?: never; }; + "/api/background-work": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Background Work */ + get: operations["background_work_api_background_work_get"]; + /** Save Background Work */ + put: operations["save_background_work_api_background_work_put"]; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/health/live": { parameters: { query?: never; @@ -6524,6 +6542,15 @@ export interface components { /** Parts Medium */ parts_medium?: string | null; }; + /** BackgroundWork */ + BackgroundWork: { + /** + * Pace + * @default normal + * @enum {string} + */ + pace: "normal" | "relaxed" | "light" | "minimal"; + }; /** BatchInput */ BatchInput: { /** Expected Content Revision */ @@ -15466,6 +15493,59 @@ export interface operations { }; }; }; + background_work_api_background_work_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["BackgroundWork"]; + }; + }; + }; + }; + save_background_work_api_background_work_put: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["BackgroundWork"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["BackgroundWork"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; liveness_api_health_live_get: { parameters: { query?: never; diff --git a/apps/web/src/pages/BackgroundWork.tsx b/apps/web/src/pages/BackgroundWork.tsx new file mode 100644 index 00000000..4e95c8d4 --- /dev/null +++ b/apps/web/src/pages/BackgroundWork.tsx @@ -0,0 +1,105 @@ +import { useState } from "react"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { api, result } from "../api/client"; +import type { components } from "../api/schema"; +import { Loading, Notice } from "../components"; + +type Pace = components["schemas"]["BackgroundWork"]["pace"]; + +const paces: { id: Pace; title: string; description: string }[] = [ + { + id: "normal", + title: "Normal", + description: "Every background job runs on its usual schedule.", + }, + { + id: "relaxed", + title: "Relaxed", + description: + "Background jobs run 5 times less often. Downloads, imports and list checks are noticed a little later.", + }, + { + id: "light", + title: "Light", + description: + "Background jobs run 20 times less often. Good for a small server that also does other work.", + }, + { + id: "minimal", + title: "Minimal", + description: + "Background jobs run 50 times less often. Finished downloads can take up to an hour to import.", + }, +]; + +export default function BackgroundWork() { + const cache = useQueryClient(); + const setting = useQuery({ + queryKey: ["background-work"], + queryFn: async () => result(await api.GET("/api/background-work")), + }); + const [draft, setDraft] = useState(null); + const pace = draft ?? setting.data?.pace ?? "normal"; + const save = useMutation({ + mutationFn: async (submitted: Pace) => + result( + await api.PUT("/api/background-work", { body: { pace: submitted } }), + ), + onSuccess: (saved, submitted) => { + setDraft((current) => (current === submitted ? null : current)); + cache.setQueryData(["background-work"], saved); + }, + }); + return ( +
+

Background work

+

+ Dewarr checks downloads, imports, lists and connections on a schedule. + On a small or shared server, a slower pace saves CPU and database work. + Searching and requesting books are not affected. +

+

+ New in this version. Normal keeps the schedule Dewarr has always used. +

+ {setting.isPending ? ( + + ) : setting.data ? ( +
{ + event.preventDefault(); + save.mutate(pace); + }} + > +
+ How often background jobs run + {paces.map((option) => ( + + ))} +
+ + {save.isSuccess && ( +

+ Background work saved. It applies within a minute. +

+ )} +
+ ) : null} + +
+ ); +} diff --git a/apps/web/src/pages/SettingsSections.tsx b/apps/web/src/pages/SettingsSections.tsx index 58a84b9f..cd51f69d 100644 --- a/apps/web/src/pages/SettingsSections.tsx +++ b/apps/web/src/pages/SettingsSections.tsx @@ -1,3 +1,4 @@ +import BackgroundWork from "./BackgroundWork"; import { lazy } from "react"; import SettingsGroup from "../components/SettingsGroup"; @@ -102,6 +103,15 @@ export function settingsSections(role: string, permissions: string[] = []) { title: "Notifications", content: , }, + ...(admin + ? [ + { + id: "background-work", + title: "Background work", + content: , + }, + ] + : []), { id: "logs", title: "Logs", content: }, ]; } diff --git a/apps/web/tests/ui/background-work.spec.ts b/apps/web/tests/ui/background-work.spec.ts new file mode 100644 index 00000000..838ac533 --- /dev/null +++ b/apps/web/tests/ui/background-work.spec.ts @@ -0,0 +1,47 @@ +import { expect, test } from "../fixtures"; + +test("admins choose how often background jobs run", async ({ page }) => { + const saved: unknown[] = []; + let pace = "normal"; + await page.route("**/api/**", async (route) => { + const request = route.request(); + const path = new URL(request.url()).pathname; + let data: unknown = { items: [], total: 0, offset: 0, limit: 40 }; + if (path === "/api/auth/me") + data = { + user: { + id: "admin", + role: "admin", + username: "admin", + display_name: "Admin", + }, + csrf_token: "test", + }; + else if (path === "/api/setup/onboarding") data = { status: "completed" }; + else if (path === "/api/library/libraries") data = []; + else if (path === "/api/background-work") { + if (request.method() === "PUT") { + saved.push(request.postDataJSON()); + pace = (request.postDataJSON() as { pace: string }).pace; + } + data = { pace }; + } + return route.fulfill({ json: data }); + }); + await page.goto("/settings#background-work"); + const section = page.getByRole("region", { name: "Background work" }); + await expect(section.getByText("New in this version.")).toBeVisible(); + await expect(section.getByRole("radio", { name: /^Normal/ })).toBeChecked(); + await section.getByRole("radio", { name: /^Light/ }).check(); + await section.getByRole("button", { name: "Save background work" }).click(); + await expect(section.getByRole("status")).toHaveText( + "Background work saved. It applies within a minute.", + ); + expect(saved).toEqual([{ pace: "light" }]); + await page.reload(); + await expect( + page + .getByRole("region", { name: "Background work" }) + .getByRole("radio", { name: /^Light/ }), + ).toBeChecked(); +}); diff --git a/docs/openapi.json b/docs/openapi.json index fca1ef4c..88a1a149 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -321,6 +321,66 @@ } } }, + "/api/background-work": { + "get": { + "tags": [ + "background work" + ], + "summary": "Background Work", + "operationId": "background_work_api_background_work_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BackgroundWork" + } + } + } + } + } + }, + "put": { + "tags": [ + "background work" + ], + "summary": "Save Background Work", + "operationId": "save_background_work_api_background_work_put", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BackgroundWork" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BackgroundWork" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/api/health/live": { "get": { "tags": [ @@ -20562,6 +20622,23 @@ "type": "object", "title": "Availability" }, + "BackgroundWork": { + "properties": { + "pace": { + "type": "string", + "enum": [ + "normal", + "relaxed", + "light", + "minimal" + ], + "title": "Pace", + "default": "normal" + } + }, + "type": "object", + "title": "BackgroundWork" + }, "BatchInput": { "properties": { "expected_content_revision": { diff --git a/services/app/api/background_work.py b/services/app/api/background_work.py new file mode 100644 index 00000000..0f910539 --- /dev/null +++ b/services/app/api/background_work.py @@ -0,0 +1,40 @@ +"""Settings -> System -> Background work: how often scheduled jobs do their work.""" + +from typing import Literal + +from fastapi import APIRouter +from pydantic import BaseModel +from sqlalchemy.dialects.postgresql import insert + +from app.api.dependencies import Admin, Database +from app.db.models import AuditEvent, BackgroundWorkSettings +from app.jobs import pace + +router = APIRouter(prefix="/background-work", tags=["background work"]) + +Pace = Literal["normal", "relaxed", "light", "minimal"] + + +class BackgroundWork(BaseModel): + pace: Pace = "normal" + + +@router.get("", response_model=BackgroundWork) +async def background_work(admin: Admin, db: Database): + row = await db.get(BackgroundWorkSettings, 1) + return BackgroundWork(pace=row.pace if row else "normal") + + +@router.put("", response_model=BackgroundWork) +async def save_background_work(body: BackgroundWork, admin: Admin, db: Database): + await db.execute( + insert(BackgroundWorkSettings) + .values(id=1, pace=body.pace) + .on_conflict_do_update(index_elements=[BackgroundWorkSettings.id], set_={"pace": body.pace}) + ) + db.add( + AuditEvent(actor_id=admin.id, action="background_work.changed", detail=body.model_dump()) + ) + await db.commit() + pace.remember(body.pace) + return body diff --git a/services/app/db/migrations/versions/0081_operation_lookups.py b/services/app/db/migrations/versions/0081_operation_lookups.py new file mode 100644 index 00000000..558bc570 --- /dev/null +++ b/services/app/db/migrations/versions/0081_operation_lookups.py @@ -0,0 +1,33 @@ +""" +Indexes for the most frequent operation lookups. Background jobs and request pages look operations +up by kind and status, and by the request id inside the command payload; without these indexes, +each lookup read the whole table. The request counts went from about 4 minutes to under a second. +""" + +import sqlalchemy as sa +from alembic import op + +revision = "0081_operation_lookups" +down_revision = "0080_member_setup" +branch_labels = depends_on = None + + +def upgrade(): + op.create_index( + "ix_operations_kind_status_created", + "operations", + ["kind", "status", "created_at"], + ) + op.create_index( + "ix_operations_command_intent", + "operations", + # Written with subscripts, exactly as the app's queries compile, or Postgres ignores it. Not + # a partial index: Postgres cannot tell that an equality on this value implies the key + # exists, so a partial index would never be used. + [sa.text("(payload['command'] ->> 'intent_id')"), "kind", "created_at"], + ) + + +def downgrade(): + op.drop_index("ix_operations_command_intent", table_name="operations") + op.drop_index("ix_operations_kind_status_created", table_name="operations") diff --git a/services/app/db/migrations/versions/0082_background_work.py b/services/app/db/migrations/versions/0082_background_work.py new file mode 100644 index 00000000..c2e03c2f --- /dev/null +++ b/services/app/db/migrations/versions/0082_background_work.py @@ -0,0 +1,28 @@ +""" +How often background jobs run. One install-wide row; "normal" keeps every job on its built-in +schedule, and the slower paces skip runs to save CPU and database work on small servers. +""" + +import sqlalchemy as sa +from alembic import op + +revision = "0082_background_work" +down_revision = "0081_operation_lookups" +branch_labels = depends_on = None + + +def upgrade(): + op.create_table( + "background_work_settings", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column("pace", sa.String(length=20), nullable=False, server_default="normal"), + sa.CheckConstraint("id = 1", name="background_work_settings_id_check"), + sa.CheckConstraint( + "pace IN ('normal', 'relaxed', 'light', 'minimal')", + name="background_work_settings_pace_check", + ), + ) + + +def downgrade(): + op.drop_table("background_work_settings") diff --git a/services/app/db/models.py b/services/app/db/models.py index e57cb4dc..8f273ba1 100644 --- a/services/app/db/models.py +++ b/services/app/db/models.py @@ -713,6 +713,15 @@ class Operation(Identity, Base): "created_at", postgresql_where=text("kind = 'lists.writeback'"), ), + # Background jobs look operations up by kind and status, and request pages by the + # request id inside the command payload. + Index("ix_operations_kind_status_created", "kind", "status", "created_at"), + Index( + "ix_operations_command_intent", + text("(payload['command'] ->> 'intent_id')"), + "kind", + "created_at", + ), ) owner_id: Mapped[UUID] = mapped_column(ForeignKey("users.id"), index=True) kind: Mapped[str] = mapped_column(String(60)) @@ -1213,6 +1222,19 @@ class OnboardingSettings(Base): ) +class BackgroundWorkSettings(Base): + __tablename__ = "background_work_settings" + __table_args__ = ( + CheckConstraint("id = 1", name="background_work_settings_id_check"), + CheckConstraint( + "pace IN ('normal', 'relaxed', 'light', 'minimal')", + name="background_work_settings_pace_check", + ), + ) + id: Mapped[int] = mapped_column(Integer, primary_key=True) + pace: Mapped[str] = mapped_column(String(20), default="normal", server_default="normal") + + class OrganizationSettings(Base): __tablename__ = "organization_settings" id: Mapped[int] = mapped_column(Integer, primary_key=True) diff --git a/services/app/db/session.py b/services/app/db/session.py index 11a8180c..e6ece7c0 100644 --- a/services/app/db/session.py +++ b/services/app/db/session.py @@ -20,7 +20,8 @@ def get_engine() -> AsyncEngine: pool_size=settings.db_pool_size, max_overflow=2, hide_parameters=True, - connect_args={"options": "-csearch_path=public,book_queue"}, + # JIT compiling the large status queries cost more than it saved (17 s on one count). + connect_args={"options": "-csearch_path=public,book_queue -cjit=off"}, ) diff --git a/services/app/disconnect.py b/services/app/disconnect.py new file mode 100644 index 00000000..e6985205 --- /dev/null +++ b/services/app/disconnect.py @@ -0,0 +1,64 @@ +"""Stop a page's database work when the browser leaves before the answer arrives.""" + +import asyncio + +# Pages whose reads only query the database. Reads that call outside services (searches on MAM, +# Prowlarr or Hardcover, for example) are never cancelled: cancelling one part-way can leave that +# service marked busy for the next request. +DATABASE_READS = ("/api/requests", "/api/following", "/api/library") + + +class CancelOnDisconnect: + """ + For reads (GET and HEAD) on the pages in DATABASE_READS, cancel the request when the browser + disconnects, for example after moving to another page. Cancelling the task makes the + database driver cancel its running query, so an abandoned page no longer keeps a database + connection busy. Writes always run to the end, so a change is never left half done. Once the + answer has started, the request also runs to the end: servers report a disconnect as soon as + a response is complete, and the clean-up after it (closing the database session, background + tasks) must still happen. + """ + + def __init__(self, app): + self.app = app + + async def __call__(self, scope, receive, send): + if ( + scope["type"] != "http" + or scope["method"] not in {"GET", "HEAD"} + or not any( + scope["path"] == prefix or scope["path"].startswith(prefix + "/") + for prefix in DATABASE_READS + ) + ): + await self.app(scope, receive, send) + return + messages: asyncio.Queue = asyncio.Queue() + gone = asyncio.Event() + answered = asyncio.Event() + + async def listen(): + while True: + message = await receive() + await messages.put(message) + if message["type"] == "http.disconnect": + if not answered.is_set(): + gone.set() + handler.cancel() + return + + async def answer(message): + if message["type"] == "http.response.start": + answered.set() + await send(message) + + handler = asyncio.ensure_future(self.app(scope, messages.get, answer)) + listener = asyncio.ensure_future(listen()) + try: + await handler + except asyncio.CancelledError: + if not gone.is_set(): + raise + # The browser has gone, so there is no one to answer. + finally: + listener.cancel() diff --git a/services/app/jobs/pace.py b/services/app/jobs/pace.py new file mode 100644 index 00000000..498d50fe --- /dev/null +++ b/services/app/jobs/pace.py @@ -0,0 +1,43 @@ +""" +How often scheduled background jobs do their work, from Settings -> System -> Background work. + +Every scheduled job keeps its built-in schedule and receives the time it was scheduled for. At a +slower pace, a job skips the runs that are not due: at Light (x20), a job scheduled every minute +does its work every 20 minutes. The decision depends only on that time, so every worker agrees on +it without storing anything, and a new pace applies within a minute without a restart. +""" + +import time + +from app.db.models import BackgroundWorkSettings +from app.db.session import session_factory + +PACES = {"normal": 1, "relaxed": 5, "light": 20, "minimal": 50} +REFRESH_SECONDS = 60 +_cache: dict[str, float | str] = {"pace": "normal", "read_at": float("-inf")} + + +def remember(pace: str) -> None: + _cache.update(pace=pace, read_at=time.monotonic()) + + +async def current() -> str: + if time.monotonic() - float(_cache["read_at"]) >= REFRESH_SECONDS: + try: + async with session_factory()() as db: + row = await db.get(BackgroundWorkSettings, 1) + remember(row.pace if row and row.pace in PACES else "normal") + except Exception: + # Never stop scheduled work because the setting could not be read. + remember(str(_cache["pace"])) + return str(_cache["pace"]) + + +def is_due(timestamp: int, every_seconds: int, pace: str) -> bool: + factor = PACES.get(pace, 1) + return factor == 1 or (int(timestamp) // every_seconds) % factor == 0 + + +async def due(timestamp: int, every_seconds: int) -> bool: + """Whether a scheduled run at this time should do its work at the current pace.""" + return is_due(timestamp, every_seconds, await current()) diff --git a/services/app/jobs/tasks.py b/services/app/jobs/tasks.py index dff12a26..aa352a9b 100644 --- a/services/app/jobs/tasks.py +++ b/services/app/jobs/tasks.py @@ -8,6 +8,7 @@ from app.config import get_settings from app.db.models import AuditEvent, Integration, Operation, User from app.db.session import session_factory +from app.jobs import pace from app.jobs.queue import tasks from app.jobs.retry import ( CatalogRetryStrategy, @@ -58,6 +59,8 @@ async def confirm_batch(operation_ids: list[str]) -> None: @tasks.periodic(cron="* * * * *") @tasks.task(name="organization.confirm", queue="system", retry=3) async def schedule_import_confirmation(timestamp: int) -> None: + if not await pace.due(timestamp, 60): + return from sqlalchemy import text from app.db.models import ImportDestination, ImportEntry, Library @@ -279,6 +282,8 @@ async def evaluate_acquisition(operation_id: str) -> None: @tasks.periodic(cron="*/5 * * * *") @tasks.task(name="acquisition.reconcile", queue="acquisition", retry=3) async def reconcile_acquisition(timestamp: int) -> None: + if not await pace.due(timestamp, 300): + return from app.domain.acquisition import reconcile_requests await reconcile_requests() @@ -287,6 +292,8 @@ async def reconcile_acquisition(timestamp: int) -> None: @tasks.periodic(cron="*/5 * * * *") @tasks.task(name="library.schedule", queue="system", retry=3) async def schedule_inventory(timestamp: int) -> None: + if not await pace.due(timestamp, 300): + return if get_settings().recovery_mode: return from app.domain.operations import enqueue_sync @@ -342,6 +349,8 @@ async def reconcile_fulfillment(work_id: str) -> None: @tasks.periodic(cron="* * * * * */10") @tasks.task(name="acquisition.downloads.schedule", queue="acquisition", retry=3) async def schedule_downloads(timestamp: int) -> None: + if not await pace.due(timestamp, 10): + return from sqlalchemy import or_, text from app.db.models import AutomaticImport, AutomaticImportContinuation, DownloadAttempt @@ -448,6 +457,8 @@ async def compare_list_membership(operation_id: str) -> None: @tasks.periodic(cron="* * * * *") @tasks.task(name="lists.schedule", queue="lists", retry=3) async def schedule_shelves(timestamp: int) -> None: + if not await pace.due(timestamp, 60): + return from app.domain.list_subscriptions import schedule await schedule() @@ -481,6 +492,8 @@ async def select_best_release(operation_id: str) -> None: @tasks.periodic(cron="* * * * *") @tasks.task(name="lists.acquisition.schedule", queue="lists", retry=3) async def schedule_list_acquisition(timestamp: int) -> None: + if not await pace.due(timestamp, 60): + return from app.domain.list_automation import schedule await schedule() @@ -543,6 +556,8 @@ async def reuse_completed_transfer(continuation_id: str) -> None: @tasks.periodic(cron="*/10 * * * *") @tasks.task(name="releases.schedule", queue="metadata", retry=3) async def schedule_releases(timestamp: int) -> None: + if not await pace.due(timestamp, 600): + return from app.domain.libro_library import schedule as schedule_enrichment from app.domain.release_monitor import schedule as schedule_monitors @@ -553,6 +568,8 @@ async def schedule_releases(timestamp: int) -> None: @tasks.periodic(cron="*/10 * * * *") @tasks.task(name="discovery.schedule", queue="metadata", retry=3) async def schedule_discovery(timestamp: int) -> None: + if not await pace.due(timestamp, 600): + return from app.domain.discovery_catalog import schedule await schedule() @@ -568,6 +585,8 @@ async def refresh_discovery(user_id: str, collection_id: str, generation: int) - @tasks.periodic(cron="*/15 * * * *") @tasks.task(name="series.gap-schedule", queue="metadata", retry=3) async def schedule_series_gaps(timestamp: int) -> None: + if not await pace.due(timestamp, 900): + return from app.domain.series_gap_watch import schedule await schedule() @@ -590,6 +609,8 @@ async def quick_add_book(operation_id: str) -> None: @tasks.periodic(cron="* * * * *") @tasks.task(name="sources.mam.account", queue="system", retry=3) async def schedule_mam_account(timestamp: int) -> None: + if not await pace.due(timestamp, 60): + return if get_settings().recovery_mode: return from app.domain.account_automation import run @@ -600,6 +621,8 @@ async def schedule_mam_account(timestamp: int) -> None: @tasks.periodic(cron="* * * * *") @tasks.task(name="notifications.dispatch", queue="notifications", lock="notifications.dispatch") async def dispatch_notifications(timestamp: int = 0) -> None: + if not await pace.due(timestamp, 60): + return from app.notifications.delivery import tick await tick() @@ -622,6 +645,8 @@ async def reject_failed_import(attempt_id: str) -> None: @tasks.periodic(cron="*/5 * * * *") @tasks.task(name="connections.health", queue="system", lock="connections.health") async def check_connection_health(timestamp: int = 0) -> None: + if not await pace.due(timestamp, 300): + return from app.domain.connection_health import run await run() diff --git a/services/app/main.py b/services/app/main.py index ec2179bf..f3ddcd7e 100644 --- a/services/app/main.py +++ b/services/app/main.py @@ -19,6 +19,7 @@ auth, automatic_imports, automatic_selection, + background_work, book_sources, capacity, catalog, @@ -76,6 +77,7 @@ from app.config import get_settings from app.db.session import get_engine, session_factory from app.diagnostics import auth_configuration +from app.disconnect import CancelOnDisconnect from app.jobs.queue import get_queue from app.recovery import active_restore, restore_pending, runtime_lease @@ -112,6 +114,7 @@ def create_app() -> FastAPI: app = FastAPI(title="Dewarr", version="0.3.0", lifespan=lifespan) app.include_router(application_release.router, prefix="/api") app.include_router(notifications.router, prefix="/api") + app.include_router(background_work.router, prefix="/api") @app.exception_handler(RequestValidationError) async def invalid_request(request: Request, exception: RequestValidationError): @@ -234,6 +237,8 @@ async def frontend(path: str): return JSONResponse({"detail": "Not found"}, 404) return FileResponse(dist / "index.html") + # Added last so it is the outermost layer and sees the browser's disconnect first. + app.add_middleware(CancelOnDisconnect) return app diff --git a/services/app/state_bundle.py b/services/app/state_bundle.py index ead83b55..9d0b98aa 100644 --- a/services/app/state_bundle.py +++ b/services/app/state_bundle.py @@ -24,7 +24,7 @@ from app.db.models import Base from app.recovery import MAINTENANCE_LOCK -SCHEMA = "0080_member_setup" +SCHEMA = "0082_background_work" CONFIG_FIELDS = { "public_url", "cookie_secure", diff --git a/tests/integration/test_background_work.py b/tests/integration/test_background_work.py new file mode 100644 index 00000000..7a06674e --- /dev/null +++ b/tests/integration/test_background_work.py @@ -0,0 +1,95 @@ +from importlib import import_module + +import pytest +from alembic.migration import MigrationContext +from alembic.operations import Operations +from sqlalchemy import create_engine, select, text + +from app.config import get_settings +from app.db.models import AuditEvent +from app.jobs import pace + +pytestmark = pytest.mark.integration +INDEXES = {"ix_operations_kind_status_created", "ix_operations_command_intent"} + + +async def test_admins_choose_the_background_work_pace(client, admin, database): + assert (await client.get("/api/background-work")).json() == {"pace": "normal"} + saved = await client.put("/api/background-work", json={"pace": "light"}) + assert saved.status_code == 200, saved.text + assert saved.json() == {"pace": "light"} + assert (await client.get("/api/background-work")).json() == {"pace": "light"} + assert await pace.current() == "light" + assert (await client.put("/api/background-work", json={"pace": "turbo"})).status_code == 422 + async with database() as db: + actions = list(await db.scalars(select(AuditEvent.action))) + assert "background_work.changed" in actions + pace.remember("normal") + + +async def test_background_work_is_admin_only(client, admin): + created = await client.post( + "/api/auth/users", + json={ + "username": "pace-member", + "display_name": "Member", + "role": "member", + "password": "a separate long password", + }, + ) + assert created.status_code == 201 + login = await client.post( + "/api/auth/login", + json={"username": "pace-member", "password": "a separate long password"}, + ) + client.headers["X-CSRF-Token"] = login.json()["csrf_token"] + assert (await client.get("/api/background-work")).status_code == 403 + assert (await client.put("/api/background-work", json={"pace": "light"})).status_code == 403 + + +def indexes(connection): + return set( + connection.execute( + text("SELECT indexname FROM pg_indexes WHERE tablename = 'operations'") + ).scalars() + ) + + +def test_both_migrations_can_be_removed_and_restored(migrated_database): + lookups = import_module("app.db.migrations.versions.0081_operation_lookups") + background = import_module("app.db.migrations.versions.0082_background_work") + engine = create_engine(get_settings().database_url.get_secret_value()) + try: + with engine.connect() as connection, connection.begin(): + assert INDEXES <= indexes(connection) + with Operations.context(MigrationContext.configure(connection)): + background.downgrade() + assert ( + connection.execute( + text("SELECT to_regclass('public.background_work_settings')") + ).scalar() + is None + ) + lookups.downgrade() + assert not INDEXES & indexes(connection) + lookups.upgrade() + background.upgrade() + assert INDEXES <= indexes(connection) + assert connection.execute( + text("SELECT to_regclass('public.background_work_settings')") + ).scalar() + # The planner can use the request index for the expression the app's queries use. + connection.execute(text("SET LOCAL enable_seqscan = off")) + plan = "\n".join( + connection.execute( + text( + "EXPLAIN SELECT id FROM operations " + "WHERE payload['command'] ->> 'intent_id' = 'x' " + "AND kind = 'acquisition.auto-select'" + ) + ).scalars() + ) + assert "ix_operations_command_intent" in plan + connection.rollback() + finally: + engine.dispose() diff --git a/tests/unit/test_background_pace.py b/tests/unit/test_background_pace.py new file mode 100644 index 00000000..6cde563e --- /dev/null +++ b/tests/unit/test_background_pace.py @@ -0,0 +1,45 @@ +import pytest + +from app.jobs import pace, tasks + + +def test_normal_runs_every_scheduled_time(): + assert all(pace.is_due(minute * 60, 60, "normal") for minute in range(100)) + + +@pytest.mark.parametrize(("name", "factor"), [("relaxed", 5), ("light", 20), ("minimal", 50)]) +def test_slower_paces_run_one_scheduled_time_in_each_group(name, factor): + for every in (10, 60, 300): + runs = [step for step in range(factor * 4) if pace.is_due(step * every, every, name)] + assert runs == [0, factor, factor * 2, factor * 3] + + +def test_an_unknown_pace_falls_back_to_normal(): + assert pace.is_due(61, 60, "turbo") + + +async def test_a_job_that_is_not_due_skips_its_work(monkeypatch): + calls = [] + + async def reconcile_requests(): + calls.append("reconcile") + + monkeypatch.setattr("app.domain.acquisition.reconcile_requests", reconcile_requests) + pace.remember("light") + try: + await tasks.reconcile_acquisition.func(timestamp=300) # Run 1 of each 20: skipped. + assert calls == [] + await tasks.reconcile_acquisition.func(timestamp=0) # Run 0: due. + assert calls == ["reconcile"] + finally: + pace.remember("normal") + + +async def test_scheduled_work_continues_when_the_setting_cannot_be_read(monkeypatch): + def broken(): + raise RuntimeError("database unavailable") + + monkeypatch.setattr(pace, "session_factory", broken) + pace.remember("normal") + monkeypatch.setitem(pace._cache, "read_at", float("-inf")) + assert await pace.current() == "normal" diff --git a/tests/unit/test_disconnect.py b/tests/unit/test_disconnect.py new file mode 100644 index 00000000..779c3cf6 --- /dev/null +++ b/tests/unit/test_disconnect.py @@ -0,0 +1,79 @@ +import asyncio + +from app.disconnect import CancelOnDisconnect + + +def scope(method="GET", path="/api/requests"): + return {"type": "http", "method": method, "path": path} + + +async def run(method, path="/api/requests"): + state = {"finished": False, "cancelled": False} + + async def slow(scope, receive, send): + await receive() + try: + await asyncio.sleep(5) + state["finished"] = True + except asyncio.CancelledError: + state["cancelled"] = True + raise + + sent = [{"type": "http.request", "body": b"", "more_body": False}] + + async def receive(): + if sent: + return sent.pop() + await asyncio.sleep(0.05) # The browser leaves shortly after asking. + return {"type": "http.disconnect"} + + async def send(message): + pass + + await asyncio.wait_for(CancelOnDisconnect(slow)(scope(method, path), receive, send), 1) + return state + + +async def test_a_page_read_stops_when_the_browser_leaves(): + assert await run("GET") == {"finished": False, "cancelled": True} + + +async def test_writes_and_non_api_requests_are_left_alone(): + for method, path in ( + ("POST", "/api/requests"), + ("GET", "/assets/app.js"), + ("GET", "/api/books/1/sources"), # Searches outside services. + ("GET", "/api/requestsx"), + ): + try: + await asyncio.wait_for(run(method, path), 0.2) + except TimeoutError: + continue # Still running after the browser left: not cancelled. + raise AssertionError(f"{method} {path} was cancelled") + + +async def test_clean_up_after_the_answer_is_not_cancelled(): + state = {"cleaned_up": False} + + async def answers_then_cleans_up(scope, receive, send): + await receive() + await send({"type": "http.response.start", "status": 200, "headers": []}) + await send({"type": "http.response.body", "body": b"", "more_body": False}) + await asyncio.sleep(0.1) # Closing the database session after the answer. + state["cleaned_up"] = True + + sent = [{"type": "http.request", "body": b"", "more_body": False}] + complete = asyncio.Event() + + async def receive(): + if sent: + return sent.pop() + await complete.wait() + return {"type": "http.disconnect"} # Servers report this once the answer is complete. + + async def send(message): + if message["type"] == "http.response.body" and not message.get("more_body"): + complete.set() + + await asyncio.wait_for(CancelOnDisconnect(answers_then_cleans_up)(scope(), receive, send), 1) + assert state["cleaned_up"]