From 7ab8f319d7e4154c1729747860a04de83d25a923 Mon Sep 17 00:00:00 2001 From: InstaZDLL Date: Sun, 25 Oct 2026 02:09:27 +0200 Subject: [PATCH 01/13] docs: RFC-021 proposes the library lifecycle, exclusions, links, missing tracks and watching Signed-off-by: InstaZDLL --- .../RFC-021-library-lifecycle-and-scan.md | 341 ++++++++++++++++++ 1 file changed, 341 insertions(+) create mode 100644 docs/rfcs/RFC-021-library-lifecycle-and-scan.md diff --git a/docs/rfcs/RFC-021-library-lifecycle-and-scan.md b/docs/rfcs/RFC-021-library-lifecycle-and-scan.md new file mode 100644 index 00000000..685dd99d --- /dev/null +++ b/docs/rfcs/RFC-021-library-lifecycle-and-scan.md @@ -0,0 +1,341 @@ +# RFC-021 — Libraries and scanning: exclusions, links, missing tracks, the library lifecycle and watching + +- **Status**: Proposed +- **Implemented by**: in progress (this RFC and its implementation, in one + pull request). The *Status* field above never flips in this project; this + line is the one that says what is running. +- **Date**: 2026-10-25 +- **Authors**: the WaveFlow project +- **Depends on**: [RFC-002](RFC-002-waveflow-server-v2.md) (the scanner, track + identity, tenancy), [RFC-018](RFC-018-incremental-scan.md) (the incremental + scan and its tolerant walk, whose guarantees all hold here) + +## Problem + +A comparative audit of 2026-10-24 found five gaps around libraries and how they +are scanned. Each was confirmed in the code. + +1. **The walk excludes nothing.** `discover_audio` (`src/scanner.rs`) indexes + everything with an audio extension, so Synology's `@eaDir` thumbnails, the + `._` AppleDouble files a Mac leaves on a share, `.Trash-1000`, + `$RECYCLE.BIN` and every hidden folder are read as music. There is no way + for an operator to say "not this folder" (Navidrome has `.ndignore`). +2. **Symbolic links are dropped in silence.** The walk does not follow links + and does not say it skipped one, so a library built of links appears empty + and nothing explains why. +3. **A track whose file is gone is kept for ever and shown nowhere.** It goes + to `is_available = 0` and stays, holding its favourites, playlist entries + and plays, with no page listing it and no way to remove it. +4. **A library can be added and never changed.** Nothing renames one, moves + its root or deletes it: `src/` holds no `UPDATE library` touching the name + or the path and no `DELETE FROM library`, and the CLI has only `add`. +5. **Changes are seen at the next interval only.** The only trigger besides a + manual one is the periodic scan, six hours by default + (`WAVEFLOW_SCAN_INTERVAL_SECS`). + +## Decision 1 — what the walk leaves out + +**Built-in exclusions**, applied to every entry below the library root (never +to the root itself or to the folders above it), by name: + +- any name beginning with `.` — hidden files and folders, which covers + `.Trash-*`, `.AppleDouble`, `._*` and `.DS_Store` — **except + `.waveflow-managed` directly under the root**, the folder where received files + land (RFC-008), which the scan must keep finding; +- `@eaDir`, `#recycle`, `#snapshot` (Synology), `$RECYCLE.BIN`, + `System Volume Information` (Windows) and `lost+found`, compared without + regard to case. + +Built-in exclusions cannot be overridden: a dot-folder of music is rare, and a +rule that a file can switch off is a rule every library has to check. + +**An ignore file**, `.waveflowignore`, in any folder of the library. Its syntax +is `.gitignore`'s, read by the `ignore` crate's matcher, so the rules are the +ones every operator already knows: one pattern per line, `#` comments, a +leading `/` anchors to the file's folder, a trailing `/` matches folders only, +`**` crosses folders, and `!` re-includes. A file applies to its folder and +everything below it, and a deeper file's rules win over a shallower one's, as in +git. A folder that is excluded is not entered, so `!` cannot re-include +something under it, as in git. + +An **empty** `.waveflowignore` (nothing but white space) excludes the folder it +sits in and everything below it, as an empty `.ndignore` does in Navidrome. +That rule does not apply at the library root, where an empty file excludes +nothing: one stray empty file must not mark a whole library missing. + +The rules are matched against the path the walk sees, so a link (decision 2) +is excluded or not by its own name, whatever its target is called. + +**An ignore file that cannot be used**, because it cannot be read, is larger +than 64 KiB, is not a regular file, or holds a pattern the matcher rejects, +makes its folder count as **unreadable**, exactly as RFC-018 decision 3 defines +the word: it is not entered, it is reported in the scan's errors with its path +relative to the library, and the tracks under it are left as they are. The +scan cannot know what the operator meant to exclude, so it neither indexes the +folder nor marks it missing. At the library root the same failure **fails the +scan**, since then nothing at all can be told apart; that is the RFC-018 rule +for a root that cannot be opened. + +## Decision 2 — an excluded file is a file the walk did not find + +**An excluded file counts as not discovered**, and that is the whole of its +treatment. It is *not* one of the files RFC-018 decision 3 protects, because +the walk did look: it saw the file and was told to leave it out. So: + +- a track already in the catalogue whose file is now excluded **becomes + unavailable at the end of the scan, exactly like a deleted one**, with the + `delete` event the library feed already sends for that; +- it may be the source of a relocation like a deleted one: a file elsewhere + with the same content (or, failing that, the same relocation hint) takes its + identity, favourites and playlist entries included; +- removing the rule makes it available again on the next scan, under the same + identifier, as a file that comes back does today. + +Only an unreadable folder (decision 1) and an unreadable entry (RFC-018) keep +their tracks as they are. The scan job counts what was left out in a new +column, `excluded`: one per excluded entry, a folder counting once whatever it +holds. + +## Decision 3 — symbolic links, followed inside the library only + +**Following links is a per-library setting, `follow_links`, off by default**, +set by an administrator (API, CLI or the web client). Per library rather than +global because a link farm is a property of one tree, and because a server +holding several libraries should not change what all of them read for the sake +of one. + +**With it off**, the walk does not follow any link, as today, but it no longer +does so silently: each link it passes is **counted** in the scan job's new +`skipped_links` column and **logged** (at `info`, by its path relative to the +library, the first twenty of a scan one by one and the rest as a count). + +**With it on**, a link is followed only when its canonical target — every link +on the way resolved — **stays inside the library root**. Anything else is +skipped, counted and logged in the same way: + +- a link whose target lies outside the root; +- a broken link; +- a loop (a link to one of its own ancestors), which `walkdir` detects. + +The rule exists because a link is a way for anyone who can write into the +library folder to make the server read and serve any file the server can read. +Confinement keeps the library the boundary it is everywhere else in this +server (streaming already refuses any path that leaves the root). + +**A file reached through a link is indexed under its canonical path**, the path +of its target relative to the root, not under the link's name. Three things +follow, all deliberate: + +- **one file is one track** however many links name it, and a file reached both + directly and through a link is not indexed twice; +- **the stored path never contains a link**, so the streaming path check + (`secure_track_path`, which refuses any link component) is unchanged; +- **renaming a link moves nothing**: identity follows the file. + +The cost is that the disk tree (RFC-012) shows the targets' layout, not the +links'. The use this serves is a link farm whose files live inside the library, +for example in a store folder the walk does not enter directly: + +```text +/music ← library root, follow_links on +/music/.waveflowignore ← "/store/" +/music/store/… ← the files, excluded from the direct walk +/music/Artist/Album/01.flac → ../../store/ab/cd.flac +``` + +A link farm whose files live **outside** the root (a beets `link: yes` import +into a separate folder, say) is not followed. The operator either makes the +root a common ancestor of both, as above, or adds the files' folder as a +library of its own. + +## Decision 4 — missing tracks: a page, a purge, and an optional retention + +**`unavailable_since`**, a new nullable column on `track`, records when a track +stopped being available. It is set when a scan marks a track missing (excluded +or deleted), cleared when one comes back, and filled from `updated_at` for the +tracks already unavailable when the migration runs. + +**A view**, for the library's owners and managers and for administrators: +`GET /api/v2/libraries/{id}/unavailable-tracks` lists the unavailable tracks, +oldest first, each with its path, title, artist, album, when it went missing, +and **what purging it would cost**: how many playlist entries, favourites and +recorded plays hold it. The web client shows it on its own page, "Missing +files", beside "Needs attention". + +**A purge**, explicit: `POST /api/v2/libraries/{id}/unavailable-tracks/purge` +with either `{"track_ids": […]}` or `{"all": true}`, and never neither. Only +tracks that are **still** unavailable when the purge runs are removed: one that +came back between the listing and the click is kept. It is also offered by the +CLI (`library unavailable`, `library purge`). + +**What a purge loses**, stated here because none of it comes back: + +- the track rows, and everything that cascades from them: playlist entries, + favourites and ratings of the track, play history, queue entries, shares' + entries, bookmarks, lyrics (embedded, sidecar and uploaded), corrections + (`track_override`), credits, genre links, folder placements, canvas links and + quality measurements; +- the albums and artists that hold no track any more, with their favourites + and ratings; a pass of the scan's own consolidation runs in the same + transaction, so the catalogue is never left with an empty album; +- nothing on disk but what the server made: audio files are never touched, + covers nothing references any more are left to the existing artwork sweep, + and a canvas (RFC-009) whose last link went with these tracks is erased after + the commit, exactly as removing that canvas would erase it. + +A file that comes back after its track was purged is a new track with a new +identifier. + +The purge takes the writer gate and runs in **one transaction**: tracks, +favourites, search index rows and the consolidation land or fail together. It +**refuses with 409 while the library is being scanned** (it takes the +library's scan lock without waiting), because a running scan may be about to +relocate one of those tracks onto a file it just found. It is audited. + +The library feed is not written again: it already said `delete` for each of +these tracks when they went missing. The user sync journal is not written +either; a device holding a favourite or a playlist entry of a track that no +longer exists drops it with the track (see "What is still open"). + +**Retention, off by default**: `WAVEFLOW_UNAVAILABLE_RETENTION_DAYS` (unset or +`0`: never). When set, a background pass once an hour purges, through the same +code, the tracks unavailable for longer than that, in every library that is not +being scanned. Off by default because the purge is not reversible and a share +that stays unmounted for a week is not a deletion. + +## Decision 5 — the library lifecycle + +Three operations, each in the native API, the CLI and the web client, all +through one implementation in `src/services/`. + +| Operation | Who | API | +| --- | --- | --- | +| Rename | the library's owner, or an administrator | `PATCH /api/v2/libraries/{id}` `{"name": …}` | +| Change the path | an administrator | `PATCH /api/v2/libraries/{id}` `{"path": …}` | +| Follow links on or off | an administrator | `PATCH /api/v2/libraries/{id}` `{"follow_links": …}` | +| Delete | an administrator | `DELETE /api/v2/libraries/{id}` `{"confirm": ""}` | + +The path and the links setting are an administrator's for the reason creating a +library is: they decide which part of the server's file system the server +reads. A `PATCH` naming either needs an administrative credential, not only an +administrator's account; a rename needs a credential that may write. + +**Changing the path** validates the new one as creating a library does (an +existing directory, not itself a link, stored canonical, not another library's +root), then queues a scan with the new trigger `library_moved`. Tracks are +matched by their path relative to the root, so **a root moved with its tree +intact keeps every identifier**, favourite and playlist entry. What the new +root does not hold is marked missing, and relocation by content still applies +to what moved within it. Changing `follow_links` queues a scan too. + +**Deleting** removes the library from the catalogue and nothing from the disk. +Its cascade, all in one transaction under the writer gate: + +- every track, with everything decision 4 lists for a purge — for **every + member**, not only the one deleting; +- every album, artist and genre of the library, their favourites and ratings, + and their search index rows; +- the memberships, the scan history and its errors, the library feed and its + acknowledgements, folders (RFC-012), credit decisions, upload sessions; +- audited as `library.deleted`. + +Received files stay where they are: the server wrote them, but they are the +library's files now. The staging files of upload sessions still open are +removed, after the commit, and so are the canvases no other library still links +to; covers are left to the artwork sweep. A client polling the library's feed +gets 404 from then on, which is its signal to drop the library. + +**Confirmation is in the request's shape.** The body must carry `confirm`, equal +to the library's current name; anything else is 422 and deletes nothing. The +web client asks for the name to be typed; the CLI takes `--confirm `. + +All three refuse with 409 while the library is being scanned, for the reason +the purge does. Every refusal of an operation the caller may not perform on a +library they cannot see is a 404, as everywhere else. + +## Decision 6 — watching the file system + +**A watcher**, on the `notify` crate, observes every library root recursively +and triggers an **incremental scan of the affected library once its changes +have settled**: no event for `WAVEFLOW_WATCH_DELAY_SECS` (10 by default). A +stream of changes that never settles, a long copy for instance, still triggers +a scan once thirty delays have passed since its first event, and the next quiet +moment triggers another. The debouncing is the server's own, per library, on +top of the raw events: one crate rather than two, and a rule this RFC can state. + +- **The periodic scan stays**, as the safety net for everything a watcher can + miss. +- **`WAVEFLOW_WATCH`** (`true` by default) turns it off. +- **It never starts a scan while one runs for that library.** A library whose + changes settle during a scan stays pending and is scanned once that one ends, + so a change the running scan walked past is not lost. Scans keep everything + else they had: the per-library lock, the writer gate, concurrency between + libraries and never between writers. +- **Events that cannot be changes to the music are ignored**: reads and access + times (the scanner reads files, and would otherwise trigger itself for ever), + and anything under `.waveflow-managed`, which an upload writes and applies + itself. +- **A queue overflow** (the kernel dropping events) marks every library pending. +- **The watched set follows the libraries**: it is refreshed when a library is + added, moved or deleted through the server, and every five minutes for the + changes the CLI makes from another process. + +**A watch that cannot be established is not an error.** If the watcher cannot +start, or a root cannot be watched (a mount that does not support it, the +inotify limit `fs.inotify.max_user_watches` reached), that is logged once with +the library's identifier and the library is left to the periodic scan. Network +mounts are the known case: NFS and SMB report changes made by this machine and +usually not those made by another, so a library on a share should not count on +the watcher. `docs/operations.md` says so. + +## Decision 7 — what the scan job says + +The scan job gains two counters, `excluded` and `skipped_links` (decisions 2 +and 3), in the table, `GET /api/v2/scans/{id}` and both frames of the progress +stream, and two triggers, `watch` and `library_moved`. SQLite cannot alter the +`CHECK` on `scan_job.trigger`, so the table is rebuilt; migrations run with +foreign keys off, so the rows that reference it (`scan_error`, +`track.last_seen_scan_id`) are carried and checked by the boot's +`PRAGMA foreign_key_check`. + +## What does not change + +- Track identity, relocation, the incremental pre-check, ordered streaming and + RFC-018's rules for what the walk could not see. +- Audio files are read-only; nothing here writes to a library. +- Creating a library, its membership rules, and the Subsonic façade, whose + frozen contract has no library management. + +## The order of the work + +1. **One PR**, holding this RFC and: + - the migration (`library.follow_links`, `track.unavailable_since`, the + rebuilt `scan_job`); + - the exclusions, `.waveflowignore` and link handling in the walk; + - the missing-tracks listing and purge, and the retention pass; + - rename, path change, links setting and deletion, in the services, the + API, the CLI and the web client; + - the watcher and its two settings; + - tests for each rule, among them: an excluded track becomes unavailable + and an unreadable ignore file keeps its tracks; a link outside the root is + skipped and counted; a purge removes only what is still unavailable and + leaves no empty album; a moved root keeps identities; a deletion leaves no + row behind; a watched change triggers a scan, and none starts while one + runs; + - `docs/operations.md`. + +## What is still open + +- **The user sync journal on a purge or a deletion.** A favourite or a + playlist entry removed by the cascade is not written to the journal of the + account that held it, so a device keeps its copy until that playlist or + favourite next changes. The library feed's `delete` for the track is the + signal a client has today. +- **Following links outside the root**, to an allowlist of further roots, for + link farms that live elsewhere. +- **Undoing a purge**, for instance by keeping purged rows for a grace period. +- **A per-library watcher switch**, for a library on a share where watching + costs and finds nothing. +- **Folder-level skipping** (RFC-018) and watching would combine well: a + watcher that names the changed folder could let a scan walk only that + folder. Not done; every watched change is a full incremental walk. From 8b60695e26a3447e786a9cb5a5a20ad5cd6daef6 Mon Sep 17 00:00:00 2001 From: InstaZDLL Date: Sun, 25 Oct 2026 02:09:27 +0200 Subject: [PATCH 02/13] =?UTF-8?q?feat:=20libraries=20and=20scanning=20?= =?UTF-8?q?=E2=80=94=20exclusions,=20links,=20missing=20tracks,=20lifecycl?= =?UTF-8?q?e=20and=20watching=20(RFC-021)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: InstaZDLL --- .env.example | 8 + Cargo.lock | 208 +++- Cargo.toml | 9 + docs/api-v2-guide.md | 21 +- docs/operations.md | 120 +++ .../20261026010000_library_lifecycle.sql | 60 ++ src/api/libraries.rs | 120 +++ src/api/mod.rs | 16 + src/catalog.rs | 78 +- src/cli.rs | 285 ++++++ src/config.rs | 39 + src/lib.rs | 15 + src/main.rs | 10 + src/scanner.rs | 530 ++++++++++- src/services/canvas.rs | 2 +- src/services/library_lifecycle.rs | 848 +++++++++++++++++ src/services/mod.rs | 9 + src/watcher.rs | 353 +++++++ tests/cli.rs | 151 +++ tests/migrations.rs | 68 +- tests/native_api.rs | 263 +++++ tests/scanner.rs | 898 +++++++++++++++++- webapp/src/api/index.ts | 8 + webapp/src/api/libraries.ts | 82 ++ webapp/src/api/queries.ts | 31 + webapp/src/main.tsx | 17 + webapp/src/pages/admin-page.tsx | 203 ++++ webapp/src/pages/missing-page.tsx | 222 +++++ webapp/src/state/i18n.tsx | 91 ++ webapp/src/styles/styles.css | 54 ++ 30 files changed, 4760 insertions(+), 59 deletions(-) create mode 100644 migrations-v2/20261026010000_library_lifecycle.sql create mode 100644 src/services/library_lifecycle.rs create mode 100644 src/watcher.rs create mode 100644 webapp/src/pages/missing-page.tsx diff --git a/.env.example b/.env.example index 894b07e6..06ccfc71 100644 --- a/.env.example +++ b/.env.example @@ -52,6 +52,14 @@ WAVEFLOW_STREAM_TICKET_TTL_SECS=3600 # remain available. Extraction is parallel, SQLite writes stay serialized. WAVEFLOW_SCAN_INTERVAL_SECS=21600 WAVEFLOW_SCAN_PARALLELISM=4 +# Watch library folders and scan a library once its changes have been quiet +# for the delay. Unreliable on network mounts; the interval above stays as the +# safety net (RFC-021). +WAVEFLOW_WATCH=true +WAVEFLOW_WATCH_DELAY_SECS=10 +# Purge tracks whose file has been missing for longer than this many days. +# 0 never purges on its own; a purge is not reversible. +WAVEFLOW_UNAVAILABLE_RETENTION_DAYS=0 # FFmpeg and ffprobe are mandatory from M2 onward. Override these when the # executables are not discoverable through PATH. diff --git a/Cargo.lock b/Cargo.lock index 5dc589f3..2e784871 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -610,6 +610,25 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-queue" version = "0.3.14" @@ -913,6 +932,15 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fsevent-sys" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2" +dependencies = [ + "libc", +] + [[package]] name = "futures-channel" version = "0.3.34" @@ -1394,6 +1422,22 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "ignore" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b69833ed729dc5aa7d19541d96d6cf8e9137194207a04916d658e43168402f" +dependencies = [ + "crossbeam-deque", + "globset", + "log", + "memchr", + "regex-automata", + "same-file", + "walkdir", + "winapi-util", +] + [[package]] name = "image" version = "0.25.10" @@ -1441,6 +1485,26 @@ dependencies = [ "serde_core", ] +[[package]] +name = "inotify" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cc00ea907cab49550b7da656f80ebb97be1b997d931fbcd28d39734e17ce592" +dependencies = [ + "bitflags", + "inotify-sys", + "libc", +] + +[[package]] +name = "inotify-sys" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c033f80b2c113cdf91ab7a33faa9cbc014726dcad99880c8609af2a370edf37d" +dependencies = [ + "libc", +] + [[package]] name = "inout" version = "0.2.2" @@ -1479,6 +1543,26 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "kqueue" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d763e5b24120b4ddf50de6c92308156765aabfbbccebf401da7cff2d70a41ea" +dependencies = [ + "kqueue-sys", + "libc", +] + +[[package]] +name = "kqueue-sys" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07293a4e297ac234359b510362495713f75ea345d5307140414f20c69ffeb087" +dependencies = [ + "bitflags", + "libc", +] + [[package]] name = "kurbo" version = "0.13.1" @@ -1662,6 +1746,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" dependencies = [ "libc", + "log", "wasi", "windows-sys 0.61.2", ] @@ -1682,6 +1767,33 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" +[[package]] +name = "notify" +version = "8.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d3d07927151ff8575b7087f245456e549fea62edf0ec4e565a5ee50c8402bc3" +dependencies = [ + "bitflags", + "fsevent-sys", + "inotify", + "kqueue", + "libc", + "log", + "mio", + "notify-types", + "walkdir", + "windows-sys 0.60.2", +] + +[[package]] +name = "notify-types" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42b8cfee0e339a0337359f3c88165702ac6e600dc01c0cc9579a92d62b08477a" +dependencies = [ + "bitflags", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -2975,7 +3087,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix", "windows-sys 0.61.2", @@ -3654,10 +3766,12 @@ dependencies = [ "futures-util", "hex", "http-body-util", + "ignore", "ipnet", "libc", "lofty", "md-5", + "notify", "percent-encoding", "rand 0.10.3", "realfft", @@ -3803,7 +3917,16 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", ] [[package]] @@ -3821,14 +3944,31 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", ] [[package]] @@ -3837,48 +3977,96 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + [[package]] name = "wit-bindgen" version = "0.57.1" diff --git a/Cargo.toml b/Cargo.toml index 7c900ba6..9deb5f2a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,6 +39,11 @@ encoding_rs = "0.8.42" futures-util = "0.3" hex = "0.4" ipnet = "2" +# `.waveflowignore` (RFC-021 decision 1): gitignore rules through the matcher +# ripgrep uses. Only its `gitignore` module is used, beside the `walkdir` walk +# the scanner already has; `globset` and `walkdir` it builds on are in the +# tree already. +ignore = "0.4" lofty = "0.25" # TTML lyrics uploads (RFC-012 decision 6). Already in the lock file through # the artwork renderer; named here because the server now parses XML itself. @@ -46,6 +51,10 @@ lofty = "0.25" roxmltree = "0.21" libc = "0.2" md-5 = "0.11" +# The library watcher (RFC-021 decision 6). Default features only, which is the +# FSEvents backend on macOS and nothing else: inotify on Linux and +# ReadDirectoryChangesW on Windows need no feature. +notify = "8" # Already in the tree through `url`; declared because `playlist_file` reads # percent escapes out of playlist entries and a transitive dependency is not a # promise. diff --git a/docs/api-v2-guide.md b/docs/api-v2-guide.md index d0467e99..a9448480 100644 --- a/docs/api-v2-guide.md +++ b/docs/api-v2-guide.md @@ -1135,6 +1135,7 @@ Admin Bearer tokens can manage: | Users | `GET/POST /api/v2/admin/users`, `PATCH/DELETE /api/v2/admin/users/{username}` | | Subsonic credential | `PUT/DELETE /api/v2/admin/users/{username}/subsonic-credential` | | Libraries | `GET/POST /api/v2/libraries` | +| A library's name, folder and links setting, and its deletion | `GET /api/v2/libraries/{library_id}/settings`, `PATCH/DELETE /api/v2/libraries/{library_id}` | | Membership | `PUT/DELETE /api/v2/libraries/{library_id}/members/{user_id}` | | Scans | `POST /api/v2/libraries/{library_id}/scans`, `GET /api/v2/scans/{scan_id}`, `GET /api/v2/scans/{scan_id}/events` | | API tokens | `GET/POST /api/v2/admin/users/{username}/tokens`, `DELETE /api/v2/admin/users/{username}/tokens/{token_id}` | @@ -1149,6 +1150,24 @@ takes the instance's write lock, so `POST /api/v2/libraries/{library_id}/scans` is reserved to the `owner` and `manager` roles; a `listener` gets `404`, the same answer as for a library that does not exist. Reading the catalogue is unaffected. + +A library's lifecycle ([RFC-021](rfcs/RFC-021-library-lifecycle-and-scan.md) +decision 5) splits the same way. `PATCH /api/v2/libraries/{library_id}` with a +`name` is open to the library's owner; with a `path` or `follow_links` it needs +an administrator on an administrative credential, and it queues a scan whose id +it answers. `DELETE` is an administrator's, and its body must carry +`{"confirm": ""}` or nothing is deleted (`422`). +Every one of them answers `409` while the library is being scanned. + +The tracks a library's scans can no longer find are listed, oldest first and +with what purging each would cost, by +`GET /api/v2/libraries/{library_id}/unavailable-tracks`, and purged by +`POST /api/v2/libraries/{library_id}/unavailable-tracks/purge` with +`{"track_ids": […]}` or `{"all": true}`. Both are for the library's owners, +managers and administrators; a purge is not reversible (RFC-021 decision 4). + +A scan job also says what its walk left out: `excluded` (built-in exclusions +and `.waveflowignore`) and `skipped_links` (symbolic links not followed). | FFmpeg status and transcode headroom | `GET /api/v2/transcode/status` | | Library change feed | `GET /api/v2/libraries/{library_id}/events` | @@ -1166,7 +1185,7 @@ tell the two apart. event: snapshot data: {"id":"…","library_id":"…","status":"running","total_files":164,"processed_files":0, "added":0,"updated":0,"moved":0,"skipped":0,"unavailable":0,"errors":0, - "current_path":null,"message":null} + "excluded":0,"skipped_links":0,"current_path":null,"message":null} event: progress data: {"id":"…","library_id":"…","status":"running","total_files":164,"processed_files":61, …} diff --git a/docs/operations.md b/docs/operations.md index 5c6bc56a..fb7fb43e 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -287,6 +287,126 @@ the scan goes on. The tracks under it are left as they are, not marked missing, so a folder that is briefly unreadable does not make its albums vanish and come back. Only the library root itself failing to open fails the scan. +[RFC-021](rfcs/RFC-021-library-lifecycle-and-scan.md) carries the reasoning +for everything below. + +### What a scan leaves out + +Some folders are never scanned, whatever else says: anything whose name begins +with a dot (hidden folders and files, `.Trash-*`, the `._` files a Mac leaves +on a share), `@eaDir`, `#recycle` and `#snapshot` (Synology), `$RECYCLE.BIN` +and `System Volume Information` (Windows), and `lost+found`. The one exception +is `.waveflow-managed` at the library root, where received files land. + +To leave out more, put a **`.waveflowignore`** file in any folder. Its syntax is +`.gitignore`'s and applies to that folder and below: + +```text +# no live recordings, no demos anywhere, no AAC except this one +/Live/ +**/demos/ +*.m4a +!Keep me.m4a +``` + +As in git, a folder that is left out is not entered, so a `!` cannot bring back +something inside it. + +An **empty** `.waveflowignore` leaves out the whole folder it sits in, as an +empty `.ndignore` does in Navidrome — except at the library root, where it +leaves out nothing. A file the scan cannot use (unreadable, over 64 KiB, or +holding a pattern it rejects) makes its folder count as unreadable: reported in +the scan's errors and left as it is. At the library root, it fails the scan. + +**What you leave out is treated as gone.** A track whose file becomes excluded +turns unavailable at the end of the scan, exactly as if the file had been +deleted, and comes back under the same identity when the rule is removed. The +scan's `excluded` counter says how many entries were left out. + +### Symbolic links + +By default a scan does not follow symbolic links. It no longer skips them in +silence: each one is counted in the scan's `skipped_links` and logged by its +path in the library. To follow them, per library: + +```bash +waveflow-server library set-links --actor admin --library-id --follow +``` + +A followed link must lead **inside the library**: one whose target is outside +the root, broken, or a loop back to a folder above it is skipped and counted. A +file reached through links is one track, under its own path, however many links +name it. A link farm whose files live in the same tree works best with the +files' folder excluded from the direct walk: + +```text +/music/.waveflowignore /store/ +/music/store/… the files +/music/Artist/Album/… links into ../../store/ +``` + +A link farm whose files live outside the root is not followed; make the root a +common ancestor, or add the files' folder as a library of its own. + +### Missing files + +A track whose file is gone stays in the catalogue, unavailable, with its +favourites, playlist entries and plays, so a share that comes back brings it +back whole. Owners and managers see them on the **Missing files** page, which +says for each what a purge would take; the CLI lists them too: + +```bash +waveflow-server library unavailable --actor admin --library-id +waveflow-server library purge --actor admin --library-id --all +waveflow-server library purge --actor admin --library-id --track-id +``` + +**A purge is not reversible.** It removes the tracks and everything that hangs +off them for every member — playlist entries, favourites, ratings, plays, queue +entries, corrections, lyrics — and the albums and artists left with nothing. A +file that comes back afterwards is a new track. Only tracks still missing when +the purge runs are removed, and a purge waits for no scan: it answers that the +library is busy instead. + +To purge on a schedule, set **`WAVEFLOW_UNAVAILABLE_RETENTION_DAYS`**: once an +hour, tracks missing for longer than that are purged. It is off by default, +because a share that stays unmounted for a week is not a deletion. + +### Renaming, moving and deleting a library + +From the administration page, or: + +```bash +waveflow-server library rename --actor admin --library-id --name "Lossless" +waveflow-server library set-path --actor admin --library-id --path /mnt/music +waveflow-server library delete --actor admin --library-id --confirm "Lossless" +``` + +**Moving** points the library at a new folder and scans it. Tracks are matched +by their path inside the library, so a tree moved intact keeps every identity, +favourite and playlist entry. The owner of a library may rename it; moving it, +its links setting and deleting it are an administrator's. + +**Deleting** removes the library from the catalogue for every member — its +tracks, albums and artists with everything that hangs off them, its scans, +folders and memberships — and nothing from the disk. `--confirm` must be the +library's current name. None of these runs while the library is being scanned. + +### Watching for changes + +The server watches every library folder and scans a library once its changes +have been quiet for `WAVEFLOW_WATCH_DELAY_SECS` (10 by default); a copy that +never pauses is still scanned every thirty delays. The periodic scan stays as +the safety net. Set `WAVEFLOW_WATCH=false` to turn watching off. + +**Do not count on it for network mounts.** NFS and SMB report the changes this +machine makes and usually not those another machine makes, so a library on a +share is found up to date at the next periodic scan. A folder that cannot be +watched is logged once and left to the periodic scan. On Linux, a large +library can exhaust the inotify watch limit; raise it with +`sysctl fs.inotify.max_user_watches=524288` (and in `/etc/sysctl.d/` to keep +it), or the library falls back to the periodic scan. + ## Probes | Path | Answers | diff --git a/migrations-v2/20261026010000_library_lifecycle.sql b/migrations-v2/20261026010000_library_lifecycle.sql new file mode 100644 index 00000000..cfae9827 --- /dev/null +++ b/migrations-v2/20261026010000_library_lifecycle.sql @@ -0,0 +1,60 @@ +-- Libraries and scanning (RFC-021). +-- +-- `library.follow_links`: whether the walk follows symbolic links whose target +-- stays inside the library root (decision 3). Off for every existing library: +-- upgrading must not change what a library reads. +ALTER TABLE library ADD COLUMN follow_links INTEGER NOT NULL DEFAULT 0 + CHECK (follow_links IN (0, 1)); + +-- `track.unavailable_since`: when a track stopped being available, so the +-- missing-tracks page can say since when and the optional retention can purge +-- by age (decision 4). NULL for every available track. The tracks already +-- unavailable take the time their row was last written, which is when the scan +-- marked them missing unless something touched them since. +ALTER TABLE track ADD COLUMN unavailable_since INTEGER; +UPDATE track SET unavailable_since = updated_at WHERE is_available = 0; + +-- `scan_job` gains two counters and two triggers (decision 7). SQLite cannot +-- alter a CHECK, so the table is rebuilt. Migrations run with foreign keys +-- off, so `scan_error.scan_id` and `track.last_seen_scan_id`, which name this +-- table, are neither cascaded nor nulled by the drop; they resolve to the +-- rebuilt table by name, and the boot's foreign_key_check answers for it. +CREATE TABLE scan_job_rebuilt ( + id TEXT PRIMARY KEY NOT NULL, + library_id TEXT NOT NULL REFERENCES library(id) ON DELETE CASCADE, + requested_by TEXT REFERENCES account(id) ON DELETE SET NULL, + trigger TEXT NOT NULL CHECK ( + trigger IN ('manual', 'boot', 'scheduled', 'library_added', 'watch', 'library_moved') + ), + status TEXT NOT NULL CHECK (status IN ('queued', 'running', 'completed', 'failed')), + total_files INTEGER NOT NULL DEFAULT 0, + processed_files INTEGER NOT NULL DEFAULT 0, + added INTEGER NOT NULL DEFAULT 0, + updated INTEGER NOT NULL DEFAULT 0, + moved INTEGER NOT NULL DEFAULT 0, + skipped INTEGER NOT NULL DEFAULT 0, + unavailable INTEGER NOT NULL DEFAULT 0, + errors INTEGER NOT NULL DEFAULT 0, + current_path TEXT, + message TEXT, + created_at INTEGER NOT NULL, + started_at INTEGER, + completed_at INTEGER, + full_scan INTEGER NOT NULL DEFAULT 0 CHECK (full_scan IN (0, 1)), + excluded INTEGER NOT NULL DEFAULT 0 CHECK (excluded >= 0), + skipped_links INTEGER NOT NULL DEFAULT 0 CHECK (skipped_links >= 0) +) STRICT; + +INSERT INTO scan_job_rebuilt + (id, library_id, requested_by, trigger, status, total_files, processed_files, + added, updated, moved, skipped, unavailable, errors, current_path, message, + created_at, started_at, completed_at, full_scan) +SELECT id, library_id, requested_by, trigger, status, total_files, processed_files, + added, updated, moved, skipped, unavailable, errors, current_path, message, + created_at, started_at, completed_at, full_scan +FROM scan_job; + +DROP TABLE scan_job; +ALTER TABLE scan_job_rebuilt RENAME TO scan_job; + +CREATE INDEX scan_job_library_idx ON scan_job(library_id, created_at DESC); diff --git a/src/api/libraries.rs b/src/api/libraries.rs index c753bc45..b361532e 100644 --- a/src/api/libraries.rs +++ b/src/api/libraries.rs @@ -84,6 +84,7 @@ pub async fn create_library( ) .await .map_err(db_error)?; + state.scanner.libraries_changed(); let scan_id = state .scanner .trigger( @@ -384,3 +385,122 @@ pub async fn library_tree( .map(Json) .map_err(service_error) } + +/// What deleting a library must carry: its current name, exactly. A body that +/// lacks it, or names anything else, deletes nothing and answers `422`. +#[derive(Debug, Deserialize, ToSchema)] +pub struct DeleteLibraryRequest { + #[serde(default)] + pub confirm: String, +} + +/// Renames a library, moves its root or changes whether its walk follows +/// links (RFC-021 decision 5). A name is the owner's or an administrator's to +/// change; the path and `follow_links` are an administrator's, on a credential +/// that carries administrative authority. A new root or links setting queues a +/// scan, whose id is answered. `409` while the library is being scanned, or +/// when another library already has that root. +#[utoipa::path(patch, path = "/api/v2/libraries/{library_id}", tag = "administration", params(("library_id" = Uuid, Path)), request_body = crate::services::LibraryUpdate, responses((status = 200, body = crate::services::LibraryUpdated), (status = 401, body = ErrorResponse), (status = 403, body = ErrorResponse), (status = 404, body = ErrorResponse), (status = 409, body = ErrorResponse), (status = 422, body = ErrorResponse)))] +pub async fn update_library( + State(state): State, + Path(library_id): Path, + headers: HeaderMap, + Json(request): Json, +) -> Result, ApiError> { + let access = if request.is_administrative() { + Access::Admin + } else { + Access::Write + }; + let actor = authenticated(&state, &headers, access).await?; + state + .services + .update_library(actor.id, library_id, request) + .await + .map(Json) + .map_err(service_error) +} + +/// Deletes a library from the catalogue, and nothing from the disk (RFC-021 +/// decision 5): every track, album and artist, with everything that cascades +/// from them for every member. The body must carry `confirm`, equal to the +/// library's current name. `409` while the library is being scanned. +#[utoipa::path(delete, path = "/api/v2/libraries/{library_id}", tag = "administration", params(("library_id" = Uuid, Path)), request_body = DeleteLibraryRequest, responses((status = 200, body = crate::services::LibraryDeleted), (status = 401, body = ErrorResponse), (status = 403, body = ErrorResponse), (status = 404, body = ErrorResponse), (status = 409, body = ErrorResponse), (status = 422, body = ErrorResponse)))] +pub async fn delete_library( + State(state): State, + Path(library_id): Path, + headers: HeaderMap, + Json(request): Json, +) -> Result, ApiError> { + let actor = authenticated(&state, &headers, Access::Admin).await?; + state + .services + .delete_library(actor.id, library_id, &request.confirm) + .await + .map(Json) + .map_err(service_error) +} + +/// A library as an administrator manages it, its root included. +#[utoipa::path(get, path = "/api/v2/libraries/{library_id}/settings", tag = "administration", params(("library_id" = Uuid, Path)), responses((status = 200, body = crate::services::LibrarySettings), (status = 401, body = ErrorResponse), (status = 403, body = ErrorResponse), (status = 404, body = ErrorResponse)))] +pub async fn library_settings( + State(state): State, + Path(library_id): Path, + headers: HeaderMap, +) -> Result, ApiError> { + let actor = authenticated(&state, &headers, Access::Admin).await?; + state + .services + .library_settings(actor.id, library_id) + .await + .map(Json) + .map_err(service_error) +} + +#[derive(Debug, Deserialize)] +pub struct UnavailableTracksQuery { + pub offset: Option, + pub limit: Option, +} + +/// The tracks a library's scans can no longer find, oldest first, with what +/// purging each would cost (RFC-021 decision 4). For the library's owners and +/// managers, and administrators; anyone else gets the `404` a library they are +/// not in would. +#[utoipa::path(get, path = "/api/v2/libraries/{library_id}/unavailable-tracks", tag = "catalog", params(("library_id" = Uuid, Path), ("offset" = Option, Query), ("limit" = Option, Query)), responses((status = 200, body = crate::services::UnavailableTracks), (status = 401, body = ErrorResponse), (status = 404, body = ErrorResponse), (status = 422, body = ErrorResponse)))] +pub async fn unavailable_tracks( + State(state): State, + Path(library_id): Path, + Query(query): Query, + headers: HeaderMap, +) -> Result, ApiError> { + let user = authenticated(&state, &headers, Access::Read).await?; + let page = + crate::services::BrowsePage::new(query.offset, query.limit).map_err(service_error)?; + state + .services + .unavailable_tracks(user.id, library_id, page) + .await + .map(Json) + .map_err(service_error) +} + +/// Purges missing tracks: those named in `track_ids`, or every one with +/// `"all": true`, and never neither (`422`). Only tracks still unavailable are +/// removed. Not reversible: see RFC-021 decision 4 for what is lost. `409` +/// while the library is being scanned. +#[utoipa::path(post, path = "/api/v2/libraries/{library_id}/unavailable-tracks/purge", tag = "catalog", params(("library_id" = Uuid, Path)), request_body = crate::services::PurgeRequest, responses((status = 200, body = crate::services::PurgeOutcome), (status = 401, body = ErrorResponse), (status = 403, body = ErrorResponse), (status = 404, body = ErrorResponse), (status = 409, body = ErrorResponse), (status = 422, body = ErrorResponse)))] +pub async fn purge_unavailable_tracks( + State(state): State, + Path(library_id): Path, + headers: HeaderMap, + Json(request): Json, +) -> Result, ApiError> { + let user = authenticated(&state, &headers, Access::Write).await?; + state + .services + .purge_unavailable_tracks(user.id, library_id, request) + .await + .map(Json) + .map_err(service_error) +} diff --git a/src/api/mod.rs b/src/api/mod.rs index 3f9cbaa8..962db211 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -108,6 +108,22 @@ pub fn router(state: AppState) -> Router { "/api/v2/libraries", get(list_libraries).post(create_library), ) + .route( + "/api/v2/libraries/{library_id}", + axum::routing::patch(update_library).delete(delete_library), + ) + .route( + "/api/v2/libraries/{library_id}/settings", + get(library_settings), + ) + .route( + "/api/v2/libraries/{library_id}/unavailable-tracks", + get(unavailable_tracks), + ) + .route( + "/api/v2/libraries/{library_id}/unavailable-tracks/purge", + post(purge_unavailable_tracks), + ) .route( "/api/v2/libraries/{library_id}/members", get(list_library_members), diff --git a/src/catalog.rs b/src/catalog.rs index d085c886..d55d3e95 100644 --- a/src/catalog.rs +++ b/src/catalog.rs @@ -16,6 +16,14 @@ pub struct LibraryRecord { pub root_path: PathBuf, } +/// A library and the one setting that changes how it is walked. +#[derive(Debug, Clone)] +pub struct LibraryScanSettings { + pub library: LibraryRecord, + /// Follow symbolic links that stay inside the root (RFC-021 decision 3). + pub follow_links: bool, +} + #[derive(Debug, Clone, Serialize, ToSchema)] pub struct LibraryAccess { pub id: Uuid, @@ -36,6 +44,10 @@ pub struct LibraryAccess { /// read `accepts_uploads` for both would hide the canvas exactly where the /// flag was split to allow it. pub accepts_canvas: bool, + /// Whether the walk follows symbolic links that stay inside the library + /// (RFC-021 decision 3). An administrator's setting, shown to every + /// member because it explains what the library holds. + pub follow_links: bool, } /// What an incremental scan compares before reading a file (RFC-018 @@ -196,6 +208,12 @@ pub struct ScanJobRecord { pub skipped: i64, pub unavailable: i64, pub errors: i64, + /// Entries the walk left out by an exclusion or an ignore file, a folder + /// counting once whatever it holds (RFC-021 decision 2). + pub excluded: i64, + /// Symbolic links the walk did not follow, for whatever reason + /// (RFC-021 decision 3). + pub skipped_links: i64, pub current_path: Option, pub message: Option, } @@ -227,7 +245,8 @@ impl Database { ) -> Result, sqlx::Error> { let rows = sqlx::query( "SELECT l.id, l.name, l.visibility, m.role, l.last_scan_started_at, \ - l.last_scan_completed_at, l.accepts_uploads, l.accepts_canvas \ + l.last_scan_completed_at, l.accepts_uploads, l.accepts_canvas, \ + l.follow_links \ FROM library l JOIN library_member m ON m.library_id=l.id \ WHERE m.user_id=? ORDER BY l.name COLLATE NOCASE, l.id", ) @@ -249,6 +268,7 @@ impl Database { last_scan_completed_at: row.try_get("last_scan_completed_at")?, accepts_uploads: row.try_get::("accepts_uploads")? != 0, accepts_canvas: row.try_get::("accepts_canvas")? != 0, + follow_links: row.try_get::("follow_links")? != 0, }) }) .collect() @@ -261,6 +281,27 @@ impl Database { rows.into_iter().map(library_from_row).collect() } + /// One library as a scan needs it, read when the scan starts rather than + /// when it was queued: its root may have moved and its links setting + /// changed in between, and it may be gone (RFC-021 decision 5). + pub async fn library_scan_settings( + &self, + library_id: Uuid, + ) -> Result, sqlx::Error> { + let row = sqlx::query("SELECT id, name, root_path, follow_links FROM library WHERE id = ?") + .bind(library_id.to_string()) + .fetch_optional(self.pool()) + .await?; + row.map(|row| { + let follow_links = row.try_get::("follow_links")? != 0; + Ok(LibraryScanSettings { + library: library_from_row(row)?, + follow_links, + }) + }) + .transpose() + } + /// Whether any library the user can see is being scanned, and how many /// available tracks they can currently reach. /// @@ -628,15 +669,30 @@ impl Database { scan_id: Uuid, total: i64, full: bool, + ) -> Result<(), sqlx::Error> { + self.start_walked_scan_job(scan_id, total, full, 0, 0).await + } + + /// Marks a job running, with what its walk found: how many files it will + /// look at, and what it left out (RFC-021 decision 7). + pub async fn start_walked_scan_job( + &self, + scan_id: Uuid, + total: i64, + full: bool, + excluded: i64, + skipped_links: i64, ) -> Result<(), sqlx::Error> { let _writer = self.writer_guard().await; let now = now_ms(); sqlx::query( "UPDATE scan_job SET status = 'running', total_files = ?, full_scan = ?, \ - started_at = ? WHERE id = ?", + excluded = ?, skipped_links = ?, started_at = ? WHERE id = ?", ) .bind(total) .bind(i64::from(full)) + .bind(excluded) + .bind(skipped_links) .bind(now) .bind(scan_id.to_string()) .execute(self.pool()) @@ -1020,6 +1076,16 @@ impl Database { row.map(scan_job_from_row).transpose() } + /// A scan job by its id alone, for the operator's CLI, which acts as the + /// instance rather than as a member of the library. + pub async fn scan_job(&self, scan_id: Uuid) -> Result, sqlx::Error> { + let row = sqlx::query("SELECT * FROM scan_job WHERE id = ?") + .bind(scan_id.to_string()) + .fetch_optional(self.pool()) + .await?; + row.map(scan_job_from_row).transpose() + } + pub async fn existing_track_by_path( &self, library_id: Uuid, @@ -1448,7 +1514,8 @@ impl Database { explicit_status=excluded.explicit_status, \ lyrics_hash=excluded.lyrics_hash, disc_subtitle=excluded.disc_subtitle, \ pid=excluded.pid, \ - is_available=1, last_seen_scan_id=excluded.last_seen_scan_id, \ + is_available=1, unavailable_since=NULL, \ + last_seen_scan_id=excluded.last_seen_scan_id, \ updated_at=excluded.updated_at", ) .bind(track_id.to_string()).bind(library_id.to_string()) @@ -1632,7 +1699,7 @@ impl Database { // scan to walk past a received file stamps it, and it rejoins the // ordinary case for good. let gone: Vec = sqlx::query_scalar( - "UPDATE track SET is_available = 0, updated_at = ? \ + "UPDATE track SET is_available = 0, unavailable_since = ?, updated_at = ? \ WHERE library_id = ? AND is_available = 1 \ AND ((last_seen_scan_id IS NOT NULL AND last_seen_scan_id <> ?) \ OR (last_seen_scan_id IS NULL \ @@ -1641,6 +1708,7 @@ impl Database { RETURNING id", ) .bind(now) + .bind(now) .bind(library_id.to_string()) .bind(scan_id.to_string()) .bind(scan_id.to_string()) @@ -2353,6 +2421,8 @@ fn scan_job_from_row(row: sqlx::sqlite::SqliteRow) -> Result, } #[derive(Debug, Args)] @@ -361,6 +444,12 @@ pub async fn execute(command: Command, state: &AppState) -> anyhow::Result<()> { LibraryCommand::RemoveMember(args) => remove_member(db, args).await, LibraryCommand::SetUploads(args) => set_uploads(db, args).await, LibraryCommand::SetCanvas(args) => set_canvas(db, args).await, + LibraryCommand::Rename(args) => rename_library(state, args).await, + LibraryCommand::SetPath(args) => set_library_path(state, args).await, + LibraryCommand::SetLinks(args) => set_library_links(state, args).await, + LibraryCommand::Delete(args) => delete_library(state, args).await, + LibraryCommand::Unavailable(args) => list_unavailable(state, args).await, + LibraryCommand::Purge(args) => purge_unavailable(state, args).await, }, Command::Credential { command } => match command { CredentialCommand::Set(args) => set_credential(db, &state.secret_box, args).await, @@ -879,6 +968,7 @@ async fn add_library(state: &AppState, args: AddLibraryArgs) -> anyhow::Result<( .create_library(owner.id, &args.name, &canonical, visibility, now_ms()) .await .context("register library")?; + state.scanner.libraries_changed(); println!("Registered library {} ({id})", args.name); let scan_id = state .scanner @@ -916,6 +1006,201 @@ async fn add_library(state: &AppState, args: AddLibraryArgs) -> anyhow::Result<( Ok(()) } +/// What a refused library operation says on a terminal. The services answer +/// in their own vocabulary; an operator needs a sentence. +fn library_refusal(error: crate::services::ServiceError, library_id: uuid::Uuid) -> anyhow::Error { + use crate::services::ServiceError; + match error { + ServiceError::NotFound | ServiceError::Forbidden => { + anyhow::anyhow!("library not found: {library_id}") + } + ServiceError::Conflict => anyhow::anyhow!( + "library {library_id} is being scanned, or another library already has that path; \ + try again once the scan is over" + ), + ServiceError::Invalid => anyhow::anyhow!("the request was refused as invalid"), + other => anyhow::anyhow!(other), + } +} + +/// Waits for a scan this command queued, and prints how it ended. The process +/// is about to exit, and the scan is a task of this process: leaving it would +/// kill it halfway. +async fn wait_for_scan(state: &AppState, scan_id: uuid::Uuid) -> anyhow::Result<()> { + loop { + let job = state + .db + .scan_job(scan_id) + .await? + .context("the scan disappeared")?; + match job.status.as_str() { + "completed" => { + println!( + "Scan complete: {} added, {} updated, {} moved, {} missing, {} errors", + job.added, job.updated, job.moved, job.unavailable, job.errors + ); + return Ok(()); + } + "failed" => anyhow::bail!( + "scan failed: {}", + job.message.unwrap_or_else(|| "unknown error".into()) + ), + _ => tokio::time::sleep(std::time::Duration::from_millis(50)).await, + } + } +} + +async fn rename_library(state: &AppState, args: RenameLibraryArgs) -> anyhow::Result<()> { + let actor = require_admin(&state.db, &args.actor).await?; + state + .services + .update_library( + actor.id, + args.library_id, + crate::services::LibraryUpdate { + name: Some(args.name.clone()), + ..Default::default() + }, + ) + .await + .map_err(|error| library_refusal(error, args.library_id))?; + println!( + "Library {} is now named {}", + args.library_id, + args.name.trim() + ); + Ok(()) +} + +async fn set_library_path(state: &AppState, args: SetPathArgs) -> anyhow::Result<()> { + let actor = require_admin(&state.db, &args.actor).await?; + let updated = state + .services + .update_library( + actor.id, + args.library_id, + crate::services::LibraryUpdate { + path: Some(args.path.to_string_lossy().into_owned()), + ..Default::default() + }, + ) + .await + .map_err(|error| match error { + crate::services::ServiceError::Invalid => anyhow::anyhow!( + "library path must be an existing directory and not a symbolic link: {}", + args.path.display() + ), + other => library_refusal(other, args.library_id), + })?; + match updated.scan_id { + Some(scan_id) => { + println!("Library {} moved; scanning it", args.library_id); + wait_for_scan(state, scan_id).await + } + None => { + println!("Library {} already has that path", args.library_id); + Ok(()) + } + } +} + +async fn set_library_links(state: &AppState, args: SetLinksArgs) -> anyhow::Result<()> { + let actor = require_admin(&state.db, &args.actor).await?; + let updated = state + .services + .update_library( + actor.id, + args.library_id, + crate::services::LibraryUpdate { + follow_links: Some(args.follow), + ..Default::default() + }, + ) + .await + .map_err(|error| library_refusal(error, args.library_id))?; + println!( + "Library {} {} symbolic links", + args.library_id, + if args.follow { + "follows" + } else { + "does not follow" + } + ); + match updated.scan_id { + Some(scan_id) => wait_for_scan(state, scan_id).await, + None => Ok(()), + } +} + +async fn delete_library(state: &AppState, args: DeleteLibraryArgs) -> anyhow::Result<()> { + let actor = require_admin(&state.db, &args.actor).await?; + let deleted = state + .services + .delete_library(actor.id, args.library_id, &args.confirm) + .await + .map_err(|error| match error { + crate::services::ServiceError::Invalid => { + anyhow::anyhow!("--confirm must be the library's current name; nothing was deleted") + } + other => library_refusal(other, args.library_id), + })?; + println!( + "Deleted library {}: {} tracks, {} albums, {} artists removed from the catalogue; \ + no file was touched", + args.library_id, deleted.tracks, deleted.albums, deleted.artists + ); + Ok(()) +} + +async fn list_unavailable(state: &AppState, args: UnavailableArgs) -> anyhow::Result<()> { + let actor = require_admin(&state.db, &args.actor).await?; + let mut offset = 0; + loop { + let page = crate::services::BrowsePage::new(Some(offset), Some(200)) + .map_err(|error| library_refusal(error, args.library_id))?; + let listed = state + .services + .unavailable_tracks(actor.id, args.library_id, page) + .await + .map_err(|error| library_refusal(error, args.library_id))?; + if offset == 0 { + println!("{} missing track(s)", listed.total); + } + for track in &listed.tracks { + println!( + "{}\t{}\t{} playlist entries, {} favourites, {} plays", + track.id, track.relative_path, track.playlist_entries, track.favorites, track.plays + ); + } + if listed.tracks.len() < 200 { + return Ok(()); + } + offset += 200; + } +} + +async fn purge_unavailable(state: &AppState, args: PurgeArgs) -> anyhow::Result<()> { + let actor = require_admin(&state.db, &args.actor).await?; + if !args.all && args.track_id.is_empty() { + anyhow::bail!("name the tracks to purge with --track-id, or pass --all"); + } + let outcome = state + .services + .purge_unavailable_tracks( + actor.id, + args.library_id, + crate::services::PurgeRequest { + track_ids: (!args.all).then_some(args.track_id), + all: args.all, + }, + ) + .await + .map_err(|error| library_refusal(error, args.library_id))?; + println!("Purged {} missing track(s)", outcome.purged); + Ok(()) +} + async fn set_member(db: &Database, args: SetMemberArgs) -> anyhow::Result<()> { let actor = require_admin(db, &args.actor).await?; let member = db diff --git a/src/config.rs b/src/config.rs index 296aec55..41b8061b 100644 --- a/src/config.rs +++ b/src/config.rs @@ -328,6 +328,21 @@ pub struct Config { pub api_token_ttl: Option, pub scan_interval: Option, pub scan_parallelism: usize, + /// Whether the server watches every library root and scans a library once + /// its changes settle (RFC-021 decision 6). `WAVEFLOW_WATCH`, on by + /// default. The periodic scan stays either way, as the safety net for + /// what a watcher misses — network mounts, above all. + pub watch: bool, + /// How long a library's changes must have been quiet before the watcher + /// scans it. `WAVEFLOW_WATCH_DELAY_SECS`, ten seconds by default. A stream + /// of changes that never settles is still scanned once thirty of these + /// have passed since its first event. + pub watch_delay: Duration, + /// How long a track may stay unavailable before a background pass purges + /// it (RFC-021 decision 4). `WAVEFLOW_UNAVAILABLE_RETENTION_DAYS`; unset or + /// `0`, the default, never purges on its own: a purge is not reversible, + /// and a share that stays unmounted for a week is not a deletion. + pub unavailable_retention: Option, pub ffmpeg_path: PathBuf, pub ffprobe_path: PathBuf, pub transcode_cache_dir: PathBuf, @@ -479,6 +494,9 @@ impl std::fmt::Debug for Config { .field("api_token_ttl", &self.api_token_ttl) .field("scan_interval", &self.scan_interval) .field("scan_parallelism", &self.scan_parallelism) + .field("watch", &self.watch) + .field("watch_delay", &self.watch_delay) + .field("unavailable_retention", &self.unavailable_retention) .field("ffmpeg_path", &self.ffmpeg_path) .field("ffprobe_path", &self.ffprobe_path) .field("transcode_cache_dir", &self.transcode_cache_dir) @@ -546,6 +564,18 @@ impl Config { .unwrap_or(4) .clamp(1, 16), )?; + let watch = parse_bool_env("WAVEFLOW_WATCH", true)?; + let watch_delay = + Duration::from_secs(parse_positive_env("WAVEFLOW_WATCH_DELAY_SECS", 10u64)?); + // Zero is a setting, the default one: never purge on its own. + let unavailable_retention_days = std::env::var("WAVEFLOW_UNAVAILABLE_RETENTION_DAYS") + .unwrap_or_else(|_| "0".to_owned()) + .parse::() + .map_err(|error| { + anyhow::anyhow!("invalid WAVEFLOW_UNAVAILABLE_RETENTION_DAYS: {error}") + })?; + let unavailable_retention = (unavailable_retention_days > 0) + .then(|| Duration::from_secs(unavailable_retention_days.saturating_mul(24 * 60 * 60))); let ffmpeg_path = std::env::var_os("WAVEFLOW_FFMPEG_PATH") .map(PathBuf::from) .unwrap_or_else(|| PathBuf::from("ffmpeg")); @@ -766,6 +796,9 @@ impl Config { scan_interval: (scan_interval_secs > 0) .then(|| Duration::from_secs(scan_interval_secs)), scan_parallelism, + watch, + watch_delay, + unavailable_retention, ffmpeg_path, ffprobe_path, transcode_cache_dir, @@ -818,6 +851,12 @@ impl Config { api_token_ttl: Some(Duration::from_secs(365 * 24 * 60 * 60)), scan_interval: None, scan_parallelism: 2, + // Off, like the periodic scan: a test that writes a file and then + // scans must not find a second scan racing its own. The watcher's + // tests start one themselves. + watch: false, + watch_delay: Duration::from_millis(200), + unavailable_retention: None, ffmpeg_path: PathBuf::from("ffmpeg"), ffprobe_path: PathBuf::from("ffprobe"), transcode_cache_dir, diff --git a/src/lib.rs b/src/lib.rs index f8b35f42..3ca265a7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -26,6 +26,7 @@ pub mod stream_ticket; pub mod subsonic; pub mod sync; pub mod tags; +pub mod watcher; pub mod webui; use std::{sync::Arc, time::Duration}; @@ -134,6 +135,11 @@ fn chunk_body_limit(limits: &config::UploadLimits) -> usize { api::start_scan, api::list_libraries, api::create_library, + api::update_library, + api::delete_library, + api::library_settings, + api::unavailable_tracks, + api::purge_unavailable_tracks, api::list_library_members, api::set_library_member, api::remove_library_member, @@ -272,6 +278,15 @@ fn chunk_body_limit(limits: &config::UploadLimits) -> usize { catalog::ScanJobRecord, catalog::TrackRecord, catalog::LibraryAccess, + api::DeleteLibraryRequest, + services::LibraryUpdate, + services::LibraryUpdated, + services::LibrarySettings, + services::LibraryDeleted, + services::UnavailableTrack, + services::UnavailableTracks, + services::PurgeRequest, + services::PurgeOutcome, services::AlbumItem, services::ArtistItem, services::GenreItem, diff --git a/src/main.rs b/src/main.rs index 5bfd9047..8a5548ab 100644 --- a/src/main.rs +++ b/src/main.rs @@ -53,6 +53,16 @@ async fn serve(config: Config, state: waveflow_server::AppState) -> anyhow::Resu } } state.scanner.spawn_background(config.scan_interval); + // A change scanned once it settles, on top of the interval (RFC-021). + if config.watch { + waveflow_server::watcher::spawn( + state.scanner.clone(), + state.db.clone(), + config.watch_delay, + ); + } + // Off unless WAVEFLOW_UNAVAILABLE_RETENTION_DAYS says otherwise. + state.services.spawn_unavailable_purge(); // At boot, then whenever a scan finishes or a library is asked to be // analysed again; one core at most by default, and off at zero (RFC-016). state.services.spawn_quality_analyzer(); diff --git a/src/scanner.rs b/src/scanner.rs index ce054a69..8184535d 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -52,6 +52,10 @@ pub struct ScanProgress { pub skipped: i64, pub unavailable: i64, pub errors: i64, + /// Entries the walk left out (RFC-021 decision 2). + pub excluded: i64, + /// Symbolic links the walk did not follow (RFC-021 decision 3). + pub skipped_links: i64, pub current_path: Option, pub message: Option, } @@ -72,6 +76,8 @@ impl ScanProgress { skipped: self.skipped, unavailable: self.unavailable, errors: self.errors, + excluded: self.excluded, + skipped_links: self.skipped_links, current_path: self.current_path.clone(), message: self.message.clone(), } @@ -93,6 +99,9 @@ pub struct ScanManager { /// quality page says "waiting for the scan" only to someone who can see /// the library being scanned. scanning: Arc>, + /// Moves whenever a library is added, moved or deleted through this + /// process, so the watcher (RFC-021 decision 6) re-reads what to watch. + libraries: Arc>, } /// One scan counted as running, for as long as this lives. @@ -147,9 +156,39 @@ impl ScanManager { library_locks: Arc::new(DashMap::new()), in_flight: Arc::new(watch::Sender::new(0)), scanning: Arc::new(DashMap::new()), + libraries: Arc::new(watch::Sender::new(0)), } } + /// The lock every scan of one library holds for its whole run. + fn library_lock(&self, library_id: Uuid) -> Arc> { + self.library_locks + .entry(library_id) + .or_insert_with(|| Arc::new(Mutex::new(()))) + .clone() + } + + /// That lock, if no scan of the library holds it now. + /// + /// For what must not interleave with a scan and must not wait for one + /// either (RFC-021 decisions 4 and 5): a purge, a move, a deletion. A scan + /// queued behind it waits until it is dropped, and then reads the library + /// as it was left. + pub fn try_lock_library(&self, library_id: Uuid) -> Option> { + self.library_lock(library_id).try_lock_owned().ok() + } + + /// Says that the set of libraries, or a root, changed. + pub fn libraries_changed(&self) { + self.libraries + .send_modify(|version| *version = version.wrapping_add(1)); + } + + /// A way to wait for [`Self::libraries_changed`]. + pub fn library_changes(&self) -> watch::Receiver { + self.libraries.subscribe() + } + /// Counts a scan of `library_id` as running until the guard is dropped. /// Every job takes one for its whole run, success or failure. pub fn begin_scan(&self, library_id: Uuid) -> ScanInFlight { @@ -215,6 +254,8 @@ impl ScanManager { skipped: 0, unavailable: 0, errors: 1, + excluded: 0, + skipped_links: 0, current_path: None, message: Some(message), }); @@ -254,23 +295,34 @@ impl ScanManager { } async fn run(&self, scan_id: Uuid, library: LibraryRecord) -> anyhow::Result<()> { - let lock = self - .library_locks - .entry(library.id) - .or_insert_with(|| Arc::new(Mutex::new(()))) - .clone(); + let lock = self.library_lock(library.id); let _library_guard = lock.lock().await; + // Read again under the lock rather than trusted from the queue: the + // library may have been moved, had its links setting changed, or been + // deleted since this job was queued (RFC-021 decision 5). + let Some(settings) = self.db.library_scan_settings(library.id).await? else { + tracing::info!(library_id = %library.id, %scan_id, "the library was deleted before its scan ran"); + return Ok(()); + }; + let library = settings.library; tokio::fs::create_dir_all(&self.artwork_dir).await?; let root = tokio::fs::canonicalize(&library.root_path).await?; + let follow_links = settings.follow_links; let Discovery { paths, unreadable, incomplete, + excluded, + skipped_links, } = tokio::task::spawn_blocking({ let root = root.clone(); - move || discover_audio(&root) + move || discover_audio(&root, follow_links) }) .await??; + let unreadable_folders: Vec = unreadable + .iter() + .map(|folder| folder.folder.clone()) + .collect(); let discovered: HashSet = paths .iter() .filter_map(|path| relative_path(&root, path).ok()) @@ -283,7 +335,13 @@ impl ScanManager { tracing::info!(library_id = %library.id, "scanning every file, skipping nothing"); } self.db - .start_scan_job(scan_id, paths.len() as i64, full) + .start_walked_scan_job( + scan_id, + paths.len() as i64, + full, + excluded as i64, + skipped_links as i64, + ) .await?; // What the catalogue already knows, read once: the pre-check compares @@ -326,17 +384,20 @@ impl ScanManager { skipped: 0, unavailable: 0, errors: 0, + excluded: excluded as i64, + skipped_links: skipped_links as i64, current_path: None, message: None, }; // A folder the walk could not read is reported, and its tracks are left - // as they are below (RFC-018 decision 3). + // as they are below (RFC-018 decision 3). So is one whose ignore file + // the walk could not use (RFC-021 decision 1), with its own reason. if !unreadable.is_empty() { progress.errors += unreadable.len() as i64; let errors: Vec<(String, String)> = unreadable .iter() - .map(|folder| (folder.clone(), "folder could not be read".to_owned())) + .map(|folder| (folder.folder.clone(), folder.reason.to_owned())) .collect(); self.db.add_scan_errors(scan_id, &errors).await?; // Written now, not left to the batches: a library whose only @@ -346,7 +407,7 @@ impl ScanManager { let walked = Walked { discovered: &discovered, - unreadable: &unreadable, + unreadable: &unreadable_folders, incomplete, }; while let Some(batch) = scanned.next().await { @@ -498,7 +559,7 @@ impl ScanManager { } self.db - .mark_tracks_under_seen(library.id, &unreadable, scan_id) + .mark_tracks_under_seen(library.id, &unreadable_folders, scan_id) .await?; // A walk error that named no path leaves no folder to spare, so // nothing this walk missed can be called missing. @@ -665,52 +726,358 @@ fn sidecar_options() -> fs::OpenOptions { options } +/// The ignore file a folder may hold (RFC-021 decision 1). +pub const IGNORE_FILE: &str = ".waveflowignore"; + +/// Beyond this an ignore file is refused rather than read: a list of +/// exclusions has no business being larger, and the walk reads one per folder. +const MAX_IGNORE_BYTES: u64 = 64 * 1024; + +/// Names the walk never enters, whatever an ignore file says (RFC-021 +/// decision 1), beside every name beginning with a dot. Compared without +/// regard to case: `$RECYCLE.BIN` is spelled several ways by the systems that +/// write it. +const BUILT_IN_EXCLUSIONS: [&str; 6] = [ + "@eaDir", + "#recycle", + "#snapshot", + "$RECYCLE.BIN", + "System Volume Information", + "lost+found", +]; + +/// How many skipped links one scan names in the log, one line each, before it +/// only counts the rest. +const LOGGED_LINKS: usize = 20; + +/// Whether an entry below the root is left out by name. The root and the +/// folders above it are never asked: a library may well live under a dot +/// folder, and a temporary directory always does. +fn built_in_exclusion(name: &std::ffi::OsStr, depth: usize, is_dir: bool) -> bool { + let name = name.to_string_lossy(); + if name.starts_with('.') { + // Received files land here (RFC-008), and the scan is what keeps them + // in the catalogue after the upload that applied them. + return !(depth == 1 && is_dir && name == crate::services::uploads::MANAGED_DIR); + } + BUILT_IN_EXCLUSIONS + .iter() + .any(|excluded| excluded.eq_ignore_ascii_case(&name)) +} + +/// What one folder's ignore file says. +enum Rules { + /// No file, or a root file with nothing in it. + None, + Matcher(ignore::gitignore::Gitignore), + /// An empty file below the root: the folder and everything under it. + ExcludeAll, + /// A file the walk cannot use, and why. Its folder counts as unreadable. + Unusable(&'static str), +} + +/// Reads `dir`'s ignore file, if it has one. Never through a link, and never +/// in a way that can block, as the lyrics sidecars are read. +fn load_rules(dir: &Path, at_root: bool) -> Rules { + let file = dir.join(IGNORE_FILE); + let metadata = match fs::symlink_metadata(&file) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Rules::None, + Err(_) => return Rules::Unusable("its .waveflowignore could not be read"), + }; + if !metadata.is_file() { + return Rules::Unusable("its .waveflowignore is not a regular file"); + } + if metadata.len() > MAX_IGNORE_BYTES { + return Rules::Unusable("its .waveflowignore is larger than 64 KiB"); + } + let Some(text) = sidecar_options().open(&file).ok().and_then(|opened| { + let mut bytes = Vec::new(); + opened + .take(MAX_IGNORE_BYTES + 1) + .read_to_end(&mut bytes) + .ok()?; + (bytes.len() as u64 <= MAX_IGNORE_BYTES) + .then(|| String::from_utf8_lossy(&bytes).into_owned()) + }) else { + return Rules::Unusable("its .waveflowignore could not be read"); + }; + let text = text.trim_start_matches('\u{feff}'); + if text.trim().is_empty() { + // At the root an empty file excludes nothing: one stray file must not + // mark a whole library missing. + return if at_root { + Rules::None + } else { + Rules::ExcludeAll + }; + } + let mut builder = ignore::gitignore::GitignoreBuilder::new(dir); + for line in text.lines() { + if builder.add_line(None, line).is_err() { + return Rules::Unusable("its .waveflowignore holds a pattern that cannot be read"); + } + } + match builder.build() { + Ok(matcher) => Rules::Matcher(matcher), + Err(_) => Rules::Unusable("its .waveflowignore holds a pattern that cannot be read"), + } +} + +/// Why the walk left a symbolic link behind (RFC-021 decision 3). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SkippedLink { + /// The library does not follow links. + NotFollowed, + /// Its target lies outside the library root. + Outside, + /// Its target does not exist, or cannot be read. + Broken, + /// It leads back to one of its own ancestors. + Loop, +} + +impl SkippedLink { + fn reason(self) -> &'static str { + match self { + Self::NotFollowed => "the library does not follow links", + Self::Outside => "its target is outside the library", + Self::Broken => "its target cannot be read", + Self::Loop => "it leads back to a folder above it", + } + } +} + +/// A folder the walk could not read, relative to the root, and why. +struct Unreadable { + folder: String, + reason: &'static str, +} + /// What the walk found: the audio files, in path order, and the folders it /// could not read, relative to the root. struct Discovery { paths: Vec, - unreadable: Vec, + unreadable: Vec, /// An error named no path inside the root, so the walk cannot say which /// tracks it missed: none may be marked missing. incomplete: bool, + /// Entries left out by an exclusion, a folder counting once whatever it + /// holds (RFC-021 decision 2). They count as not discovered. + excluded: usize, + /// Symbolic links the walk did not follow (RFC-021 decision 3). + skipped_links: usize, +} + +/// What the walk accumulates while `walkdir` decides, entry by entry, what to +/// enter. Shared between the filter and the loop, hence the cell. +#[derive(Default)] +struct WalkState { + /// Each folder's ignore rules, by the path the walk sees it under. + rules: std::collections::HashMap, + unreadable: Vec, + excluded: usize, + skipped_links: usize, +} + +impl WalkState { + /// Whether an ignore file of a folder above `path` leaves it out. The + /// deepest file that has an opinion decides, as in git. + fn ignored(&self, root: &Path, path: &Path, is_dir: bool) -> bool { + let mut folder = path.parent(); + while let Some(current) = folder { + if let Some(rules) = self.rules.get(current) { + let matched = rules.matched(path, is_dir); + if matched.is_ignore() { + return true; + } + if matched.is_whitelist() { + return false; + } + } + if current == root { + break; + } + folder = current.parent(); + } + false + } + + fn skip_link(&mut self, path: &str, why: SkippedLink) { + self.skipped_links += 1; + if self.skipped_links <= LOGGED_LINKS { + tracing::info!(path = %path, reason = why.reason(), "a symbolic link in the library was skipped"); + } + } +} + +/// A path relative to the root, as the catalogue and the log spell it. +fn relative_to(root: &Path, path: &Path) -> String { + path.strip_prefix(root) + .map(|path| path.to_string_lossy().replace('\\', "/")) + .unwrap_or_else(|_| "".to_owned()) +} + +/// The walk's decision on one entry below the root: enter it, or leave it. +fn admit( + entry: &walkdir::DirEntry, + root: &Path, + follow_links: bool, + walk: &std::cell::RefCell, +) -> bool { + if entry.depth() == 0 { + return true; + } + let mut walk = walk.borrow_mut(); + let path = entry.path(); + // Followed, `file_type` is the target's; not followed, it is the link's. + let is_dir = entry.file_type().is_dir(); + let is_link = if follow_links { + entry.path_is_symlink() + } else { + entry.file_type().is_symlink() + }; + // The rules themselves: read with their folder, never counted as left out. + if entry.file_name() == IGNORE_FILE { + return false; + } + // By the name the walk sees, a link's own included (decision 1). + if built_in_exclusion(entry.file_name(), entry.depth(), is_dir) + || walk.ignored(root, path, is_dir) + { + walk.excluded += 1; + tracing::debug!(path = %relative_to(root, path), "left out of the scan"); + return false; + } + if is_link { + let skipped = if !follow_links { + Some(SkippedLink::NotFollowed) + } else { + match fs::canonicalize(path) { + Ok(target) if target.starts_with(root) => None, + Ok(_) => Some(SkippedLink::Outside), + Err(_) => Some(SkippedLink::Broken), + } + }; + if let Some(why) = skipped { + walk.skip_link(&relative_to(root, path), why); + return false; + } + } + if is_dir { + match load_rules(path, false) { + Rules::None => {} + Rules::Matcher(rules) => { + walk.rules.insert(path.to_path_buf(), rules); + } + Rules::ExcludeAll => { + walk.excluded += 1; + tracing::debug!(path = %relative_to(root, path), "left out of the scan by an empty ignore file"); + return false; + } + Rules::Unusable(reason) => { + let folder = relative_to(root, path); + tracing::warn!( + path = %folder, + reason, + "a folder's ignore file cannot be used; its tracks are left as they are" + ); + walk.unreadable.push(Unreadable { folder, reason }); + return false; + } + } + } + true } -/// Walks the library (RFC-018 decision 3). An entry the walk cannot read is -/// logged by its path relative to the root, never the absolute one, and the -/// walk goes on; only the root itself failing fails the scan, since then -/// nothing can be told apart. -fn discover_audio(root: &Path) -> anyhow::Result { +/// Walks the library (RFC-018 decision 3, RFC-021 decisions 1 to 3). +/// +/// An entry the walk cannot read is logged by its path relative to the root, +/// never the absolute one, and the walk goes on; only the root itself failing +/// fails the scan, since then nothing can be told apart — and that includes an +/// ignore file at the root the walk cannot use. +/// +/// With `follow_links`, a file reached through a link is returned under its +/// canonical path, so it is one track however many links name it. +fn discover_audio(root: &Path, follow_links: bool) -> anyhow::Result { + let walk = std::cell::RefCell::new(WalkState::default()); + match load_rules(root, true) { + Rules::Matcher(rules) => { + walk.borrow_mut().rules.insert(root.to_path_buf(), rules); + } + Rules::Unusable(reason) => { + anyhow::bail!("the .waveflowignore at the library root cannot be used: {reason}") + } + Rules::None | Rules::ExcludeAll => {} + } let mut paths = Vec::new(); - let mut unreadable = Vec::new(); let mut incomplete = false; - for entry in WalkDir::new(root).follow_links(false) { + let entries = WalkDir::new(root) + .follow_links(follow_links) + .into_iter() + .filter_entry(|entry| admit(entry, root, follow_links, &walk)); + for entry in entries { let entry = match entry { Ok(entry) => entry, Err(error) if error.depth() == 0 => return Err(error.into()), Err(error) => { - let Some(folder) = error - .path() - .and_then(|path| path.strip_prefix(root).ok()) - .map(|path| path.to_string_lossy().replace('\\', "/")) - else { - tracing::warn!("the library walk failed somewhere it could not name"); - incomplete = true; - unreadable.push("".to_owned()); - continue; + let mut walk = walk.borrow_mut(); + // A link the walk was told to follow and could not: a loop, or + // a target it cannot read. A link, not a folder, so nothing + // under a folder is spared for it. + let link = if error.loop_ancestor().is_some() { + Some(SkippedLink::Loop) + } else if follow_links + && error.path().is_some_and(|path| { + fs::symlink_metadata(path).is_ok_and(|meta| meta.file_type().is_symlink()) + }) + { + Some(SkippedLink::Broken) + } else { + None }; - tracing::warn!( - path = %folder, - "a folder in the library could not be read; its tracks are left as they are" - ); - unreadable.push(folder); + let named = error + .path() + .filter(|path| path.starts_with(root)) + .map(|path| relative_to(root, path)); + match (link, named) { + (Some(why), Some(path)) => walk.skip_link(&path, why), + (_, None) => { + tracing::warn!("the library walk failed somewhere it could not name"); + incomplete = true; + walk.unreadable.push(Unreadable { + folder: "".to_owned(), + reason: "folder could not be read", + }); + } + (None, Some(folder)) => { + tracing::warn!( + path = %folder, + "a folder in the library could not be read; its tracks are left as they are" + ); + walk.unreadable.push(Unreadable { + folder, + reason: "folder could not be read", + }); + } + } continue; } }; - if entry.file_type().is_symlink() || !entry.file_type().is_file() { + // A link is only ever yielded here when it is followed, and then its + // type is its target's. + if !entry.file_type().is_file() { continue; } - let supported = entry - .path() + let path = if follow_links { + match fs::canonicalize(entry.path()) { + Ok(canonical) if canonical.starts_with(root) => canonical, + _ => continue, + } + } else { + entry.into_path() + }; + let supported = path .extension() .and_then(|value| value.to_str()) .is_some_and(|ext| { @@ -719,14 +1086,26 @@ fn discover_audio(root: &Path) -> anyhow::Result { .any(|candidate| candidate.eq_ignore_ascii_case(ext)) }); if supported { - paths.push(entry.into_path()); + paths.push(path); } } paths.sort(); + // Followed links can name one file twice, or name a file the walk also + // found directly; sorted, the copies are adjacent. + paths.dedup(); + let walk = walk.into_inner(); + if walk.skipped_links > LOGGED_LINKS { + tracing::info!( + count = walk.skipped_links - LOGGED_LINKS, + "further symbolic links in the library were skipped" + ); + } Ok(Discovery { paths, - unreadable, + unreadable: walk.unreadable, incomplete, + excluded: walk.excluded, + skipped_links: walk.skipped_links, }) } @@ -1263,7 +1642,80 @@ fn lyrics_hash(entries: &[LyricsInput]) -> String { #[cfg(test)] mod tests { - use super::{extended_tags, read_sidecar, Walked, MAX_LYRICS_BYTES}; + use super::{ + built_in_exclusion, extended_tags, load_rules, read_sidecar, Rules, Walked, IGNORE_FILE, + MAX_IGNORE_BYTES, MAX_LYRICS_BYTES, + }; + + /// RFC-021 decision 1: what is left out by name, below the root only, and + /// the one dot folder that is not. + #[test] + fn built_in_exclusions_spare_the_folder_uploads_land_in() { + use std::ffi::OsStr; + for name in [ + ".hidden", + "._Track.flac", + ".Trash-1000", + "@eaDir", + "#recycle", + "$Recycle.Bin", + "System Volume Information", + "LOST+FOUND", + ] { + assert!(built_in_exclusion(OsStr::new(name), 1, true), "{name}"); + } + assert!(!built_in_exclusion(OsStr::new("Album"), 1, true)); + assert!(!built_in_exclusion(OsStr::new("eaDir"), 1, true)); + // Received files: kept directly under the root, and only there. + assert!(!built_in_exclusion( + OsStr::new(".waveflow-managed"), + 1, + true + )); + assert!(built_in_exclusion(OsStr::new(".waveflow-managed"), 2, true)); + assert!(built_in_exclusion( + OsStr::new(".waveflow-managed"), + 1, + false + )); + } + + /// RFC-021 decision 1: an empty file excludes its folder except at the + /// root, and a file the walk cannot use says why. + #[test] + fn ignore_files_are_read_as_the_rfc_says() { + let temp = tempfile::tempdir().unwrap(); + let dir = temp.path(); + assert!(matches!(load_rules(dir, false), Rules::None)); + + std::fs::write(dir.join(IGNORE_FILE), " \n\t\n").unwrap(); + assert!(matches!(load_rules(dir, false), Rules::ExcludeAll)); + assert!( + matches!(load_rules(dir, true), Rules::None), + "an empty file at the root must exclude nothing" + ); + + std::fs::write(dir.join(IGNORE_FILE), "# scans\n*.wav\n!Keep.wav\n").unwrap(); + let Rules::Matcher(rules) = load_rules(dir, false) else { + panic!("a valid file is a matcher"); + }; + assert!(rules.matched(dir.join("Drop.wav"), false).is_ignore()); + assert!(rules.matched(dir.join("Keep.wav"), false).is_whitelist()); + + std::fs::write(dir.join(IGNORE_FILE), "[z-a]\n").unwrap(); + assert!(matches!(load_rules(dir, false), Rules::Unusable(_))); + + std::fs::write( + dir.join(IGNORE_FILE), + vec![b'#'; MAX_IGNORE_BYTES as usize + 1], + ) + .unwrap(); + assert!(matches!(load_rules(dir, false), Rules::Unusable(_))); + + std::fs::remove_file(dir.join(IGNORE_FILE)).unwrap(); + std::fs::create_dir(dir.join(IGNORE_FILE)).unwrap(); + assert!(matches!(load_rules(dir, false), Rules::Unusable(_))); + } /// A track under a folder the walk could not read is not missing, so no /// file elsewhere may claim it; nor may any track when the walk could not diff --git a/src/services/canvas.rs b/src/services/canvas.rs index 1a0c9311..130a0a90 100644 --- a/src/services/canvas.rs +++ b/src/services/canvas.rs @@ -522,7 +522,7 @@ impl DomainServices { /// names, and [`DomainServices::sweep_canvas_store`] is what collects those /// — the store is content-addressed and therefore enumerable, which is what /// made the sweep writable at all. - async fn release_canvas_blob(&self, hash: &str) { + pub(super) async fn release_canvas_blob(&self, hash: &str) { // Scoped for the same reason as the placing path: the entry goes back // to the map only once this call holds nothing. { diff --git a/src/services/library_lifecycle.rs b/src/services/library_lifecycle.rs new file mode 100644 index 00000000..075ff9d5 --- /dev/null +++ b/src/services/library_lifecycle.rs @@ -0,0 +1,848 @@ +//! A library's lifecycle and its missing tracks (RFC-021 decisions 4 and 5): +//! renaming, moving and deleting a library, listing the tracks its scans could +//! no longer find, and purging them. +//! +//! The one implementation behind the native API, the CLI and the web client. + +use super::*; + +/// How long the retention pass waits between two looks at the libraries. +const RETENTION_INTERVAL: std::time::Duration = std::time::Duration::from_secs(60 * 60); + +/// What a library may be changed into. A field left out is left as it is. +#[derive(Debug, Default, Clone, Deserialize, ToSchema)] +pub struct LibraryUpdate { + /// A new name, 1 to 160 characters once trimmed. The library's owner or an + /// administrator. + pub name: Option, + /// A new root: an existing directory, not itself a link, and not another + /// library's. An administrator only; queues a scan. + pub path: Option, + /// Follow symbolic links that stay inside the root. An administrator only; + /// queues a scan. + pub follow_links: Option, +} + +impl LibraryUpdate { + /// Whether this needs an administrator, and an administrative credential: + /// the path and the links setting decide which part of the server's file + /// system the server reads. + pub fn is_administrative(&self) -> bool { + self.path.is_some() || self.follow_links.is_some() + } +} + +/// What an update set in motion. +#[derive(Debug, Clone, Serialize, ToSchema)] +pub struct LibraryUpdated { + /// The scan a new path or links setting queued, if it queued one. + pub scan_id: Option, +} + +/// A library as an administrator manages it, its root included. Never shown +/// to a member who is not one: where the server keeps its files is the +/// operator's business. +#[derive(Debug, Clone, Serialize, ToSchema)] +pub struct LibrarySettings { + pub id: Uuid, + pub name: String, + pub path: String, + pub follow_links: bool, +} + +/// What a deletion removed from the catalogue. Nothing was removed from disk. +#[derive(Debug, Clone, Serialize, ToSchema)] +pub struct LibraryDeleted { + pub tracks: i64, + pub albums: i64, + pub artists: i64, +} + +/// A track whose file the scans can no longer find. +#[derive(Debug, Clone, Serialize, ToSchema)] +pub struct UnavailableTrack { + pub id: Uuid, + /// Where the file was, relative to the library. + pub relative_path: String, + pub title: String, + pub artist: Option, + pub album: Option, + pub duration_ms: i64, + /// Since when, in epoch milliseconds. + pub unavailable_since: i64, + /// What a purge would lose, across every account: playlist entries, + /// favourites and recorded plays holding this track. + pub playlist_entries: i64, + pub favorites: i64, + pub plays: i64, +} + +/// One page of a library's missing tracks, oldest first. +#[derive(Debug, Clone, Serialize, ToSchema)] +pub struct UnavailableTracks { + /// How many there are in all, not on this page. + pub total: i64, + pub tracks: Vec, +} + +/// Which missing tracks to purge: named ones, or all of them, and never +/// neither — a purge is explicit or it does not happen. +#[derive(Debug, Clone, Default, Deserialize, ToSchema)] +pub struct PurgeRequest { + pub track_ids: Option>, + #[serde(default)] + pub all: bool, +} + +/// What a purge removed. +#[derive(Debug, Clone, Serialize, ToSchema)] +pub struct PurgeOutcome { + pub purged: i64, +} + +/// Which tracks one purge is about. +enum Selection { + All, + Named(String), + OlderThan(i64), +} + +/// What the caller is to one library. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Standing { + Admin, + Owner, + Manager, + Listener, +} + +impl Standing { + /// Owners, managers and administrators look after a library's missing + /// tracks, as they look after what needs attention in it. + fn curates(self) -> bool { + matches!(self, Self::Admin | Self::Owner | Self::Manager) + } +} + +/// The caller's standing on a library, read on the connection the operation +/// runs on. A library that does not exist, and one the caller is neither a +/// member of nor an administrator over, are the same answer: not found. +async fn standing_on( + connection: &mut SqliteConnection, + actor_id: Uuid, + library_id: Uuid, +) -> Result { + let row = sqlx::query( + "SELECT (SELECT role FROM library_member WHERE library_id = l.id AND user_id = ?) AS role, \ + EXISTS (SELECT 1 FROM account a WHERE a.id = ? AND a.role = 'admin' \ + AND a.disabled = 0) AS admin \ + FROM library l WHERE l.id = ?", + ) + .bind(actor_id.to_string()) + .bind(actor_id.to_string()) + .bind(library_id.to_string()) + .fetch_optional(&mut *connection) + .await? + .ok_or(ServiceError::NotFound)?; + if row.try_get::("admin")? != 0 { + return Ok(Standing::Admin); + } + match row.try_get::, _>("role")?.as_deref() { + Some("owner") => Ok(Standing::Owner), + Some("manager") => Ok(Standing::Manager), + Some("listener") => Ok(Standing::Listener), + _ => Err(ServiceError::NotFound), + } +} + +/// A root as creating a library accepts one: an existing directory, not +/// itself a link, stored canonical. +async fn validated_root(path: &str) -> Result { + let path = PathBuf::from(path); + let metadata = tokio::fs::symlink_metadata(&path) + .await + .map_err(|_| ServiceError::Invalid)?; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + return Err(ServiceError::Invalid); + } + tokio::fs::canonicalize(&path) + .await + .map_err(|_| ServiceError::Invalid) +} + +/// An audit row with what it is about, in the transaction of the change. +async fn audit( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + actor_id: Option, + kind: &str, + library_id: Uuid, + details: serde_json::Value, +) -> Result<(), sqlx::Error> { + sqlx::query( + "INSERT INTO audit_event (actor_user_id, kind, subject_id, occurred_at, details_json) \ + VALUES (?, ?, ?, ?, ?)", + ) + .bind(actor_id.map(|id| id.to_string())) + .bind(kind) + .bind(library_id.to_string()) + .bind(now_ms()) + .bind(details.to_string()) + .execute(&mut **tx) + .await?; + Ok(()) +} + +/// Deletes the favourites and ratings of the albums and artists among +/// `albums` and `artists` (JSON arrays of ids) that no longer exist. +async fn forget_vanished_entities( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + albums: &str, + artists: &str, +) -> Result<(), sqlx::Error> { + sqlx::query( + "DELETE FROM user_star WHERE entity_type = 'album' \ + AND entity_id IN (SELECT value FROM json_each(?)) \ + AND NOT EXISTS (SELECT 1 FROM album WHERE album.id = user_star.entity_id)", + ) + .bind(albums) + .execute(&mut **tx) + .await?; + sqlx::query( + "DELETE FROM user_rating WHERE entity_type = 'album' \ + AND entity_id IN (SELECT value FROM json_each(?)) \ + AND NOT EXISTS (SELECT 1 FROM album WHERE album.id = user_rating.entity_id)", + ) + .bind(albums) + .execute(&mut **tx) + .await?; + sqlx::query( + "DELETE FROM user_star WHERE entity_type = 'artist' \ + AND entity_id IN (SELECT value FROM json_each(?)) \ + AND NOT EXISTS (SELECT 1 FROM artist WHERE artist.id = user_star.entity_id)", + ) + .bind(artists) + .execute(&mut **tx) + .await?; + sqlx::query( + "DELETE FROM user_rating WHERE entity_type = 'artist' \ + AND entity_id IN (SELECT value FROM json_each(?)) \ + AND NOT EXISTS (SELECT 1 FROM artist WHERE artist.id = user_rating.entity_id)", + ) + .bind(artists) + .execute(&mut **tx) + .await?; + Ok(()) +} + +/// What one purge removed, and the canvases it may have left with no link: +/// those are released after the commit, under their own locks, as removing a +/// canvas releases them. +#[derive(Default)] +struct Purged { + tracks: i64, + canvases: Vec, +} + +/// The purge itself, in the caller's transaction under the writer gate it +/// holds: the tracks that are **still** unavailable among those selected, what +/// cascades from them, their favourites and search rows, then the scan's own +/// consolidation, so no album or artist is left holding nothing. +async fn purge_in( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + library_id: Uuid, + selection: &Selection, +) -> Result { + let library = library_id.to_string(); + let targets: Vec<(String, Option)> = match selection { + Selection::All => { + sqlx::query_as( + "SELECT id, album_id FROM track WHERE library_id = ? AND is_available = 0", + ) + .bind(&library) + .fetch_all(&mut **tx) + .await? + } + Selection::Named(ids) => { + sqlx::query_as( + "SELECT id, album_id FROM track WHERE library_id = ? AND is_available = 0 \ + AND id IN (SELECT value FROM json_each(?))", + ) + .bind(&library) + .bind(ids) + .fetch_all(&mut **tx) + .await? + } + Selection::OlderThan(cutoff) => { + sqlx::query_as( + "SELECT id, album_id FROM track WHERE library_id = ? AND is_available = 0 \ + AND COALESCE(unavailable_since, updated_at) < ?", + ) + .bind(&library) + .bind(cutoff) + .fetch_all(&mut **tx) + .await? + } + }; + if targets.is_empty() { + return Ok(Purged::default()); + } + let tracks = serde_json::to_string( + &targets + .iter() + .map(|(id, _)| id.as_str()) + .collect::>(), + ) + .expect("a list of strings serializes"); + // What may be left holding nothing once these go, read before they do. + let albums: Vec = targets + .iter() + .filter_map(|(_, album)| album.clone()) + .collect::>() + .into_iter() + .collect(); + let mut artists: std::collections::BTreeSet = sqlx::query_scalar( + "SELECT DISTINCT artist_id FROM track_participant \ + WHERE track_id IN (SELECT value FROM json_each(?))", + ) + .bind(&tracks) + .fetch_all(&mut **tx) + .await? + .into_iter() + .collect(); + let albums = serde_json::to_string(&albums).expect("a list of strings serializes"); + artists.extend( + sqlx::query_scalar::<_, String>( + "SELECT album_artist_id FROM album WHERE album_artist_id IS NOT NULL \ + AND id IN (SELECT value FROM json_each(?))", + ) + .bind(&albums) + .fetch_all(&mut **tx) + .await?, + ); + let artists = serde_json::to_string(&artists).expect("a list of strings serializes"); + // The loops these tracks carry: their links cascade, the blobs do not. + let canvases: Vec = sqlx::query_scalar( + "SELECT DISTINCT canvas_hash FROM track_canvas \ + WHERE track_id IN (SELECT value FROM json_each(?))", + ) + .bind(&tracks) + .fetch_all(&mut **tx) + .await?; + + // Neither has a foreign key to cascade through: an entity id names a + // track, an album or an artist, whichever the type says. + sqlx::query( + "DELETE FROM user_star WHERE entity_type = 'track' \ + AND entity_id IN (SELECT value FROM json_each(?))", + ) + .bind(&tracks) + .execute(&mut **tx) + .await?; + sqlx::query( + "DELETE FROM user_rating WHERE entity_type = 'track' \ + AND entity_id IN (SELECT value FROM json_each(?))", + ) + .bind(&tracks) + .execute(&mut **tx) + .await?; + // A virtual table: nothing cascades into it. + sqlx::query("DELETE FROM track_fts WHERE track_id IN (SELECT value FROM json_each(?))") + .bind(&tracks) + .execute(&mut **tx) + .await?; + // `is_available = 0` again, in the statement that deletes: the selection + // above was read in this transaction under the writer gate, so nothing can + // have brought a track back since, and this says so where it matters. + let purged = sqlx::query( + "DELETE FROM track WHERE library_id = ? AND is_available = 0 \ + AND id IN (SELECT value FROM json_each(?))", + ) + .bind(&library) + .bind(&tracks) + .execute(&mut **tx) + .await? + .rows_affected() as i64; + // The scan's own consolidation: the album and artist sweeps, the credits + // and the counts, the artist search index — over what is left. + Database::consolidate_sort_names_in(tx, library_id).await?; + Database::consolidate_catalog_derivations_in(tx, library_id).await?; + forget_vanished_entities(tx, &albums, &artists).await?; + sqlx::query( + "DELETE FROM genre WHERE library_id = ? \ + AND NOT EXISTS (SELECT 1 FROM track_genre tg WHERE tg.genre_id = genre.id)", + ) + .bind(&library) + .execute(&mut **tx) + .await?; + Ok(Purged { + tracks: purged, + canvases, + }) +} + +impl DomainServices { + /// The settings an administrator manages, the root included. + pub async fn library_settings( + &self, + actor_id: Uuid, + library_id: Uuid, + ) -> Result { + self.require_admin(actor_id).await?; + let settings = self + .db + .library_scan_settings(library_id) + .await? + .ok_or(ServiceError::NotFound)?; + Ok(LibrarySettings { + id: settings.library.id, + name: settings.library.name, + path: settings.library.root_path.to_string_lossy().into_owned(), + follow_links: settings.follow_links, + }) + } + + /// Renames a library, moves its root, or changes whether its walk follows + /// links (RFC-021 decision 5). + /// + /// A new name is the owner's or an administrator's to give; the path and + /// the links setting are an administrator's. A new root, or a changed links + /// setting, queues a scan; since tracks are matched by their path relative + /// to the root, a root moved with its tree intact keeps every identifier. + /// Either refuses while the library is being scanned. + pub async fn update_library( + &self, + actor_id: Uuid, + library_id: Uuid, + update: LibraryUpdate, + ) -> Result { + if update.name.is_none() && !update.is_administrative() { + return Err(ServiceError::Invalid); + } + let standing = { + let mut connection = self.db.pool().acquire().await?; + standing_on(&mut connection, actor_id, library_id).await? + }; + let allowed = if update.is_administrative() { + standing == Standing::Admin + } else { + matches!(standing, Standing::Admin | Standing::Owner) + }; + if !allowed { + return Err(ServiceError::Forbidden); + } + let name = match update.name.as_deref().map(str::trim) { + Some(name) if name.is_empty() || name.chars().count() > 160 => { + return Err(ServiceError::Invalid) + } + other => other.map(str::to_owned), + }; + let root = match update.path.as_deref() { + Some(path) => Some(validated_root(path).await?), + None => None, + }; + // Only what changes how the library is walked has to wait for no scan; + // a new name is a label. + let _scan_lock = if update.is_administrative() { + Some( + self.scanner + .try_lock_library(library_id) + .ok_or(ServiceError::Conflict)?, + ) + } else { + None + }; + let before = self + .db + .library_scan_settings(library_id) + .await? + .ok_or(ServiceError::NotFound)?; + let moved = root + .as_ref() + .is_some_and(|root| *root != before.library.root_path); + let relinked = update + .follow_links + .is_some_and(|follow| follow != before.follow_links); + { + let _writer = self.db.writer_guard().await; + let mut tx = self.db.pool().begin().await?; + let written = sqlx::query( + "UPDATE library SET name = COALESCE(?, name), \ + root_path = COALESCE(?, root_path), \ + follow_links = COALESCE(?, follow_links), updated_at = ? \ + WHERE id = ?", + ) + .bind(name.as_deref()) + .bind( + root.as_ref() + .map(|root| root.to_string_lossy().into_owned()), + ) + .bind(update.follow_links.map(i64::from)) + .bind(now_ms()) + .bind(library_id.to_string()) + .execute(&mut *tx) + .await; + match written { + Ok(result) if result.rows_affected() == 0 => return Err(ServiceError::NotFound), + Ok(_) => {} + // Another library already has that root. + Err(sqlx::Error::Database(error)) if error.is_unique_violation() => { + return Err(ServiceError::Conflict) + } + Err(error) => return Err(error.into()), + } + if let Some(name) = name.as_deref().filter(|name| *name != before.library.name) { + audit( + &mut tx, + Some(actor_id), + "library.renamed", + library_id, + serde_json::json!({ "name": name }), + ) + .await?; + } + if moved { + audit( + &mut tx, + Some(actor_id), + "library.moved", + library_id, + serde_json::json!({}), + ) + .await?; + } + if relinked { + audit( + &mut tx, + Some(actor_id), + "library.follow_links_set", + library_id, + serde_json::json!({ "follow_links": update.follow_links }), + ) + .await?; + } + tx.commit().await?; + } + drop(_scan_lock); + if moved { + self.scanner.libraries_changed(); + } + if !(moved || relinked) { + return Ok(LibraryUpdated { scan_id: None }); + } + let after = self + .db + .library_scan_settings(library_id) + .await? + .ok_or(ServiceError::NotFound)?; + let scan_id = self + .db + .create_scan_job( + library_id, + Some(actor_id), + if moved { "library_moved" } else { "manual" }, + ) + .await?; + self.scanner.spawn(scan_id, after.library); + Ok(LibraryUpdated { + scan_id: Some(scan_id), + }) + } + + /// Deletes a library from the catalogue, and nothing from the disk + /// (RFC-021 decision 5). An administrator's, confirmed by the library's + /// current name, and refused while the library is being scanned. + /// + /// Every track goes, with everything that cascades from it for every + /// member — playlist entries, favourites, plays, corrections — and every + /// album, artist and genre, with their favourites and search rows; then the + /// library row, which takes the rest with it. Received files stay where + /// they are; the staging files of transfers still open are removed after + /// the commit. + pub async fn delete_library( + &self, + actor_id: Uuid, + library_id: Uuid, + confirm: &str, + ) -> Result { + let standing = { + let mut connection = self.db.pool().acquire().await?; + standing_on(&mut connection, actor_id, library_id).await? + }; + if standing != Standing::Admin { + return Err(ServiceError::Forbidden); + } + let settings = self + .db + .library_scan_settings(library_id) + .await? + .ok_or(ServiceError::NotFound)?; + if confirm != settings.library.name { + return Err(ServiceError::Invalid); + } + let _scan_lock = self + .scanner + .try_lock_library(library_id) + .ok_or(ServiceError::Conflict)?; + let library = library_id.to_string(); + let deleted; + let staging: Vec; + let canvases: Vec; + { + let _writer = self.db.writer_guard().await; + let mut tx = self.db.pool().begin().await?; + let (tracks, albums, artists): (i64, i64, i64) = sqlx::query_as( + "SELECT (SELECT COUNT(*) FROM track WHERE library_id = ?1), \ + (SELECT COUNT(*) FROM album WHERE library_id = ?1), \ + (SELECT COUNT(*) FROM artist WHERE library_id = ?1)", + ) + .bind(&library) + .fetch_one(&mut *tx) + .await?; + staging = sqlx::query_scalar("SELECT id FROM upload_session WHERE library_id = ?") + .bind(&library) + .fetch_all(&mut *tx) + .await?; + // The loops the library's tracks carry: their links cascade, the + // blobs do not, and another library may share them. + canvases = sqlx::query_scalar( + "SELECT DISTINCT canvas_hash FROM track_canvas WHERE library_id = ?", + ) + .bind(&library) + .fetch_all(&mut *tx) + .await?; + for statement in [ + "DELETE FROM user_star WHERE \ + (entity_type = 'track' AND entity_id IN (SELECT id FROM track WHERE library_id = ?1)) \ + OR (entity_type = 'album' AND entity_id IN (SELECT id FROM album WHERE library_id = ?1)) \ + OR (entity_type = 'artist' AND entity_id IN (SELECT id FROM artist WHERE library_id = ?1))", + "DELETE FROM user_rating WHERE \ + (entity_type = 'track' AND entity_id IN (SELECT id FROM track WHERE library_id = ?1)) \ + OR (entity_type = 'album' AND entity_id IN (SELECT id FROM album WHERE library_id = ?1)) \ + OR (entity_type = 'artist' AND entity_id IN (SELECT id FROM artist WHERE library_id = ?1))", + // Virtual tables: nothing cascades into them. + "DELETE FROM track_fts WHERE library_id = ?1", + "DELETE FROM artist_fts WHERE library_id = ?1", + // In this order, because a track restricts the deletion of its + // album and an album that of its album artist: left to the + // library's cascade, which visits them in no promised order, + // the deletion could trip over its own rows. + "DELETE FROM track WHERE library_id = ?1", + "DELETE FROM album_participant WHERE library_id = ?1", + "DELETE FROM album WHERE library_id = ?1", + "DELETE FROM artist WHERE library_id = ?1", + "DELETE FROM genre WHERE library_id = ?1", + ] { + sqlx::query(statement) + .bind(&library) + .execute(&mut *tx) + .await?; + } + audit( + &mut tx, + Some(actor_id), + "library.deleted", + library_id, + serde_json::json!({ + "name": settings.library.name, + "tracks": tracks, + "albums": albums, + "artists": artists, + }), + ) + .await?; + sqlx::query("DELETE FROM library WHERE id = ?") + .bind(&library) + .execute(&mut *tx) + .await?; + tx.commit().await?; + deleted = LibraryDeleted { + tracks, + albums, + artists, + }; + } + // After the commit, and best effort: a file left behind is litter in + // the server's own folder, and the rows that named it are gone. + let managed = settings.library.root_path.join(uploads::MANAGED_DIR); + for session in staging { + let _ = tokio::fs::remove_file(managed.join(format!("{session}.part"))).await; + } + for hash in &canvases { + self.release_canvas_blob(hash).await; + } + self.scanner.libraries_changed(); + Ok(deleted) + } + + /// One page of the tracks a library's scans can no longer find, oldest + /// first, with what purging each would cost (RFC-021 decision 4). + pub async fn unavailable_tracks( + &self, + actor_id: Uuid, + library_id: Uuid, + page: BrowsePage, + ) -> Result { + let mut connection = self.db.pool().acquire().await?; + if !standing_on(&mut connection, actor_id, library_id) + .await? + .curates() + { + return Err(ServiceError::Forbidden); + } + let total: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM track WHERE library_id = ? AND is_available = 0", + ) + .bind(library_id.to_string()) + .fetch_one(&mut *connection) + .await?; + let rows = sqlx::query( + "SELECT t.id, t.relative_path, COALESCE(ovr.title, t.title) AS title, \ + t.artist_display, t.album_title, t.duration_ms, \ + COALESCE(t.unavailable_since, t.updated_at) AS since, \ + (SELECT COUNT(*) FROM playlist_track pt WHERE pt.track_id = t.id) \ + AS playlist_entries, \ + (SELECT COUNT(*) FROM user_star us \ + WHERE us.entity_type = 'track' AND us.entity_id = t.id) AS favorites, \ + (SELECT COUNT(*) FROM play_event pe WHERE pe.track_id = t.id) AS plays \ + FROM track t LEFT JOIN track_override ovr ON ovr.track_id = t.id \ + WHERE t.library_id = ? AND t.is_available = 0 \ + ORDER BY since, t.relative_path, t.id LIMIT ? OFFSET ?", + ) + .bind(library_id.to_string()) + .bind(page.limit) + .bind(page.offset) + .fetch_all(&mut *connection) + .await?; + let tracks = rows + .into_iter() + .map(|row| { + Ok(UnavailableTrack { + id: parse_uuid(row.try_get("id")?)?, + relative_path: row.try_get("relative_path")?, + title: row.try_get("title")?, + artist: row.try_get("artist_display")?, + album: row.try_get("album_title")?, + duration_ms: row.try_get("duration_ms")?, + unavailable_since: row.try_get("since")?, + playlist_entries: row.try_get("playlist_entries")?, + favorites: row.try_get("favorites")?, + plays: row.try_get("plays")?, + }) + }) + .collect::, sqlx::Error>>()?; + Ok(UnavailableTracks { total, tracks }) + } + + /// Purges missing tracks, explicitly (RFC-021 decision 4). + /// + /// Only those still unavailable when it runs: a track that came back + /// between the listing and the request is kept. One transaction under the + /// writer gate, refused while the library is being scanned — a running + /// scan may be about to relocate one of these onto a file it just found. + pub async fn purge_unavailable_tracks( + &self, + actor_id: Uuid, + library_id: Uuid, + request: PurgeRequest, + ) -> Result { + let selection = match (request.all, request.track_ids) { + (true, None) => Selection::All, + (false, Some(ids)) if !ids.is_empty() => Selection::Named( + serde_json::to_string(&ids.iter().map(|id| id.to_string()).collect::>()) + .expect("a list of strings serializes"), + ), + _ => return Err(ServiceError::Invalid), + }; + let _scan_lock = self + .scanner + .try_lock_library(library_id) + .ok_or(ServiceError::Conflict)?; + let purged = { + let _writer = self.db.writer_guard().await; + let mut tx = self.db.pool().begin().await?; + if !standing_on(&mut tx, actor_id, library_id).await?.curates() { + return Err(ServiceError::Forbidden); + } + let purged = purge_in(&mut tx, library_id, &selection).await?; + if purged.tracks > 0 { + audit( + &mut tx, + Some(actor_id), + "library.tracks_purged", + library_id, + serde_json::json!({ "tracks": purged.tracks }), + ) + .await?; + } + tx.commit().await?; + purged + }; + // After the commit and out of the writer gate, which releasing takes. + for hash in &purged.canvases { + self.release_canvas_blob(hash).await; + } + Ok(PurgeOutcome { + purged: purged.tracks, + }) + } + + /// One retention pass: in every library not being scanned, the tracks + /// unavailable since before `cutoff` (epoch milliseconds). Public so a test + /// can run it rather than wait an hour. Answers how many went. + pub async fn purge_expired_unavailable_tracks(&self, cutoff: i64) -> Result { + let mut total = 0; + for library in self.db.all_libraries().await? { + let Some(_scan_lock) = self.scanner.try_lock_library(library.id) else { + continue; + }; + let purged = { + let _writer = self.db.writer_guard().await; + let mut tx = self.db.pool().begin().await?; + let purged = purge_in(&mut tx, library.id, &Selection::OlderThan(cutoff)).await?; + if purged.tracks > 0 { + audit( + &mut tx, + None, + "library.tracks_purged", + library.id, + serde_json::json!({ "tracks": purged.tracks, "retention": true }), + ) + .await?; + } + tx.commit().await?; + purged + }; + for hash in &purged.canvases { + self.release_canvas_blob(hash).await; + } + total += purged.tracks; + } + Ok(total) + } + + /// The retention of missing tracks, when the operator set one + /// (`WAVEFLOW_UNAVAILABLE_RETENTION_DAYS`): once at boot, then hourly. + pub fn spawn_unavailable_purge(&self) { + let Some(retention) = self.unavailable_retention else { + return; + }; + let services = self.clone(); + tokio::spawn(async move { + let mut ticker = tokio::time::interval(RETENTION_INTERVAL); + loop { + ticker.tick().await; + let cutoff = now_ms() + .saturating_sub(i64::try_from(retention.as_millis()).unwrap_or(i64::MAX)); + match services.purge_expired_unavailable_tracks(cutoff).await { + Ok(0) => {} + Ok(purged) => { + tracing::info!(purged, "missing tracks past their retention were purged") + } + Err(error) => { + tracing::warn!(%error, "could not purge missing tracks past their retention") + } + } + } + }); + } +} diff --git a/src/services/mod.rs b/src/services/mod.rs index 6276c40a..93239bfd 100644 --- a/src/services/mod.rs +++ b/src/services/mod.rs @@ -1301,6 +1301,9 @@ pub struct DomainServices { scanner: crate::scanner::ScanManager, uploads: crate::config::UploadLimits, library_event_retention: crate::config::LibraryEventRetention, + /// How long a missing track is kept before the retention pass purges it, + /// or `None`, the default, for never (RFC-021 decision 4). + unavailable_retention: Option, /// Bounds a credential an account mints for itself. See the field on /// [`crate::config::Config`] for why the administrative path is not /// bounded by it. @@ -1423,6 +1426,7 @@ mod favorites; mod lastfm; mod library_events; mod library_folders; +mod library_lifecycle; mod library_tree; mod lyrics_uploads; mod playback; @@ -1464,6 +1468,10 @@ pub use library_folders::{ FiledFolder, FiledTree, FolderPatch, FolderRemoved, Placed, MAX_FOLDERS_PER_LIBRARY, MAX_FOLDER_DEPTH, MAX_PLACEMENTS, }; +pub use library_lifecycle::{ + LibraryDeleted, LibrarySettings, LibraryUpdate, LibraryUpdated, PurgeOutcome, PurgeRequest, + UnavailableTrack, UnavailableTracks, +}; pub use library_tree::{DiskFolder, DiskListing, DiskTrack}; pub use lyrics_uploads::{LyricsFile, MAX_UPLOAD_BYTES as MAX_LYRICS_UPLOAD_BYTES}; pub use quality::{ @@ -1493,6 +1501,7 @@ impl DomainServices { scanner, uploads: config.uploads, library_event_retention: config.library_event_retention, + unavailable_retention: config.unavailable_retention, api_token_ttl: config.api_token_ttl, upload_locks: Arc::new(dashmap::DashMap::new()), canvas: config.canvas, diff --git a/src/watcher.rs b/src/watcher.rs new file mode 100644 index 00000000..a66ab052 --- /dev/null +++ b/src/watcher.rs @@ -0,0 +1,353 @@ +//! Watching library roots, so a change is scanned in seconds rather than at +//! the next interval (RFC-021 decision 6). +//! +//! One `notify` watcher observes every root recursively. Its events are only a +//! hint that something moved: they name a library, never what to read, and the +//! scan they trigger is the ordinary incremental one, which works out by itself +//! what changed. The debouncing is this module's own, per library: a library is +//! scanned once its changes have been quiet for the configured delay, or once +//! thirty delays have passed since the first of them when they never are. +//! +//! The periodic scan stays the safety net. A watch that cannot be established — +//! a mount that does not support it, the inotify limit reached, a backend that +//! will not start — is logged once and the library is left to that net. + +use std::{ + collections::{HashMap, HashSet}, + path::{Component, Path, PathBuf}, + time::Duration, +}; + +use notify::{ + event::{MetadataKind, ModifyKind}, + Event, EventKind, RecursiveMode, Watcher, +}; +use tokio::{sync::mpsc, time::Instant}; +use uuid::Uuid; + +use crate::{catalog::LibraryRecord, database::Database, scanner::ScanManager}; + +/// How often the watched set is re-read even when nothing in this process said +/// it changed: the CLI adds and removes libraries from another process. +const REFRESH_INTERVAL: Duration = Duration::from_secs(5 * 60); + +/// How many delays a stream of changes that never settles waits before it is +/// scanned anyway. +const MAX_WAIT_DELAYS: u32 = 30; + +/// Libraries with changes waiting, and when they arrived. +#[derive(Debug)] +pub struct Settle { + delay: Duration, + max_wait: Duration, + pending: HashMap, +} + +impl Settle { + pub fn new(delay: Duration) -> Self { + Self { + delay, + max_wait: delay * MAX_WAIT_DELAYS, + pending: HashMap::new(), + } + } + + /// Records a change in one library. + pub fn touch(&mut self, library_id: Uuid, now: Instant) { + self.pending + .entry(library_id) + .and_modify(|(_, last)| *last = now) + .or_insert((now, now)); + } + + /// The libraries whose changes have settled and that no scan is running + /// for, now no longer pending. One that has settled while a scan of it + /// runs stays pending, and is answered here once that scan is over: the + /// running scan may have walked past the change. + pub fn due(&mut self, now: Instant, scanning: impl Fn(Uuid) -> bool) -> Vec { + let mut due = Vec::new(); + let (delay, max_wait) = (self.delay, self.max_wait); + self.pending.retain(|library_id, (first, last)| { + let settled = + now.duration_since(*last) >= delay || now.duration_since(*first) >= max_wait; + if settled && !scanning(*library_id) { + due.push(*library_id); + false + } else { + true + } + }); + due.sort(); + due + } + + pub fn is_pending(&self, library_id: Uuid) -> bool { + self.pending.contains_key(&library_id) + } +} + +/// Whether an event can be a change to the music. Reads are not, and the +/// scanner reads every file it hashes: answering them would have the watcher +/// trigger itself for ever. +pub fn relevant(event: &Event) -> bool { + !matches!( + event.kind, + EventKind::Access(_) | EventKind::Modify(ModifyKind::Metadata(MetadataKind::AccessTime)) + ) +} + +/// The library a changed path belongs to, the deepest root first. Nothing +/// under `.waveflow-managed`: an upload writes there, and applies what it +/// wrote itself. +pub fn library_of(libraries: &HashMap, path: &Path) -> Option { + libraries + .values() + .filter_map(|library| { + let relative = path.strip_prefix(&library.root_path).ok()?; + let managed = matches!( + relative.components().next(), + Some(Component::Normal(first)) if first == crate::services::uploads::MANAGED_DIR + ); + (!managed).then_some((library.root_path.components().count(), library.id)) + }) + .max() + .map(|(_, id)| id) +} + +/// Starts watching every library, and keeps the watched set in step with the +/// catalogue. Answers at once; the watcher runs until the process ends. +pub fn spawn(scanner: ScanManager, db: Database, delay: Duration) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + let (sender, mut events) = mpsc::unbounded_channel::>(); + let mut watcher = match notify::recommended_watcher(move |event| { + let _ = sender.send(event); + }) { + Ok(watcher) => watcher, + Err(error) => { + tracing::warn!( + %error, + "the file system watcher could not start; libraries are scanned on the interval only" + ); + return; + } + }; + let mut watched: HashMap = HashMap::new(); + let mut refused: HashSet<(Uuid, PathBuf)> = HashSet::new(); + let mut changes = scanner.library_changes(); + let mut refresh = tokio::time::interval(REFRESH_INTERVAL); + let mut tick = tokio::time::interval(delay.min(Duration::from_secs(1))); + tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + let mut settle = Settle::new(delay); + loop { + tokio::select! { + _ = refresh.tick() => { + reconcile(&db, &mut watcher, &mut watched, &mut refused).await; + } + changed = changes.changed() => { + if changed.is_err() { + return; + } + reconcile(&db, &mut watcher, &mut watched, &mut refused).await; + } + Some(event) = events.recv() => { + let now = Instant::now(); + match event { + // The kernel dropped events: anything may have changed. + Ok(event) if event.need_rescan() => { + for library_id in watched.keys() { + settle.touch(*library_id, now); + } + } + Ok(event) if relevant(&event) => { + for path in &event.paths { + if let Some(library_id) = library_of(&watched, path) { + settle.touch(library_id, now); + } + } + } + Ok(_) => {} + Err(error) => { + for path in &error.paths { + if let Some(library_id) = library_of(&watched, path) { + settle.touch(library_id, now); + } + } + tracing::debug!(%error, "the file system watcher reported an error"); + } + } + } + _ = tick.tick() => { + for library_id in settle.due(Instant::now(), |id| scanner.is_scanning(id)) { + let Some(library) = watched.get(&library_id).cloned() else { + continue; + }; + if let Err(error) = scanner.trigger(library, None, "watch").await { + tracing::warn!(%library_id, %error, "could not queue a scan for a watched change"); + } + } + } + } + } + }) +} + +/// Watches what the catalogue holds and nothing else. +async fn reconcile( + db: &Database, + watcher: &mut notify::RecommendedWatcher, + watched: &mut HashMap, + refused: &mut HashSet<(Uuid, PathBuf)>, +) { + let libraries = match db.all_libraries().await { + Ok(libraries) => libraries, + Err(error) => { + tracing::warn!(%error, "could not list the libraries to watch"); + return; + } + }; + let current: HashMap = libraries + .into_iter() + .map(|library| (library.id, library)) + .collect(); + let gone: Vec = watched + .iter() + .filter(|(id, library)| { + current + .get(id) + .is_none_or(|now| now.root_path != library.root_path) + }) + .map(|(id, _)| *id) + .collect(); + for library_id in gone { + if let Some(library) = watched.remove(&library_id) { + let _ = watcher.unwatch(&library.root_path); + } + } + refused.retain(|(id, root)| current.get(id).is_some_and(|now| now.root_path == *root)); + for (library_id, library) in current { + if watched.contains_key(&library_id) + || refused.contains(&(library_id, library.root_path.clone())) + { + continue; + } + match watcher.watch(&library.root_path, RecursiveMode::Recursive) { + Ok(()) => { + tracing::debug!(%library_id, "watching the library for changes"); + watched.insert(library_id, library); + } + Err(error) => { + // A recursive watch can fail halfway; take back what it set. + let _ = watcher.unwatch(&library.root_path); + tracing::warn!( + %library_id, + %error, + "the library cannot be watched; it is scanned on the interval only" + ); + refused.insert((library_id, library.root_path)); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn library(id: Uuid, root: &str) -> LibraryRecord { + LibraryRecord { + id, + name: "watched".into(), + root_path: PathBuf::from(root), + } + } + + #[test] + fn a_library_is_due_once_its_changes_are_quiet() { + let delay = Duration::from_secs(10); + let mut settle = Settle::new(delay); + let library = Uuid::new_v4(); + let start = Instant::now(); + settle.touch(library, start); + settle.touch(library, start + Duration::from_secs(8)); + assert!(settle + .due(start + Duration::from_secs(17), |_| false) + .is_empty()); + assert_eq!( + settle.due(start + Duration::from_secs(18), |_| false), + vec![library] + ); + assert!( + !settle.is_pending(library), + "a due library is no longer pending" + ); + } + + #[test] + fn changes_that_never_settle_are_scanned_after_thirty_delays() { + let delay = Duration::from_secs(1); + let mut settle = Settle::new(delay); + let library = Uuid::new_v4(); + let start = Instant::now(); + // A change every second, for thirty seconds: never quiet for a delay. + for second in 0..30 { + settle.touch(library, start + Duration::from_secs(second)); + assert!(settle + .due(start + Duration::from_millis(second * 1000 + 100), |_| { + false + }) + .is_empty()); + } + assert_eq!( + settle.due(start + Duration::from_secs(30), |_| false), + vec![library] + ); + } + + /// Never a second scan while one runs: the library stays pending, and is + /// due as soon as the scan is over. + #[test] + fn no_scan_starts_while_one_runs_for_the_library() { + let delay = Duration::from_secs(1); + let mut settle = Settle::new(delay); + let (busy, idle) = (Uuid::new_v4(), Uuid::new_v4()); + let start = Instant::now(); + settle.touch(busy, start); + settle.touch(idle, start); + let later = start + Duration::from_secs(5); + assert_eq!(settle.due(later, |id| id == busy), vec![idle]); + assert!(settle.is_pending(busy)); + assert_eq!(settle.due(later, |_| false), vec![busy]); + } + + #[test] + fn reads_and_received_files_are_not_changes() { + let read = Event::new(EventKind::Access(notify::event::AccessKind::Any)); + assert!(!relevant(&read)); + let atime = Event::new(EventKind::Modify(ModifyKind::Metadata( + MetadataKind::AccessTime, + ))); + assert!(!relevant(&atime)); + let created = Event::new(EventKind::Create(notify::event::CreateKind::File)); + assert!(relevant(&created)); + + let (outer, inner) = (Uuid::new_v4(), Uuid::new_v4()); + let libraries = HashMap::from([ + (outer, library(outer, "/music")), + (inner, library(inner, "/music/classical")), + ]); + assert_eq!( + library_of(&libraries, Path::new("/music/rock/a.flac")), + Some(outer) + ); + assert_eq!( + library_of(&libraries, Path::new("/music/classical/b.flac")), + Some(inner), + "the deepest root owns the path" + ); + assert_eq!( + library_of(&libraries, Path::new("/music/.waveflow-managed/c.part")), + None + ); + assert_eq!(library_of(&libraries, Path::new("/elsewhere/d.flac")), None); + } +} diff --git a/tests/cli.rs b/tests/cli.rs index 0b7f7c98..7eee94e5 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -1191,3 +1191,154 @@ async fn database_rescan_full_asks_every_library_to_read_everything() { assert!(state.db.full_scan_requested(library).await.unwrap()); } + +/// RFC-021 decision 5 from the command line: rename, move with its scan, +/// purge the missing, and a deletion that refuses until the name is typed. +#[tokio::test] +async fn the_cli_renames_moves_purges_and_deletes_a_library() { + let (temp, config, state) = test_app().await; + let admin = inserted_account(&state, "lifecycle-admin", "admin").await; + let music = config.data_dir.join("lifecycle-music"); + std::fs::create_dir_all(&music).unwrap(); + write_test_wav(&music.join("Kept.wav")); + write_test_wav_of_len(&music.join("Gone.wav"), 900); + let root = std::fs::canonicalize(&music).unwrap(); + let library = state + .db + .create_library( + admin, + "Lifecycle", + &root, + waveflow_server::database::LibraryVisibility::Private, + now_ms(), + ) + .await + .unwrap(); + let id = library.to_string(); + scan_once( + &state, + admin, + waveflow_server::catalog::LibraryRecord { + id: library, + name: "Lifecycle".into(), + root_path: root.clone(), + }, + ) + .await; + + run_cli( + &state, + &[ + "library", + "rename", + "--actor", + "lifecycle-admin", + "--library-id", + &id, + "--name", + "Renamed", + ], + ) + .await + .unwrap(); + assert_eq!( + count( + &state, + "SELECT COUNT(*) FROM library WHERE name = 'Renamed'" + ) + .await, + 1 + ); + + // Moved with one file fewer: the scan the move queues finds it missing. + let moved = temp.path().join("lifecycle-moved"); + std::fs::rename(&music, &moved).unwrap(); + std::fs::remove_file(moved.join("Gone.wav")).unwrap(); + let moved_text = moved.to_string_lossy().into_owned(); + run_cli( + &state, + &[ + "library", + "set-path", + "--actor", + "lifecycle-admin", + "--library-id", + &id, + "--path", + &moved_text, + ], + ) + .await + .unwrap(); + assert_eq!( + count(&state, "SELECT COUNT(*) FROM track WHERE is_available = 0").await, + 1 + ); + + let unnamed = run_cli( + &state, + &[ + "library", + "purge", + "--actor", + "lifecycle-admin", + "--library-id", + &id, + ], + ) + .await; + assert!(unnamed.is_err(), "a purge naming nothing must refuse"); + run_cli( + &state, + &[ + "library", + "purge", + "--actor", + "lifecycle-admin", + "--library-id", + &id, + "--all", + ], + ) + .await + .unwrap(); + assert_eq!(count(&state, "SELECT COUNT(*) FROM track").await, 1); + + let mistyped = run_cli( + &state, + &[ + "library", + "delete", + "--actor", + "lifecycle-admin", + "--library-id", + &id, + "--confirm", + "Lifecycle", + ], + ) + .await; + assert!( + mistyped.is_err(), + "the old name must not confirm a deletion" + ); + assert_eq!(count(&state, "SELECT COUNT(*) FROM library").await, 1); + run_cli( + &state, + &[ + "library", + "delete", + "--actor", + "lifecycle-admin", + "--library-id", + &id, + "--confirm", + "Renamed", + ], + ) + .await + .unwrap(); + assert_eq!(count(&state, "SELECT COUNT(*) FROM library").await, 0); + assert_eq!(count(&state, "SELECT COUNT(*) FROM track").await, 0); + assert!(moved.join("Kept.wav").exists(), "a file was touched"); +} diff --git a/tests/migrations.rs b/tests/migrations.rs index 1a92482c..7cd2ab6f 100644 --- a/tests/migrations.rs +++ b/tests/migrations.rs @@ -81,6 +81,7 @@ const ALBUM: &str = "6f1d1b6e-0f1a-4a3d-9c21-0000000000a5"; const TRACK: &str = "6f1d1b6e-0f1a-4a3d-9c21-0000000000a6"; const GENRE: &str = "6f1d1b6e-0f1a-4a3d-9c21-0000000000a7"; const PLAYLIST: &str = "6f1d1b6e-0f1a-4a3d-9c21-0000000000a8"; +const SCAN: &str = "6f1d1b6e-0f1a-4a3d-9c21-0000000000a9"; /// One account, one library, and a catalogue with every shape a later /// migration might have to carry. @@ -163,14 +164,37 @@ async fn seed_an_installation(connection: &mut SqliteConnection) { VALUES (?, ?, 'Ambient', 'ambient', 1000)", vec![GENRE, LIBRARY], ), + ( + // The scan that last saw the track, with an error of its own: the + // two rows that point at `scan_job`, which RFC-021 rebuilds. + "INSERT INTO scan_job (id, library_id, requested_by, trigger, status, total_files, \ + processed_files, added, errors, created_at, started_at, \ + completed_at) \ + VALUES (?, ?, ?, 'boot', 'completed', 1, 1, 1, 1, 1000, 1000, 1000)", + vec![SCAN, LIBRARY, ACCOUNT], + ), + ( + "INSERT INTO scan_error (scan_id, relative_path, message, created_at) \ + VALUES (?, 'convergence/02.flac', 'unreadable', 1000)", + vec![SCAN], + ), ( "INSERT INTO track (id, library_id, album_id, relative_path, file_size, \ file_modified_at, quick_hash, full_hash, title, album_title, \ artist_display, genre_display, year, track_number, disc_number, \ - duration_ms, is_available, created_at, updated_at) \ + duration_ms, is_available, last_seen_scan_id, created_at, \ + updated_at) \ VALUES (?, ?, ?, 'convergence/01.flac', 4096, 1000, ?, ?, 'Zenith', \ - 'Convergence', 'Nova Kern', 'Ambient', 2026, 1, 1, 210000, 1, 1000, 1000)", - vec![TRACK, LIBRARY, ALBUM, &"1".repeat(64), &"2".repeat(64)], + 'Convergence', 'Nova Kern', 'Ambient', 2026, 1, 1, 210000, 1, ?, 1000, \ + 1000)", + vec![ + TRACK, + LIBRARY, + ALBUM, + &"1".repeat(64), + &"2".repeat(64), + SCAN, + ], ), ( "INSERT INTO track_participant (track_id, artist_id, library_id, role, sub_role, \ @@ -437,6 +461,44 @@ async fn every_released_schema_upgrades_with_the_rows_it_was_holding() { "{tag}: the play history did not survive the upgrade" ); assert_eq!(top[0].title, "Zenith", "{tag}"); + + // The scan history, through the table RFC-021 rebuilt to admit two + // more triggers: the job, its error, and the track that names it as + // the scan that last saw it. A rebuild run with foreign keys on would + // have cascaded the error away and nulled the track's reference — and + // a null there makes the next scan treat the track as never seen. + let scan: uuid::Uuid = SCAN.parse().expect("scan id"); + let job = state + .db + .scan_job(scan) + .await + .expect("read the job") + .unwrap_or_else(|| panic!("{tag}: the scan history did not survive")); + assert_eq!( + (job.status.as_str(), job.added, job.errors), + ("completed", 1, 1), + "{tag}" + ); + assert_eq!((job.excluded, job.skipped_links), (0, 0), "{tag}"); + let mut connection = state.db.pool().acquire().await.expect("connection"); + assert_eq!( + count( + &mut connection, + "SELECT COUNT(*) FROM scan_error WHERE scan_id = '6f1d1b6e-0f1a-4a3d-9c21-0000000000a9'" + ) + .await, + 1, + "{tag}: the scan's error did not survive" + ); + assert_eq!( + count( + &mut connection, + "SELECT COUNT(*) FROM track WHERE last_seen_scan_id = '6f1d1b6e-0f1a-4a3d-9c21-0000000000a9'" + ) + .await, + 1, + "{tag}: the track lost the scan that last saw it" + ); } } diff --git a/tests/native_api.rs b/tests/native_api.rs index 538480b5..bd532113 100644 --- a/tests/native_api.rs +++ b/tests/native_api.rs @@ -5584,3 +5584,266 @@ async fn chosen_artist_images_lead_every_reader_and_outlive_the_portrait_fill() StatusCode::NOT_FOUND ); } + +/// An account written straight into the table, with an API token minted for +/// it at run time: these routes need a bearer, never a password. +async fn account_with_token( + state: &waveflow_server::AppState, + username: &str, + role: &str, +) -> (uuid::Uuid, String) { + let id = uuid::Uuid::new_v4(); + sqlx::query( + "INSERT INTO account (id, username, password_hash, role, disabled, created_at, updated_at) \ + VALUES (?, ?, 'this-account-signs-in-by-token-only', ?, 0, ?, ?)", + ) + .bind(id.to_string()) + .bind(username) + .bind(role) + .bind(now_ms()) + .bind(now_ms()) + .execute(state.db.pool()) + .await + .unwrap(); + let token = security::generate_token("wfapi_"); + state + .db + .create_api_token( + waveflow_server::database::NewApiToken { + actor_id: id, + user_id: id, + name: "lifecycle", + token_hash: &security::token_hash(&token), + scopes: &[], + expires_at: None, + }, + now_ms(), + ) + .await + .unwrap(); + (id, token) +} + +fn bearer_json(method: &str, uri: &str, token: &str, body: serde_json::Value) -> Request { + Request::builder() + .method(method) + .uri(uri) + .header("authorization", format!("Bearer {token}")) + .header("content-type", "application/json") + .body(Body::from(body.to_string())) + .unwrap() +} + +/// RFC-021 over HTTP: the routes of a library's lifecycle and of its missing +/// tracks, who each one answers, and the confirmation a deletion carries. +#[tokio::test] +async fn the_library_lifecycle_routes_answer_by_standing() { + let (_temp, config, state) = test_app().await; + let (_admin, admin_token) = account_with_token(&state, "lifecycle-admin", "admin").await; + let (owner, owner_token) = account_with_token(&state, "lifecycle-owner", "user").await; + let (listener, listener_token) = account_with_token(&state, "lifecycle-listener", "user").await; + let music = config.data_dir.join("lifecycle-music"); + std::fs::create_dir_all(&music).unwrap(); + write_test_wav(&music.join("Only.wav")); + let root = std::fs::canonicalize(&music).unwrap(); + let library_id = state + .db + .create_library( + owner, + "Lifecycle", + &root, + LibraryVisibility::Private, + now_ms(), + ) + .await + .unwrap(); + state + .db + .add_library_member( + owner, + library_id, + listener, + waveflow_server::database::LibraryRole::Listener, + now_ms(), + ) + .await + .unwrap(); + run_scan( + &state, + owner, + LibraryRecord { + id: library_id, + name: "Lifecycle".into(), + root_path: root.clone(), + }, + ) + .await; + let router = waveflow_server::app(&config, state); + let library = format!("/api/v2/libraries/{library_id}"); + let send = |request: Request| { + let router = router.clone(); + async move { router.oneshot(request).await.unwrap() } + }; + + // The owner renames; a listener cannot even see that there is something + // to rename; the path is an administrator's. + let renamed = send(bearer_json( + "PATCH", + &library, + &owner_token, + serde_json::json!({"name": "Renamed"}), + )) + .await; + assert_eq!(renamed.status(), StatusCode::OK); + assert!(json_body(renamed).await["scan_id"].is_null()); + let refused = send(bearer_json( + "PATCH", + &library, + &listener_token, + serde_json::json!({"name": "Mine"}), + )) + .await; + assert_eq!(refused.status(), StatusCode::NOT_FOUND); + let repath = send(bearer_json( + "PATCH", + &library, + &owner_token, + serde_json::json!({"path": root.to_string_lossy()}), + )) + .await; + assert_eq!(repath.status(), StatusCode::FORBIDDEN); + let settings = send(bearer_json( + "GET", + &format!("{library}/settings"), + &owner_token, + serde_json::json!({}), + )) + .await; + assert_eq!(settings.status(), StatusCode::FORBIDDEN); + let settings = send(bearer_json( + "GET", + &format!("{library}/settings"), + &admin_token, + serde_json::json!({}), + )) + .await; + assert_eq!(settings.status(), StatusCode::OK); + let settings = json_body(settings).await; + assert_eq!(settings["name"], "Renamed"); + assert_eq!(settings["follow_links"], false); + + // Missing tracks: the owner reads them, a listener does not, and a purge + // that names nothing purges nothing. + std::fs::remove_file(root.join("Only.wav")).unwrap(); + let scan = send(bearer_json( + "POST", + &format!("{library}/scans"), + &owner_token, + serde_json::json!({}), + )) + .await; + assert_eq!(scan.status(), StatusCode::ACCEPTED); + let scan_id = json_body(scan).await["scan_id"] + .as_str() + .unwrap() + .to_owned(); + for _ in 0..200 { + let job = send(bearer_json( + "GET", + &format!("/api/v2/scans/{scan_id}"), + &owner_token, + serde_json::json!({}), + )) + .await; + if json_body(job).await["status"] == "completed" { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + let missing = send(bearer_json( + "GET", + &format!("{library}/unavailable-tracks"), + &owner_token, + serde_json::json!({}), + )) + .await; + assert_eq!(missing.status(), StatusCode::OK); + let missing = json_body(missing).await; + assert_eq!(missing["total"], 1); + assert_eq!(missing["tracks"][0]["relative_path"], "Only.wav"); + let hidden = send(bearer_json( + "GET", + &format!("{library}/unavailable-tracks"), + &listener_token, + serde_json::json!({}), + )) + .await; + assert_eq!(hidden.status(), StatusCode::NOT_FOUND); + let neither = send(bearer_json( + "POST", + &format!("{library}/unavailable-tracks/purge"), + &owner_token, + serde_json::json!({}), + )) + .await; + assert_eq!(neither.status(), StatusCode::UNPROCESSABLE_ENTITY); + let purged = send(bearer_json( + "POST", + &format!("{library}/unavailable-tracks/purge"), + &owner_token, + serde_json::json!({"all": true}), + )) + .await; + assert_eq!(purged.status(), StatusCode::OK); + assert_eq!(json_body(purged).await["purged"], 1); + + // A deletion is an administrator's, and carries the name it deletes. + let by_owner = send(bearer_json( + "DELETE", + &library, + &owner_token, + serde_json::json!({"confirm": "Renamed"}), + )) + .await; + assert_eq!(by_owner.status(), StatusCode::FORBIDDEN); + let unconfirmed = send(bearer_json( + "DELETE", + &library, + &admin_token, + serde_json::json!({}), + )) + .await; + assert_eq!(unconfirmed.status(), StatusCode::UNPROCESSABLE_ENTITY); + let mistyped = send(bearer_json( + "DELETE", + &library, + &admin_token, + serde_json::json!({"confirm": "renamed"}), + )) + .await; + assert_eq!(mistyped.status(), StatusCode::UNPROCESSABLE_ENTITY); + let deleted = send(bearer_json( + "DELETE", + &library, + &admin_token, + serde_json::json!({"confirm": "Renamed"}), + )) + .await; + assert_eq!(deleted.status(), StatusCode::OK); + let listed = send(bearer_json( + "GET", + "/api/v2/libraries", + &owner_token, + serde_json::json!({}), + )) + .await; + assert_eq!(json_body(listed).await, serde_json::json!([])); + let again = send(bearer_json( + "DELETE", + &library, + &admin_token, + serde_json::json!({"confirm": "Renamed"}), + )) + .await; + assert_eq!(again.status(), StatusCode::NOT_FOUND); +} diff --git a/tests/scanner.rs b/tests/scanner.rs index c0e0f87d..01fd65b6 100644 --- a/tests/scanner.rs +++ b/tests/scanner.rs @@ -1024,6 +1024,8 @@ fn scan_progress_converts_into_the_shape_the_stream_sends() { skipped: 4, unavailable: 5, errors: 6, + excluded: 7, + skipped_links: 8, current_path: Some("Rue Delacour/Nocturne.flac".into()), message: Some("still reading".into()), } @@ -1045,6 +1047,8 @@ fn scan_progress_converts_into_the_shape_the_stream_sends() { assert_eq!(record.skipped, 4); assert_eq!(record.unavailable, 5); assert_eq!(record.errors, 6); + assert_eq!(record.excluded, 7); + assert_eq!(record.skipped_links, 8); assert_eq!( record.current_path.as_deref(), Some("Rue Delacour/Nocturne.flac") @@ -1052,7 +1056,7 @@ fn scan_progress_converts_into_the_shape_the_stream_sends() { assert_eq!(record.message.as_deref(), Some("still reading")); // And the names themselves, since they are the whole point: a client binds - // these thirteen and nothing else. + // these fifteen and nothing else — the last two RFC-021's. let json = serde_json::to_value(&record).unwrap(); let mut keys: Vec<&str> = json .as_object() @@ -1067,12 +1071,14 @@ fn scan_progress_converts_into_the_shape_the_stream_sends() { "added", "current_path", "errors", + "excluded", "id", "library_id", "message", "moved", "processed_files", "skipped", + "skipped_links", "status", "total_files", "unavailable", @@ -1444,3 +1450,893 @@ async fn a_fifo_beside_an_audio_file_does_not_hang_the_scan() { let second = job(&state, owner, scan_once(&state, owner, library).await).await; assert_eq!(second.skipped, 1); } + +// ── Libraries and scanning: exclusions, links, missing tracks (RFC-021) ── + +/// An account inserted straight into the table: these tests need an owner, a +/// manager or a listener, never a login. +async fn insert_account(state: &waveflow_server::AppState, name: &str, role: &str) -> uuid::Uuid { + let id = uuid::Uuid::new_v4(); + sqlx::query( + "INSERT INTO account (id, username, password_hash, role, disabled, created_at, updated_at) \ + VALUES (?, ?, 'this-account-never-authenticates', ?, 0, ?, ?)", + ) + .bind(id.to_string()) + .bind(name) + .bind(role) + .bind(now_ms()) + .bind(now_ms()) + .execute(state.db.pool()) + .await + .unwrap(); + id +} + +async fn count(state: &waveflow_server::AppState, query: &'static str) -> i64 { + sqlx::query_scalar(query) + .fetch_one(state.db.pool()) + .await + .unwrap() +} + +async fn available_paths(state: &waveflow_server::AppState) -> Vec { + sqlx::query_scalar( + "SELECT relative_path FROM track WHERE is_available = 1 ORDER BY relative_path", + ) + .fetch_all(state.db.pool()) + .await + .unwrap() +} + +/// Waits for one job to end, whichever way. +async fn settled( + state: &waveflow_server::AppState, + owner: uuid::Uuid, + scan: uuid::Uuid, +) -> waveflow_server::catalog::ScanJobRecord { + for _ in 0..500 { + let record = job(state, owner, scan).await; + if record.status == "completed" || record.status == "failed" { + return record; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + panic!("scan timed out"); +} + +/// RFC-021 decision 1: the built-in exclusions and `.waveflowignore`, with its +/// gitignore rules, its empty-file rule and its `!`. Received files are kept, +/// and the ignore files themselves are not counted as left out. +#[tokio::test] +async fn built_in_exclusions_and_ignore_files_leave_files_out() { + let (_temp, state, owner, library, _root) = incremental_library("exclusions", |music| { + for (index, path) in [ + "Album/One.wav", + "@eaDir/Thumb.wav", + ".hidden/Two.wav", + "._One.wav", + "Skip/Three.wav", + "Nested/Four.wav", + "Keep/Unwanted.wav", + "Keep/Wanted.wav", + ".waveflow-managed/Received.wav", + ] + .into_iter() + .enumerate() + { + let path = music.join(path); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + write_test_wav_of_len(&path, 800 + index * 10); + } + std::fs::write(music.join(".waveflowignore"), "# not these\n/Skip/\n").unwrap(); + std::fs::write(music.join("Nested/.waveflowignore"), "").unwrap(); + std::fs::write(music.join("Keep/.waveflowignore"), "*.wav\n!Wanted.wav\n").unwrap(); + }) + .await; + let scan = job(&state, owner, scan_once(&state, owner, library).await).await; + assert_eq!( + available_paths(&state).await, + [ + ".waveflow-managed/Received.wav", + "Album/One.wav", + "Keep/Wanted.wav" + ] + ); + // @eaDir, .hidden, ._One.wav, Skip, Nested, Keep/Unwanted.wav. + assert_eq!(scan.excluded, 6); + assert_eq!(scan.errors, 0); +} + +/// RFC-021 decision 2: an excluded file counts as not discovered. Its track +/// becomes unavailable like a deleted one, with the time it went, and comes +/// back under the same identifier when the rule is lifted. +#[tokio::test] +async fn an_excluded_track_becomes_unavailable_like_a_deleted_one() { + let (_temp, state, owner, library, root) = incremental_library("excluded-later", |music| { + std::fs::create_dir_all(music.join("Live")).unwrap(); + write_test_wav(&music.join("Live/One.wav")); + write_test_wav_of_len(&music.join("Studio.wav"), 900); + }) + .await; + scan_once(&state, owner, library.clone()).await; + let id = track_id(&state, "Live/One.wav").await; + + std::fs::write(root.join(".waveflowignore"), "Live/\n").unwrap(); + let excluded = job( + &state, + owner, + scan_once(&state, owner, library.clone()).await, + ) + .await; + assert_eq!( + excluded.unavailable, 1, + "an excluded file must count as missing, not as unseen" + ); + let (available, since): (i64, Option) = + sqlx::query_as("SELECT is_available, unavailable_since FROM track WHERE id = ?") + .bind(&id) + .fetch_one(state.db.pool()) + .await + .unwrap(); + assert_eq!(available, 0); + assert!(since.is_some(), "the time it went missing is recorded"); + + std::fs::remove_file(root.join(".waveflowignore")).unwrap(); + scan_once(&state, owner, library).await; + let (available, since): (i64, Option) = + sqlx::query_as("SELECT is_available, unavailable_since FROM track WHERE id = ?") + .bind(&id) + .fetch_one(state.db.pool()) + .await + .unwrap(); + assert_eq!((available, since), (1, None)); +} + +/// RFC-021 decision 1: a folder whose ignore file cannot be used is an +/// unreadable folder — reported, and its tracks left as they are — while the +/// same failure at the root fails the scan. +#[tokio::test] +async fn an_ignore_file_that_cannot_be_used_keeps_its_tracks() { + let (_temp, state, owner, library, root) = incremental_library("ignore-unusable", |music| { + std::fs::create_dir_all(music.join("Sub")).unwrap(); + write_test_wav(&music.join("Sub/One.wav")); + write_test_wav_of_len(&music.join("Two.wav"), 900); + }) + .await; + scan_once(&state, owner, library.clone()).await; + + std::fs::write(root.join("Sub/.waveflowignore"), "[z-a]\n").unwrap(); + let scan = scan_once(&state, owner, library.clone()).await; + let record = job(&state, owner, scan).await; + assert_eq!(record.errors, 1); + assert_eq!( + record.unavailable, 0, + "a track under a folder whose rules are unknown was marked missing" + ); + assert_eq!(available_paths(&state).await, ["Sub/One.wav", "Two.wav"]); + let message: String = sqlx::query_scalar("SELECT message FROM scan_error WHERE scan_id = ?") + .bind(scan.to_string()) + .fetch_one(state.db.pool()) + .await + .unwrap(); + assert!(message.contains(".waveflowignore"), "{message}"); + + std::fs::write(root.join(".waveflowignore"), "[z-a]\n").unwrap(); + let failing = state + .scanner + .trigger(library, Some(owner), "manual") + .await + .unwrap(); + assert_eq!( + settled(&state, owner, failing).await.status, + "failed", + "a root whose ignore file cannot be used must fail the scan" + ); + assert_eq!(available_paths(&state).await, ["Sub/One.wav", "Two.wav"]); +} + +/// A symbolic link to `target` at `link`, or `false` where this system will +/// not let the test make one (Windows without the privilege). +fn make_link(target: &std::path::Path, link: &std::path::Path, directory: bool) -> bool { + #[cfg(unix)] + { + let _ = directory; + std::os::unix::fs::symlink(target, link).is_ok() + } + #[cfg(windows)] + { + if directory { + std::os::windows::fs::symlink_dir(target, link).is_ok() + } else { + std::os::windows::fs::symlink_file(target, link).is_ok() + } + } +} + +/// RFC-021 decision 3: links are skipped and counted unless the library +/// follows them, and then only those whose target stays inside the root; a +/// followed file is one track under its canonical path, however many links +/// name it. +#[tokio::test] +async fn links_are_followed_only_inside_the_library() { + let outside = tempfile::tempdir().unwrap(); + write_test_wav_of_len(&outside.path().join("Elsewhere.wav"), 950); + let mut linked = true; + let (_temp, state, owner, library, _root) = incremental_library("links", |music| { + std::fs::create_dir_all(music.join("store")).unwrap(); + std::fs::create_dir_all(music.join("Album")).unwrap(); + write_test_wav(&music.join("store/a.wav")); + // The store is reached through the links only. + std::fs::write(music.join(".waveflowignore"), "/store/\n").unwrap(); + linked &= make_link( + &music.join("store/a.wav"), + &music.join("Album/a.wav"), + false, + ); + linked &= make_link(&music.join("store"), &music.join("Mirror"), true); + linked &= make_link( + &outside.path().join("Elsewhere.wav"), + &music.join("Outside.wav"), + false, + ); + linked &= make_link(&music.join("missing.wav"), &music.join("Broken.wav"), false); + linked &= make_link(music, &music.join("Loop"), true); + }) + .await; + if !linked { + eprintln!("skipped: this system will not let the test make a symbolic link"); + return; + } + + let unfollowed = job( + &state, + owner, + scan_once(&state, owner, library.clone()).await, + ) + .await; + assert_eq!(unfollowed.added, 0); + assert_eq!( + unfollowed.skipped_links, 5, + "every link must be counted, not skipped in silence" + ); + + let updated = state + .services + .update_library( + owner, + library.id, + waveflow_server::services::LibraryUpdate { + follow_links: Some(true), + ..Default::default() + }, + ) + .await + .unwrap(); + let scan = updated + .scan_id + .expect("changing the links setting queues a scan"); + let followed = settled(&state, owner, scan).await; + assert_eq!(followed.status, "completed", "{:?}", followed.message); + assert_eq!( + available_paths(&state).await, + ["store/a.wav"], + "one file, under its canonical path, whatever links name it" + ); + // Outside the root, broken, and a loop back to the root. + assert_eq!(followed.skipped_links, 3); +} + +/// A FLAC file tagged with a title, an album and an artist, its tone chosen by +/// `frequency` so no two fixtures share their bytes. +fn write_tagged_flac(path: &std::path::Path, title: &str, album: &str, frequency: usize) { + let output = std::process::Command::new("ffmpeg") + .args([ + "-hide_banner", + "-loglevel", + "error", + "-y", + "-f", + "lavfi", + "-i", + &format!("sine=frequency={frequency}:duration=0.1"), + "-metadata", + &format!("title={title}"), + "-metadata", + &format!("album={album}"), + "-metadata", + "artist=Lifecycle Artist", + "-c:a", + "flac", + ]) + .arg(path) + .output() + .expect("FFmpeg is required for tagged fixtures"); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); +} + +/// Two tracks in an album, one in another, scanned. +async fn library_with_albums( + name: &str, +) -> ( + tempfile::TempDir, + waveflow_server::AppState, + uuid::Uuid, + LibraryRecord, + std::path::PathBuf, +) { + let fixture = incremental_library(name, |music| { + for (index, album) in ["First", "First", "Second"].into_iter().enumerate() { + std::fs::create_dir_all(music.join(album)).unwrap(); + write_tagged_flac( + &music.join(format!("{album}/{index}.flac")), + &format!("Track {index}"), + album, + 300 + index * 110, + ); + } + }) + .await; + scan_once(&fixture.1, fixture.2, fixture.3.clone()).await; + fixture +} + +async fn track_id(state: &waveflow_server::AppState, path: &str) -> String { + sqlx::query_scalar("SELECT id FROM track WHERE relative_path = ?") + .bind(path) + .fetch_one(state.db.pool()) + .await + .unwrap() +} + +/// RFC-021 decision 4: the listing says what a purge would cost, and the +/// purge removes only what is still missing, with what hangs off it, and +/// leaves no album, favourite or search row behind with nothing to stand on. +#[tokio::test] +async fn a_purge_removes_only_what_is_still_missing() { + let (temp, state, owner, library, root) = library_with_albums("purge").await; + let first = track_id(&state, "First/0.flac").await; + let second = track_id(&state, "Second/2.flac").await; + // A loop on the track that goes: its link cascades, and the blob it was + // the last to name must go with it rather than sit in the store for ever. + let canvas = "c".repeat(64); + let canvas_file = temp + .path() + .join("data/canvas") + .join(format!("{canvas}.mp4")); + std::fs::create_dir_all(canvas_file.parent().unwrap()).unwrap(); + std::fs::write(&canvas_file, b"mp4").unwrap(); + sqlx::query("INSERT INTO canvas (hash, format, byte_size, created_at) VALUES (?, 'mp4', 3, 1)") + .bind(&canvas) + .execute(state.db.pool()) + .await + .unwrap(); + sqlx::query( + "INSERT INTO track_canvas (track_id, library_id, canvas_hash, created_at) \ + VALUES (?, ?, ?, 1)", + ) + .bind(&second) + .bind(library.id.to_string()) + .bind(&canvas) + .execute(state.db.pool()) + .await + .unwrap(); + let second_album: String = sqlx::query_scalar("SELECT album_id FROM track WHERE id = ?") + .bind(&second) + .fetch_one(state.db.pool()) + .await + .unwrap(); + let playlist = uuid::Uuid::new_v4().to_string(); + for (statement, binds) in [ + ( + "INSERT INTO user_star (user_id, entity_type, entity_id, starred_at) \ + VALUES (?, 'track', ?, 1)", + vec![owner.to_string(), second.clone()], + ), + ( + "INSERT INTO user_star (user_id, entity_type, entity_id, starred_at) \ + VALUES (?, 'album', ?, 1)", + vec![owner.to_string(), second_album.clone()], + ), + ( + "INSERT INTO playlist (id, owner_user_id, name, public, created_at, updated_at) \ + VALUES (?, ?, 'Kept', 0, 1, 1)", + vec![playlist.clone(), owner.to_string()], + ), + ( + "INSERT INTO playlist_track (playlist_id, track_id, position, added_at) \ + VALUES (?, ?, 0, 1)", + vec![playlist.clone(), second.clone()], + ), + ( + "INSERT INTO play_event (user_id, track_id, submission, played_at) \ + VALUES (?, ?, 1, 1)", + vec![owner.to_string(), second.clone()], + ), + ] { + let mut query = sqlx::query(statement); + for bind in binds { + query = query.bind(bind); + } + query.execute(state.db.pool()).await.unwrap(); + } + + let first_bytes = std::fs::read(root.join("First/0.flac")).unwrap(); + std::fs::remove_file(root.join("First/0.flac")).unwrap(); + std::fs::remove_file(root.join("Second/2.flac")).unwrap(); + scan_once(&state, owner, library.clone()).await; + + let listed = state + .services + .unavailable_tracks( + owner, + library.id, + waveflow_server::services::BrowsePage::default(), + ) + .await + .unwrap(); + assert_eq!(listed.total, 2); + let costly = listed + .tracks + .iter() + .find(|track| track.id.to_string() == second) + .unwrap(); + assert_eq!( + (costly.playlist_entries, costly.favorites, costly.plays), + (1, 1, 1) + ); + + // One of them comes back between the listing and the purge. + std::fs::write(root.join("First/0.flac"), first_bytes).unwrap(); + scan_once(&state, owner, library.clone()).await; + + let purged = state + .services + .purge_unavailable_tracks( + owner, + library.id, + waveflow_server::services::PurgeRequest { + track_ids: Some(vec![first.parse().unwrap(), second.parse().unwrap()]), + all: false, + }, + ) + .await + .unwrap(); + assert_eq!(purged.purged, 1, "a track that came back must be kept"); + assert_eq!( + count(&state, "SELECT COUNT(*) FROM track WHERE is_available = 0").await, + 0 + ); + assert_eq!( + available_paths(&state).await, + ["First/0.flac", "First/1.flac"] + ); + for (query, what) in [ + ("SELECT COUNT(*) FROM playlist_track", "the playlist entry"), + ("SELECT COUNT(*) FROM play_event", "the play"), + ("SELECT COUNT(*) FROM user_star", "the favourites"), + ( + "SELECT COUNT(*) FROM album \ + WHERE NOT EXISTS (SELECT 1 FROM track t WHERE t.album_id = album.id)", + "an empty album", + ), + ( + "SELECT COUNT(*) FROM track_fts WHERE track_id NOT IN (SELECT id FROM track)", + "a search row", + ), + ] { + assert_eq!(count(&state, query).await, 0, "{what} outlived the purge"); + } + assert_eq!( + count(&state, "SELECT COUNT(*) FROM playlist").await, + 1, + "the playlist itself is not the track's to take" + ); + assert_eq!(count(&state, "SELECT COUNT(*) FROM canvas").await, 0); + assert!( + !canvas_file.exists(), + "the purged track's canvas stayed in the store" + ); +} + +/// RFC-021 decision 4: a purge is explicit, refused while the library is +/// scanned, and for curators only; the retention pass purges by age. +#[tokio::test] +async fn a_purge_is_explicit_and_waits_for_no_scan() { + use waveflow_server::services::{PurgeRequest, ServiceError}; + + let (_temp, state, owner, library, root) = library_with_albums("purge-rules").await; + std::fs::remove_file(root.join("First/0.flac")).unwrap(); + std::fs::remove_file(root.join("Second/2.flac")).unwrap(); + scan_once(&state, owner, library.clone()).await; + + let neither = state + .services + .purge_unavailable_tracks(owner, library.id, PurgeRequest::default()) + .await; + assert!(matches!(neither, Err(ServiceError::Invalid)), "{neither:?}"); + + let listener = insert_account(&state, "purge-listener", "user").await; + state + .db + .add_library_member( + owner, + library.id, + listener, + waveflow_server::database::LibraryRole::Listener, + now_ms(), + ) + .await + .unwrap(); + let all = || PurgeRequest { + track_ids: None, + all: true, + }; + assert!(matches!( + state + .services + .purge_unavailable_tracks(listener, library.id, all()) + .await, + Err(ServiceError::Forbidden) + )); + assert!(matches!( + state + .services + .unavailable_tracks(listener, library.id, Default::default()) + .await, + Err(ServiceError::Forbidden) + )); + + let scanning = state.scanner.try_lock_library(library.id).unwrap(); + assert!(matches!( + state + .services + .purge_unavailable_tracks(owner, library.id, all()) + .await, + Err(ServiceError::Conflict) + )); + assert_eq!( + state + .services + .purge_expired_unavailable_tracks(i64::MAX) + .await + .unwrap(), + 0, + "the retention pass must leave a library being scanned alone" + ); + drop(scanning); + + // One went missing long ago, the other just now. + sqlx::query("UPDATE track SET unavailable_since = 1000 WHERE relative_path = 'First/0.flac'") + .execute(state.db.pool()) + .await + .unwrap(); + assert_eq!( + state + .services + .purge_expired_unavailable_tracks(now_ms() - 60_000) + .await + .unwrap(), + 1 + ); + assert_eq!( + count(&state, "SELECT COUNT(*) FROM track WHERE is_available = 0").await, + 1 + ); +} + +/// RFC-021 decision 5: a root moved with its tree intact keeps every +/// identifier, and the scan it queues says why it ran. +#[tokio::test] +async fn a_moved_root_keeps_every_identity() { + let (temp, state, owner, library, root) = library_with_albums("moved").await; + let before: Vec<(String, String)> = + sqlx::query_as("SELECT relative_path, id FROM track ORDER BY relative_path") + .fetch_all(state.db.pool()) + .await + .unwrap(); + let moved = temp.path().join("moved-elsewhere"); + std::fs::rename(&root, &moved).unwrap(); + + let updated = state + .services + .update_library( + owner, + library.id, + waveflow_server::services::LibraryUpdate { + path: Some(moved.to_string_lossy().into_owned()), + ..Default::default() + }, + ) + .await + .unwrap(); + let scan = updated.scan_id.expect("a new root queues a scan"); + let record = settled(&state, owner, scan).await; + assert_eq!(record.status, "completed", "{:?}", record.message); + assert_eq!((record.added, record.unavailable), (0, 0)); + let trigger: String = sqlx::query_scalar("SELECT trigger FROM scan_job WHERE id = ?") + .bind(scan.to_string()) + .fetch_one(state.db.pool()) + .await + .unwrap(); + assert_eq!(trigger, "library_moved"); + let after: Vec<(String, String)> = sqlx::query_as( + "SELECT relative_path, id FROM track WHERE is_available = 1 ORDER BY relative_path", + ) + .fetch_all(state.db.pool()) + .await + .unwrap(); + assert_eq!(after, before); + let stored: String = sqlx::query_scalar("SELECT root_path FROM library WHERE id = ?") + .bind(library.id.to_string()) + .fetch_one(state.db.pool()) + .await + .unwrap(); + assert_eq!( + std::path::PathBuf::from(stored), + std::fs::canonicalize(&moved).unwrap() + ); +} + +/// RFC-021 decision 5: who may do what. The owner renames; the path, the +/// links setting and deletion are an administrator's; a listener does +/// nothing; and nothing that changes the walk happens during a scan. +#[tokio::test] +async fn the_lifecycle_follows_the_role_model() { + use waveflow_server::services::{LibraryUpdate, ServiceError}; + + let (temp, state, admin, library, _root) = library_with_albums("roles").await; + let owner = insert_account(&state, "roles-owner", "user").await; + let listener = insert_account(&state, "roles-listener", "user").await; + sqlx::query("UPDATE library_member SET user_id = ? WHERE library_id = ?") + .bind(owner.to_string()) + .bind(library.id.to_string()) + .execute(state.db.pool()) + .await + .unwrap(); + state + .db + .add_library_member( + owner, + library.id, + listener, + waveflow_server::database::LibraryRole::Listener, + now_ms(), + ) + .await + .unwrap(); + let rename = |name: &str| LibraryUpdate { + name: Some(name.into()), + ..Default::default() + }; + + state + .services + .update_library(owner, library.id, rename(" Renamed ")) + .await + .unwrap(); + let name: String = sqlx::query_scalar("SELECT name FROM library WHERE id = ?") + .bind(library.id.to_string()) + .fetch_one(state.db.pool()) + .await + .unwrap(); + assert_eq!(name, "Renamed"); + assert!(matches!( + state + .services + .update_library(listener, library.id, rename("Mine now")) + .await, + Err(ServiceError::Forbidden) + )); + assert!(matches!( + state + .services + .update_library(owner, library.id, rename(" ")) + .await, + Err(ServiceError::Invalid) + )); + let elsewhere = temp.path().join("elsewhere"); + std::fs::create_dir_all(&elsewhere).unwrap(); + let repath = LibraryUpdate { + path: Some(elsewhere.to_string_lossy().into_owned()), + ..Default::default() + }; + assert!(matches!( + state + .services + .update_library(owner, library.id, repath.clone()) + .await, + Err(ServiceError::Forbidden) + )); + assert!(matches!( + state + .services + .delete_library(owner, library.id, "Renamed") + .await, + Err(ServiceError::Forbidden) + )); + let scanning = state.scanner.try_lock_library(library.id).unwrap(); + assert!(matches!( + state + .services + .update_library(admin, library.id, repath) + .await, + Err(ServiceError::Conflict) + )); + assert!(matches!( + state + .services + .delete_library(admin, library.id, "Renamed") + .await, + Err(ServiceError::Conflict) + )); + drop(scanning); + assert!(matches!( + state + .services + .update_library( + admin, + library.id, + LibraryUpdate { + path: Some(temp.path().join("absent").to_string_lossy().into_owned()), + ..Default::default() + } + ) + .await, + Err(ServiceError::Invalid) + )); + // Somebody else's library, by an outsider: as missing as no library. + let outsider = insert_account(&state, "roles-outsider", "user").await; + assert!(matches!( + state + .services + .update_library(outsider, library.id, rename("Ours")) + .await, + Err(ServiceError::NotFound) + )); +} + +/// RFC-021 decision 5: deleting a library is confirmed by its name, removes +/// every row it held and nothing on disk, and leaves the other libraries +/// alone. +#[tokio::test] +async fn a_deleted_library_leaves_no_row_behind() { + use waveflow_server::services::ServiceError; + + let (temp, state, admin, library, root) = library_with_albums("deleted").await; + let kept_root = temp.path().join("kept"); + std::fs::create_dir_all(&kept_root).unwrap(); + write_test_wav_of_len(&kept_root.join("Kept.wav"), 1234); + let kept_root = std::fs::canonicalize(&kept_root).unwrap(); + let kept_id = state + .db + .create_library( + admin, + "kept", + &kept_root, + LibraryVisibility::Private, + now_ms(), + ) + .await + .unwrap(); + scan_once( + &state, + admin, + LibraryRecord { + id: kept_id, + name: "kept".into(), + root_path: kept_root, + }, + ) + .await; + let doomed = track_id(&state, "First/0.flac").await; + sqlx::query( + "INSERT INTO user_star (user_id, entity_type, entity_id, starred_at) \ + VALUES (?, 'track', ?, 1)", + ) + .bind(admin.to_string()) + .bind(&doomed) + .execute(state.db.pool()) + .await + .unwrap(); + + assert!(matches!( + state + .services + .delete_library(admin, library.id, "Deleted") + .await, + Err(ServiceError::Invalid) + )); + assert_eq!(count(&state, "SELECT COUNT(*) FROM library").await, 2); + + let deleted = state + .services + .delete_library(admin, library.id, "deleted") + .await + .unwrap(); + assert_eq!(deleted.tracks, 3); + assert_eq!(deleted.albums, 2); + + for table in [ + "SELECT COUNT(*) FROM track WHERE library_id = ?", + "SELECT COUNT(*) FROM album WHERE library_id = ?", + "SELECT COUNT(*) FROM artist WHERE library_id = ?", + "SELECT COUNT(*) FROM genre WHERE library_id = ?", + "SELECT COUNT(*) FROM track_fts WHERE library_id = ?", + "SELECT COUNT(*) FROM artist_fts WHERE library_id = ?", + "SELECT COUNT(*) FROM library_member WHERE library_id = ?", + "SELECT COUNT(*) FROM scan_job WHERE library_id = ?", + "SELECT COUNT(*) FROM library_event WHERE library_id = ?", + ] { + let left: i64 = sqlx::query_scalar(table) + .bind(library.id.to_string()) + .fetch_one(state.db.pool()) + .await + .unwrap(); + assert_eq!(left, 0, "{table}"); + } + assert_eq!( + count(&state, "SELECT COUNT(*) FROM user_star").await, + 0, + "a favourite of a deleted track outlived it" + ); + let violations = sqlx::query("PRAGMA foreign_key_check") + .fetch_all(state.db.pool()) + .await + .unwrap(); + assert!(violations.is_empty()); + assert!(root.join("First/0.flac").exists(), "a file was touched"); + assert_eq!( + count(&state, "SELECT COUNT(*) FROM track WHERE is_available = 1").await, + 1, + "the other library lost its track" + ); + let audited: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM audit_event WHERE kind = 'library.deleted' AND subject_id = ?", + ) + .bind(library.id.to_string()) + .fetch_one(state.db.pool()) + .await + .unwrap(); + assert_eq!(audited, 1); +} + +/// RFC-021 decision 6: a change in a watched library is scanned once it +/// settles, by a scan that says it came from the watcher. +#[tokio::test] +async fn a_watched_change_is_scanned_once_it_settles() { + let (_temp, state, owner, library, root) = incremental_library("watched", |music| { + write_test_wav(&music.join("First.wav")); + }) + .await; + scan_once(&state, owner, library.clone()).await; + let watcher = waveflow_server::watcher::spawn( + state.scanner.clone(), + state.db.clone(), + std::time::Duration::from_millis(200), + ); + // Long enough for the watch to be placed before the change it must see. + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + write_test_wav_of_len(&root.join("Second.wav"), 900); + + let mut watched = None; + for _ in 0..300 { + let found: Option = sqlx::query_scalar( + "SELECT added FROM scan_job WHERE trigger = 'watch' AND status = 'completed'", + ) + .fetch_optional(state.db.pool()) + .await + .unwrap(); + if found.is_some() { + watched = found; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + watcher.abort(); + assert_eq!(watched, Some(1), "no scan followed the change"); + assert_eq!(available_paths(&state).await, ["First.wav", "Second.wav"]); +} diff --git a/webapp/src/api/index.ts b/webapp/src/api/index.ts index 26f6ce5b..8e59d2ec 100644 --- a/webapp/src/api/index.ts +++ b/webapp/src/api/index.ts @@ -110,17 +110,25 @@ export { export type { Library, LibraryMember, + LibrarySettings, ScanJob, + UnavailableTrack, } from "./libraries"; export { addLibrary, + deleteLibrary, drainEventStream, + getLibrarySettings, getScan, listLibraries, listLibraryMembers, + listUnavailableTracks, + purgeUnavailableTracks, removeLibraryMember, setLibraryMember, startScan, + UNAVAILABLE_PAGE, + updateLibrary, watchScan, } from "./libraries"; export type { AuthorizeRequest } from "./oauth"; diff --git a/webapp/src/api/libraries.ts b/webapp/src/api/libraries.ts index f2213563..9d5aca28 100644 --- a/webapp/src/api/libraries.ts +++ b/webapp/src/api/libraries.ts @@ -17,10 +17,88 @@ export type Library = { * A library closed to files may be open to loops, and the reverse. */ accepts_canvas: boolean; + /** + * Whether the scan follows symbolic links that stay inside the library + * (RFC-021). An administrator's setting; shown to every member because it + * explains what the library holds. + */ + follow_links: boolean; }; export const listLibraries = () => call("/api/v2/libraries"); +/** A library as an administrator manages it, its root on the server included. */ +export type LibrarySettings = { + id: string; + name: string; + path: string; + follow_links: boolean; +}; + +export const getLibrarySettings = (libraryId: string) => + call(`/api/v2/libraries/${libraryId}/settings`); + +/** + * Renames a library, moves it, or changes whether its scan follows links. A + * name is its owner's to change; the path and the links setting are an + * administrator's. A new path or links setting queues a scan. + */ +export const updateLibrary = ( + libraryId: string, + update: { name?: string; path?: string; follow_links?: boolean }, +) => + call<{ scan_id: string | null }>(`/api/v2/libraries/${libraryId}`, { + method: "PATCH", + body: JSON.stringify(update), + }); + +/** + * Removes a library from the catalogue, for every member, and nothing from the + * disk. `confirm` must be its current name, or nothing is deleted. + */ +export const deleteLibrary = (libraryId: string, confirm: string) => + call<{ tracks: number; albums: number; artists: number }>( + `/api/v2/libraries/${libraryId}`, + { method: "DELETE", body: JSON.stringify({ confirm }) }, + ); + +/** A track whose file the scans can no longer find, with what a purge costs. */ +export type UnavailableTrack = { + id: string; + relative_path: string; + title: string; + artist: string | null; + album: string | null; + duration_ms: number; + unavailable_since: number; + playlist_entries: number; + favorites: number; + plays: number; +}; + +export const UNAVAILABLE_PAGE = 100; + +export const listUnavailableTracks = ( + libraryId: string, + offset: number, + limit: number, + signal?: AbortSignal, +) => + call<{ total: number; tracks: UnavailableTrack[] }>( + `/api/v2/libraries/${libraryId}/unavailable-tracks?offset=${offset}&limit=${limit}`, + { signal }, + ); + +/** Purges named missing tracks, or all of them. Never neither. */ +export const purgeUnavailableTracks = ( + libraryId: string, + selection: { track_ids: string[] } | { all: true }, +) => + call<{ purged: number }>( + `/api/v2/libraries/${libraryId}/unavailable-tracks/purge`, + { method: "POST", body: JSON.stringify(selection) }, + ); + export const addLibrary = ( name: string, path: string, @@ -43,6 +121,10 @@ export type ScanJob = { skipped: number; unavailable: number; errors: number; + /** Entries the scan left out: built-in exclusions and `.waveflowignore`. */ + excluded: number; + /** Symbolic links the scan did not follow. */ + skipped_links: number; current_path: string | null; message: string | null; }; diff --git a/webapp/src/api/queries.ts b/webapp/src/api/queries.ts index 06930fb2..72f08660 100644 --- a/webapp/src/api/queries.ts +++ b/webapp/src/api/queries.ts @@ -10,6 +10,7 @@ import { getAlbum, getArtist, getAttentionSummary, + getLibrarySettings, getLyrics, getQualitySummary, getTrack, @@ -43,6 +44,7 @@ import { listScrobbleLinks, listShares, listSongs, + listUnavailableTracks, listUncertainScrobbles, listUsers, QUALITY_TRACKS_PAGE, @@ -52,6 +54,7 @@ import { type SongSort, search, type TrackCategory, + UNAVAILABLE_PAGE, } from "./index"; /** @@ -474,6 +477,34 @@ export const libraryMembersQuery = (libraryId: string, open: boolean) => enabled: open, }); +/** + * A library as an administrator manages it (RFC-021), asked for only once its + * panel is open: the admin screen draws one per library. + */ +export const librarySettingsQuery = (libraryId: string, open: boolean) => + queryOptions({ + queryKey: ["library-settings", libraryId], + queryFn: () => getLibrarySettings(libraryId), + enabled: open, + }); + +/** + * The files a library's scans can no longer find (RFC-021), a page at a time, + * oldest first. Keyed under `missing` and the library, so one invalidation + * after a purge refreshes the page. + */ +export const unavailableTracksQuery = (libraryId: string) => + infiniteQueryOptions({ + queryKey: ["missing", libraryId], + queryFn: ({ pageParam, signal }) => + listUnavailableTracks(libraryId, pageParam, UNAVAILABLE_PAGE, signal), + initialPageParam: 0, + getNextPageParam: (last, _pages, lastParam) => + last.tracks.length < UNAVAILABLE_PAGE + ? undefined + : lastParam + UNAVAILABLE_PAGE, + }); + export const administrationQuery = () => queryOptions({ queryKey: ["administration"], diff --git a/webapp/src/main.tsx b/webapp/src/main.tsx index 0a4d6685..d790f7b7 100644 --- a/webapp/src/main.tsx +++ b/webapp/src/main.tsx @@ -64,6 +64,7 @@ import { GenrePage } from "./pages/genre-page"; import { GenresPage } from "./pages/genres-page"; import { HistoryPage } from "./pages/history-page"; import { LoginPage } from "./pages/login-page"; +import { MissingPage } from "./pages/missing-page"; import { NotFoundPage } from "./pages/not-found-page"; import { PlayingPage } from "./pages/playing-page"; import { PlaylistsPage } from "./pages/playlists-page"; @@ -120,6 +121,7 @@ const navigation: Array<{ | "/songs" | "/folders" | "/attention" + | "/missing" | "/quality" | "/genres" | "/search" @@ -169,6 +171,13 @@ const navigation: Array<{ curate: true, group: "library", }, + { + to: "/missing", + labelKey: "nav.missing", + icon: "inbox", + curate: true, + group: "library", + }, { to: "/quality", labelKey: "nav.quality", @@ -320,6 +329,7 @@ async function prepareTab( case "/folders": // Its lists are asked for once the page knows the active library. case "/attention": + case "/missing": case "/quality": break; // A fresh draw on every ask, held for no time at all: a draw made here @@ -913,6 +923,12 @@ const attentionRoute = createRoute({ }, }); +const missingRoute = createRoute({ + getParentRoute: () => authedRoute, + path: "/missing", + component: MissingPage, +}); + // The verdict filter is a search parameter, so a filtered list can be linked // to, as the lists of "Needs attention" can. const qualityRoute = createRoute({ @@ -957,6 +973,7 @@ const routeTree = rootRoute.addChildren([ songsRoute, foldersRoute, attentionRoute, + missingRoute, qualityRoute, artistsRoute, artistRoute, diff --git a/webapp/src/pages/admin-page.tsx b/webapp/src/pages/admin-page.tsx index b46fda95..bcc70a9a 100644 --- a/webapp/src/pages/admin-page.tsx +++ b/webapp/src/pages/admin-page.tsx @@ -2,10 +2,13 @@ import { useQuery } from "@tanstack/react-query"; import { type FormEvent, Fragment, useEffect, useState } from "react"; import { + ApiError, addLibrary, createApiToken, createUser, currentUser, + deleteLibrary, + type Library, removeLibraryMember, revokeApiToken, type ScanJob, @@ -14,12 +17,14 @@ import { setUserDisabled, startScan, type User, + updateLibrary, watchScan, } from "../api"; import { administrationQuery, apiTokensQuery, libraryMembersQuery, + librarySettingsQuery, nowPlayingQuery, } from "../api/queries"; import { @@ -138,6 +143,16 @@ function ScanProgress({ scanId }: { scanId: string }) { errors: job.errors, })}

+ {/* What the walk left out (RFC-021): only when there is something to + say, since most libraries have neither. */} + {job.excluded > 0 || job.skipped_links > 0 ? ( +

+ {t("scan.leftOut", { + excluded: job.excluded, + links: job.skipped_links, + })} +

+ ) : null} {/* The path is the only thing here that moves fast enough to read as liveness, and it is what tells a long scan from a stuck one. */} {!done && job.current_path ? ( @@ -436,6 +451,184 @@ function LibraryMembersPanel({ ); } +/** + * One library's name, folder and links setting, and its deletion (RFC-021 + * decision 5). A disclosure of its own per library, for the reason the + * membership panel has one: the settings, which carry the folder on the + * server, are asked for only once it is open. + * + * Deleting asks for the library's name to be typed, and the server checks it + * again: a deletion takes every track, album and artist out of the catalogue + * for every member, with their playlist entries, favourites and plays. + */ +function LibrarySettingsPanel({ + library, + onScan, + onDeleted, +}: { + library: Library; + onScan: (scanId: string) => void; + onDeleted: (message: string) => void; +}) { + const { t } = useI18n(); + const [open, setOpen] = useState(false); + const reloadSettings = useReload( + librarySettingsQuery(library.id, true).queryKey, + ); + const reloadAdministration = useReload(administrationQuery().queryKey); + const { data: settings, error } = useQuery( + librarySettingsQuery(library.id, open), + ); + // `null` until edited: the form shows what the server holds until then. + const [name, setName] = useState(null); + const [path, setPath] = useState(null); + const [follow, setFollow] = useState(null); + const [confirm, setConfirm] = useState(""); + const [busy, setBusy] = useState(false); + const [notice, setNotice] = useState<{ error: boolean; text: string } | null>( + null, + ); + + function refusal(error: unknown, fallback: TranslationKey): string { + return error instanceof ApiError && error.status === 409 + ? t("admin.saveBusy") + : t(fallback); + } + + async function save(event: FormEvent) { + event.preventDefault(); + if (!settings) return; + const update: { name?: string; path?: string; follow_links?: boolean } = {}; + if (name !== null && name.trim() !== settings.name) update.name = name; + if (path !== null && path !== settings.path) update.path = path; + if (follow !== null && follow !== settings.follow_links) { + update.follow_links = follow; + } + if (Object.keys(update).length === 0) return; + setBusy(true); + setNotice(null); + try { + const result = await updateLibrary(library.id, update); + setNotice({ + error: false, + text: result.scan_id + ? t("admin.savedScan", { id: result.scan_id }) + : t("admin.saved"), + }); + if (result.scan_id) onScan(result.scan_id); + setName(null); + setPath(null); + setFollow(null); + reloadSettings(); + reloadAdministration(); + } catch (error) { + setNotice({ error: true, text: refusal(error, "admin.saveError") }); + } finally { + setBusy(false); + } + } + + async function remove() { + setBusy(true); + setNotice(null); + try { + const deleted = await deleteLibrary(library.id, confirm); + onDeleted( + t("admin.deleted", { name: library.name, tracks: deleted.tracks }), + ); + reloadAdministration(); + } catch (error) { + setNotice({ error: true, text: refusal(error, "admin.deleteError") }); + setBusy(false); + } + } + + return ( +
+

+ +

+ {open ? ( + error ? ( +

+ {t("common.loadError")} +

+ ) : !settings ? ( + + ) : ( + <> +
void save(event)} + > + + + + +
+ {notice ? ( +

+ {notice.text} +

+ ) : null} +
+ {t("admin.delete")} +

{t("admin.deleteNote")}

+ setConfirm(event.target.value)} + placeholder={library.name} + aria-label={t("admin.deleteConfirm")} + autoComplete="off" + /> + +
+ + ) + ) : null} +
+ ); +} + export function AdminPage() { const signedInUser = currentUser(); const { t } = useI18n(); @@ -573,6 +766,16 @@ export function AdminPage() {
    • +
  • + { + setAdminError(null); + setNotice(message); + }} + /> +
    • ))} diff --git a/webapp/src/pages/missing-page.tsx b/webapp/src/pages/missing-page.tsx new file mode 100644 index 00000000..8c4a30e0 --- /dev/null +++ b/webapp/src/pages/missing-page.tsx @@ -0,0 +1,222 @@ +import { useInfiniteQuery, useQueryClient } from "@tanstack/react-query"; +import { useEffect, useMemo, useState } from "react"; + +import { ApiError, purgeUnavailableTracks } from "../api"; +import { unavailableTracksQuery } from "../api/queries"; +import { Loading, PageHeader } from "../components/page-shell"; +import { useI18n } from "../state/i18n"; +import { mayCorrectTracks, useLibraryScope } from "../state/library-scope"; + +/** How long a purge waits for its second click, as Split does. */ +const CONFIRM_MS = 4000; + +/** + * "Missing files" (RFC-021 decision 4): the tracks a library's scans can no + * longer find, oldest first, for its owners and managers. + * + * Each row says what purging it would cost — playlist entries, favourites and + * plays, across every account — because that is what a purge takes, and it + * does not come back. A purge is two clicks: the first arms it, the second + * within four seconds runs it. The server removes only what is still missing + * when it runs, so a file that came back in between keeps its track. + */ +export function MissingPage() { + const { t, locale } = useI18n(); + const { active, ready } = useLibraryScope(); + const libraryId = active?.id ?? ""; + const may = mayCorrectTracks(active ?? undefined); + const missing = useInfiniteQuery({ + ...unavailableTracksQuery(libraryId), + enabled: active !== null && may, + staleTime: 0, + }); + const client = useQueryClient(); + const tracks = useMemo( + () => missing.data?.pages.flatMap((page) => page.tracks) ?? [], + [missing.data], + ); + const total = missing.data?.pages[0]?.total; + const [selected, setSelected] = useState>(new Set()); + const [armed, setArmed] = useState<"selected" | "all" | null>(null); + const [busy, setBusy] = useState(false); + const [notice, setNotice] = useState<{ error: boolean; text: string } | null>( + null, + ); + useEffect(() => { + if (!armed) return; + const timer = window.setTimeout(() => setArmed(null), CONFIRM_MS); + return () => window.clearTimeout(timer); + }, [armed]); + // A selection belongs to one library's list. + useEffect(() => { + void libraryId; + setSelected(new Set()); + setArmed(null); + }, [libraryId]); + + const number = new Intl.NumberFormat(locale); + const day = new Intl.DateTimeFormat(locale, { dateStyle: "medium" }); + + const header = ( + + ); + + if (ready && (!active || !may)) { + return ( +
    + {header} +

    + {t(active ? "missing.notYours" : "folders.noLibrary")} +

    +
    + ); + } + + function toggle(id: string) { + setSelected((current) => { + const next = new Set(current); + if (next.has(id)) next.delete(id); + else next.add(id); + return next; + }); + setArmed(null); + } + + async function purge(which: "selected" | "all") { + setBusy(true); + setNotice(null); + try { + const outcome = await purgeUnavailableTracks( + libraryId, + which === "all" ? { all: true } : { track_ids: [...selected] }, + ); + setSelected(new Set()); + setNotice({ + error: false, + text: t("missing.purged", { count: outcome.purged }), + }); + // Albums and artists left with nothing went with them, anywhere they + // were drawn. + await client.invalidateQueries(); + } catch (error) { + setNotice({ + error: true, + text: + error instanceof ApiError && error.status === 409 + ? t("missing.scanning") + : t("missing.failed"), + }); + } finally { + setBusy(false); + setArmed(null); + } + } + + // What the armed button would purge, said on the button itself. + const count = armed === "all" ? (total ?? 0) : selected.size; + + return ( +
    + {header} +

    {t("missing.note")}

    + {notice ? ( +

    + {notice.text} +

    + ) : null} + {!missing.data ? ( + + ) : tracks.length === 0 ? null : ( + <> +
    + + +
    +
      + {tracks.map((track) => ( +
    • + +
      + + {t("missing.since", { + date: day.format(new Date(track.unavailable_since)), + })} + + + {t("missing.cost", { + playlists: number.format(track.playlist_entries), + favorites: number.format(track.favorites), + plays: number.format(track.plays), + })} + +
      +
      • + ))} +
    + {missing.hasNextPage ? ( + + ) : null} + + )} +
    + ); +} diff --git a/webapp/src/state/i18n.tsx b/webapp/src/state/i18n.tsx index ac083b6c..1c8cd5b8 100644 --- a/webapp/src/state/i18n.tsx +++ b/webapp/src/state/i18n.tsx @@ -758,6 +758,51 @@ const en = { "admin.memberAdd": "Give access to", "admin.memberChoose": "Choose an account…", "admin.memberError": "That membership could not be changed.", + "admin.manage": "Name, folder and deletion", + "admin.rename": "Name", + "admin.path": "Folder on the server", + "admin.followLinks": + "Follow symbolic links whose target stays inside the library", + "admin.save": "Save", + "admin.saved": "Saved.", + "admin.savedScan": "Saved. Scan {id} queued.", + "admin.saveError": + "That change could not be saved. A folder must exist on the server and must not be a symbolic link.", + "admin.saveBusy": + "The library is being scanned, or another library already uses that folder. Try again once the scan is over.", + "admin.delete": "Delete this library", + "admin.deleteNote": + "Removes the library from the catalogue for every member: its tracks, albums and artists, with their playlist entries, favourites and plays. No file is touched on disk. Type the library’s name to confirm.", + "admin.deleteConfirm": "The library’s name, to confirm", + "admin.deleteButton": "Delete for good", + "admin.deleted": "{name} was deleted: {tracks} tracks left the catalogue.", + "admin.deleteError": "The library could not be deleted.", + "scan.leftOut": "{excluded} left out · {links} links not followed", + "nav.missing": "Missing files", + "missing.total": { + one: "{count} file the scans can no longer find", + other: "{count} files the scans can no longer find", + }, + "missing.nothing": "Every file is where the catalogue expects it.", + "missing.notYours": + "Only the owners and managers of this library see its missing files.", + "missing.note": + "These tracks stay in the catalogue, with their favourites, playlist entries and plays, until they are purged; a file that comes back first keeps its track. A purge is not reversible: the track, and everything that hangs off it for every member, goes for good.", + "missing.since": "missing since {date}", + "missing.cost": + "{playlists} in playlists · {favorites} favourites · {plays} plays", + "missing.select": "Select {title}", + "missing.purgeSelected": "Purge the selected ({count})", + "missing.purgeAll": "Purge them all", + "missing.confirm": "Confirm: purge {count}", + "missing.purged": { + one: "{count} track purged.", + other: "{count} tracks purged.", + }, + "missing.scanning": + "The library is being scanned. Try again once the scan is over.", + "missing.failed": "The purge did not work. Try again.", + "missing.more": "Show more", "scrobbling.title": "External scrobbling", "scrobbling.detail": "Forward what you listen to here to ListenBrainz, Maloja or Last.fm.", @@ -1598,6 +1643,52 @@ const fr: Record = { "admin.memberAdd": "Donner accès à", "admin.memberChoose": "Choisir un compte…", "admin.memberError": "Cette appartenance n’a pas pu être modifiée.", + "admin.manage": "Nom, dossier et suppression", + "admin.rename": "Nom", + "admin.path": "Dossier sur le serveur", + "admin.followLinks": + "Suivre les liens symboliques dont la cible reste dans la bibliothèque", + "admin.save": "Enregistrer", + "admin.saved": "Enregistré.", + "admin.savedScan": "Enregistré. Scan {id} ajouté à la file.", + "admin.saveError": + "Ce changement n’a pas pu être enregistré. Le dossier doit exister sur le serveur et ne pas être un lien symbolique.", + "admin.saveBusy": + "La bibliothèque est en cours d’analyse, ou une autre bibliothèque utilise déjà ce dossier. Réessayez une fois l’analyse terminée.", + "admin.delete": "Supprimer cette bibliothèque", + "admin.deleteNote": + "Retire la bibliothèque du catalogue pour tous ses membres : ses titres, albums et artistes, avec leurs entrées de playlist, favoris et écoutes. Aucun fichier n’est touché sur le disque. Tapez le nom de la bibliothèque pour confirmer.", + "admin.deleteConfirm": "Le nom de la bibliothèque, pour confirmer", + "admin.deleteButton": "Supprimer définitivement", + "admin.deleted": + "{name} a été supprimée : {tracks} titres ont quitté le catalogue.", + "admin.deleteError": "La bibliothèque n’a pas pu être supprimée.", + "scan.leftOut": "{excluded} écartés · {links} liens non suivis", + "nav.missing": "Fichiers manquants", + "missing.total": { + one: "{count} fichier que les scans ne trouvent plus", + other: "{count} fichiers que les scans ne trouvent plus", + }, + "missing.nothing": "Chaque fichier est là où le catalogue l’attend.", + "missing.notYours": + "Seuls les propriétaires et gestionnaires de cette bibliothèque voient ses fichiers manquants.", + "missing.note": + "Ces titres restent au catalogue, avec leurs favoris, entrées de playlist et écoutes, jusqu’à leur purge ; un fichier qui revient avant garde son titre. Une purge est irréversible : le titre, et tout ce qui s’y rattache pour chaque membre, disparaît pour de bon.", + "missing.since": "manquant depuis le {date}", + "missing.cost": + "{playlists} en playlist · {favorites} favoris · {plays} écoutes", + "missing.select": "Sélectionner {title}", + "missing.purgeSelected": "Purger la sélection ({count})", + "missing.purgeAll": "Tout purger", + "missing.confirm": "Confirmer : purger {count}", + "missing.purged": { + one: "{count} titre purgé.", + other: "{count} titres purgés.", + }, + "missing.scanning": + "La bibliothèque est en cours d’analyse. Réessayez une fois l’analyse terminée.", + "missing.failed": "La purge n’a pas abouti. Réessayez.", + "missing.more": "Afficher plus", "scrobbling.title": "Scrobbling externe", "scrobbling.detail": "Transmettre ce que vous écoutez ici à ListenBrainz, Maloja ou Last.fm.", diff --git a/webapp/src/styles/styles.css b/webapp/src/styles/styles.css index 9d88c91e..3860e772 100644 --- a/webapp/src/styles/styles.css +++ b/webapp/src/styles/styles.css @@ -3619,3 +3619,57 @@ button.danger { .quality-spectrogram-action { margin-top: var(--space-3); } + +/* "Missing files" (RFC-021): a row is a checkbox and what it names, then when + it went and what a purge would take. */ +.missing-actions { + display: flex; + flex-wrap: wrap; + gap: var(--space-2); + margin: var(--space-3) 0; +} + +.missing-list li { + grid-template-columns: minmax(12rem, 1fr) auto; +} + +.missing-row { + display: flex; + align-items: flex-start; + gap: var(--space-3); + min-width: 0; +} + +.missing-text, +.missing-facts { + display: grid; + min-width: 0; +} + +.missing-facts { + justify-items: end; + text-align: end; +} + +.missing-path { + overflow-wrap: anywhere; +} + +/* The deletion of a library sits apart from the form that edits it, so the + button that saves a name is never the one beside the one that deletes. */ +.danger-zone { + margin-top: var(--space-4); + padding-top: var(--space-3); + border-top: 1px solid var(--line); +} + +.danger-note { + margin: 0; + font-size: var(--text-sm); +} + +.control.checkbox { + display: flex; + align-items: center; + gap: var(--space-2); +} From 80245d2356bee7e003d29343dc95515a7dd4aed8 Mon Sep 17 00:00:00 2001 From: InstaZDLL Date: Sun, 25 Oct 2026 02:32:29 +0200 Subject: [PATCH 03/13] fix(api): take administrative standing over a library from the credential, not the account Signed-off-by: InstaZDLL --- src/api/access.rs | 15 ++++ src/api/libraries.rs | 10 +-- src/cli.rs | 20 +++-- src/services/library_lifecycle.rs | 67 ++++++++++++----- src/services/mod.rs | 4 +- tests/native_api.rs | 117 ++++++++++++++++++++++++++++++ tests/scanner.rs | 94 +++++++++++++++++++----- 7 files changed, 277 insertions(+), 50 deletions(-) diff --git a/src/api/access.rs b/src/api/access.rs index 0801c1da..a100fbd2 100644 --- a/src/api/access.rs +++ b/src/api/access.rs @@ -91,6 +91,21 @@ pub(crate) async fn authenticated( } } +/// The caller of a library operation as the services weigh it (RFC-021): its +/// account, and whether the credential it came in on carries administrative +/// authority — the same two halves [`authenticated`] checks for +/// [`Access::Admin`]. A route that admits a narrower credential must not let +/// the account's role stand in for what the credential lacks. +pub(crate) fn library_actor( + user: &crate::authentication::AuthUser, +) -> crate::services::LibraryActor { + crate::services::LibraryActor { + id: user.id, + admin_credential: user.role == crate::database::AccountRole::Admin + && Access::Admin.granted_by(&user.scopes), + } +} + /// The bearer token, whatever case the client spelled the scheme in. /// /// RFC 7235 §2.1 makes the scheme name case-insensitive, so `bearer` is as diff --git a/src/api/libraries.rs b/src/api/libraries.rs index b361532e..f20754f8 100644 --- a/src/api/libraries.rs +++ b/src/api/libraries.rs @@ -415,7 +415,7 @@ pub async fn update_library( let actor = authenticated(&state, &headers, access).await?; state .services - .update_library(actor.id, library_id, request) + .update_library(library_actor(&actor), library_id, request) .await .map(Json) .map_err(service_error) @@ -435,7 +435,7 @@ pub async fn delete_library( let actor = authenticated(&state, &headers, Access::Admin).await?; state .services - .delete_library(actor.id, library_id, &request.confirm) + .delete_library(library_actor(&actor), library_id, &request.confirm) .await .map(Json) .map_err(service_error) @@ -451,7 +451,7 @@ pub async fn library_settings( let actor = authenticated(&state, &headers, Access::Admin).await?; state .services - .library_settings(actor.id, library_id) + .library_settings(library_actor(&actor), library_id) .await .map(Json) .map_err(service_error) @@ -479,7 +479,7 @@ pub async fn unavailable_tracks( crate::services::BrowsePage::new(query.offset, query.limit).map_err(service_error)?; state .services - .unavailable_tracks(user.id, library_id, page) + .unavailable_tracks(library_actor(&user), library_id, page) .await .map(Json) .map_err(service_error) @@ -499,7 +499,7 @@ pub async fn purge_unavailable_tracks( let user = authenticated(&state, &headers, Access::Write).await?; state .services - .purge_unavailable_tracks(user.id, library_id, request) + .purge_unavailable_tracks(library_actor(&user), library_id, request) .await .map(Json) .map_err(service_error) diff --git a/src/cli.rs b/src/cli.rs index 50c5bd59..2453f6ea 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -1055,7 +1055,7 @@ async fn rename_library(state: &AppState, args: RenameLibraryArgs) -> anyhow::Re state .services .update_library( - actor.id, + crate::services::LibraryActor::operator(actor.id), args.library_id, crate::services::LibraryUpdate { name: Some(args.name.clone()), @@ -1077,7 +1077,7 @@ async fn set_library_path(state: &AppState, args: SetPathArgs) -> anyhow::Result let updated = state .services .update_library( - actor.id, + crate::services::LibraryActor::operator(actor.id), args.library_id, crate::services::LibraryUpdate { path: Some(args.path.to_string_lossy().into_owned()), @@ -1109,7 +1109,7 @@ async fn set_library_links(state: &AppState, args: SetLinksArgs) -> anyhow::Resu let updated = state .services .update_library( - actor.id, + crate::services::LibraryActor::operator(actor.id), args.library_id, crate::services::LibraryUpdate { follow_links: Some(args.follow), @@ -1137,7 +1137,11 @@ async fn delete_library(state: &AppState, args: DeleteLibraryArgs) -> anyhow::Re let actor = require_admin(&state.db, &args.actor).await?; let deleted = state .services - .delete_library(actor.id, args.library_id, &args.confirm) + .delete_library( + crate::services::LibraryActor::operator(actor.id), + args.library_id, + &args.confirm, + ) .await .map_err(|error| match error { crate::services::ServiceError::Invalid => { @@ -1161,7 +1165,11 @@ async fn list_unavailable(state: &AppState, args: UnavailableArgs) -> anyhow::Re .map_err(|error| library_refusal(error, args.library_id))?; let listed = state .services - .unavailable_tracks(actor.id, args.library_id, page) + .unavailable_tracks( + crate::services::LibraryActor::operator(actor.id), + args.library_id, + page, + ) .await .map_err(|error| library_refusal(error, args.library_id))?; if offset == 0 { @@ -1188,7 +1196,7 @@ async fn purge_unavailable(state: &AppState, args: PurgeArgs) -> anyhow::Result< let outcome = state .services .purge_unavailable_tracks( - actor.id, + crate::services::LibraryActor::operator(actor.id), args.library_id, crate::services::PurgeRequest { track_ids: (!args.all).then_some(args.track_id), diff --git a/src/services/library_lifecycle.rs b/src/services/library_lifecycle.rs index 075ff9d5..bec377c2 100644 --- a/src/services/library_lifecycle.rs +++ b/src/services/library_lifecycle.rs @@ -124,14 +124,42 @@ impl Standing { } } +/// Who is asking, and with what authority. +/// +/// An administrator's account is not enough to act as one: the credential the +/// request came in on must carry administrative authority too. A token whose +/// scopes stop short of `admin` holds only what its account's memberships +/// give it, as `AuthUser::scopes` says a narrowed credential must. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct LibraryActor { + pub id: Uuid, + /// Whether the credential grants administrative authority. The account's + /// role is checked again on every operation; this can only narrow it. + pub admin_credential: bool, +} + +impl LibraryActor { + /// The operator at the CLI, whose authority is the account's own. + pub fn operator(id: Uuid) -> Self { + Self { + id, + admin_credential: true, + } + } +} + /// The caller's standing on a library, read on the connection the operation /// runs on. A library that does not exist, and one the caller is neither a /// member of nor an administrator over, are the same answer: not found. +/// +/// Administrative standing needs both halves: an active administrator's +/// account, and a credential that carries that authority. async fn standing_on( connection: &mut SqliteConnection, - actor_id: Uuid, + actor: LibraryActor, library_id: Uuid, ) -> Result { + let actor_id = actor.id; let row = sqlx::query( "SELECT (SELECT role FROM library_member WHERE library_id = l.id AND user_id = ?) AS role, \ EXISTS (SELECT 1 FROM account a WHERE a.id = ? AND a.role = 'admin' \ @@ -144,7 +172,7 @@ async fn standing_on( .fetch_optional(&mut *connection) .await? .ok_or(ServiceError::NotFound)?; - if row.try_get::("admin")? != 0 { + if actor.admin_credential && row.try_get::("admin")? != 0 { return Ok(Standing::Admin); } match row.try_get::, _>("role")?.as_deref() { @@ -384,10 +412,13 @@ impl DomainServices { /// The settings an administrator manages, the root included. pub async fn library_settings( &self, - actor_id: Uuid, + actor: LibraryActor, library_id: Uuid, ) -> Result { - self.require_admin(actor_id).await?; + if !actor.admin_credential { + return Err(ServiceError::Forbidden); + } + self.require_admin(actor.id).await?; let settings = self .db .library_scan_settings(library_id) @@ -411,7 +442,7 @@ impl DomainServices { /// Either refuses while the library is being scanned. pub async fn update_library( &self, - actor_id: Uuid, + actor: LibraryActor, library_id: Uuid, update: LibraryUpdate, ) -> Result { @@ -420,7 +451,7 @@ impl DomainServices { } let standing = { let mut connection = self.db.pool().acquire().await?; - standing_on(&mut connection, actor_id, library_id).await? + standing_on(&mut connection, actor, library_id).await? }; let allowed = if update.is_administrative() { standing == Standing::Admin @@ -493,7 +524,7 @@ impl DomainServices { if let Some(name) = name.as_deref().filter(|name| *name != before.library.name) { audit( &mut tx, - Some(actor_id), + Some(actor.id), "library.renamed", library_id, serde_json::json!({ "name": name }), @@ -503,7 +534,7 @@ impl DomainServices { if moved { audit( &mut tx, - Some(actor_id), + Some(actor.id), "library.moved", library_id, serde_json::json!({}), @@ -513,7 +544,7 @@ impl DomainServices { if relinked { audit( &mut tx, - Some(actor_id), + Some(actor.id), "library.follow_links_set", library_id, serde_json::json!({ "follow_links": update.follow_links }), @@ -538,7 +569,7 @@ impl DomainServices { .db .create_scan_job( library_id, - Some(actor_id), + Some(actor.id), if moved { "library_moved" } else { "manual" }, ) .await?; @@ -560,13 +591,13 @@ impl DomainServices { /// the commit. pub async fn delete_library( &self, - actor_id: Uuid, + actor: LibraryActor, library_id: Uuid, confirm: &str, ) -> Result { let standing = { let mut connection = self.db.pool().acquire().await?; - standing_on(&mut connection, actor_id, library_id).await? + standing_on(&mut connection, actor, library_id).await? }; if standing != Standing::Admin { return Err(ServiceError::Forbidden); @@ -639,7 +670,7 @@ impl DomainServices { } audit( &mut tx, - Some(actor_id), + Some(actor.id), "library.deleted", library_id, serde_json::json!({ @@ -678,12 +709,12 @@ impl DomainServices { /// first, with what purging each would cost (RFC-021 decision 4). pub async fn unavailable_tracks( &self, - actor_id: Uuid, + actor: LibraryActor, library_id: Uuid, page: BrowsePage, ) -> Result { let mut connection = self.db.pool().acquire().await?; - if !standing_on(&mut connection, actor_id, library_id) + if !standing_on(&mut connection, actor, library_id) .await? .curates() { @@ -741,7 +772,7 @@ impl DomainServices { /// scan may be about to relocate one of these onto a file it just found. pub async fn purge_unavailable_tracks( &self, - actor_id: Uuid, + actor: LibraryActor, library_id: Uuid, request: PurgeRequest, ) -> Result { @@ -760,14 +791,14 @@ impl DomainServices { let purged = { let _writer = self.db.writer_guard().await; let mut tx = self.db.pool().begin().await?; - if !standing_on(&mut tx, actor_id, library_id).await?.curates() { + if !standing_on(&mut tx, actor, library_id).await?.curates() { return Err(ServiceError::Forbidden); } let purged = purge_in(&mut tx, library_id, &selection).await?; if purged.tracks > 0 { audit( &mut tx, - Some(actor_id), + Some(actor.id), "library.tracks_purged", library_id, serde_json::json!({ "tracks": purged.tracks }), diff --git a/src/services/mod.rs b/src/services/mod.rs index 93239bfd..e7365ae0 100644 --- a/src/services/mod.rs +++ b/src/services/mod.rs @@ -1469,8 +1469,8 @@ pub use library_folders::{ MAX_FOLDER_DEPTH, MAX_PLACEMENTS, }; pub use library_lifecycle::{ - LibraryDeleted, LibrarySettings, LibraryUpdate, LibraryUpdated, PurgeOutcome, PurgeRequest, - UnavailableTrack, UnavailableTracks, + LibraryActor, LibraryDeleted, LibrarySettings, LibraryUpdate, LibraryUpdated, PurgeOutcome, + PurgeRequest, UnavailableTrack, UnavailableTracks, }; pub use library_tree::{DiskFolder, DiskListing, DiskTrack}; pub use lyrics_uploads::{LyricsFile, MAX_UPLOAD_BYTES as MAX_LYRICS_UPLOAD_BYTES}; diff --git a/tests/native_api.rs b/tests/native_api.rs index bd532113..3a016ad2 100644 --- a/tests/native_api.rs +++ b/tests/native_api.rs @@ -5847,3 +5847,120 @@ async fn the_library_lifecycle_routes_answer_by_standing() { .await; assert_eq!(again.status(), StatusCode::NOT_FOUND); } + +/// An administrator's account is not enough: a token whose scopes stop short +/// of `admin` acts with the account's memberships only, so it cannot rename, +/// purge or delete a library its account is not a member of — while the same +/// account on an unrestricted credential can. +#[tokio::test] +async fn a_narrowed_admin_token_has_no_say_over_a_foreign_library() { + let (_temp, config, state) = test_app().await; + let (admin, full_token) = account_with_token(&state, "scoped-admin", "admin").await; + let (owner, _) = account_with_token(&state, "scoped-owner", "user").await; + let write_token = security::generate_token("wfapi_"); + state + .db + .create_api_token( + waveflow_server::database::NewApiToken { + actor_id: admin, + user_id: admin, + name: "write only", + token_hash: &security::token_hash(&write_token), + scopes: &["write".to_owned()], + expires_at: None, + }, + now_ms(), + ) + .await + .unwrap(); + let music = config.data_dir.join("scoped-music"); + std::fs::create_dir_all(&music).unwrap(); + let root = std::fs::canonicalize(&music).unwrap(); + let library_id = state + .db + .create_library( + owner, + "Foreign", + &root, + LibraryVisibility::Private, + now_ms(), + ) + .await + .unwrap(); + let router = waveflow_server::app(&config, state.clone()); + let library = format!("/api/v2/libraries/{library_id}"); + let send = |request: Request| { + let router = router.clone(); + async move { router.oneshot(request).await.unwrap().status() } + }; + + assert_eq!( + send(bearer_json( + "PATCH", + &library, + &write_token, + serde_json::json!({"name": "Taken"}), + )) + .await, + StatusCode::NOT_FOUND, + "a narrowed token renamed a library its account is not a member of" + ); + assert_eq!( + send(bearer_json( + "POST", + &format!("{library}/unavailable-tracks/purge"), + &write_token, + serde_json::json!({"all": true}), + )) + .await, + StatusCode::NOT_FOUND + ); + assert_eq!( + send(bearer_json( + "GET", + &format!("{library}/unavailable-tracks"), + &write_token, + serde_json::json!({}), + )) + .await, + StatusCode::NOT_FOUND + ); + assert_eq!( + send(bearer_json( + "DELETE", + &library, + &write_token, + serde_json::json!({"confirm": "Foreign"}), + )) + .await, + StatusCode::FORBIDDEN + ); + let name: String = sqlx::query_scalar("SELECT name FROM library WHERE id = ?") + .bind(library_id.to_string()) + .fetch_one(state.db.pool()) + .await + .unwrap(); + assert_eq!(name, "Foreign"); + + // The same account, unrestricted: its authority is an administrator's. + assert_eq!( + send(bearer_json( + "PATCH", + &library, + &full_token, + serde_json::json!({"name": "Renamed by the operator"}), + )) + .await, + StatusCode::OK + ); + assert_eq!( + send(bearer_json( + "DELETE", + &library, + &full_token, + serde_json::json!({"confirm": "Renamed by the operator"}), + )) + .await, + StatusCode::OK + ); +} diff --git a/tests/scanner.rs b/tests/scanner.rs index 01fd65b6..88302a8d 100644 --- a/tests/scanner.rs +++ b/tests/scanner.rs @@ -1703,7 +1703,7 @@ async fn links_are_followed_only_inside_the_library() { let updated = state .services .update_library( - owner, + waveflow_server::services::LibraryActor::operator(owner), library.id, waveflow_server::services::LibraryUpdate { follow_links: Some(true), @@ -1872,7 +1872,7 @@ async fn a_purge_removes_only_what_is_still_missing() { let listed = state .services .unavailable_tracks( - owner, + waveflow_server::services::LibraryActor::operator(owner), library.id, waveflow_server::services::BrowsePage::default(), ) @@ -1896,7 +1896,7 @@ async fn a_purge_removes_only_what_is_still_missing() { let purged = state .services .purge_unavailable_tracks( - owner, + waveflow_server::services::LibraryActor::operator(owner), library.id, waveflow_server::services::PurgeRequest { track_ids: Some(vec![first.parse().unwrap(), second.parse().unwrap()]), @@ -1955,7 +1955,11 @@ async fn a_purge_is_explicit_and_waits_for_no_scan() { let neither = state .services - .purge_unavailable_tracks(owner, library.id, PurgeRequest::default()) + .purge_unavailable_tracks( + waveflow_server::services::LibraryActor::operator(owner), + library.id, + PurgeRequest::default(), + ) .await; assert!(matches!(neither, Err(ServiceError::Invalid)), "{neither:?}"); @@ -1978,14 +1982,22 @@ async fn a_purge_is_explicit_and_waits_for_no_scan() { assert!(matches!( state .services - .purge_unavailable_tracks(listener, library.id, all()) + .purge_unavailable_tracks( + waveflow_server::services::LibraryActor::operator(listener), + library.id, + all() + ) .await, Err(ServiceError::Forbidden) )); assert!(matches!( state .services - .unavailable_tracks(listener, library.id, Default::default()) + .unavailable_tracks( + waveflow_server::services::LibraryActor::operator(listener), + library.id, + Default::default() + ) .await, Err(ServiceError::Forbidden) )); @@ -1994,7 +2006,11 @@ async fn a_purge_is_explicit_and_waits_for_no_scan() { assert!(matches!( state .services - .purge_unavailable_tracks(owner, library.id, all()) + .purge_unavailable_tracks( + waveflow_server::services::LibraryActor::operator(owner), + library.id, + all() + ) .await, Err(ServiceError::Conflict) )); @@ -2044,7 +2060,7 @@ async fn a_moved_root_keeps_every_identity() { let updated = state .services .update_library( - owner, + waveflow_server::services::LibraryActor::operator(owner), library.id, waveflow_server::services::LibraryUpdate { path: Some(moved.to_string_lossy().into_owned()), @@ -2115,7 +2131,11 @@ async fn the_lifecycle_follows_the_role_model() { state .services - .update_library(owner, library.id, rename(" Renamed ")) + .update_library( + waveflow_server::services::LibraryActor::operator(owner), + library.id, + rename(" Renamed "), + ) .await .unwrap(); let name: String = sqlx::query_scalar("SELECT name FROM library WHERE id = ?") @@ -2127,14 +2147,22 @@ async fn the_lifecycle_follows_the_role_model() { assert!(matches!( state .services - .update_library(listener, library.id, rename("Mine now")) + .update_library( + waveflow_server::services::LibraryActor::operator(listener), + library.id, + rename("Mine now") + ) .await, Err(ServiceError::Forbidden) )); assert!(matches!( state .services - .update_library(owner, library.id, rename(" ")) + .update_library( + waveflow_server::services::LibraryActor::operator(owner), + library.id, + rename(" ") + ) .await, Err(ServiceError::Invalid) )); @@ -2147,14 +2175,22 @@ async fn the_lifecycle_follows_the_role_model() { assert!(matches!( state .services - .update_library(owner, library.id, repath.clone()) + .update_library( + waveflow_server::services::LibraryActor::operator(owner), + library.id, + repath.clone() + ) .await, Err(ServiceError::Forbidden) )); assert!(matches!( state .services - .delete_library(owner, library.id, "Renamed") + .delete_library( + waveflow_server::services::LibraryActor::operator(owner), + library.id, + "Renamed" + ) .await, Err(ServiceError::Forbidden) )); @@ -2162,14 +2198,22 @@ async fn the_lifecycle_follows_the_role_model() { assert!(matches!( state .services - .update_library(admin, library.id, repath) + .update_library( + waveflow_server::services::LibraryActor::operator(admin), + library.id, + repath + ) .await, Err(ServiceError::Conflict) )); assert!(matches!( state .services - .delete_library(admin, library.id, "Renamed") + .delete_library( + waveflow_server::services::LibraryActor::operator(admin), + library.id, + "Renamed" + ) .await, Err(ServiceError::Conflict) )); @@ -2178,7 +2222,7 @@ async fn the_lifecycle_follows_the_role_model() { state .services .update_library( - admin, + waveflow_server::services::LibraryActor::operator(admin), library.id, LibraryUpdate { path: Some(temp.path().join("absent").to_string_lossy().into_owned()), @@ -2193,7 +2237,11 @@ async fn the_lifecycle_follows_the_role_model() { assert!(matches!( state .services - .update_library(outsider, library.id, rename("Ours")) + .update_library( + waveflow_server::services::LibraryActor::operator(outsider), + library.id, + rename("Ours") + ) .await, Err(ServiceError::NotFound) )); @@ -2246,7 +2294,11 @@ async fn a_deleted_library_leaves_no_row_behind() { assert!(matches!( state .services - .delete_library(admin, library.id, "Deleted") + .delete_library( + waveflow_server::services::LibraryActor::operator(admin), + library.id, + "Deleted" + ) .await, Err(ServiceError::Invalid) )); @@ -2254,7 +2306,11 @@ async fn a_deleted_library_leaves_no_row_behind() { let deleted = state .services - .delete_library(admin, library.id, "deleted") + .delete_library( + waveflow_server::services::LibraryActor::operator(admin), + library.id, + "deleted", + ) .await .unwrap(); assert_eq!(deleted.tracks, 3); From f1aebeafa8ea70e63c27c6ff35a6ad311023e5ba Mon Sep 17 00:00:00 2001 From: InstaZDLL Date: Sun, 25 Oct 2026 02:33:23 +0200 Subject: [PATCH 04/13] docs(rfc): say that administrative standing over a library comes from the credential Signed-off-by: InstaZDLL --- docs/rfcs/RFC-021-library-lifecycle-and-scan.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/rfcs/RFC-021-library-lifecycle-and-scan.md b/docs/rfcs/RFC-021-library-lifecycle-and-scan.md index 685dd99d..2292bb03 100644 --- a/docs/rfcs/RFC-021-library-lifecycle-and-scan.md +++ b/docs/rfcs/RFC-021-library-lifecycle-and-scan.md @@ -220,6 +220,14 @@ library is: they decide which part of the server's file system the server reads. A `PATCH` naming either needs an administrative credential, not only an administrator's account; a rename needs a credential that may write. +**Administrative standing comes from the credential as well as the account**, +on every operation here, the missing-tracks view and purge included. An +administrator whose token's scopes stop short of `admin` holds only the +memberships of the account behind it: such a token cannot rename, purge or +list the missing tracks of a library that account is not a member of, and the +deletion refuses it outright. That is the rule `AuthUser::scopes` states for +every narrowed credential. + **Changing the path** validates the new one as creating a library does (an existing directory, not itself a link, stored canonical, not another library's root), then queues a scan with the new trigger `library_moved`. Tracks are From 639bf72a8532fb4188e07933ffd1d25247db3c0a Mon Sep 17 00:00:00 2001 From: InstaZDLL Date: Sun, 25 Oct 2026 02:57:37 +0200 Subject: [PATCH 05/13] test(scanner): a queued scan walks the library as it is when it runs Signed-off-by: InstaZDLL --- tests/scanner.rs | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/scanner.rs b/tests/scanner.rs index 88302a8d..74ccb6a9 100644 --- a/tests/scanner.rs +++ b/tests/scanner.rs @@ -2396,3 +2396,35 @@ async fn a_watched_change_is_scanned_once_it_settles() { assert_eq!(watched, Some(1), "no scan followed the change"); assert_eq!(available_paths(&state).await, ["First.wav", "Second.wav"]); } + +/// RFC-021 decision 5: a scan queued before a move walks the library as it is +/// when it runs, not as it was when it was queued — the watcher and the +/// interval queue scans that a move can overtake. +#[tokio::test] +async fn a_queued_scan_reads_the_library_as_it_is_when_it_runs() { + let (temp, state, owner, library, root) = library_with_albums("queued").await; + let held = state.scanner.try_lock_library(library.id).unwrap(); + // Queued with the old root, and waiting for the lock. + let queued = state + .scanner + .trigger(library.clone(), Some(owner), "manual") + .await + .unwrap(); + let moved = temp.path().join("queued-elsewhere"); + std::fs::rename(&root, &moved).unwrap(); + sqlx::query("UPDATE library SET root_path = ? WHERE id = ?") + .bind( + std::fs::canonicalize(&moved) + .unwrap() + .to_string_lossy() + .into_owned(), + ) + .bind(library.id.to_string()) + .execute(state.db.pool()) + .await + .unwrap(); + drop(held); + let record = settled(&state, owner, queued).await; + assert_eq!(record.status, "completed", "{:?}", record.message); + assert_eq!((record.added, record.unavailable), (0, 0)); +} From cefda815a218121e4f36b5c84653717c9399d9bd Mon Sep 17 00:00:00 2001 From: InstaZDLL Date: Sun, 25 Oct 2026 02:08:31 +0100 Subject: [PATCH 06/13] fix(webapp): draw the follow-links setting as a sentence beside its box Signed-off-by: InstaZDLL --- webapp/src/pages/admin-page.tsx | 3 ++- webapp/src/styles/styles.css | 12 +++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/webapp/src/pages/admin-page.tsx b/webapp/src/pages/admin-page.tsx index bcc70a9a..c9615ea7 100644 --- a/webapp/src/pages/admin-page.tsx +++ b/webapp/src/pages/admin-page.tsx @@ -583,9 +583,10 @@ function LibrarySettingsPanel({ required /> -