From 720b6ab8e1807b0e79983e6864b681cd1c7142bf Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 19:36:29 +0000 Subject: [PATCH 01/10] Add Missing Search Previews: fill search-page previews without re-indexing Photos indexed before the backend stored search-page previews had no way to get one short of sending them through Analyze & Index again. Backend: POST /v1/photos/thumbnails/missing lists the indexed photos (of a selection or a catalog) without a preview; POST /v1/photos/thumbnail stores one for an indexed photo from any rendered JPEG, scaled to the indexing size, and refuses photos that have no index row. Plugin: Library > Plug-in Extras > Add Missing Search Previews asks which photos lack one, takes Lightroom's cached preview (requestJpegThumbnail) when it is at least 256 px, and exports small JPEGs in chunked export sessions for the rest. LightroomPreviews.lua wraps requestJpegThumbnail (deadline, size check via Util.jpegDimensions); export settings are now built per session. Docs-Reviewed: Dev-Backend-API.md Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HaMMvqfgsKcoNWQaKy3c9C --- .luacheckrc | 1 + docs/wiki/Dev-Backend-API.md | 43 ++- docs/wiki/Dev-Plugin-README.md | 5 +- docs/wiki/Help-Advanced-Search.md | 11 +- docs/wiki/Help-Analyze-and-Index.md | 14 +- docs/wiki/Plugin-Guide.md | 1 + .../LrGeniusAI.lrdevplugin/APISearchIndex.lua | 125 ++++++- plugin/LrGeniusAI.lrdevplugin/Info.lua | 8 + plugin/LrGeniusAI.lrdevplugin/Init.lua | 1 + .../LightroomPreviews.lua | 124 +++++++ .../TaskFillSearchPreviews.lua | 309 ++++++++++++++++++ plugin/LrGeniusAI.lrdevplugin/Util.lua | 89 +++++ plugin/README.md | 5 +- plugin/spec/util_preview_render_spec.lua | 71 ++++ server-rs/crates/lrg-api/src/routes/index.rs | 182 ++++++++++- server-rs/crates/lrg-api/src/ui/search.html | 9 +- server-rs/crates/lrg-api/tests/contract.rs | 97 ++++++ 17 files changed, 1053 insertions(+), 42 deletions(-) create mode 100644 plugin/LrGeniusAI.lrdevplugin/LightroomPreviews.lua create mode 100644 plugin/LrGeniusAI.lrdevplugin/TaskFillSearchPreviews.lua create mode 100644 plugin/spec/util_preview_render_spec.lua diff --git a/.luacheckrc b/.luacheckrc index 5c00913c..3ce3a6f1 100644 --- a/.luacheckrc +++ b/.luacheckrc @@ -22,6 +22,7 @@ globals = { "UpdateCheck", "Defaults", "KeywordConfigProvider", + "LightroomPreviews", "LocalModelCatalog", "MetadataManager", "OnboardingWizard", diff --git a/docs/wiki/Dev-Backend-API.md b/docs/wiki/Dev-Backend-API.md index 8b182d7f..f2a4079b 100644 --- a/docs/wiki/Dev-Backend-API.md +++ b/docs/wiki/Dev-Backend-API.md @@ -247,10 +247,11 @@ preview for the browser search page — …"`). `/v1/index/unprocessed` does **not** count a missing preview as outstanding: the cull pre-flight and the agent's indexing command ask it too, and would offer -to prepare photos that lack nothing they need. A photo indexed before previews -were stored gets one from the next run that *sends* it — any scope of Analyze & -Index but *New or unprocessed photos*, with *Skip photos with existing data*, so -nothing else is made again. +to prepare photos that lack nothing they need. Photos indexed before previews +were stored get theirs from *Add Missing Search Previews* +(`POST /v1/photos/thumbnails/missing`, then `POST /v1/photos/thumbnail` per +photo), which renders only the preview and touches nothing else; the next run +that *sends* a photo makes one too. **The computer stays awake.** Both index routes run through `process_batch`, which holds `AppState::keep_awake` for the batch (`crate::keep_awake`). On macOS @@ -308,7 +309,7 @@ route; the search page fetches it with the header and shows the bytes as a |---|---|---| | 200 | the JPEG | a preview is stored | | 400 | `{"error"}` | no `photo_id` | -| 404 | `{"error", "error_code": "no_thumbnail"}` | none is stored: the photo was indexed before previews were, or not at all. An answer, not a failure — the page draws a placeholder and counts these | +| 404 | `{"error", "error_code": "no_thumbnail"}` | none is stored: the photo was indexed before previews were, or not at all. An answer, not a failure — the page draws a placeholder and counts these, and names *Add Missing Search Previews* as the fix | **Disk cost and lifetime.** One file per photo, `/thumbnails//.jpg` (the lance root is @@ -322,6 +323,38 @@ cleanup's purge (`POST /v1/photos/catalogs/cleanup`), only for rows that are really gone; left out of `POST /v1/db/backups`, since any indexing run makes them again. +### `POST /v1/photos/thumbnails/missing` +Which **indexed** photos have no search-page preview — what the plug-in's +*Library ▸ Plug-in Extras ▸ Add Missing Search Previews* asks before it renders +anything. Body (JSON, every field optional): + +| Field | Meaning | +|---|---| +| `photo_ids` | Only these photos (the selection). Left out: every indexed photo. Not a list → 400 | +| `catalog_id` | Only photos this catalog has (`catalog_ids`) | + +Returns `{"photo_ids": [...], "indexed": , "missing": }` — `indexed` +counts the photos the question covered that have an index row, `photo_ids` +those of them without a preview. A photo that is not indexed is never listed: +a preview belongs to an index row, and the next indexing run makes one anyway. +No database bound → the empty answer. Costs a metadata scan (like +`GET /v1/photos/ids`) plus a file stat per photo. + +### `POST /v1/photos/thumbnail` +Stores the search-page preview of an already indexed photo, without indexing it +again. Multipart: `photo_id` and one `image` part — any JPEG/PNG; the plug-in +sends Lightroom's cached preview or a small export. The backend scales it down +to the size indexing stores (256 px long edge, quality 75) and writes it where +`GET /v1/photos/thumbnail` reads it, replacing any earlier one. Nothing else +about the photo changes. + +| Status | Body | When | +|---|---|---| +| 200 | `{"status": "success", "photo_id"}` | stored | +| 400 | `{"error"}` | no `photo_id`, no `image`, or an image that cannot be decoded | +| 404 | `{"error", "error_code": "not_indexed"}` | the photo has no index row — refused, since nothing would ever delete a preview of a photo the index does not know | +| 503 | `{"error"}` | no database is bound | + ### `GET /v1/photos/ids` Returns a list of all indexed `photo_id` values, optionally filtered by catalog. diff --git a/docs/wiki/Dev-Plugin-README.md b/docs/wiki/Dev-Plugin-README.md index bc61489b..cd867f8a 100644 --- a/docs/wiki/Dev-Plugin-README.md +++ b/docs/wiki/Dev-Plugin-README.md @@ -41,7 +41,10 @@ The plugin is designed to work with local and cloud providers, while keeping Lig - Optional face detection and clustering - Optional species identification (BioCLIP 2, on-device) - Every processed photo also gets a small preview (256 px, ~13 KB, stored by the backend beside - the index) for the browser search page + the index) for the browser search page. Photos indexed before previews were stored get theirs + from *Add Missing Search Previews...* (`TaskFillSearchPreviews.lua`): it asks the backend which + indexed photos lack one, takes Lightroom's cached preview where it is big enough + (`LightroomPreviews.lua`) and a small export otherwise, and touches nothing else ### Advanced Search diff --git a/docs/wiki/Help-Advanced-Search.md b/docs/wiki/Help-Advanced-Search.md index 23a0104b..edef34ca 100644 --- a/docs/wiki/Help-Advanced-Search.md +++ b/docs/wiki/Help-Advanced-Search.md @@ -164,12 +164,11 @@ The previews are small copies (256 pixels on the long side) that **Analyze & Index Photos** stores for each photo it processes, whatever you ask it to do; they take about 13 KB a photo on the backend's disk, roughly 130 MB per 10,000 photos, beside the search index. Photos indexed before this version have none -yet: they show *No preview yet*, and the page says how many. To add them, click -**Show in Lightroom**, select the photos of the new collection and run **Analyze -& Index Photos** with the scope *Selected photos only* and *Skip photos with -existing data* — the previews are added and nothing that is already there is made again. -The scope *New or unprocessed photos* does not pick them up, because nothing -they need for searching is missing. +yet: they show *No preview yet*, and the page says how many. To add them, run +**Library ▸ Plug-in Extras ▸ Add Missing Search Previews...** — for the whole +catalog, or click **Show in Lightroom** first and choose *Selected photos*. It +only adds the previews (from Lightroom's own previews where it can, from a small +export otherwise); nothing is analyzed again. A preview is deleted with its photo's index data (the index cleanup, or removing the photo from the index), and is not part of a database backup — diff --git a/docs/wiki/Help-Analyze-and-Index.md b/docs/wiki/Help-Analyze-and-Index.md index 8b911fe8..f38056b8 100644 --- a/docs/wiki/Help-Analyze-and-Index.md +++ b/docs/wiki/Help-Analyze-and-Index.md @@ -107,12 +107,14 @@ photos with existing data* skips everything else for a photo. If one cannot be saved, the completion dialog lists it as a warning; the photo is indexed all the same. -Photos indexed before previews existed get theirs the next time a run processes -them: with the scope *Selected photos only*, *Current view* or *All photos in -catalog* and *Skip photos with existing data*, nothing else about them is made -again. The scope -*New or unprocessed photos* leaves them out, because nothing a search, a cull or -a face lookup needs is missing. +Photos indexed before previews existed get theirs from **Library ▸ Plug-in +Extras ▸ Add Missing Search Previews...**: choose selected photos, the current +view or the whole catalog, and it adds a preview to every indexed photo that has +none — from Lightroom's own previews where they are big enough, from a small +export otherwise — without analyzing anything again. (A run of Analyze & Index +that processes a photo adds one too, but *New or unprocessed photos* leaves +these photos out, because nothing a search, a cull or a face lookup needs is +missing.) ### Turning off AI metadata makes the rest of the run faster diff --git a/docs/wiki/Plugin-Guide.md b/docs/wiki/Plugin-Guide.md index 94c9ec44..1be4bc59 100644 --- a/docs/wiki/Plugin-Guide.md +++ b/docs/wiki/Plugin-Guide.md @@ -17,6 +17,7 @@ All actions are available under `Library → Plug-in Extras`: | Advanced Search... | Semantic free-text photo search | | Cull Similar Photos... | Burst grouping, ranking, Picks/Alternates/Rejects | | Retrieve Metadata from Backend... | Pull AI-generated metadata back into Lightroom | +| Add Missing Search Previews... | Give photos indexed before previews existed their picture on the browser search page, without re-indexing | | Import Metadata from Catalog... | Sync existing Lightroom metadata into backend | | People... | Browse face clusters, assign names, open person collections | | Find Similar Faces... | Find photos with the same face as a selected photo | diff --git a/plugin/LrGeniusAI.lrdevplugin/APISearchIndex.lua b/plugin/LrGeniusAI.lrdevplugin/APISearchIndex.lua index 1ed3e872..76222b8d 100644 --- a/plugin/LrGeniusAI.lrdevplugin/APISearchIndex.lua +++ b/plugin/LrGeniusAI.lrdevplugin/APISearchIndex.lua @@ -201,6 +201,10 @@ local ENDPOINTS = { IMPORT_METADATA = "/v1/photos/metadata/import", SYNC_CLAIM = "/v1/photos/catalogs/claim", SYNC_CLEANUP = "/v1/photos/catalogs/cleanup", + -- The search page's per-photo preview: which indexed photos lack one, and + -- storing one for a photo that is already indexed. + THUMBNAIL = "/v1/photos/thumbnail", + THUMBNAILS_MISSING = "/v1/photos/thumbnails/missing", -- Indexing pipeline INDEX = "/v1/index/photos", @@ -300,6 +304,24 @@ local EXPORT_SETTINGS = { LR_embeddedMetadataOption = "all", } +--- The export settings for one session: the defaults above with the current +--- size and quality preferences (which can change after this file loaded), +--- the temp folder as destination, and `overrides` on top. A fresh table each +--- time, so one caller's override can never leak into the next export. +local function exportSettingsFor(overrides) + local settings = {} + for k, v in pairs(EXPORT_SETTINGS) do + settings[k] = v + end + settings.LR_jpeg_quality = tonumber(prefs.exportQuality) or 60 + settings.LR_size_maxHeight = tonumber(prefs.exportSize) or 1024 + settings.LR_export_destinationPathPrefix = LrPathUtils.getStandardFilePath("temp") + for k, v in pairs(overrides or {}) do + settings[k] = v + end + return settings +end + --- The two generation settings for a request: the caller's value, else the --- saved setting, else the default. Each provider uses only one of them (see --- AiProviders.appliesTemperature); both are always sent and the backend @@ -870,14 +892,11 @@ function SearchIndexAPI.exportPhotoForIndexing(photo) return nil end - local tempDir = LrPathUtils.getStandardFilePath("temp") local photoName = LrPathUtils.leafName(photo:getFormattedMetadata("fileName")) - EXPORT_SETTINGS.LR_export_destinationPathPrefix = tempDir - local exportSession = LrExportSession({ photosToExport = { photo }, - exportSettings = EXPORT_SETTINGS, + exportSettings = exportSettingsFor(), }) local resultPath = nil @@ -902,28 +921,49 @@ function SearchIndexAPI.exportPhotoForIndexing(photo) return resultPath end -function SearchIndexAPI.exportPhotosForIndexing(photos) +--- +-- Exports several photos in one export session. Lightroom renders the photos +-- of one session in parallel, so this is much faster than one session per +-- photo — but keep the batches small (see Util.exportChunkSize): a session +-- holds on to what it rendered until it ends, which is how one session over a +-- whole selection let Lightroom's memory climb through a long run. +-- @param photos table Array of LrPhoto. +-- @param overrides table|nil Export settings to change, e.g. a smaller +-- LR_size_maxHeight for a preview. +-- @param shouldCancel function|nil Returns true to stop: the renditions not +-- yet rendered are cancelled. +-- @return table photo -> path of its JPEG (photos that failed are absent), +-- table photo -> error message for those that failed. +-- +function SearchIndexAPI.exportPhotosForIndexing(photos, overrides, shouldCancel) if not photos or #photos == 0 then - return {} + return {}, {} end - local tempDir = LrPathUtils.getStandardFilePath("temp") - - EXPORT_SETTINGS.LR_export_destinationPathPrefix = tempDir - local exportSession = LrExportSession({ photosToExport = photos, - exportSettings = EXPORT_SETTINGS, + exportSettings = exportSettingsFor(overrides), }) local photoPaths = {} + local failures = {} for _, rendition in exportSession:renditions() do - local success, path = rendition:waitForRender() + local canceled = shouldCancel ~= nil and shouldCancel() + local success, path + if canceled then + LrTasks.pcall(function() + rendition:skipRender() + end) + else + success, path = rendition:waitForRender() + end -- Same reason as in analyzeAndIndexSelectedPhotos: the rendition knows its -- own photo, and a loop counter would mis-key every later path as soon as -- Lightroom skips one rendition. local photo = rendition.photo - if photo ~= nil then + if canceled then + log:trace("Export canceled before rendering a photo") + elseif photo ~= nil then local photoName = LrPathUtils.leafName(photo:getFormattedMetadata("fileName")) log:trace( "Export completed for photo: " @@ -937,13 +977,13 @@ function SearchIndexAPI.exportPhotosForIndexing(photos) photoPaths[photo] = path else log:error("Failed to export photo for indexing. " .. (path or "unknown error")) - photoPaths[photo] = nil + failures[photo] = path or "unknown error" end else log:error("Photo is nil in exportPhotosForIndexing, probably it got deleted in the meantime.") end end - return photoPaths + return photoPaths, failures end --- @@ -5695,6 +5735,61 @@ function SearchIndexAPI.styleEdit(photoId, filepath, options) return false, response.error or "Unexpected response" end +--- +-- Which indexed photos have no preview for the browser search page. +-- @param photoIds table|nil Only these photo ids (the selection); nil asks for +-- every indexed photo of this catalog. +-- @return table|nil { photo_ids = {...}, indexed = n, missing = n }, string|nil error +--- +function SearchIndexAPI.getPhotosMissingThumbnails(photoIds) + local body = { catalog_id = getCatalogId() } + if photoIds ~= nil then + body.photo_ids = photoIds + end + local response, err = _request("POST", SearchIndexAPI.url("THUMBNAILS_MISSING"), body, 300) + if not response then + return nil, err or "Unknown error" + end + if type(response.photo_ids) ~= "table" then + return nil, response.error or "Unexpected response" + end + return response, nil +end + +--- +-- Stores the search-page preview of an indexed photo from a rendered image. +-- The backend scales it to its own preview size, so any JPEG will do. +-- @param photoId string +-- @param filepath string A JPEG on disk. +-- @return boolean success, string|nil error The backend's own words; a photo +-- that is not indexed is refused with a message that says so. +--- +function SearchIndexAPI.uploadPhotoThumbnail(photoId, filepath) + if not photoId or photoId == "" then + return false, "No photo ID provided" + end + if not filepath or not LrFileUtils.exists(filepath) then + return false, "No rendered image to upload" + end + local mimeChunks = { + { name = "photo_id", value = photoId }, + { + name = "image", + fileName = LrPathUtils.leafName(filepath), + filePath = filepath, + contentType = "image/jpeg", + }, + } + local response, err = _requestMultipart(SearchIndexAPI.url("THUMBNAIL"), mimeChunks, 60) + if not response then + return false, err or "Unknown error" + end + if response.status == "success" then + return true, nil + end + return false, response.error or "Unexpected response" +end + --- -- The aesthetic score (1..9, higher is better) of one rendered image. -- diff --git a/plugin/LrGeniusAI.lrdevplugin/Info.lua b/plugin/LrGeniusAI.lrdevplugin/Info.lua index 17e0ee95..1647adf0 100644 --- a/plugin/LrGeniusAI.lrdevplugin/Info.lua +++ b/plugin/LrGeniusAI.lrdevplugin/Info.lua @@ -50,6 +50,10 @@ return { title = LOC("$$$/LrGeniusAI/Menu/RetrieveMetadata=Retrieve Metadata from Backend..."), file = "TaskRetrieveMetadata.lua", }, + { + title = "Add Missing Search Previews...", + file = "TaskFillSearchPreviews.lua", + }, { title = LOC("$$$/LrGeniusAI/Menu/People=People..."), file = "TaskPeople.lua", @@ -101,6 +105,10 @@ return { title = LOC("$$$/LrGeniusAI/Menu/RetrieveMetadata=Retrieve Metadata from Backend..."), file = "TaskRetrieveMetadata.lua", }, + { + title = "Add Missing Search Previews...", + file = "TaskFillSearchPreviews.lua", + }, { title = LOC("$$$/LrGeniusAI/Menu/People=People..."), file = "TaskPeople.lua", diff --git a/plugin/LrGeniusAI.lrdevplugin/Init.lua b/plugin/LrGeniusAI.lrdevplugin/Init.lua index 54466028..d346997b 100644 --- a/plugin/LrGeniusAI.lrdevplugin/Init.lua +++ b/plugin/LrGeniusAI.lrdevplugin/Init.lua @@ -40,6 +40,7 @@ require("PromptConfigProvider") require("UpdateCheck") require("ErrorHandler") require("APISearchIndex") +require("LightroomPreviews") require("IndexCleanup") require("SpeciesLinks") require("PhotoSelector") diff --git a/plugin/LrGeniusAI.lrdevplugin/LightroomPreviews.lua b/plugin/LrGeniusAI.lrdevplugin/LightroomPreviews.lua new file mode 100644 index 00000000..8a1f20b0 --- /dev/null +++ b/plugin/LrGeniusAI.lrdevplugin/LightroomPreviews.lua @@ -0,0 +1,124 @@ +--[[ + LightroomPreviews.lua + + Rendered JPEGs of photos without an export: Lightroom's own previews, + through `photo:requestJpegThumbnail`. + + An export renders the photo from scratch, which is what makes indexing a + large selection slow and what lets Lightroom's memory climb over a long + run. A preview is already rendered — with the photo's current develop + settings — whenever Lightroom has one cached, so handing it over costs + almost nothing. Two things make it unreliable, and both are handled here + rather than by every caller: + + * Lightroom answers with whatever it has cached, which can be far smaller + than what was asked for (a 160 px grid thumbnail when only that was ever + built). The size is read from the JPEG itself (`Util.jpegDimensions`) + and the caller decides what is big enough; anything smaller is the + caller's cue to export instead. + * The callback may never come (a photo whose preview cannot be built), or + come more than once (a small preview first, a better one after). The + wait has a deadline, and a better answer arriving shortly after a first + one is still taken. + + Previews can lag behind the latest edit until Lightroom re-renders them; + for a search preview that is fine, for an aesthetic score it is the + trade-off the "Use Lightroom previews" setting names. +]] + +LightroomPreviews = {} + +--- How long to keep waiting for a better answer once a first one arrived. +local SETTLE_SECONDS = 0.4 + +--- +-- Asks Lightroom for a rendered JPEG of `photo`, at most `longEdge` px on its +-- long edge. +-- @param photo LrPhoto +-- @param longEdge number The size wanted. +-- @param timeoutSeconds number|nil How long to wait for Lightroom (default 10). +-- @return string|nil jpegData, number longEdgeReceived (0 when none), +-- string|nil reason Why there is no data, for the log. +-- +function LightroomPreviews.requestJpeg(photo, longEdge, timeoutSeconds) + local deadline = LrDate.currentTime() + (timeoutSeconds or 10) + local best, bestEdge, failure = nil, 0, nil + local firstAt = nil + + local ok, request = LrTasks.pcall(function() + return photo:requestJpegThumbnail(longEdge, longEdge, function(data, err) + if type(data) == "string" and #data > 0 then + local w, h = Util.jpegDimensions(data) + local edge = math.max(w or 0, h or 0) + if edge > bestEdge then + best, bestEdge = data, edge + end + firstAt = firstAt or LrDate.currentTime() + else + failure = err or "Lightroom returned no preview" + end + end) + end) + if not ok then + return nil, 0, "requestJpegThumbnail failed: " .. tostring(request) + end + + -- `request` must stay referenced until the callback has run: Lightroom + -- cancels a request whose handle is garbage-collected. + while LrDate.currentTime() < deadline do + if bestEdge >= longEdge then + break + end + if firstAt and LrDate.currentTime() - firstAt >= SETTLE_SECONDS then + break + end + if failure and not best then + break + end + LrTasks.sleep(0.05) + end + request = nil -- luacheck: ignore 311 + + if best == nil then + return nil, 0, failure or "Lightroom did not answer in time" + end + return best, bestEdge, nil +end + +--- +-- Writes JPEG bytes to a new file in the temp folder. +-- @return string|nil path, string|nil error +-- +function LightroomPreviews.writeTemp(data, nameHint) + local name = "lrgenius-preview-" + .. tostring(nameHint or "photo"):gsub("[^%w%-_]", "_") + .. "-" + .. tostring(math.floor(LrDate.currentTime() * 1000)) + .. ".jpg" + local path = LrPathUtils.child(LrPathUtils.getStandardFilePath("temp"), name) + local fh, err = io.open(path, "wb") + if not fh then + return nil, "could not write " .. path .. ": " .. tostring(err) + end + fh:write(data) + fh:close() + return path, nil +end + +--- +-- A rendered JPEG of `photo` on disk, from its Lightroom preview when that is +-- at least `minLongEdge` px, else nil (the caller exports instead). +-- @return string|nil path, string|nil reason Why the preview was not used. +-- +function LightroomPreviews.previewFile(photo, wantedLongEdge, minLongEdge, timeoutSeconds) + local data, edge, reason = LightroomPreviews.requestJpeg(photo, wantedLongEdge, timeoutSeconds) + if data == nil then + return nil, reason + end + if edge < minLongEdge then + return nil, "the cached preview is " .. tostring(edge) .. " px, " .. tostring(minLongEdge) .. " px needed" + end + return LightroomPreviews.writeTemp(data, photo.localIdentifier) +end + +return LightroomPreviews diff --git a/plugin/LrGeniusAI.lrdevplugin/TaskFillSearchPreviews.lua b/plugin/LrGeniusAI.lrdevplugin/TaskFillSearchPreviews.lua new file mode 100644 index 00000000..d43cd181 --- /dev/null +++ b/plugin/LrGeniusAI.lrdevplugin/TaskFillSearchPreviews.lua @@ -0,0 +1,309 @@ +--[[ + Add Missing Search Previews: gives photos that were indexed before + LrGeniusAI stored previews the small picture the browser search page shows, + without indexing them again. + + The backend says which photos of the chosen scope are indexed but have no + preview (/v1/photos/thumbnails/missing). For each of those this task takes + Lightroom's own cached preview when it is big enough (LightroomPreviews), + and exports a small JPEG — a few dozen photos per export session — for the + rest. Each picture goes to /v1/photos/thumbnail, which scales it to the + preview size. Nothing else about a photo is touched. +]] + +--- Long edge asked of Lightroom and of the fallback export. The backend +--- stores 256 px, so this leaves room for its own downscale. +local RENDER_LONG_EDGE = 512 + +--- Smallest cached preview that is still used: the stored size itself. +local MIN_PREVIEW_LONG_EDGE = 256 + +--- How long to wait for Lightroom's answer per photo before exporting it. +local PREVIEW_TIMEOUT_SECONDS = 5 + +--- Misses in a row after which Lightroom's previews are not asked any more: +--- a catalog whose previews were never built would otherwise cost the full +--- timeout per photo before each export. +local MAX_PREVIEW_MISSES_IN_A_ROW = 10 + +--- Photo ids per question to the backend, so a whole catalog does not go out +--- as one request body. +local LOOKUP_CHUNK = 2000 + +--- Failures named one by one in the final dialog; the rest are counted. +local MAX_LISTED_FAILURES = 8 + +local function chooseScope(context) + local f = LrView.osFactory() + local props = LrBinding.makePropertyTable(context) + props.scope = prefs.fillPreviewsScope or "selected" + local contents = f:column({ + bind_to_object = props, + spacing = f:control_spacing(), + f:static_text({ + title = "Photos indexed before LrGeniusAI stored previews have no picture on the\n" + .. "browser search page. This adds one to each of them, from Lightroom's\n" + .. "previews where possible. Nothing else about the photos changes.", + }), + f:row({ + f:static_text({ title = "Photos:" }), + f:popup_menu({ + value = LrView.bind("scope"), + items = { + { title = "Selected photos", value = "selected" }, + { title = "Current view", value = "view" }, + { title = "All photos in the catalog", value = "all" }, + }, + }), + }), + }) + local result = LrDialogs.presentModalDialog({ + title = "Add Missing Search Previews", + contents = contents, + actionVerb = "Add Previews", + }) + if result ~= "ok" then + return nil + end + prefs.fillPreviewsScope = props.scope + return props.scope +end + +--- photoId -> photo for `photos`; photos whose id cannot be worked out are +--- counted. Several photos can share an id (virtual copies of one file): the +--- first one stands for the file, which is what the index stores. +local function photosById(photos, scope) + local byId, ids, unresolved = {}, {}, 0 + for i, photo in ipairs(photos) do + if scope:isCanceled() then + return nil + end + local photoId = SearchIndexAPI.getPhotoIdForPhoto(photo) + if photoId then + if byId[photoId] == nil then + byId[photoId] = photo + table.insert(ids, photoId) + end + else + unresolved = unresolved + 1 + end + if i % 200 == 0 then + scope:setCaption("Identifying photos (" .. i .. " of " .. #photos .. ")...") + LrTasks.yield() + end + end + return byId, ids, unresolved +end + +local function missingIds(ids, scope) + local missing, indexed = {}, 0 + for start = 1, #ids, LOOKUP_CHUNK do + if scope:isCanceled() then + return nil + end + local chunk = {} + for i = start, math.min(start + LOOKUP_CHUNK - 1, #ids) do + table.insert(chunk, ids[i]) + end + local response, err = SearchIndexAPI.getPhotosMissingThumbnails(chunk) + if not response then + return nil, err + end + indexed = indexed + (tonumber(response.indexed) or 0) + for _, id in ipairs(response.photo_ids) do + table.insert(missing, id) + end + end + return missing, nil, indexed +end + +local function run(context) + local chosen = chooseScope(context) + if not chosen then + return + end + if not SearchIndexAPI.pingServer() then + ErrorHandler.handleError( + "The backend server is not responding", + "Open Plug-in Manager > LrGeniusAI and use Restart Backend, then try again." + ) + return + end + + local scope = LrProgressScope({ title = "Adding search previews", functionContext = context }) + scope:setCaption("Collecting photos...") + local photos, selectErr = PhotoSelector.getPhotosInScope(chosen) + if not photos then + scope:done() + ErrorHandler.handleError("Could not collect the photos", selectErr) + return + end + if #photos == 0 then + scope:done() + LrDialogs.message("Add Missing Search Previews", "There are no photos in this scope.", "info") + return + end + + local byId, ids, unresolved = photosById(photos, scope) + if not byId then + scope:done() + return + end + scope:setCaption("Asking the backend which photos have no preview...") + local missing, lookupErr, indexed = missingIds(ids, scope) + if not missing then + scope:done() + if lookupErr then + ErrorHandler.handleError("Could not ask the backend which photos lack a preview", lookupErr) + end + return + end + if #missing == 0 then + scope:done() + local note = indexed == 0 and "None of these photos is indexed yet. Analyze & Index Photos adds the preview." + or "All " .. indexed .. " indexed photos in this scope already have a preview." + LrDialogs.message("Add Missing Search Previews", note, "info") + return + end + + local total = #missing + local added, failures = 0, {} + local fromPreview, exported = 0, 0 + local needExport = {} + + local function upload(photoId, path, how) + local ok, err = SearchIndexAPI.uploadPhotoThumbnail(photoId, path) + LrTasks.pcall(function() + LrFileUtils.delete(path) + end) + if ok then + added = added + 1 + if how == "preview" then + fromPreview = fromPreview + 1 + else + exported = exported + 1 + end + else + local photo = byId[photoId] + local name = photo and photo:getFormattedMetadata("fileName") or photoId + table.insert(failures, name .. ": " .. tostring(err)) + end + end + + local done = 0 + local function step(caption) + done = done + 1 + scope:setPortionComplete(done, total) + scope:setCaption(caption .. " (" .. done .. " of " .. total .. ")") + end + + -- Pass 1: Lightroom's cached previews. + local missesInARow = 0 + for _, photoId in ipairs(missing) do + if scope:isCanceled() then + break + end + if missesInARow >= MAX_PREVIEW_MISSES_IN_A_ROW then + table.insert(needExport, photoId) + else + local photo = byId[photoId] + local path, reason = + LightroomPreviews.previewFile(photo, RENDER_LONG_EDGE, MIN_PREVIEW_LONG_EDGE, PREVIEW_TIMEOUT_SECONDS) + if path then + missesInARow = 0 + upload(photoId, path, "preview") + step("Adding previews") + else + missesInARow = missesInARow + 1 + log:trace( + "No usable Lightroom preview for " .. photoId .. " (" .. tostring(reason) .. "); exporting it" + ) + table.insert(needExport, photoId) + if missesInARow == MAX_PREVIEW_MISSES_IN_A_ROW then + log:info("Search previews: no usable Lightroom preview for 10 photos in a row; exporting the rest") + end + end + end + end + + -- Pass 2: small exports for the rest, a few dozen per session. + local chunkSize = Util.exportChunkSize(prefs.exportChunkSize) + local index = 1 + while index <= #needExport and not scope:isCanceled() do + local chunkIds, chunkPhotos = {}, {} + for i = index, math.min(index + chunkSize - 1, #needExport) do + table.insert(chunkIds, needExport[i]) + table.insert(chunkPhotos, byId[needExport[i]]) + end + index = index + #chunkIds + local paths, exportFailures = SearchIndexAPI.exportPhotosForIndexing(chunkPhotos, { + LR_size_maxHeight = RENDER_LONG_EDGE, + LR_jpeg_quality = 80, + }, function() + return scope:isCanceled() + end) + for i, photoId in ipairs(chunkIds) do + local photo = chunkPhotos[i] + if paths[photo] then + upload(photoId, paths[photo], "export") + elseif not scope:isCanceled() then + table.insert( + failures, + photo:getFormattedMetadata("fileName") + .. ": the export failed (" + .. tostring(exportFailures[photo] or "Lightroom gave no reason") + .. ")" + ) + end + if not scope:isCanceled() then + step("Exporting previews") + end + end + end + + local canceled = scope:isCanceled() + scope:done() + + local summary = added + .. " of " + .. total + .. " photos got a preview (" + .. fromPreview + .. " from Lightroom's previews, " + .. exported + .. " exported)." + if canceled then + summary = "Stopped. " .. summary + end + if unresolved > 0 then + summary = summary .. "\n" .. unresolved .. " photo(s) were skipped: their ID could not be worked out." + end + + if #failures > 0 then + local listed = {} + for i = 1, math.min(#failures, MAX_LISTED_FAILURES) do + table.insert(listed, failures[i]) + end + if #failures > MAX_LISTED_FAILURES then + table.insert(listed, "... and " .. (#failures - MAX_LISTED_FAILURES) .. " more (see the plug-in log).") + end + for _, f in ipairs(failures) do + log:error("Search preview not added: " .. f) + end + ErrorHandler.handleError( + #failures .. " photo(s) did not get a search preview", + summary .. "\n\n" .. table.concat(listed, "\n") + ) + else + LrDialogs.message("Add Missing Search Previews", summary, "info") + end +end + +LrTasks.startAsyncTask(function() + LrFunctionContext.callWithContext("TaskFillSearchPreviews", function(context) + local ok, err = LrTasks.pcall(run, context) + if not ok then + ErrorHandler.handleError("Adding search previews failed", tostring(err)) + end + end) +end) diff --git a/plugin/LrGeniusAI.lrdevplugin/Util.lua b/plugin/LrGeniusAI.lrdevplugin/Util.lua index 74e72e7e..972f0c52 100644 --- a/plugin/LrGeniusAI.lrdevplugin/Util.lua +++ b/plugin/LrGeniusAI.lrdevplugin/Util.lua @@ -3363,4 +3363,93 @@ function Util.cullFlagSummaryLines(counts) return lines end +--- +-- Width and height of a JPEG, read from its frame header (SOFn) without +-- decoding it. Used to tell whether a preview Lightroom handed back is big +-- enough to stand in for an export: `requestJpegThumbnail` answers with +-- whatever it has cached, which can be far smaller than what was asked for. +-- @param data string The JPEG's bytes. +-- @return number|nil width, number|nil height nil, nil for anything that is not +-- a JPEG or has no frame header before the image data. +-- +function Util.jpegDimensions(data) + if type(data) ~= "string" or #data < 4 then + return nil, nil + end + if data:byte(1) ~= 0xFF or data:byte(2) ~= 0xD8 then + return nil, nil + end + local i = 3 + local n = #data + while i + 3 <= n do + if data:byte(i) ~= 0xFF then + return nil, nil + end + -- Any number of 0xFF fill bytes may precede a marker. + while i <= n and data:byte(i) == 0xFF do + i = i + 1 + end + if i > n then + return nil, nil + end + local marker = data:byte(i) + i = i + 1 + if marker == 0xD9 or marker == 0xDA then + -- End of image, or start of scan: no frame header came first. + return nil, nil + end + -- Markers without a length: TEM and the restart markers. + if not (marker == 0x01 or (marker >= 0xD0 and marker <= 0xD7)) then + if i + 1 > n then + return nil, nil + end + local length = data:byte(i) * 256 + data:byte(i + 1) + -- SOF0..SOF15, except DHT (C4), JPG (C8) and DAC (CC). + if marker >= 0xC0 and marker <= 0xCF and marker ~= 0xC4 and marker ~= 0xC8 and marker ~= 0xCC then + if i + 6 > n then + return nil, nil + end + local height = data:byte(i + 3) * 256 + data:byte(i + 4) + local width = data:byte(i + 5) * 256 + data:byte(i + 6) + return width, height + end + if length < 2 then + return nil, nil + end + i = i + length + end + end + return nil, nil +end + +--- Photos per export session when indexing exports JPEGs. +Util.DEFAULT_EXPORT_CHUNK_SIZE = 25 + +--- +-- How many photos go into one export session. +-- +-- Lightroom renders the photos of one session in parallel, so a session per +-- photo leaves most of the computer idle. One session over the whole +-- selection is the other extreme: Lightroom keeps what a session rendered +-- until the session ends, and on a run of thousands of photos its memory +-- climbs until the computer swaps. A few dozen photos per session keeps the +-- parallel rendering and gives the memory back between sessions. +-- @param configured any The saved preference (prefs.exportChunkSize), if any. +-- @return number Between 1 and 200. +-- +function Util.exportChunkSize(configured) + local n = tonumber(configured) + if n == nil or n ~= n then + return Util.DEFAULT_EXPORT_CHUNK_SIZE + end + n = math.floor(n) + if n < 1 then + return 1 + end + if n > 200 then + return 200 + end + return n +end + return Util diff --git a/plugin/README.md b/plugin/README.md index e710a1df..c7d133ae 100644 --- a/plugin/README.md +++ b/plugin/README.md @@ -37,7 +37,10 @@ The plugin is designed to work with local and cloud providers, while keeping Lig - Optional face detection and clustering - Optional species identification (BioCLIP 2, on-device) - Every processed photo also gets a small preview (256 px, ~13 KB, stored by the backend beside - the index) for the browser search page + the index) for the browser search page. Photos indexed before previews were stored get theirs + from *Add Missing Search Previews...* (`TaskFillSearchPreviews.lua`): it asks the backend which + indexed photos lack one, takes Lightroom's cached preview where it is big enough + (`LightroomPreviews.lua`) and a small export otherwise, and touches nothing else ### Advanced Search diff --git a/plugin/spec/util_preview_render_spec.lua b/plugin/spec/util_preview_render_spec.lua new file mode 100644 index 00000000..9b6304aa --- /dev/null +++ b/plugin/spec/util_preview_render_spec.lua @@ -0,0 +1,71 @@ +-- Util.jpegDimensions: reads a JPEG's size from its frame header, which is how +-- a Lightroom preview is checked against the size an export would have had. + +local Util = require("Util") + +local function u16(n) + return string.char(math.floor(n / 256), n % 256) +end + +-- SOI, an APP0 segment, then a frame header of the given marker and size. +local function jpeg(marker, width, height) + local app0 = "\255\224" .. u16(16) .. "JFIF\0" .. string.rep("\0", 9) + local sof = "\255" .. string.char(marker) .. u16(17) .. "\8" .. u16(height) .. u16(width) .. string.rep("\0", 10) + return "\255\216" .. app0 .. sof .. "\255\218" +end + +describe("Util.jpegDimensions", function() + it("reads a baseline frame header after other segments", function() + local w, h = Util.jpegDimensions(jpeg(0xC0, 1024, 683)) + assert.are.equal(1024, w) + assert.are.equal(683, h) + end) + + it("reads a progressive frame header", function() + local w, h = Util.jpegDimensions(jpeg(0xC2, 640, 960)) + assert.are.equal(640, w) + assert.are.equal(960, h) + end) + + it("skips a Huffman table, which shares the SOF marker range", function() + local dht = "\255\196" .. u16(5) .. "\0\0\0" + local data = "\255\216" .. dht .. jpeg(0xC0, 300, 200):sub(3) + local w, h = Util.jpegDimensions(data) + assert.are.equal(300, w) + assert.are.equal(200, h) + end) + + it("reads a real file", function() + local fh = assert(io.open("server-rs/testdata/astronaut.jpg", "rb")) + local data = fh:read("*a") + fh:close() + local w, h = Util.jpegDimensions(data) + assert.are.equal(512, w) + assert.are.equal(512, h) + end) + + it("answers nil for anything that is not a usable JPEG", function() + assert.is_nil(Util.jpegDimensions(nil)) + assert.is_nil(Util.jpegDimensions("")) + assert.is_nil(Util.jpegDimensions("\137PNG\r\n\26\n")) + -- Truncated inside the frame header. + assert.is_nil(Util.jpegDimensions(jpeg(0xC0, 10, 10):sub(1, 26))) + -- Image data before any frame header. + assert.is_nil(Util.jpegDimensions("\255\216\255\218\0\0")) + end) +end) + +describe("Util.exportChunkSize", function() + it("defaults when nothing usable is saved", function() + assert.are.equal(Util.DEFAULT_EXPORT_CHUNK_SIZE, Util.exportChunkSize(nil)) + assert.are.equal(Util.DEFAULT_EXPORT_CHUNK_SIZE, Util.exportChunkSize("lots")) + assert.are.equal(Util.DEFAULT_EXPORT_CHUNK_SIZE, Util.exportChunkSize(0 / 0)) + end) + + it("keeps a saved size within bounds", function() + assert.are.equal(40, Util.exportChunkSize("40")) + assert.are.equal(7, Util.exportChunkSize(7.9)) + assert.are.equal(1, Util.exportChunkSize(-3)) + assert.are.equal(200, Util.exportChunkSize(5000)) + end) +end) diff --git a/server-rs/crates/lrg-api/src/routes/index.rs b/server-rs/crates/lrg-api/src/routes/index.rs index 5166ff29..e452c519 100644 --- a/server-rs/crates/lrg-api/src/routes/index.rs +++ b/server-rs/crates/lrg-api/src/routes/index.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use axum::{ - extract::{Query, State}, + extract::{Multipart, Query, State}, http::StatusCode, response::{IntoResponse, Response}, routing::{get, post}, @@ -20,7 +20,8 @@ use crate::state::AppState; pub fn router() -> Router> { Router::new() .route("/photos/lookup", post(get_photo_data)) - .route("/photos/thumbnail", get(get_thumbnail)) + .route("/photos/thumbnail", get(get_thumbnail).post(put_thumbnail)) + .route("/photos/thumbnails/missing", post(missing_thumbnails)) .route("/photos/ids", get(get_ids)) .route("/photos/remove", post(remove_image)) .route("/photos/metadata/remove", post(remove_metadata)) @@ -589,7 +590,7 @@ async fn get_thumbnail( None => ( StatusCode::NOT_FOUND, Json(json!({ - "error": "No preview is stored for this photo. Run Analyze & Index Photos on it to add one.", + "error": "No preview is stored for this photo. Run Library > Plug-in Extras > Add Missing Search Previews to add one.", "error_code": "no_thumbnail", })), ) @@ -597,6 +598,181 @@ async fn get_thumbnail( } } +/// Which indexed photos have no search-page preview (`crate::thumbnails`). +/// +/// Photos indexed before previews were stored have every signal but no +/// picture, and nothing in `/v1/index/unprocessed` asks for one (see the note +/// there). This is the question the plug-in's "Add Missing Search Previews" +/// asks instead, before it renders anything: `photo_ids` narrows it to those +/// photos (the selection), its absence means every photo of `catalog_id` (or +/// of the database). Photos that are not indexed are never listed — a preview +/// belongs to an index row, and `POST /v1/photos/thumbnail` refuses one +/// without it. +async fn missing_thumbnails( + State(state): State>, + body: Option>, +) -> Response { + let data = body.map(|Json(v)| v).unwrap_or_else(|| json!({})); + let catalog_id = data + .get("catalog_id") + .and_then(Value::as_str) + .map(str::trim) + .filter(|s| !s.is_empty()); + let wanted: Option> = match data.get("photo_ids") { + None | Some(Value::Null) => None, + Some(Value::Array(list)) => Some( + list.iter() + .filter_map(Value::as_str) + .filter_map(|s| meta::normalize_photo_id(Some(s), None)) + .collect(), + ), + Some(_) => { + return ( + StatusCode::BAD_REQUEST, + Json(json!({"error": "photo_ids must be a list, or left out for every photo"})), + ) + .into_response() + } + }; + + let (Some(store), Some(root)) = (state.store(), state.lance_root()) else { + return Json(json!({"photo_ids": [], "indexed": 0, "missing": 0})).into_response(); + }; + let rows = match &wanted { + Some(ids) if ids.is_empty() => Ok(Vec::new()), + Some(ids) => store.get_meta(IMAGE_TABLE, ids).await, + None => store.scan_meta(IMAGE_TABLE).await, + }; + let rows = match rows { + Ok(rows) => rows, + Err(e) => { + log::error!("Listing photos without a preview failed: {e}"); + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({"error": e.to_string()})), + ) + .into_response(); + } + }; + + let indexed: Vec = rows + .into_iter() + .filter(|(_, m)| catalog_id.is_none_or(|cat| meta::parse_catalog_ids(m).contains(cat))) + .map(|(id, _)| id) + .collect(); + // A stat per photo: about a second per 100,000 on a local disk, which is + // what one press of the button costs — not something the page polls. + let missing: Vec = indexed + .iter() + .filter(|id| !crate::thumbnails::exists(&root, id)) + .cloned() + .collect(); + log::info!( + "Search previews: {} of {} indexed photo(s) have none", + missing.len(), + indexed.len() + ); + Json(json!({ + "photo_ids": missing, + "indexed": indexed.len(), + "missing": missing.len(), + })) + .into_response() +} + +/// Stores the search-page preview of an already indexed photo from an image +/// the plug-in rendered (multipart: `photo_id` and `image`). +/// +/// The image is scaled down to the same size indexing stores +/// (`lrg_imaging::convert::THUMBNAIL_LONG_EDGE`), so the plug-in can send +/// whatever Lightroom hands it — a cached preview or a small export — without +/// knowing that size. Nothing else about the photo is touched: this is how +/// photos indexed before previews existed get one without being indexed again. +/// +/// A photo with no index row is refused (`error_code: "not_indexed"`): the +/// preview would have no photo to belong to, and nothing would ever delete it. +async fn put_thumbnail(State(state): State>, mut multipart: Multipart) -> Response { + let bad_request = + |msg: String| (StatusCode::BAD_REQUEST, Json(json!({"error": msg}))).into_response(); + let form = match crate::routes::route_util::parse_multipart(&mut multipart).await { + Ok(form) => form.into_single_photo(), + Err(e) => return bad_request(e), + }; + let Some(photo_id) = form + .photo_ids + .first() + .and_then(|id| meta::normalize_photo_id(Some(id), None)) + else { + return bad_request("photo_id is required".to_string()); + }; + let Some(image_bytes) = form.image_bytes.filter(|b| !b.is_empty()) else { + return bad_request("an image part is required".to_string()); + }; + let (Some(store), Some(root)) = (state.store(), state.lance_root()) else { + return ( + StatusCode::SERVICE_UNAVAILABLE, + Json(json!({"error": "No database is open. Restart the backend from Plug-in Manager and try again."})), + ) + .into_response(); + }; + + match store + .get_meta(IMAGE_TABLE, std::slice::from_ref(&photo_id)) + .await + { + Ok(rows) if rows.is_empty() => { + return ( + StatusCode::NOT_FOUND, + Json(json!({ + "error": "This photo is not indexed, so it gets its preview when Analyze & Index Photos runs on it.", + "error_code": "not_indexed", + })), + ) + .into_response(); + } + Ok(_) => {} + Err(e) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({"error": e.to_string()})), + ) + .into_response() + } + } + + let encoded = tokio::task::spawn_blocking(move || { + let decoded = image::load_from_memory(&image_bytes) + .map_err(|e| format!("could not decode the image: {e}"))? + .to_rgb8(); + let (width, height) = (decoded.width(), decoded.height()); + lrg_imaging::convert::thumbnail_jpeg( + decoded.as_raw(), + width, + height, + lrg_imaging::convert::THUMBNAIL_LONG_EDGE, + lrg_imaging::convert::THUMBNAIL_JPEG_QUALITY, + ) + .map_err(|e| e.to_string()) + }) + .await + .map_err(|e| format!("preview task failed: {e}")) + .and_then(|r| r); + let jpeg = match encoded { + Ok(jpeg) => jpeg, + Err(e) => return bad_request(e), + }; + if let Err(e) = crate::thumbnails::write(&root, &photo_id, &jpeg) { + log::error!("Photo {photo_id}: could not save its preview: {e}"); + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({"error": format!("The preview could not be saved: {e}")})), + ) + .into_response(); + } + log::debug!("Photo {photo_id}: preview stored ({} bytes)", jpeg.len()); + Json(json!({"status": "success", "photo_id": photo_id})).into_response() +} + fn body_photo_id(body: &Option>) -> Option { let data = body.as_ref()?.0.as_object()?; meta::normalize_photo_id( diff --git a/server-rs/crates/lrg-api/src/ui/search.html b/server-rs/crates/lrg-api/src/ui/search.html index 6cc2628b..8c5d7e89 100644 --- a/server-rs/crates/lrg-api/src/ui/search.html +++ b/server-rs/crates/lrg-api/src/ui/search.html @@ -712,11 +712,10 @@

Search

banner( missing + (missing === 1 ? " of these photos has" : " of these photos have") + - " no preview yet: indexed before LrGeniusAI stored previews. To add them, click “Show in " + - "Lightroom”, select the photos of the new collection and run Library ▸ Plug-in Extras ▸ " + - "Analyze & Index Photos with the scope “Selected photos only” and “Skip photos with existing data”: " + - "the previews are added, and nothing that is already there is made again. (“New or " + - "unprocessed photos” does not pick them up.)", + " no preview yet: indexed before LrGeniusAI stored previews. To add them, run Library ▸ " + + "Plug-in Extras ▸ Add Missing Search Previews in Lightroom — for the whole catalog, or click " + + "“Show in Lightroom” first and choose “Selected photos”. It only adds the previews; nothing " + + "else is analyzed again.", "note", "warnings" ); diff --git a/server-rs/crates/lrg-api/tests/contract.rs b/server-rs/crates/lrg-api/tests/contract.rs index e972cb86..32839072 100644 --- a/server-rs/crates/lrg-api/tests/contract.rs +++ b/server-rs/crates/lrg-api/tests/contract.rs @@ -3629,3 +3629,100 @@ async fn indexing_stores_a_preview_that_search_reports_and_remove_deletes() { "astro" )); } + +/// Photos indexed before previews were stored get one without being indexed +/// again: `/v1/photos/thumbnails/missing` lists them (catalog-scoped, and only +/// photos that are indexed), `POST /v1/photos/thumbnail` stores a preview +/// scaled to the indexing size, and a photo with no index row is refused. +#[tokio::test] +async fn missing_previews_are_listed_and_filled_without_reindexing() { + let (app, state) = fresh_app(); + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("lrgenius.db").to_str().unwrap().to_string(); + state.ensure_db_path(&db_path).await.unwrap(); + seed_searchable( + &state, + &[ + ("old", None, "", None, "indexed before previews"), + ("other", None, "", None, "another catalog's photo"), + ("done", None, "", None, "already has one"), + ], + ) + .await; + let store = state.store().unwrap(); + let mut rows = store + .get( + lrg_store::IMAGE_TABLE, + &["old".to_string(), "other".to_string(), "done".to_string()], + ) + .await + .unwrap(); + for row in &mut rows { + let catalog = if row.id == "other" { "cat-2" } else { "cat-1" }; + row.metadata + .insert("catalog_ids".into(), serde_json::json!([catalog])); + } + store.upsert(lrg_store::IMAGE_TABLE, &rows).await.unwrap(); + let root = state.lance_root().unwrap(); + lrg_api::thumbnails::write(&root, "done", b"jpeg").unwrap(); + + let (status, listed) = post_json( + &app, + "/v1/photos/thumbnails/missing", + serde_json::json!({"catalog_id": "cat-1", "db_path": db_path}), + ) + .await; + assert_eq!(status, StatusCode::OK, "{listed}"); + assert_eq!(listed["photo_ids"], serde_json::json!(["old"]), "{listed}"); + assert_eq!(listed["indexed"], 2, "{listed}"); + + // Narrowed to a selection: a photo that is not indexed is not listed. + let (_, selection) = post_json( + &app, + "/v1/photos/thumbnails/missing", + serde_json::json!({"photo_ids": ["other", "never-indexed"], "db_path": db_path}), + ) + .await; + assert_eq!( + selection["photo_ids"], + serde_json::json!(["other"]), + "{selection}" + ); + + let jpeg = std::fs::read(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../testdata/astronaut.jpg" + )) + .unwrap(); + let upload = |id: &'static str| { + let (content_type, body) = multipart_body( + &[("photo_id", id), ("db_path", db_path.as_str())], + Some(("preview.jpg", &jpeg)), + ); + Request::post("/v1/photos/thumbnail") + .header("content-type", content_type) + .body(Body::from(body)) + .unwrap() + }; + let stored = app.clone().oneshot(upload("old")).await.unwrap(); + assert_eq!(stored.status(), StatusCode::OK); + let preview = + image::load_from_memory(&lrg_api::thumbnails::read(&root, "old").unwrap()).unwrap(); + assert_eq!( + preview.width().max(preview.height()), + lrg_imaging::convert::THUMBNAIL_LONG_EDGE + ); + + let refused = app.clone().oneshot(upload("never-indexed")).await.unwrap(); + assert_eq!(refused.status(), StatusCode::NOT_FOUND); + assert_eq!(body_json(refused).await["error_code"], "not_indexed"); + assert!(!lrg_api::thumbnails::exists(&root, "never-indexed")); + + let (_, after) = post_json( + &app, + "/v1/photos/thumbnails/missing", + serde_json::json!({"catalog_id": "cat-1", "db_path": db_path}), + ) + .await; + assert_eq!(after["photo_ids"], serde_json::json!([]), "{after}"); +} From 481edff5640f93284b302f5a4f054b4aeb674983 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 19:38:43 +0000 Subject: [PATCH 02/10] Indexing: export in batches, optionally use Lightroom's previews One LrExportSession over the whole selection (the macOS pipeline) kept every rendition until the run ended, so Lightroom's memory climbed through long runs; on Windows each photo got its own session, so nothing rendered in parallel. Both paths now export in batches (25 by default, 'Photos per export batch' in Plug-in Manager): the macOS producer opens one session per batch and waits while a batch's worth of JPEGs is still queued, the sequential path renders the next batch in one session and consumes it photo by photo. New opt-in 'Use Lightroom's previews where they are large enough': indexing takes the cached preview (requestJpegThumbnail) when its long edge is at least 3/4 of the export size and exports the rest; the completion dialog says how many were exported and how to build previews first. Docs-Reviewed: Dev-Backend-API.md Docs-Reviewed: Getting-Started.md Docs-Reviewed: Help-Local-AI-Models.md Docs-Reviewed: Help-Other-AI-Server.md Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HaMMvqfgsKcoNWQaKy3c9C --- docs/wiki/Dev-Plugin-README.md | 5 +- docs/wiki/Help-Performance-Tips.md | 39 +++- .../LrGeniusAI.lrdevplugin/APISearchIndex.lua | 191 +++++++++++++++--- plugin/LrGeniusAI.lrdevplugin/Init.lua | 4 + .../PluginInfoDialogSections.lua | 55 +++++ plugin/LrGeniusAI.lrdevplugin/Util.lua | 19 ++ plugin/README.md | 5 +- plugin/spec/util_preview_render_spec.lua | 9 + 8 files changed, 287 insertions(+), 40 deletions(-) diff --git a/docs/wiki/Dev-Plugin-README.md b/docs/wiki/Dev-Plugin-README.md index cd867f8a..61e16c38 100644 --- a/docs/wiki/Dev-Plugin-README.md +++ b/docs/wiki/Dev-Plugin-README.md @@ -316,7 +316,10 @@ In the plugin settings dialog you can configure: (`LocalModelCatalog.engineFor`). The section reports why it is unavailable when the host cannot use it — a source build without the `llamacpp` feature, or a missing MLX helper. -- Export size and quality used for AI processing +- Export size and quality used for AI processing, photos per export session (indexing exports + in batches of that size, `Util.exportChunkSize`, so Lightroom frees its memory between them), + and whether indexing may use Lightroom's cached previews instead of exporting + (`LightroomPreviews.lua`; photos without a large-enough preview are exported) - Prompt presets — a preset is the *system* prompt: which expert is looking at the photo, which vocabulary they use, how specific they may be, and what they must never invent. **Default** (the analytical voice) and **Family & diff --git a/docs/wiki/Help-Performance-Tips.md b/docs/wiki/Help-Performance-Tips.md index 0485abde..eac93351 100644 --- a/docs/wiki/Help-Performance-Tips.md +++ b/docs/wiki/Help-Performance-Tips.md @@ -97,6 +97,32 @@ where you mainly want keywords, not fine detail; go up to **2048px+** only if you need the model to read small text or catch fine detail, and only on smaller batches. +### Photos per export batch + +Lightroom renders the photos of one export session in parallel, but keeps +what a session rendered until it ends. LrGeniusAI therefore exports in +batches — **25 photos** by default, *Photos per export batch* in the same +section (10, 25, 50, 100). Lightroom's memory goes back down after every +batch, so a run over thousands of photos no longer makes it swell until the +computer slows to a crawl. Lower it if Lightroom still grows on your computer; +raise it on a machine with plenty of memory for slightly faster runs. + +### Use Lightroom's previews where they are large enough + +With **Use Lightroom's previews where they are large enough (faster)** on, +indexing takes Lightroom's cached preview of a photo instead of exporting it, +whenever that preview is at least three quarters of the export size (768 px +for the default 1024 px). Photos without such a preview are exported as usual, +and the run's completion dialog says how many that were. Previews show your +develop edits, so crops and adjustments still count — with one caveat: a +preview Lightroom has not re-rendered since your last edit shows the photo as +it was, and the aesthetic score is made from what the backend sees. + +To get the most out of it, select the photos and run *Library ▸ Previews ▸ +Build Standard-Sized Previews* first, with *Standard Preview Size* in +*Catalog Settings ▸ File Handling* at least as large as the export size. The +option is off by default and has no effect when *Submit original files* is on. + --- ## 4. "Submit original files for fastest indexing" (experimental) @@ -173,12 +199,13 @@ parallelism in a batched run is entirely on the backend's side, inside one request: it is free to use several cores, while the plugin still waits for a single answer. -On macOS, when you're *not* using "submit originals," the plugin pipelines -export and upload — the next photo's JPEG export overlaps with the previous -photo's backend round-trip, so you're not paying export time and network -time back-to-back. This pipelining isn't available on Windows, so on Windows -the "submit originals" fast path (§4) or a smaller export size (§3) buys you -more. +On macOS, when you're *not* using "submit originals" or Lightroom's +previews, the plugin pipelines export and upload — the next batch's JPEG +export overlaps with the previous photos' backend round-trips, so you're not +paying export time and network time back-to-back. On Windows the exports of a +batch are still rendered together, in one export session, and then sent one +by one; there the "submit originals" fast path (§4), Lightroom's previews +(§3) or a smaller export size buy you more. --- diff --git a/plugin/LrGeniusAI.lrdevplugin/APISearchIndex.lua b/plugin/LrGeniusAI.lrdevplugin/APISearchIndex.lua index 76222b8d..251352e2 100644 --- a/plugin/LrGeniusAI.lrdevplugin/APISearchIndex.lua +++ b/plugin/LrGeniusAI.lrdevplugin/APISearchIndex.lua @@ -304,6 +304,10 @@ local EXPORT_SETTINGS = { LR_embeddedMetadataOption = "all", } +--- How long indexing waits for Lightroom's cached preview of one photo +--- before it exports the photo instead. +local PREVIEW_TIMEOUT_SECONDS = 10 + --- The export settings for one session: the defaults above with the current --- size and quality preferences (which can change after this file loaded), --- the temp folder as destination, and `overrides` on top. A fresh table each @@ -2578,52 +2582,84 @@ function SearchIndexAPI.analyzeAndIndexSelectedPhotos(selectedPhotos, progressSc -- Opt-in fast path: submit the original file instead of exporting a JPEG. -- Local backend gets just the path; remote backends get the raw upload. local useOriginals = (prefs and prefs.indexSubmitOriginals == true) + -- Opt-in: take Lightroom's cached preview instead of exporting, when it is + -- big enough (LightroomPreviews). Only meaningful in export mode. + local usePreviews = (not useOriginals) and prefs ~= nil and prefs.indexUseLightroomPreviews == true + local exportLongEdge = tonumber(prefs and prefs.exportSize) or 1024 + -- A preview a quarter smaller than the export would have been is still + -- plenty for the models (SigLIP2 and the aesthetic scorer look at far + -- less); anything smaller is exported. + local minPreviewEdge = math.floor(exportLongEdge * 0.75) + local chunkSize = Util.exportChunkSize(prefs and prefs.exportChunkSize) + local previewStats = { used = 0, exported = 0 } local errorMessages = {} local warningsList = {} - if MAC_ENV and numPhotos > 1 and not useOriginals then - -- Pipelined export/index path (macOS only). A producer drives a single - -- LrExportSession and pushes rendered JPEG paths into a queue; a consumer - -- POSTs each photo to the backend as soon as its export completes, so the - -- next export overlaps with the previous photo's backend round-trip. + if MAC_ENV and numPhotos > 1 and not useOriginals and not usePreviews then + -- Pipelined export/index path (macOS only). A producer drives the export + -- sessions and pushes rendered JPEG paths into a queue; a consumer POSTs + -- each photo to the backend as soon as its export completes, so the next + -- export overlaps with the previous photo's backend round-trip. -- Disabled on Windows because earlier multi-worker attempts crashed Lightroom. -- Skipped when originals are submitted directly: there is nothing to export. - local tempDir = LrPathUtils.getStandardFilePath("temp") - EXPORT_SETTINGS.LR_export_destinationPathPrefix = tempDir - - local exportSession = LrExportSession({ - photosToExport = selectedPhotos, - exportSettings = EXPORT_SETTINGS, - }) - + -- + -- One session per `chunkSize` photos, not one for the whole selection: + -- Lightroom keeps what a session rendered until the session ends, and a + -- single session over thousands of photos is what made its memory climb + -- through a long run. The producer also waits while a chunk's worth of + -- rendered files is still queued, so it never runs far ahead of the + -- backend and temp files cannot pile up. local queue = {} local producerDone = false local consumerDone = false + local function stopping() + return progressScope:isCanceled() or not keepRunning + end + LrTasks.startAsyncTask(function() local idx = 0 - for _, rendition in exportSession:renditions() do - idx = idx + 1 - if progressScope:isCanceled() or not keepRunning then - LrTasks.pcall(function() - rendition:cancel() - end) - else - local success, path = rendition:waitForRender() - -- The rendition carries its own photo. Pairing by loop counter - -- instead would silently shift every later photo's analysis onto - -- the wrong file as soon as Lightroom skips one rendition. - local photo = rendition.photo - if photo == nil then - log:error("Rendition #" .. tostring(idx) .. " carried no photo; skipping it.") - table.insert(queue, { exportFailed = true, error = "rendition carried no photo" }) - elseif success and path then - table.insert(queue, { photo = photo, path = path }) + local chunkStart = 1 + while chunkStart <= numPhotos and not stopping() do + while #queue >= chunkSize and not stopping() do + LrTasks.sleep(0.05) + end + if stopping() then + break + end + local chunk = {} + for i = chunkStart, math.min(chunkStart + chunkSize - 1, numPhotos) do + table.insert(chunk, selectedPhotos[i]) + end + chunkStart = chunkStart + #chunk + local exportSession = LrExportSession({ + photosToExport = chunk, + exportSettings = exportSettingsFor(), + }) + for _, rendition in exportSession:renditions() do + idx = idx + 1 + if stopping() then + LrTasks.pcall(function() + rendition:skipRender() + end) else - table.insert(queue, { photo = photo, exportFailed = true, error = path }) + local success, path = rendition:waitForRender() + -- The rendition carries its own photo. Pairing by loop counter + -- instead would silently shift every later photo's analysis onto + -- the wrong file as soon as Lightroom skips one rendition. + local photo = rendition.photo + if photo == nil then + log:error("Rendition #" .. tostring(idx) .. " carried no photo; skipping it.") + table.insert(queue, { exportFailed = true, error = "rendition carried no photo" }) + elseif success and path then + table.insert(queue, { photo = photo, path = path }) + else + table.insert(queue, { photo = photo, exportFailed = true, error = path }) + end end end + log:trace("Indexing export chunk done (" .. tostring(idx) .. " of " .. tostring(numPhotos) .. ")") end producerDone = true log:trace("Indexing export producer finished (" .. tostring(idx) .. " renditions)") @@ -2837,6 +2873,44 @@ function SearchIndexAPI.analyzeAndIndexSelectedPhotos(selectedPhotos, progressSc end end + -- photo -> path of its already exported JPEG. Filled a chunk at a time + -- by prefillExports, consumed one photo at a time below. + local exportMemo = {} + + --- + -- Exports `photo` together with the next photos on the stack in one + -- export session (Lightroom renders a session's photos in parallel). + -- Bounded by `chunkSize`, so the session's memory is given back long + -- before a large run ends. A photo whose export failed has no memo and + -- gets one more try on its own. + -- + local function prefillExports(photo) + local chunk = { photo } + for i = 1, math.min(chunkSize - 1, #photoToProcessStack) do + local nextPhoto = photoToProcessStack[i] + if nextPhoto ~= nil and exportMemo[nextPhoto] == nil then + table.insert(chunk, nextPhoto) + end + end + if #chunk > 1 then + progressScope:setCaption( + "Exporting photos " + .. tostring(stats.processed + 1) + .. "-" + .. tostring(stats.processed + #chunk) + .. " of " + .. tostring(numPhotos) + .. "..." + ) + end + local paths = SearchIndexAPI.exportPhotosForIndexing(chunk, nil, function() + return progressScope:isCanceled() or not keepRunning + end) + for p, path in pairs(paths) do + exportMemo[p] = path + end + end + local analyzeWorker = function() while #photoToProcessStack > 0 do if progressScope:isCanceled() then @@ -2920,8 +2994,41 @@ function SearchIndexAPI.analyzeAndIndexSelectedPhotos(selectedPhotos, progressSc end end + if not success and usePreviews then + local previewPath, why = LightroomPreviews.previewFile( + photo, + exportLongEdge, + minPreviewEdge, + PREVIEW_TIMEOUT_SECONDS + ) + if previewPath then + log:trace("Using Lightroom's preview for " .. filename) + previewStats.used = previewStats.used + 1 + success, indexResponse = + SearchIndexAPI.analyzeAndIndexPhoto(photoId, previewPath, photoOptions) + LrTasks.pcall(function() + LrFileUtils.delete(previewPath) + end) + else + log:trace( + "No usable Lightroom preview for " .. filename .. " (" .. tostring(why) .. ")" + ) + previewStats.exported = previewStats.exported + 1 + end + end + if not success then - local exportedPhotoPath = SearchIndexAPI.exportPhotoForIndexing(photo) + -- Pure export mode renders the next photos with this one, + -- in one session (see prefillExports). In the other modes + -- an export is the exception, so it stays a single one. + if not useOriginals and not usePreviews and exportMemo[photo] == nil then + prefillExports(photo) + end + local exportedPhotoPath = exportMemo[photo] + exportMemo[photo] = nil + if exportedPhotoPath == nil then + exportedPhotoPath = SearchIndexAPI.exportPhotoForIndexing(photo) + end if exportedPhotoPath then log:trace("Using exported JPEG for " .. filename) success, indexResponse = @@ -3013,6 +3120,13 @@ function SearchIndexAPI.analyzeAndIndexSelectedPhotos(selectedPhotos, progressSc end end end + + -- A cancelled run leaves exported JPEGs of photos it never reached. + for _, path in pairs(exportMemo) do + LrTasks.pcall(function() + LrFileUtils.delete(path) + end) + end end if shouldCloseScope then @@ -3023,6 +3137,19 @@ function SearchIndexAPI.analyzeAndIndexSelectedPhotos(selectedPhotos, progressSc return "canceled", stats.processed, stats.failed, processedPhotos end + -- Not a failure — those photos were exported instead — but the user chose + -- previews to make the run faster, and this is why it was not. + if usePreviews and previewStats.exported > 0 then + table.insert( + warningsList, + Util.previewFallbackWarning( + previewStats.exported, + previewStats.used + previewStats.exported, + minPreviewEdge + ) + ) + end + local status, combinedError, combinedWarnings = SearchIndexAPI.summarizeRun(stats, errorMessages, warningsList) return status, stats.processed, stats.failed, processedPhotos, combinedError, combinedWarnings end diff --git a/plugin/LrGeniusAI.lrdevplugin/Init.lua b/plugin/LrGeniusAI.lrdevplugin/Init.lua index d346997b..bff773ea 100644 --- a/plugin/LrGeniusAI.lrdevplugin/Init.lua +++ b/plugin/LrGeniusAI.lrdevplugin/Init.lua @@ -138,6 +138,10 @@ if _G.prefs.indexSubmitOriginals == nil then _G.prefs.indexSubmitOriginals = false end +if _G.prefs.indexUseLightroomPreviews == nil then + _G.prefs.indexUseLightroomPreviews = false +end + if _G.prefs.exportSize == nil then _G.prefs.exportSize = Defaults.defaultExportSize end diff --git a/plugin/LrGeniusAI.lrdevplugin/PluginInfoDialogSections.lua b/plugin/LrGeniusAI.lrdevplugin/PluginInfoDialogSections.lua index 9b508d61..589aac50 100644 --- a/plugin/LrGeniusAI.lrdevplugin/PluginInfoDialogSections.lua +++ b/plugin/LrGeniusAI.lrdevplugin/PluginInfoDialogSections.lua @@ -68,6 +68,8 @@ function PluginInfoDialogSections.startDialog(propertyTable) propertyTable.exportSize = prefs.exportSize propertyTable.exportQuality = prefs.exportQuality propertyTable.indexSubmitOriginals = (prefs.indexSubmitOriginals == true) + propertyTable.indexUseLightroomPreviews = (prefs.indexUseLightroomPreviews == true) + propertyTable.exportChunkSize = tostring(Util.exportChunkSize(prefs.exportChunkSize)) propertyTable.promptTitles = Util.promptMenuItems(prefs.prompts, Defaults.defaultPromptName) @@ -1480,6 +1482,57 @@ function PluginInfoDialogSections.sectionsForTopOfDialog(f, propertyTable) width_in_chars = 5, }), }), + f:row({ + fill_horizontal = 1, + f:static_text({ + title = "Photos per export batch", + alignment = "right", + width = share("labelWidth"), + }), + f:popup_menu({ + value = bind("exportChunkSize"), + items = { "10", "25", "50", "100" }, + enabled = LrView.bind({ + key = "indexSubmitOriginals", + transform = function(v) + return v ~= true + end, + }), + }), + }), + f:row({ + fill_horizontal = 1, + f:static_text({ + title = "Lightroom renders the photos of a batch in parallel and frees its memory after each\n" + .. "batch. Smaller batches keep Lightroom's memory low on long runs; larger ones can be\n" + .. "a little faster on computers with plenty of memory.", + enabled = false, + }), + }), + f:row({ + fill_horizontal = 1, + f:checkbox({ + value = bind("indexUseLightroomPreviews"), + title = "Use Lightroom's previews where they are large enough (faster)", + enabled = LrView.bind({ + key = "indexSubmitOriginals", + transform = function(v) + return v ~= true + end, + }), + }), + }), + f:row({ + fill_horizontal = 1, + f:static_text({ + title = "Takes Lightroom's cached preview instead of exporting when it is at least three\n" + .. "quarters of the export size; other photos are exported as usual. Previews show your\n" + .. "edits, but one Lightroom has not re-rendered since the last edit shows the photo as\n" + .. "it was then — which matters for the aesthetic score. Build Standard-Sized Previews\n" + .. "(Library > Previews) first to get the most out of this.", + enabled = false, + }), + }), }), }, { @@ -1847,6 +1900,8 @@ function PluginInfoDialogSections.endDialog(propertyTable) prefs.exportSize = propertyTable.exportSize prefs.exportQuality = propertyTable.exportQuality prefs.indexSubmitOriginals = (propertyTable.indexSubmitOriginals == true) + prefs.indexUseLightroomPreviews = (propertyTable.indexUseLightroomPreviews == true) + prefs.exportChunkSize = Util.exportChunkSize(propertyTable.exportChunkSize) prefs.prompts = propertyTable.prompts prefs.prompt = Util.resolvePromptName(propertyTable.prompts, propertyTable.prompt, Defaults.defaultPromptName) diff --git a/plugin/LrGeniusAI.lrdevplugin/Util.lua b/plugin/LrGeniusAI.lrdevplugin/Util.lua index 972f0c52..e460cdc9 100644 --- a/plugin/LrGeniusAI.lrdevplugin/Util.lua +++ b/plugin/LrGeniusAI.lrdevplugin/Util.lua @@ -3452,4 +3452,23 @@ function Util.exportChunkSize(configured) return n end +--- +-- The run-end note for indexing with "Use Lightroom previews" on, when some +-- photos had no cached preview big enough and were exported instead. +-- @param exported number Photos exported instead. +-- @param total number Photos that went through the preview check. +-- @param minEdge number The smallest long edge a preview needed. +-- @return string +-- +function Util.previewFallbackWarning(exported, total, minEdge) + return tostring(exported) + .. " of " + .. tostring(total) + .. " photos had no Lightroom preview of at least " + .. tostring(minEdge) + .. " px and were exported instead, which is slower. To use previews for all of them next time, " + .. "select the photos and run Library > Previews > Build Standard-Sized Previews first " + .. "(with the standard preview size in Catalog Settings at least as large)." +end + return Util diff --git a/plugin/README.md b/plugin/README.md index c7d133ae..2a14a428 100644 --- a/plugin/README.md +++ b/plugin/README.md @@ -312,7 +312,10 @@ In the plugin settings dialog you can configure: (`LocalModelCatalog.engineFor`). The section reports why it is unavailable when the host cannot use it — a source build without the `llamacpp` feature, or a missing MLX helper. -- Export size and quality used for AI processing +- Export size and quality used for AI processing, photos per export session (indexing exports + in batches of that size, `Util.exportChunkSize`, so Lightroom frees its memory between them), + and whether indexing may use Lightroom's cached previews instead of exporting + (`LightroomPreviews.lua`; photos without a large-enough preview are exported) - Prompt presets — a preset is the *system* prompt: which expert is looking at the photo, which vocabulary they use, how specific they may be, and what they must never invent. **Default** (the analytical voice) and **Family & diff --git a/plugin/spec/util_preview_render_spec.lua b/plugin/spec/util_preview_render_spec.lua index 9b6304aa..82ace3bf 100644 --- a/plugin/spec/util_preview_render_spec.lua +++ b/plugin/spec/util_preview_render_spec.lua @@ -69,3 +69,12 @@ describe("Util.exportChunkSize", function() assert.are.equal(200, Util.exportChunkSize(5000)) end) end) + +describe("Util.previewFallbackWarning", function() + it("says how many were exported and what makes the next run use previews", function() + local text = Util.previewFallbackWarning(3, 40, 768) + assert.truthy(text:find("3 of 40 photos", 1, true)) + assert.truthy(text:find("768 px", 1, true)) + assert.truthy(text:find("Build Standard-Sized Previews", 1, true)) + end) +end) From 11ff2b95be10d5d203715efaace8fc80e881b50a Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 20:32:00 +0000 Subject: [PATCH 03/10] Face recognition: optional AuraFace recognizer, names survive the switch FaceNet stays the default. AuraFace (fal/AuraFace-v1, glintr100.onnx, Apache-2.0, ~260 MB) can be downloaded from Hugging Face and selected in Plug-in Manager > Face recognition. lrg-ml: Recognizer {FaceNet, AuraFace} with its own model id (yunet-2023mar/auraface-v1); AuraFace runs on the canonical 112x112 ArcFace crop, RGB, (v-127.5)/127.5, the recognizer input name read from the graph. detect_faces_with_model loads and runs under one lock and returns the id the embeddings were made with, so a concurrent switch cannot mislabel rows. lrg-api: GET/POST /v1/faces/recognizer (saved in face_recognizer.txt next to the models, restored at start; 409 until downloaded) and /v1/faces/recognizer/downloads (download_url_assets, generalised from the release downloader). Re-detection carries person assignments from the other recognizer's rows by box overlap (same detector, IoU >= 0.6), so names survive a switch. /v1/faces/search only ranks rows of the query's model. Health reports face_recognizer. scripts/auraface_reference.py records InsightFace's own embedding of the golden face; the AuraFace golden test compares against it and skips until that reference is committed. Docs-Reviewed: Help-Choosing-AI-Model.md Docs-Reviewed: Help-Other-AI-Server.md Docs-Reviewed: Troubleshooting.md Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HaMMvqfgsKcoNWQaKy3c9C --- docs/wiki/Credits.md | 1 + docs/wiki/Dev-Backend-API.md | 46 ++- docs/wiki/Dev-Plugin-README.md | 4 + docs/wiki/Dev-Server-README.md | 21 ++ docs/wiki/Help-People-Faces.md | 28 ++ .../LrGeniusAI.lrdevplugin/APISearchIndex.lua | 92 +++++- .../PluginInfoDialogSections.lua | 104 +++++++ plugin/LrGeniusAI.lrdevplugin/Util.lua | 26 ++ plugin/README.md | 4 + plugin/spec/util_preview_render_spec.lua | 25 ++ server-rs/Cargo.lock | 1 + server-rs/README.md | 21 ++ server-rs/crates/lrg-api/src/lib.rs | 1 + server-rs/crates/lrg-api/src/routes/clip.rs | 32 +- .../lrg-api/src/routes/face_recognizer.rs | 236 +++++++++++++++ server-rs/crates/lrg-api/src/routes/faces.rs | 7 +- .../crates/lrg-api/src/routes/index_upload.rs | 190 ++++++++++-- server-rs/crates/lrg-api/src/routes/mod.rs | 1 + server-rs/crates/lrg-api/src/routes/server.rs | 1 + server-rs/crates/lrg-api/src/state.rs | 7 +- server-rs/crates/lrg-api/tests/contract.rs | 39 +++ server-rs/crates/lrg-ml/Cargo.toml | 3 + server-rs/crates/lrg-ml/src/auraface.rs | 59 ++++ server-rs/crates/lrg-ml/src/faces.rs | 275 +++++++++++++++--- server-rs/crates/lrg-ml/src/lib.rs | 1 + server-rs/crates/lrg-ml/src/model_paths.rs | 14 +- server-rs/crates/lrg-ml/tests/face_goldens.rs | 76 ++++- server-rs/scripts/auraface_reference.py | 91 ++++++ 28 files changed, 1335 insertions(+), 71 deletions(-) create mode 100644 server-rs/crates/lrg-api/src/routes/face_recognizer.rs create mode 100644 server-rs/crates/lrg-ml/src/auraface.rs create mode 100644 server-rs/scripts/auraface_reference.py diff --git a/docs/wiki/Credits.md b/docs/wiki/Credits.md index 53d22c4b..80ce35c2 100644 --- a/docs/wiki/Credits.md +++ b/docs/wiki/Credits.md @@ -5,6 +5,7 @@ LrGeniusAI is made possible by these amazing open-source projects and AI framewo ## Core Backend Dependencies - **YuNet**: Fast, lightweight face detector from the OpenCV Model Zoo; provides the face boxes and five landmarks. [GitHub](https://github.com/opencv/opencv_zoo) - **FaceNet (facenet-pytorch)**: Inception-ResNet-v1 with VGGFace2 weights; produces the face embeddings that group photos by person. [GitHub](https://github.com/timesler/facenet-pytorch) +- **AuraFace (fal.ai)**: ArcFace-style ResNet-100 trained on commercially usable data, Apache-2.0; the optional, more accurate face recognizer. [Hugging Face](https://huggingface.co/fal/AuraFace-v1) - **SigLIP2 (NaFlex)** (Apache-2.0): Google's `siglip2-so400m-patch16-naflex` ([SigLIP 2, Tschannen et al., 2025](https://arxiv.org/abs/2502.14786)), the vision-language model behind semantic search and the photo genre; it keeps each photo's aspect ratio. Exported with Hugging Face Transformers. [Model](https://huggingface.co/google/siglip2-so400m-patch16-naflex) - **AestheticSigLIP**: Gowthami Somepalli's `AestheticSigLIP`, a SigLIP 2 So400m NaFlex encoder fine-tuned to score photos from 1 to 9, which provides the *Aesthetic score*. Apache-2.0. [Model](https://huggingface.co/somepago/AestheticSigLIP) - **CLIP-IQA**: The prompt-pair method (Wang, Chan & Loy, *Exploring CLIP for Assessing the Look and Feel of Images*, AAAI 2023) behind the moment, emotion and kiss-gate questions culling asks of a photo. [Paper](https://arxiv.org/abs/2207.12396) diff --git a/docs/wiki/Dev-Backend-API.md b/docs/wiki/Dev-Backend-API.md index f2a4079b..b03f6307 100644 --- a/docs/wiki/Dev-Backend-API.md +++ b/docs/wiki/Dev-Backend-API.md @@ -62,7 +62,8 @@ when it is `"failed"`. | `clip_image_model` / `clip_image_error` | The SigLIP2 **image** tower alone — what indexing and the training path embed with. | | `clip_text_model` / `clip_text_error` | The SigLIP2 **text** tower alone — what search, keyword lookup and the CLIP-IQA prompt sets embed with. | | `aesthetic_model` / `aesthetic_error` | AestheticSigLIP, the photo quality scorer. Its own backbone, loaded and idle-unloaded on its own clock. | -| `face_model` / `face_error` | YuNet + FaceNet. | +| `face_model` / `face_error` | YuNet + the active recognizer (FaceNet or AuraFace). | +| `face_recognizer` | Which recognizer is active: `"facenet"` or `"auraface"` (see `GET /v1/faces/recognizer`). | | `species_model` / `species_error` | BioCLIP 2. | The two SigLIP2 towers load and idle-unload **independently**, so an @@ -727,7 +728,48 @@ them decides what a re-cluster is allowed to touch: Detects faces in an uploaded image and stores their embeddings. ### `POST /v1/faces/search` -Finds photos containing faces similar to those in a reference image. +Finds photos containing faces similar to those in a reference image. Only +stored faces from the recognizer that embedded the query (`face_model`) are +ranked: after a FaceNet/AuraFace switch the other rows are vectors of another +space until their photos are re-indexed. + +### `GET /v1/faces/recognizer` +Which network makes face identity embeddings (`lrg_ml::faces::Recognizer`): + +```json +{"active": "facenet", "saved": "facenet", "model_id": "yunet-2023mar/facenet-vggface2", + "recognizers": [{"id": "facenet", "name": "FaceNet", "description": "...", "ready": true, + "active": true, "model_id": "...", "approx_bytes": 0, "downloadable": false}, + {"id": "auraface", "name": "AuraFace", "ready": false, "downloadable": true, + "approx_bytes": 261000000, ...}], + "warnings": []} +``` + +`saved` differs from `active` when the saved choice could not be restored at +start (AuraFace's file is gone); `warnings` then says so, for the Plug-in +Manager. + +### `POST /v1/faces/recognizer` +`{"recognizer": "facenet" | "auraface"}` — switches, saves the choice (restored +at start) and answers like `GET`, plus `changed` and `warnings` for the user +(the faces of the other recognizer are set aside until re-indexed; names carry +over). 400 for an unknown name; 409 `error_code: "not_downloaded"` when the +recognizer's model is not on disk. + +The switch changes `face_model` on every row written afterwards. Clustering, +`/v1/faces/search` and `/v1/index/unprocessed` only treat rows of the active +model as current, so the two spaces are never compared. On re-detection +(`POST /v1/index/photos` with faces), person assignments of previous rows from +the *other* recognizer are carried over by box overlap (IoU ≥ 0.6) — the +detector is the same, so a face lands on its old box — while rows of the same +recognizer keep matching by embedding. + +### `POST /v1/faces/recognizer/downloads` / `GET /v1/faces/recognizer/downloads` +Starts the AuraFace download (`fal/AuraFace-v1`, `glintr100.onnx`, from +Hugging Face — `HF_ENDPOINT` is honoured) / reports its progress, in the same +shape as `/v1/models/assets/downloads` (`status`, `progress`, `total`, +`current_file`, `error`). Does not switch; the plugin calls +`POST /v1/faces/recognizer` once it completed. ### `POST /v1/faces/cluster` Re-clusters all stored face embeddings into person groups. The policy lives in diff --git a/docs/wiki/Dev-Plugin-README.md b/docs/wiki/Dev-Plugin-README.md index 61e16c38..34511e96 100644 --- a/docs/wiki/Dev-Plugin-README.md +++ b/docs/wiki/Dev-Plugin-README.md @@ -61,6 +61,10 @@ The plugin is designed to work with local and cloud providers, while keeping Lig ### People Workflows - Cluster faces into persons +- Choose the face recognizer in Plug-in Manager (*Face recognition*): FaceNet (default) or + AuraFace (more accurate, ~260 MB download). `PluginInfoDialogSections.switchFaceRecognizer` + downloads AuraFace when needed and switches via `/v1/faces/recognizer`; names carry over when + the photos are re-indexed - Rename persons - Jump from a person directly to a Lightroom collection - The People interface opens in your web browser (served by the backend at diff --git a/docs/wiki/Dev-Server-README.md b/docs/wiki/Dev-Server-README.md index 98b93671..687c5cb5 100644 --- a/docs/wiki/Dev-Server-README.md +++ b/docs/wiki/Dev-Server-README.md @@ -419,6 +419,27 @@ which is where `POST /v1/models/assets/downloads` puts them. Assets are publishe by `.github/workflows/model-assets-face.yml` to the fixed `face-assets-v1` tag, independent of the other two families. +**AuraFace (optional recognizer).** `fal/AuraFace-v1`'s `glintr100.onnx` +(Apache-2.0, ~260 MB) can replace FaceNet: `POST /v1/faces/recognizer/downloads` +fetches it from Hugging Face (`HF_ENDPOINT` is honoured) to +`~/.cache/lrgenius/models/auraface_glintr100.onnx` (or `LRG_AURAFACE_ONNX`), +and `POST /v1/faces/recognizer` switches. The choice is saved in +`~/.cache/lrgenius/models/face_recognizer.txt` (or `LRG_FACE_RECOGNIZER_FILE`) +and restored at start. It runs on the same YuNet landmarks and the same 112x112 +ArcFace crop InsightFace makes, RGB, `(v - 127.5) / 127.5` +(`lrg-ml/src/auraface.rs`). To check that chain numerically, write +InsightFace's own embedding into the golden file and run the golden test: + +```bash +uv run --no-project server-rs/scripts/auraface_reference.py --output ~/.cache/lrgenius/models/auraface_glintr100.onnx +cargo test -p lrg-ml --test face_goldens -- auraface # LRG_REQUIRE_GOLDENS=auraface to fail instead of skip +``` + +The two recognizers' rows carry different `face_model` ids +(`yunet-2023mar/facenet-vggface2`, `yunet-2023mar/auraface-v1`), so nothing +compares across them; on re-detection, person assignments from the other +recognizer's rows are carried over by box overlap (same detector). + These replaced InsightFace's `buffalo_l` (SCRFD + ArcFace), which was resolved from `INSIGHTFACE_ROOT` and could not be redistributed — face detection was the one family the plugin could not download for you. Note the diff --git a/docs/wiki/Help-People-Faces.md b/docs/wiki/Help-People-Faces.md index f052acae..24209639 100644 --- a/docs/wiki/Help-People-Faces.md +++ b/docs/wiki/Help-People-Faces.md @@ -197,6 +197,34 @@ the whole indexed catalog. - The fastest way to judge a threshold is not to guess it but to look. Double-click your biggest person, sort by *outliers first*, and see whether the faces at the far end are still the same human. If they are, the threshold can go up; if strangers appear, it should come down. - For culling portrait sessions, face data also feeds into the **Cull Photos** scoring (eye openness, blink detection, face sharpness). See [Help: Cull Photos](Help-Cull-Photos). +## Choosing the face recognizer: FaceNet or AuraFace + +Which faces belong to the same person is decided by a *face recognizer*. Two are +available, under **Face recognition** in the plugin settings (*On-device AI +models*): + +- **FaceNet** — the default, part of **Download AI models**. Good for telling + family and friends apart. +- **AuraFace** — more accurate: it keeps similar-looking people apart more + reliably and recognises the same person across ages, angles and difficult + light better. It is a separate download of about 260 MB (from Hugging Face, + `fal/AuraFace-v1`, Apache-2.0 licensed) and takes somewhat longer per face. + Click **Use AuraFace...**; the plugin downloads it and switches. + +The two recognizers describe faces in different ways that cannot be compared, +so after a switch: + +- Faces found with the other recognizer are left **unassigned** by **Cluster + faces** and are not offered by **Find Similar Faces**. +- Their photos count as unprocessed again, so **Analyze & Index Photos** with + *New or unprocessed photos* and *Enable face detection* recognises them anew. +- **Names you gave are kept.** The face *detector* is the same for both, so a + re-detected face lands on the same spot of the photo and takes over the person + it was assigned to — including faces you detached from a person by hand. + +Then run **Cluster faces** once more. You can switch back at any time; the same +applies in the other direction. + ## Upgrading from a version before the face-model change The face pipeline used to run InsightFace's `buffalo_l` models, which could not diff --git a/plugin/LrGeniusAI.lrdevplugin/APISearchIndex.lua b/plugin/LrGeniusAI.lrdevplugin/APISearchIndex.lua index 251352e2..4dd29238 100644 --- a/plugin/LrGeniusAI.lrdevplugin/APISearchIndex.lua +++ b/plugin/LrGeniusAI.lrdevplugin/APISearchIndex.lua @@ -224,6 +224,10 @@ local ENDPOINTS = { FACES_CLUSTER = "/v1/faces/cluster", FACES_PERSONS = "/v1/faces/persons", FACES_PERSON_PHOTOS = "/v1/faces/persons", -- suffix //photos + -- Which network makes face identities (FaceNet or AuraFace), and the + -- AuraFace download. + FACE_RECOGNIZER = "/v1/faces/recognizer", + FACE_RECOGNIZER_DOWNLOADS = "/v1/faces/recognizer/downloads", -- AI develop edits and the style-training corpus EDIT = "/v1/edit/recipe", @@ -3086,6 +3090,16 @@ function SearchIndexAPI.analyzeAndIndexSelectedPhotos(selectedPhotos, progressSc log:error("Photo is nil in analyze worker, probably it got deleted in the meantime.") end end + -- A cancelled run leaves exported JPEGs of photos it never reached. + -- Removed here, by the worker, rather than by the caller after the + -- wait: on cancel the caller stops waiting while an export may still + -- be finishing. + for p, path in pairs(exportMemo) do + exportMemo[p] = nil + LrTasks.pcall(function() + LrFileUtils.delete(path) + end) + end log:trace("Analyze worker thread finished.") activeWorkers = activeWorkers - 1 end @@ -3120,13 +3134,6 @@ function SearchIndexAPI.analyzeAndIndexSelectedPhotos(selectedPhotos, progressSc end end end - - -- A cancelled run leaves exported JPEGs of photos it never reached. - for _, path in pairs(exportMemo) do - LrTasks.pcall(function() - LrFileUtils.delete(path) - end) - end end if shouldCloseScope then @@ -4866,6 +4873,77 @@ end -- "finish setting up" for someone upgrading from a version that only had the -- search model. -- +--- +-- The face recognizer in use and the ones on offer. +-- @return table|nil { active, saved, recognizers = { {id, name, description, +-- ready, active, approx_bytes, downloadable}, ... }, warnings }, string|nil error +--- +function SearchIndexAPI.getFaceRecognizer() + local response, err = _request("GET", SearchIndexAPI.url("FACE_RECOGNIZER")) + if not response then + return nil, err or "Unknown error" + end + if type(response.recognizers) ~= "table" then + return nil, response.error or "Unexpected response" + end + return response, nil +end + +--- +-- Switches the face recognizer ("facenet" or "auraface"); the backend saves +-- the choice. Refused while the chosen model is not downloaded. +-- @return table|nil The new state, with `warnings` to show the user; +-- string|nil error +--- +function SearchIndexAPI.setFaceRecognizer(recognizer) + local response, err = _request("POST", SearchIndexAPI.url("FACE_RECOGNIZER"), { recognizer = recognizer }) + if not response then + return nil, err or "Unknown error" + end + if response.error then + return nil, response.error + end + return response, nil +end + +--- +-- Downloads AuraFace with a progress bar and waits for it. Must run in an +-- async task. +-- @return boolean success, string|nil error +--- +function SearchIndexAPI.downloadAuraFace() + local _, postErr = _request("POST", SearchIndexAPI.url("FACE_RECOGNIZER_DOWNLOADS"), {}) + if postErr then + return false, postErr + end + local progressScope = LrProgressScope({ title = "Downloading AuraFace" }) + while true do + local status, err = _request("GET", SearchIndexAPI.url("FACE_RECOGNIZER_DOWNLOADS")) + if err then + progressScope:done() + return false, err + end + if status ~= nil then + if status.status == "downloading" then + progressScope:setCaption("Downloading the AuraFace face model...") + progressScope:setPortionComplete(tonumber(status.progress) or 0, tonumber(status.total) or 0) + elseif status.status == "completed" then + progressScope:done() + return true, nil + else + progressScope:done() + return false, Util.errorText(status.error, "The download stopped without saying why.") + end + end + -- The download goes on in the backend; this only stops watching it. + if progressScope:isCanceled() then + progressScope:done() + return false, "Stopped watching the download. It continues in the backend; try again in a minute." + end + LrTasks.sleep(1) + end +end + function SearchIndexAPI.startAssetDownload() local status = SearchIndexAPI.getAssetStatus() if status and status.ready then diff --git a/plugin/LrGeniusAI.lrdevplugin/PluginInfoDialogSections.lua b/plugin/LrGeniusAI.lrdevplugin/PluginInfoDialogSections.lua index 589aac50..974dd34b 100644 --- a/plugin/LrGeniusAI.lrdevplugin/PluginInfoDialogSections.lua +++ b/plugin/LrGeniusAI.lrdevplugin/PluginInfoDialogSections.lua @@ -24,6 +24,8 @@ function PluginInfoDialogSections.startDialog(propertyTable) propertyTable.bioclipReady = false propertyTable.faceReady = false propertyTable.assetsReady = false + propertyTable.faceRecognizerText = "Checking..." + propertyTable.faceRecognizerActive = "" propertyTable.keepChecksRunning = true checksGeneration = checksGeneration + 1 @@ -51,6 +53,11 @@ function PluginInfoDialogSections.startDialog(propertyTable) propertyTable.faceReady = family.ready == true end end + local recognizer = SearchIndexAPI.getFaceRecognizer() + if recognizer then + propertyTable.faceRecognizerActive = recognizer.active or "" + propertyTable.faceRecognizerText = Util.faceRecognizerSummary(recognizer) + end end LrTasks.startAsyncTask(function() @@ -1068,6 +1075,46 @@ function PluginInfoDialogSections.sectionsForTopOfDialog(f, propertyTable) ), }), }), + f:group_box({ + width = groupBoxWidth, + title = "Face recognition", + f:static_text({ + title = bind("faceRecognizerText"), + width_in_chars = 60, + height_in_lines = 3, + wrap = true, + }), + f:row({ + f:push_button({ + title = "Use FaceNet", + enabled = bind({ + key = "faceRecognizerActive", + transform = function(v) + return v ~= "facenet" + end, + }), + action = function() + LrTasks.startAsyncTask(function() + PluginInfoDialogSections.switchFaceRecognizer(propertyTable, "facenet") + end) + end, + }), + f:push_button({ + title = "Use AuraFace...", + enabled = bind({ + key = "faceRecognizerActive", + transform = function(v) + return v ~= "auraface" + end, + }), + action = function() + LrTasks.startAsyncTask(function() + PluginInfoDialogSections.switchFaceRecognizer(propertyTable, "auraface") + end) + end, + }), + }), + }), }, { bind_to_object = propertyTable, @@ -1876,6 +1923,63 @@ function PluginInfoDialogSections.sectionsForTopOfDialog(f, propertyTable) } end +--- +-- Switches face recognition to `wanted` ("facenet" or "auraface") after the +-- user confirms, downloading AuraFace first when it is not on disk yet. +-- Runs in an async task. +--- +function PluginInfoDialogSections.switchFaceRecognizer(propertyTable, wanted) + local current, err = SearchIndexAPI.getFaceRecognizer() + if not current then + ErrorHandler.handleError("Could not ask the backend about face recognition", err) + return + end + local entry + for _, r in ipairs(current.recognizers) do + if r.id == wanted then + entry = r + end + end + if not entry then + ErrorHandler.handleError( + "This backend does not offer " .. tostring(wanted), + "Update the backend and try again." + ) + return + end + + local info = "Faces already found are recognized again the next time their photos are indexed with face " + .. 'detection on (Analyze & Index Photos, "New or unprocessed photos" picks them up). Until then ' + .. "People lists them as unassigned. The names you gave carry over." + if wanted == "auraface" and entry.ready ~= true then + info = info + .. "\n\nThis first downloads AuraFace (about 260 MB, from Hugging Face: fal/AuraFace-v1, Apache-2.0)." + end + local verb = (entry.ready == true) and ("Use " .. entry.name) or "Download and Use" + if LrDialogs.confirm("Use " .. entry.name .. " for face recognition?", info, verb) ~= "ok" then + return + end + + if entry.ready ~= true then + local ok, downloadErr = SearchIndexAPI.downloadAuraFace() + if not ok then + ErrorHandler.handleError("Could not download AuraFace", downloadErr) + return + end + end + + local result, setErr = SearchIndexAPI.setFaceRecognizer(wanted) + if not result then + ErrorHandler.handleError("Could not switch face recognition to " .. entry.name, setErr) + return + end + propertyTable.faceRecognizerActive = result.active or "" + propertyTable.faceRecognizerText = Util.faceRecognizerSummary(result) + if type(result.warnings) == "table" and #result.warnings > 0 then + LrDialogs.message("Face recognition: " .. entry.name, table.concat(result.warnings, "\n\n"), "info") + end +end + function PluginInfoDialogSections.endDialog(propertyTable) -- First, before any of the preference saving below: if one of those lines -- throws, the polling loops must still stop. diff --git a/plugin/LrGeniusAI.lrdevplugin/Util.lua b/plugin/LrGeniusAI.lrdevplugin/Util.lua index e460cdc9..5ab3e594 100644 --- a/plugin/LrGeniusAI.lrdevplugin/Util.lua +++ b/plugin/LrGeniusAI.lrdevplugin/Util.lua @@ -3471,4 +3471,30 @@ function Util.previewFallbackWarning(exported, total, minEdge) .. "(with the standard preview size in Catalog Settings at least as large)." end +--- +-- One paragraph for the Plug-in Manager about the face recognizer, from +-- GET /v1/faces/recognizer. +-- @param state table The backend's answer. +-- @return string +-- +function Util.faceRecognizerSummary(state) + if type(state) ~= "table" or type(state.recognizers) ~= "table" then + return "Unknown: the backend did not say which face recognizer it uses." + end + local active + for _, r in ipairs(state.recognizers) do + if r.id == state.active then + active = r + end + end + if not active then + return "Unknown: the backend did not say which face recognizer it uses." + end + local text = "In use: " .. tostring(active.name) .. ". " .. tostring(active.description or "") + if type(state.warnings) == "table" and state.warnings[1] then + text = text .. "\n" .. tostring(state.warnings[1]) + end + return text +end + return Util diff --git a/plugin/README.md b/plugin/README.md index 2a14a428..da0b3c39 100644 --- a/plugin/README.md +++ b/plugin/README.md @@ -57,6 +57,10 @@ The plugin is designed to work with local and cloud providers, while keeping Lig ### People Workflows - Cluster faces into persons +- Choose the face recognizer in Plug-in Manager (*Face recognition*): FaceNet (default) or + AuraFace (more accurate, ~260 MB download). `PluginInfoDialogSections.switchFaceRecognizer` + downloads AuraFace when needed and switches via `/v1/faces/recognizer`; names carry over when + the photos are re-indexed - Rename persons - Jump from a person directly to a Lightroom collection - The People interface opens in your web browser (served by the backend at diff --git a/plugin/spec/util_preview_render_spec.lua b/plugin/spec/util_preview_render_spec.lua index 82ace3bf..be1e3d4e 100644 --- a/plugin/spec/util_preview_render_spec.lua +++ b/plugin/spec/util_preview_render_spec.lua @@ -78,3 +78,28 @@ describe("Util.previewFallbackWarning", function() assert.truthy(text:find("Build Standard-Sized Previews", 1, true)) end) end) + +describe("Util.faceRecognizerSummary", function() + local state = { + active = "auraface", + recognizers = { + { id = "facenet", name = "FaceNet", description = "Included." }, + { id = "auraface", name = "AuraFace", description = "More accurate." }, + }, + warnings = {}, + } + + it("names the recognizer in use", function() + assert.are.equal("In use: AuraFace. More accurate.", Util.faceRecognizerSummary(state)) + end) + + it("adds the backend's first warning", function() + state.warnings = { "AuraFace was chosen, but its model file is missing." } + assert.truthy(Util.faceRecognizerSummary(state):find("model file is missing", 1, true)) + end) + + it("says so when the answer is unusable", function() + assert.truthy(Util.faceRecognizerSummary(nil):find("Unknown", 1, true)) + assert.truthy(Util.faceRecognizerSummary({ active = "x", recognizers = {} }):find("Unknown", 1, true)) + end) +end) diff --git a/server-rs/Cargo.lock b/server-rs/Cargo.lock index 2fea4652..c469b5b0 100644 --- a/server-rs/Cargo.lock +++ b/server-rs/Cargo.lock @@ -4479,6 +4479,7 @@ dependencies = [ "ort", "serde", "serde_json", + "tempfile", "thiserror 2.0.21", "tokenizers", ] diff --git a/server-rs/README.md b/server-rs/README.md index e9d7eab4..50912e37 100644 --- a/server-rs/README.md +++ b/server-rs/README.md @@ -415,6 +415,27 @@ which is where `POST /v1/models/assets/downloads` puts them. Assets are publishe by `.github/workflows/model-assets-face.yml` to the fixed `face-assets-v1` tag, independent of the other two families. +**AuraFace (optional recognizer).** `fal/AuraFace-v1`'s `glintr100.onnx` +(Apache-2.0, ~260 MB) can replace FaceNet: `POST /v1/faces/recognizer/downloads` +fetches it from Hugging Face (`HF_ENDPOINT` is honoured) to +`~/.cache/lrgenius/models/auraface_glintr100.onnx` (or `LRG_AURAFACE_ONNX`), +and `POST /v1/faces/recognizer` switches. The choice is saved in +`~/.cache/lrgenius/models/face_recognizer.txt` (or `LRG_FACE_RECOGNIZER_FILE`) +and restored at start. It runs on the same YuNet landmarks and the same 112x112 +ArcFace crop InsightFace makes, RGB, `(v - 127.5) / 127.5` +(`lrg-ml/src/auraface.rs`). To check that chain numerically, write +InsightFace's own embedding into the golden file and run the golden test: + +```bash +uv run --no-project server-rs/scripts/auraface_reference.py --output ~/.cache/lrgenius/models/auraface_glintr100.onnx +cargo test -p lrg-ml --test face_goldens -- auraface # LRG_REQUIRE_GOLDENS=auraface to fail instead of skip +``` + +The two recognizers' rows carry different `face_model` ids +(`yunet-2023mar/facenet-vggface2`, `yunet-2023mar/auraface-v1`), so nothing +compares across them; on re-detection, person assignments from the other +recognizer's rows are carried over by box overlap (same detector). + These replaced InsightFace's `buffalo_l` (SCRFD + ArcFace), which was resolved from `INSIGHTFACE_ROOT` and could not be redistributed — face detection was the one family the plugin could not download for you. Note the diff --git a/server-rs/crates/lrg-api/src/lib.rs b/server-rs/crates/lrg-api/src/lib.rs index 8a7e1d16..d6275e67 100644 --- a/server-rs/crates/lrg-api/src/lib.rs +++ b/server-rs/crates/lrg-api/src/lib.rs @@ -45,6 +45,7 @@ pub fn build_router(state: Arc) -> Router { .merge(routes::bioclip::router()) .merge(routes::assets::router()) .merge(routes::faces::router()) + .merge(routes::face_recognizer::router()) .merge(routes::index_upload::router()) .merge(routes::index_by_reference::router()) .merge(routes::jobs::router()) diff --git a/server-rs/crates/lrg-api/src/routes/clip.rs b/server-rs/crates/lrg-api/src/routes/clip.rs index 82f103eb..73b25d64 100644 --- a/server-rs/crates/lrg-api/src/routes/clip.rs +++ b/server-rs/crates/lrg-api/src/routes/clip.rs @@ -199,15 +199,34 @@ pub(crate) async fn download_release_assets( key: &str, label: &str, assets: &[ReleaseAsset<'_>], +) { + let urls: Vec> = assets + .iter() + .map(|(tag, name, dest)| (asset_url(tag, name), *name, *dest)) + .collect(); + download_url_assets(state, key, label, &urls).await; +} + +/// One file to fetch from anywhere: its URL, the name progress reports it +/// under, and where it goes on disk. +pub(crate) type UrlAsset<'a> = (String, &'a str, &'a std::path::Path); + +/// [`download_release_assets`] for files that are not GitHub release assets +/// (AuraFace comes from Hugging Face). Same HEAD pass, `.part` staging and +/// progress reporting. +pub(crate) async fn download_url_assets( + state: &Downloads, + key: &str, + label: &str, + assets: &[UrlAsset<'_>], ) { let client = reqwest::Client::new(); // Resolve sizes up front (HEAD per asset) so total progress is known // from the start, matching Python's up-front HfApi files_metadata call. let mut total: u64 = 0; - for (tag, name, _) in assets { - let url = asset_url(tag, name); - let resp = match client.head(&url).send().await { + for (url, name, _) in assets { + let resp = match client.head(url.as_str()).send().await { Ok(r) => r, Err(e) => { return set_error(state, key, label, format!("failed to resolve {name}: {e}")) @@ -219,7 +238,7 @@ pub(crate) async fn download_release_assets( key, label, format!( - "release asset {name} not found for {tag} (HTTP {}) — the model-assets release may not exist yet", + "{name} not found at {url} (HTTP {}) — the release may not exist yet", resp.status() ), ); @@ -229,12 +248,11 @@ pub(crate) async fn download_release_assets( with_status(state, key, |s| s.total = total); let mut downloaded: u64 = 0; - for (tag, name, dest) in assets { + for (url, name, dest) in assets { with_status(state, key, |s| s.current_file = Some(name.to_string())); - let url = asset_url(tag, name); log::info!("Starting {label} asset download: {name} from {url}"); - let resp = match client.get(&url).send().await { + let resp = match client.get(url.as_str()).send().await { Ok(r) if r.status().is_success() => r, Ok(r) => { return set_error( diff --git a/server-rs/crates/lrg-api/src/routes/face_recognizer.rs b/server-rs/crates/lrg-api/src/routes/face_recognizer.rs new file mode 100644 index 00000000..512fcc1f --- /dev/null +++ b/server-rs/crates/lrg-api/src/routes/face_recognizer.rs @@ -0,0 +1,236 @@ +//! Which network makes the face identity embeddings: FaceNet (the default, +//! part of "Download AI models") or AuraFace (opt-in, its own download). +//! +//! The choice is the backend's, not a request option, because it decides what +//! every stored face row means: `lrg_ml::faces::Recognizer::model_id` is +//! stamped on each row, clustering and "Find Similar Faces" only compare rows +//! of the active model, and `/v1/index/unprocessed` re-queues photos whose +//! faces another model found. So a switch is safe — nothing is mixed — but it +//! only takes full effect once the photos with faces are indexed again (with +//! face detection on). The names the user gave survive that re-index: the +//! detector is the same, so `index_upload` carries them over by box. +//! +//! The choice is saved in `lrg_ml::model_paths::face_recognizer_setting()` +//! and restored when the backend starts ([`restore`]). + +use std::sync::Arc; + +use axum::extract::State; +use axum::http::StatusCode; +use axum::response::{IntoResponse, Response}; +use axum::{routing::get, routing::post, Json, Router}; +use serde_json::{json, Value}; + +use lrg_ml::faces::{FaceModel, Recognizer}; + +use crate::routes::clip::{download_url_assets, Downloads, ModelDownloadStatus}; +use crate::state::AppState; + +/// Key for the AuraFace download in `AppState::model_download`. +const DOWNLOAD_KEY: &str = "face_recognizer"; + +/// Where AuraFace is published, Apache-2.0. +const AURAFACE_REPO: &str = "fal/AuraFace-v1"; +/// The repository revision downloaded. `main` until the exact commit has been +/// verified against `scripts/auraface_reference.py`'s reference embedding; +/// pin it then, like YuNet's commit in `export_face_models.py`. +const AURAFACE_REVISION: &str = "main"; +/// The recognition network in that repository (the others are InsightFace's +/// detector and attribute models, which this pipeline does not use). +const AURAFACE_FILE: &str = "glintr100.onnx"; +/// Rough size for the "this downloads about N MB" line; the HEAD pass at +/// download time has the exact one. +const AURAFACE_APPROX_BYTES: u64 = 261_000_000; + +pub fn router() -> Router> { + Router::new() + .route("/faces/recognizer", get(status).post(select)) + .route( + "/faces/recognizer/downloads", + post(download_start).get(download_status), + ) +} + +/// Applies the saved choice at startup. A saved AuraFace whose file is gone +/// falls back to FaceNet, loudly in the log; `GET /v1/faces/recognizer` then +/// reports `saved: "auraface"` next to `active: "facenet"`, which the +/// Plug-in Manager shows. +pub fn restore(face: &FaceModel) { + let path = lrg_ml::model_paths::face_recognizer_setting(); + let Some(saved) = std::fs::read_to_string(&path) + .ok() + .and_then(|s| Recognizer::parse(&s)) + else { + return; + }; + if let Err(e) = face.set_recognizer(saved) { + log::warn!( + "The saved face recognizer ({}) cannot be used, staying on {}: {e}", + saved.as_str(), + face.recognizer().as_str() + ); + } +} + +fn saved_choice() -> Option { + std::fs::read_to_string(lrg_ml::model_paths::face_recognizer_setting()) + .ok() + .and_then(|s| Recognizer::parse(&s)) +} + +fn describe(state: &AppState) -> Value { + let face = &state.face; + let active = face.recognizer(); + let saved = saved_choice().unwrap_or_default(); + let paths = face.paths(); + let entry = |r: Recognizer, name: &str, about: &str, approx: u64, downloadable: bool| { + json!({ + "id": r.as_str(), + "name": name, + "description": about, + "ready": paths.ready_for(r), + "active": r == active, + "model_id": r.model_id(), + "approx_bytes": approx, + "downloadable": downloadable, + }) + }; + let mut warnings = Vec::new(); + if saved != active { + warnings.push(format!( + "{} was chosen, but its model file is missing, so {} is in use. Download it again, or choose {}.", + name_of(saved), + name_of(active), + name_of(active), + )); + } + json!({ + "active": active.as_str(), + "saved": saved.as_str(), + "model_id": active.model_id(), + "recognizers": [ + entry( + Recognizer::FaceNet, + name_of(Recognizer::FaceNet), + "Included in the AI model download. Good for telling family and friends apart.", + 0, + false, + ), + entry( + Recognizer::AuraFace, + name_of(Recognizer::AuraFace), + "More accurate, especially for similar-looking people, across ages and in \ + difficult light. A separate download of about 260 MB; slower per face.", + AURAFACE_APPROX_BYTES, + true, + ), + ], + "warnings": warnings, + }) +} + +fn name_of(r: Recognizer) -> &'static str { + match r { + Recognizer::FaceNet => "FaceNet", + Recognizer::AuraFace => "AuraFace", + } +} + +async fn status(State(state): State>) -> Json { + Json(describe(&state)) +} + +/// `{"recognizer": "facenet" | "auraface"}`: switches and saves the choice. +async fn select(State(state): State>, body: Option>) -> Response { + let data = body.map(|Json(v)| v).unwrap_or(Value::Null); + let Some(wanted) = data + .get("recognizer") + .and_then(Value::as_str) + .and_then(Recognizer::parse) + else { + return ( + StatusCode::BAD_REQUEST, + Json(json!({"error": "recognizer must be \"facenet\" or \"auraface\""})), + ) + .into_response(); + }; + let before = state.face.recognizer(); + if let Err(e) = state.face.set_recognizer(wanted) { + return ( + StatusCode::CONFLICT, + Json(json!({ + "error": format!("{} cannot be used yet: {e}. Download it first.", name_of(wanted)), + "error_code": "not_downloaded", + })), + ) + .into_response(); + } + + let setting = lrg_ml::model_paths::face_recognizer_setting(); + let saved = setting + .parent() + .map_or(Ok(()), std::fs::create_dir_all) + .and_then(|()| std::fs::write(&setting, wanted.as_str())); + + let mut body = describe(&state); + let mut warnings: Vec = Vec::new(); + if let Err(e) = saved { + log::error!( + "Could not save the face recognizer choice to {}: {e}", + setting.display() + ); + warnings.push(json!(format!( + "{} is in use now, but the choice could not be saved ({e}), so the backend \ + goes back to its previous one when it restarts.", + name_of(wanted) + ))); + } else if let Some(w) = body["warnings"].as_array() { + warnings.extend(w.iter().cloned()); + } + if before != wanted { + warnings.push(json!(format!( + "Faces found with {} are set aside until their photos are indexed again with \ + face detection on. Run Analyze & Index Photos with \"New or unprocessed photos\" \ + to do that; the names you gave carry over.", + name_of(before) + ))); + } + body["warnings"] = Value::Array(warnings); + body["changed"] = json!(before != wanted); + Json(body).into_response() +} + +async fn download_start(State(state): State>) -> Json { + let already_running = { + let mut downloads = state.model_download.lock().unwrap(); + let status = downloads.entry(DOWNLOAD_KEY.to_string()).or_default(); + if status.status == "downloading" { + true + } else { + *status = ModelDownloadStatus::downloading(); + false + } + }; + if !already_running { + tokio::spawn(run_download(state.model_download.clone())); + } + Json(json!({"download": "started"})) +} + +async fn download_status(State(state): State>) -> Json { + let status = state + .model_download + .lock() + .unwrap() + .get(DOWNLOAD_KEY) + .cloned() + .unwrap_or_default(); + Json(json!(status)) +} + +async fn run_download(downloads: Arc) { + let paths = lrg_ml::model_paths::resolve_face(); + let url = crate::llm_models::hf_url(AURAFACE_REPO, AURAFACE_REVISION, AURAFACE_FILE); + let assets = [(url, AURAFACE_FILE, paths.auraface_onnx.as_path())]; + download_url_assets(&downloads, DOWNLOAD_KEY, "AuraFace", &assets).await; +} diff --git a/server-rs/crates/lrg-api/src/routes/faces.rs b/server-rs/crates/lrg-api/src/routes/faces.rs index e978b1cd..7631cc1f 100644 --- a/server-rs/crates/lrg-api/src/routes/faces.rs +++ b/server-rs/crates/lrg-api/src/routes/faces.rs @@ -128,7 +128,7 @@ async fn query(State(state): State>, body: Option>) -> let n_results = data.get("n_results").and_then(Value::as_u64).unwrap_or(10) as usize; let face_index = data.get("face_index").and_then(Value::as_u64).unwrap_or(0) as usize; - let faces = match state.face.detect_faces( + let (faces, face_model) = match state.face.detect_faces_with_model( &pixels, w, h, @@ -171,8 +171,13 @@ async fn query(State(state): State>, body: Option>) -> } }; + // Only faces embedded by the same recognizer: after a switch between + // FaceNet and AuraFace the older rows are 512-d unit vectors of another + // space, and a distance to them ranks nonsense among real matches. They + // come back once their photos are indexed again. let mut scored: Vec<(f64, &StoreRecord)> = records .iter() + .filter(|r| r.metadata.get("face_model").and_then(Value::as_str) == Some(face_model)) .filter_map(|r| { let v = r.vector.as_ref()?; Some((cosine_distance(query_emb, v), r)) diff --git a/server-rs/crates/lrg-api/src/routes/index_upload.rs b/server-rs/crates/lrg-api/src/routes/index_upload.rs index 65ea2e4b..19867319 100644 --- a/server-rs/crates/lrg-api/src/routes/index_upload.rs +++ b/server-rs/crates/lrg-api/src/routes/index_upload.rs @@ -1963,7 +1963,7 @@ async fn finish_one( // A decode failure is reported the same way a detector failure is, // as a warning: faces are optional and must not sink the photo. let t_decode = Instant::now(); - let face_result: Result, String> = if !state.face.is_cached() { + let face_result: Result<(Vec<_>, &str), String> = if !state.face.is_cached() { // Checked before decoding: without the model files the detector // fails anyway, and the decode would be pure waste. // @@ -1971,11 +1971,19 @@ async fn finish_one( // cause is a run started before the models finished // downloading, and "model not loaded" left the user with a // symptom and nowhere to go. - Err( - "face model is not downloaded yet — run \"Download AI models\" \ - in Plug-in Manager and index these photos again" + Err(match state.face.recognizer() { + lrg_ml::faces::Recognizer::AuraFace + if state.face.paths().det_onnx.is_file() => + { + "the AuraFace face model is missing — download it again under \ + Face recognition in Plug-in Manager, or switch back to FaceNet, \ + and index these photos again" + .to_string() + } + _ => "face model is not downloaded yet — run \"Download AI models\" \ + in Plug-in Manager and index these photos again" .to_string(), - ) + }) } else { match image::load_from_memory(&image_bytes).map(|img| img.to_rgb8()) { Ok(decoded) => { @@ -1988,7 +1996,7 @@ async fn finish_one( let t0 = Instant::now(); let result = state .face - .detect_faces( + .detect_faces_with_model( &pixels, width, height, @@ -2010,7 +2018,7 @@ async fn finish_one( // full run had established. It also deliberately leaves // `faces_checked` unset, so `/v1/index/unprocessed` still // reports the photo as needing a real face pass later. - Ok(faces) if options.face_pass == FacePass::QualityOnly => { + Ok((faces, _)) if options.face_pass == FacePass::QualityOnly => { let inputs: Vec = faces .iter() .map(|f| FaceMetricsInput { @@ -2029,7 +2037,7 @@ async fn finish_one( faces.len() ); } - Ok(faces) => { + Ok((faces, face_model)) => { let prefix = format!("{photo_id}_"); // Read before deleting: detection replaces this photo's rows // wholesale, and a fresh row starts with an empty @@ -2073,14 +2081,26 @@ async fn finish_one( // tight enough that it would mostly refuse to carry // anything over — but "mostly" is how one person's name // ends up on another person's face. - let previous: Vec = previous - .into_iter() - .filter(|r| { + let (previous, other_model): (Vec, Vec) = + previous.into_iter().partition(|r| { r.metadata.get("face_model").and_then(Value::as_str) - == Some(state.face.model_id()) - }) - .collect(); - let carried = carry_over_person_ids(&previous, &embeddings); + == Some(face_model) + }); + let mut carried = carry_over_person_ids(&previous, &embeddings); + // Rows from the other recognizer (the user switched + // between FaceNet and AuraFace) are matched by where the + // face is instead: both use the same detector, so a + // re-detected face lands on its old box, and the names + // the user gave survive the switch. + let boxes: Vec<[i64; 4]> = faces.iter().map(|f| f.bbox).collect(); + for (slot, by_box) in carried + .iter_mut() + .zip(carry_over_person_ids_by_box(&other_model, &boxes)) + { + if slot.0.is_empty() && !slot.1 { + *slot = by_box; + } + } let carried_count = carried .iter() .filter(|(p, manual)| !p.is_empty() || *manual) @@ -2115,7 +2135,7 @@ async fn finish_one( m.insert("face_eye_openness".into(), json!(f.eye_openness)); m.insert("face_blink_penalty".into(), json!(f.blink_penalty)); m.insert("face_occlusion".into(), json!(f.occlusion)); - m.insert("face_model".into(), json!(state.face.model_id())); + m.insert("face_model".into(), json!(face_model)); StoreRecord { id: format!("{photo_id}_{i}"), vector: Some(f.embedding.clone()), @@ -2151,7 +2171,7 @@ async fn finish_one( // replaces the photo's rows, which is how person assignments // used to disappear on a plain re-index. main_metadata.insert("faces_checked".into(), json!(true)); - main_metadata.insert("face_model".into(), json!(state.face.model_id())); + main_metadata.insert("face_model".into(), json!(face_model)); } Err(e) => { log::warn!("Face detection/indexing failed for {photo_id}: {e}"); @@ -2428,6 +2448,92 @@ fn carry_over_person_ids(previous: &[StoreRecord], embeddings: &[&[f32]]) -> Vec out } +/// Overlap (intersection over union) above which a re-detected face counts as +/// the same detection as a previous row from the *other* recognizer. +/// +/// The detector did not change, so the same pixels give the same box and a +/// genuine match lands near 1.0; a re-export at another size or a small crop +/// change shifts it a little. Two different faces in one frame overlap far +/// less than this even in a tight group shot. +const FACE_CARRY_OVER_MIN_IOU: f64 = 0.6; + +fn box_iou(a: [i64; 4], b: [i64; 4]) -> f64 { + let ix = (a[2].min(b[2]) - a[0].max(b[0])).max(0) as f64; + let iy = (a[3].min(b[3]) - a[1].max(b[1])).max(0) as f64; + let inter = ix * iy; + let area = |r: [i64; 4]| ((r[2] - r[0]).max(0) * (r[3] - r[1]).max(0)) as f64; + let union = area(a) + area(b) - inter; + if union <= 0.0 { + 0.0 + } else { + inter / union + } +} + +fn stored_bbox(record: &StoreRecord) -> Option<[i64; 4]> { + match record.metadata.get("bbox")? { + Value::String(s) => serde_json::from_str::<[i64; 4]>(s).ok(), + other => serde_json::from_value::<[i64; 4]>(other.clone()).ok(), + } +} + +/// [`carry_over_person_ids`] for previous rows whose embeddings come from the +/// other recognizer and so cannot be compared: matched by box overlap +/// instead, globally best-first, each old row claimed at most once. Faces with +/// no match get an empty assignment. +fn carry_over_person_ids_by_box( + previous: &[StoreRecord], + boxes: &[[i64; 4]], +) -> Vec<(String, bool)> { + let mut out = vec![(String::new(), false); boxes.len()]; + let mut pairs: Vec<(f64, usize, usize)> = Vec::new(); + for (j, old) in previous.iter().enumerate() { + let manual = old + .metadata + .get("person_manual") + .and_then(Value::as_bool) + .unwrap_or(false); + let person = old + .metadata + .get("person_id") + .and_then(Value::as_str) + .unwrap_or(""); + if !manual && person.is_empty() { + continue; + } + let Some(old_box) = stored_bbox(old) else { + continue; + }; + for (i, new_box) in boxes.iter().enumerate() { + let iou = box_iou(*new_box, old_box); + if iou >= FACE_CARRY_OVER_MIN_IOU { + pairs.push((iou, i, j)); + } + } + } + pairs.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal)); + let mut new_taken = vec![false; boxes.len()]; + let mut old_taken = vec![false; previous.len()]; + for (_, i, j) in pairs { + if new_taken[i] || old_taken[j] { + continue; + } + let meta = &previous[j].metadata; + out[i] = ( + meta.get("person_id") + .and_then(Value::as_str) + .unwrap_or("") + .to_string(), + meta.get("person_manual") + .and_then(Value::as_bool) + .unwrap_or(false), + ); + new_taken[i] = true; + old_taken[j] = true; + } + out +} + fn apply_face_aggregate(metadata: &mut Map, faces: &[FaceMetricsInput]) { let agg = aggregate_face_culling_metrics(faces, &FaceMetricsConfig::defaults()); metadata.insert("cull_face_count".into(), json!(agg.cull_face_count)); @@ -2719,6 +2825,56 @@ mod face_carry_over_tests { ); } + fn boxed_row(id: &str, person: &str, bbox: [i64; 4], manual: bool) -> StoreRecord { + let mut row = face_row(id, person, vec![0.0, 0.0, 1.0]); + row.metadata + .insert("bbox".into(), json!(serde_json::to_string(&bbox).unwrap())); + if manual { + row.metadata.insert("person_manual".into(), json!(true)); + } + row + } + + /// Switching recognizer (FaceNet <-> AuraFace) leaves embeddings that + /// cannot be compared, but the detector is the same, so a face lands on + /// its old box and keeps the name the user gave it. + #[test] + fn a_recognizer_switch_carries_names_by_box() { + let previous = vec![ + boxed_row("p_0", "person_7", [100, 100, 200, 220], false), + boxed_row("p_1", "person_9", [300, 90, 390, 200], true), + ]; + // Detected in the other order, one box shifted by a few pixels. + let boxes = [[302, 92, 391, 203], [100, 100, 200, 220]]; + assert_eq!( + carry_over_person_ids_by_box(&previous, &boxes), + vec![("person_9".to_string(), true), guessed("person_7")] + ); + } + + #[test] + fn a_box_elsewhere_in_the_frame_inherits_nothing() { + let previous = vec![boxed_row("p_0", "person_7", [100, 100, 200, 220], false)]; + // Overlaps the old face a little, as a neighbour in a group shot can. + let boxes = [[170, 110, 270, 230]]; + assert_eq!( + carry_over_person_ids_by_box(&previous, &boxes), + vec![nobody()] + ); + } + + #[test] + fn a_row_without_a_box_or_a_name_is_skipped() { + let previous = vec![ + face_row("p_0", "person_7", vec![1.0]), + boxed_row("p_1", "", [0, 0, 50, 50], false), + ]; + assert_eq!( + carry_over_person_ids_by_box(&previous, &[[0, 0, 50, 50]]), + vec![nobody()] + ); + } + /// "This face is not that person" is a decision with an *empty* /// `person_id`. Re-detection has to carry it, or the correction is undone /// by the next re-index. diff --git a/server-rs/crates/lrg-api/src/routes/mod.rs b/server-rs/crates/lrg-api/src/routes/mod.rs index 7e843b39..e849c7b3 100644 --- a/server-rs/crates/lrg-api/src/routes/mod.rs +++ b/server-rs/crates/lrg-api/src/routes/mod.rs @@ -5,6 +5,7 @@ pub mod bootstrap; pub mod clip; pub mod db; pub mod edit; +pub mod face_recognizer; pub mod faces; pub mod find_similar; pub mod group_similar; diff --git a/server-rs/crates/lrg-api/src/routes/server.rs b/server-rs/crates/lrg-api/src/routes/server.rs index e2c12f08..98b866bc 100644 --- a/server-rs/crates/lrg-api/src/routes/server.rs +++ b/server-rs/crates/lrg-api/src/routes/server.rs @@ -422,6 +422,7 @@ async fn health(State(state): State>) -> Json { "aesthetic_error": aesthetic_error, "face_model": face_status, "face_error": face_error, + "face_recognizer": state.face.recognizer().as_str(), "species_model": species_status, "species_error": species_error, })) diff --git a/server-rs/crates/lrg-api/src/state.rs b/server-rs/crates/lrg-api/src/state.rs index 3ba05b94..32bdd548 100644 --- a/server-rs/crates/lrg-api/src/state.rs +++ b/server-rs/crates/lrg-api/src/state.rs @@ -122,7 +122,12 @@ impl ModelAssetPaths { impl AppState { pub fn new(initial_db_path: Option, debug: bool) -> Self { - Self::with_model_paths(initial_db_path, debug, ModelAssetPaths::from_env()) + let state = Self::with_model_paths(initial_db_path, debug, ModelAssetPaths::from_env()); + // Only here, not in `with_model_paths`: tests that supply their own + // model paths must not pick up the face recognizer saved on the + // machine they run on. + crate::routes::face_recognizer::restore(&state.face); + state } /// [`AppState::new`] with the ML model locations given explicitly instead diff --git a/server-rs/crates/lrg-api/tests/contract.rs b/server-rs/crates/lrg-api/tests/contract.rs index 32839072..0f91d853 100644 --- a/server-rs/crates/lrg-api/tests/contract.rs +++ b/server-rs/crates/lrg-api/tests/contract.rs @@ -57,6 +57,7 @@ fn models_absent() -> ModelAssetPaths { face: lrg_ml::faces::FaceModelPaths { det_onnx: dir.join("yunet_face_detection.onnx"), rec_onnx: dir.join("facenet_vggface2.onnx"), + auraface_onnx: dir.join("auraface_glintr100.onnx"), }, bioclip: lrg_ml::bioclip::BioclipModelPaths { image_onnx: dir.join("bioclip2_image.onnx"), @@ -3726,3 +3727,41 @@ async fn missing_previews_are_listed_and_filled_without_reindexing() { .await; assert_eq!(after["photo_ids"], serde_json::json!([]), "{after}"); } + +/// The face recognizer is FaceNet until the user picks AuraFace, and AuraFace +/// cannot be picked before its model is on disk: a switch that fails on the +/// next photo would leave every run reporting a missing model. +#[tokio::test] +async fn auraface_cannot_be_chosen_before_it_is_downloaded() { + let (app, state) = fresh_app(); + let status = get_json(&app, "/v1/faces/recognizer").await; + assert_eq!(status["active"], "facenet", "{status}"); + let ids: Vec<&str> = status["recognizers"] + .as_array() + .unwrap() + .iter() + .map(|r| r["id"].as_str().unwrap()) + .collect(); + assert_eq!(ids, ["facenet", "auraface"]); + assert_eq!(status["recognizers"][1]["ready"], false, "{status}"); + assert_eq!(status["recognizers"][1]["downloadable"], true, "{status}"); + + let (code, refused) = post_json( + &app, + "/v1/faces/recognizer", + serde_json::json!({"recognizer": "auraface"}), + ) + .await; + assert_eq!(code, StatusCode::CONFLICT, "{refused}"); + assert_eq!(refused["error_code"], "not_downloaded"); + assert_eq!(state.face.recognizer(), lrg_ml::faces::Recognizer::FaceNet); + assert_eq!(state.face.model_id(), lrg_ml::faces::MODEL_ID); + + let (bad, _) = post_json( + &app, + "/v1/faces/recognizer", + serde_json::json!({"recognizer": "arcface"}), + ) + .await; + assert_eq!(bad, StatusCode::BAD_REQUEST); +} diff --git a/server-rs/crates/lrg-ml/Cargo.toml b/server-rs/crates/lrg-ml/Cargo.toml index 2ceea0b1..b31ba54d 100644 --- a/server-rs/crates/lrg-ml/Cargo.toml +++ b/server-rs/crates/lrg-ml/Cargo.toml @@ -25,3 +25,6 @@ ort = { version = "2.0.0-rc.13", features = ["coreml"] } [[example]] name = "bench_image" + +[dev-dependencies] +tempfile.workspace = true diff --git a/server-rs/crates/lrg-ml/src/auraface.rs b/server-rs/crates/lrg-ml/src/auraface.rs new file mode 100644 index 00000000..5a0062ea --- /dev/null +++ b/server-rs/crates/lrg-ml/src/auraface.rs @@ -0,0 +1,59 @@ +//! AuraFace recognition (`auraface_glintr100.onnx`) — fal.ai's AuraFace-v1, +//! an ArcFace-style ResNet-100 trained on commercially usable data and +//! published under Apache-2.0 (huggingface.co/fal/AuraFace-v1, file +//! `glintr100.onnx`, renamed on download so its role is obvious next to the +//! other models). +//! +//! An alternative to FaceNet ([`crate::facenet`]) that the user opts into: it +//! separates identities more sharply — the point of offering it — at roughly +//! three times FaceNet's size and per-face cost. Both produce 512-d unit +//! vectors, so the store cannot tell them apart by shape; every row carries +//! [`crate::faces::Recognizer::model_id`] instead, exactly as for the FaceNet +//! switch. +//! +//! Preprocessing is InsightFace's for its ArcFace-family ONNX models, which is +//! how AuraFace is published to be used (`FaceAnalysis(name="auraface")`): +//! the five-landmark similarity warp onto the 112x112 ArcFace template +//! ([`crate::umeyama::norm_crop_112`] — the very crop InsightFace's +//! `norm_crop` makes), RGB order (InsightFace feeds `blobFromImages` a BGR +//! image with `swapRB=True`), and `(v - 127.5) / 127.5`, which is what +//! `ArcFaceONNX` picks for a graph that does not normalise its own input. +//! `scripts/auraface_reference.py` produces the reference embedding the +//! golden test checks this whole chain against. + +pub const INPUT_SIZE: usize = 112; +pub const EMBED_DIM: usize = 512; + +const INPUT_MEAN: f32 = 127.5; +const INPUT_STD: f32 = 127.5; + +/// Aligned 112x112 RGB crop -> NCHW blob (1x3x112x112). +pub fn to_blob(aligned_rgb: &[u8]) -> Vec { + let n = INPUT_SIZE * INPUT_SIZE; + debug_assert_eq!(aligned_rgb.len(), n * 3); + let mut out = vec![0.0f32; 3 * n]; + for i in 0..n { + out[i] = (aligned_rgb[i * 3] as f32 - INPUT_MEAN) / INPUT_STD; + out[n + i] = (aligned_rgb[i * 3 + 1] as f32 - INPUT_MEAN) / INPUT_STD; + out[2 * n + i] = (aligned_rgb[i * 3 + 2] as f32 - INPUT_MEAN) / INPUT_STD; + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn blob_is_rgb_and_scaled_to_unit_range() { + let mut px = vec![0u8; INPUT_SIZE * INPUT_SIZE * 3]; + px[0] = 255; // R + px[1] = 127; // G + px[2] = 0; // B + let blob = to_blob(&px); + let n = INPUT_SIZE * INPUT_SIZE; + assert!((blob[0] - 1.0).abs() < 1e-6); + assert!((blob[n] - (127.0 - 127.5) / 127.5).abs() < 1e-6); + assert!((blob[2 * n] + 1.0).abs() < 1e-6); + } +} diff --git a/server-rs/crates/lrg-ml/src/faces.rs b/server-rs/crates/lrg-ml/src/faces.rs index 8a3e9702..c195eee9 100644 --- a/server-rs/crates/lrg-ml/src/faces.rs +++ b/server-rs/crates/lrg-ml/src/faces.rs @@ -1,5 +1,5 @@ -//! Top-level face pipeline: YuNet detect + FaceNet embed + quality -//! metrics + base64 thumbnail. +//! Top-level face pipeline: YuNet detect + FaceNet (or, opted into, AuraFace) +//! embed + quality metrics + base64 thumbnail. //! //! Both models replaced InsightFace's `buffalo_l` pack (SCRFD + ArcFace), //! which could not be redistributed with this project. The pipeline's shape @@ -20,7 +20,7 @@ use ort::value::Tensor; use lrg_imaging::cull_config::FaceMetricsConfig; use lrg_imaging::pil_resample::{resize_plane, Filter}; -use crate::{face_quality, facenet, umeyama, yunet}; +use crate::{auraface, face_quality, facenet, umeyama, yunet}; const IDLE_UNLOAD_SECONDS: i64 = 30 * 60; @@ -39,6 +39,56 @@ const IDLE_UNLOAD_SECONDS: i64 = 30 * 60; /// alignment framing in [`crate::umeyama::norm_crop_facenet`]. pub const MODEL_ID: &str = "yunet-2023mar/facenet-vggface2"; +/// [`MODEL_ID`] for the AuraFace recognizer. Same detector, so the boxes of a +/// re-detected photo line up with its FaceNet rows — which is what lets a +/// switch carry the user's names over (see the carry-over in `lrg-api`). +pub const AURAFACE_MODEL_ID: &str = "yunet-2023mar/auraface-v1"; + +/// Which network turns an aligned face into its identity embedding. +/// +/// FaceNet ships with the default model download. AuraFace is the opt-in +/// alternative: sharper identities, a 250 MB download. The choice is global +/// to the backend and changes [`FaceModel::model_id`], so every stored face +/// from the other recognizer is set aside by clustering and its photo is +/// re-queued for detection until it is re-done. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum Recognizer { + #[default] + FaceNet, + AuraFace, +} + +impl Recognizer { + pub const ALL: [Recognizer; 2] = [Recognizer::FaceNet, Recognizer::AuraFace]; + + /// The name used in the API and in the saved setting. + #[must_use] + pub fn as_str(self) -> &'static str { + match self { + Recognizer::FaceNet => "facenet", + Recognizer::AuraFace => "auraface", + } + } + + #[must_use] + pub fn parse(value: &str) -> Option { + match value.trim().to_ascii_lowercase().as_str() { + "facenet" => Some(Recognizer::FaceNet), + "auraface" => Some(Recognizer::AuraFace), + _ => None, + } + } + + /// What a stored row from this recognizer is stamped with. + #[must_use] + pub fn model_id(self) -> &'static str { + match self { + Recognizer::FaceNet => MODEL_ID, + Recognizer::AuraFace => AURAFACE_MODEL_ID, + } + } +} + #[derive(Debug, thiserror::Error)] pub enum FaceError { #[error("ort error: {0}")] @@ -54,21 +104,47 @@ pub struct FaceModelPaths { pub det_onnx: PathBuf, /// FaceNet recognizer (`facenet_vggface2.onnx`). pub rec_onnx: PathBuf, + /// AuraFace recognizer (`auraface_glintr100.onnx`). Optional: only + /// needed once the user switches to it. + pub auraface_onnx: PathBuf, } impl FaceModelPaths { + /// The default pair (YuNet + FaceNet) — what "Download AI models" fetches. pub fn all_exist(&self) -> bool { self.det_onnx.is_file() && self.rec_onnx.is_file() } + + /// The recognizer's weights. + #[must_use] + pub fn recognizer_onnx(&self, recognizer: Recognizer) -> &Path { + match recognizer { + Recognizer::FaceNet => &self.rec_onnx, + Recognizer::AuraFace => &self.auraface_onnx, + } + } + + /// Whether the detector and `recognizer` are both on disk. + #[must_use] + pub fn ready_for(&self, recognizer: Recognizer) -> bool { + self.det_onnx.is_file() && self.recognizer_onnx(recognizer).is_file() + } } struct Loaded { det_session: Session, rec_session: Session, + /// Which recognizer `rec_session` is. Checked on every use, so a switch + /// between two photos reloads instead of embedding with the old network. + recognizer: Recognizer, + /// The recognizer's input name: FaceNet's export names it `input`, + /// InsightFace-family graphs `input.1`, so it is read from the graph. + rec_input: String, } pub struct FaceModel { paths: FaceModelPaths, + recognizer: Mutex, loaded: Mutex>, last_used_unix: AtomicI64, load_error: Mutex>, @@ -179,30 +255,85 @@ impl FaceModel { pub fn new(paths: FaceModelPaths) -> Self { FaceModel { paths, + recognizer: Mutex::new(Recognizer::default()), loaded: Mutex::new(None), last_used_unix: AtomicI64::new(0), load_error: Mutex::new(None), } } + /// Whether the detector and the *active* recognizer are on disk. pub fn is_cached(&self) -> bool { - self.paths.all_exist() + self.paths.ready_for(self.recognizer()) } - fn ensure_loaded(&self) -> Result<(), FaceError> { - let mut guard = self.loaded.lock().unwrap(); - if guard.is_some() { + /// The model files, for the routes that report or download them. + #[must_use] + pub fn paths(&self) -> &FaceModelPaths { + &self.paths + } + + /// The recognizer new embeddings are made with. + #[must_use] + pub fn recognizer(&self) -> Recognizer { + *self.recognizer.lock().unwrap() + } + + /// Switches the recognizer. The sessions of the old one are dropped, and + /// [`FaceModel::model_id`] changes with it. Refused when the recognizer's + /// weights are not on disk: a switch that fails on the next photo would + /// leave every run reporting a missing model. + pub fn set_recognizer(&self, recognizer: Recognizer) -> Result<(), String> { + if !self.paths.recognizer_onnx(recognizer).is_file() { + return Err(format!( + "the {} model is not downloaded ({} is missing)", + recognizer.as_str(), + self.paths.recognizer_onnx(recognizer).display() + )); + } + let mut current = self.recognizer.lock().unwrap(); + if *current != recognizer { + log::info!( + "Face recognizer: {} -> {}", + current.as_str(), + recognizer.as_str() + ); + *current = recognizer; + drop(current); + self.unload_with_reason("recognizer switched"); + } + Ok(()) + } + + /// Loads the sessions for the active recognizer into `guard` unless they + /// are already there. Takes the guard so [`FaceModel::detect_faces_with_model`] + /// loads and runs under one lock: a switch can then never land between + /// the two and leave a photo embedded by one network and stamped with the + /// other's id. + fn load_into(&self, guard: &mut Option) -> Result<(), FaceError> { + let recognizer = self.recognizer(); + if guard.as_ref().is_some_and(|l| l.recognizer == recognizer) { self.last_used_unix.store(now_unix(), Ordering::Relaxed); return Ok(()); } + *guard = None; match ( build_session(&self.paths.det_onnx), - build_session(&self.paths.rec_onnx), + build_session(self.paths.recognizer_onnx(recognizer)), ) { (Ok(det_session), Ok(rec_session)) => { + let rec_input = rec_session + .inputs() + .first() + .map(|i| i.name().to_string()) + .ok_or_else(|| { + FaceError::UnexpectedOutputs("recognizer has no input".to_string()) + })?; *guard = Some(Loaded { det_session, rec_session, + recognizer, + rec_input, }); *self.load_error.lock().unwrap() = None; self.last_used_unix.store(now_unix(), Ordering::Relaxed); @@ -248,7 +379,7 @@ impl FaceModel { /// of their own, and the paths are fixed filenames, so the only thing that /// can change which embeddings this build produces is this build. pub fn model_id(&self) -> &'static str { - MODEL_ID + self.recognizer().model_id() } pub fn status(&self) -> (&'static str, Option) { @@ -271,9 +402,27 @@ impl FaceModel { cfg: &FaceMetricsConfig, pass: FacePass, ) -> Result, FaceError> { - self.ensure_loaded()?; + self.detect_faces_with_model(pixels, width, height, cfg, pass) + .map(|(faces, _)| faces) + } + + /// [`FaceModel::detect_faces`], plus the [`Recognizer::model_id`] the + /// embeddings were made with — what the caller stamps on stored rows. + /// Read from the sessions that actually ran rather than from + /// [`FaceModel::model_id`] afterwards, which a concurrent switch could + /// already have changed. + pub fn detect_faces_with_model( + &self, + pixels: &[u8], + width: usize, + height: usize, + cfg: &FaceMetricsConfig, + pass: FacePass, + ) -> Result<(Vec, &'static str), FaceError> { let mut guard = self.loaded.lock().unwrap(); + self.load_into(&mut guard)?; let loaded = guard.as_mut().ok_or(FaceError::NotLoaded)?; + let model_id = loaded.recognizer.model_id(); let pre = yunet::preprocess(pixels, width, height); let input = Tensor::from_array(([1, 3, 640, 640], pre.blob))?; @@ -319,35 +468,50 @@ impl FaceModel { let embedding = match pass { FacePass::Full => { - // A second warp rather than a resize of `aligned`: FaceNet - // wants a wider framing than ArcFace did (see - // `norm_crop_facenet`), and context outside the 112 crop - // cannot be recovered by upscaling it. `QualityOnly` never - // pays for this, so the cull pass costs exactly what it did. - let wide = umeyama::norm_crop_facenet( - pixels, - width, - height, - &kps_f64, - facenet::INPUT_SIZE, - ); - let blob = facenet::to_blob(&wide); - let dim = facenet::INPUT_SIZE; + let (blob, dim, embed_dim) = match loaded.recognizer { + // A second warp rather than a resize of `aligned`: + // FaceNet wants a wider framing than ArcFace did (see + // `norm_crop_facenet`), and context outside the 112 + // crop cannot be recovered by upscaling it. + // `QualityOnly` never pays for this, so the cull pass + // costs exactly what it did. + Recognizer::FaceNet => { + let wide = umeyama::norm_crop_facenet( + pixels, + width, + height, + &kps_f64, + facenet::INPUT_SIZE, + ); + ( + facenet::to_blob(&wide), + facenet::INPUT_SIZE, + facenet::EMBED_DIM, + ) + } + // AuraFace is an ArcFace-family network: it takes + // exactly the canonical crop made above. + Recognizer::AuraFace => ( + auraface::to_blob(&aligned), + auraface::INPUT_SIZE, + auraface::EMBED_DIM, + ), + }; let rec_input = Tensor::from_array(([1, 3, dim, dim], blob))?; - let rec_out = loaded.rec_session.run(ort::inputs!["input" => rec_input])?; + let rec_out = loaded + .rec_session + .run(ort::inputs![loaded.rec_input.as_str() => rec_input])?; let (_, raw_emb) = rec_out[0].try_extract_tensor::()?; - if raw_emb.len() != facenet::EMBED_DIM { + if raw_emb.len() != embed_dim { return Err(FaceError::UnexpectedOutputs(format!( - "recognizer returned {} dims, expected {}", + "recognizer returned {} dims, expected {embed_dim}", raw_emb.len(), - facenet::EMBED_DIM ))); } let mut embedding = raw_emb.to_vec(); - // FaceNet normalizes its own output, so this is a no-op on a - // healthy model; it is kept so a re-export that drops the - // final `F.normalize` cannot quietly change what cosine - // distance means for every stored row. + // FaceNet normalizes its own output, so this is a no-op on + // it; AuraFace (like every ArcFace export) does not, and + // cosine distance downstream assumes unit vectors. crate::siglip::l2_normalize(&mut embedding); embedding } @@ -394,6 +558,51 @@ impl FaceModel { occlusion: round4(occlusion), }); } - Ok(results) + Ok((results, model_id)) + } +} + +#[cfg(test)] +mod recognizer_tests { + use super::*; + + fn paths(dir: &Path) -> FaceModelPaths { + FaceModelPaths { + det_onnx: dir.join("det.onnx"), + rec_onnx: dir.join("facenet.onnx"), + auraface_onnx: dir.join("auraface.onnx"), + } + } + + #[test] + fn names_round_trip_and_ids_differ() { + for r in Recognizer::ALL { + assert_eq!(Recognizer::parse(r.as_str()), Some(r)); + } + assert_eq!(Recognizer::parse(" AuraFace\n"), Some(Recognizer::AuraFace)); + assert_eq!(Recognizer::parse("arcface"), None); + assert_ne!( + Recognizer::FaceNet.model_id(), + Recognizer::AuraFace.model_id() + ); + } + + /// The stamp on stored rows follows the switch; that is what keeps the + /// two embedding spaces from being compared. + #[test] + fn switching_changes_the_model_id_and_needs_the_file() { + let dir = tempfile::tempdir().unwrap(); + let model = FaceModel::new(paths(dir.path())); + assert_eq!(model.model_id(), MODEL_ID); + assert!(model.set_recognizer(Recognizer::AuraFace).is_err()); + assert_eq!(model.recognizer(), Recognizer::FaceNet); + + std::fs::write(dir.path().join("auraface.onnx"), b"weights").unwrap(); + model.set_recognizer(Recognizer::AuraFace).unwrap(); + assert_eq!(model.model_id(), AURAFACE_MODEL_ID); + // Readiness is the active recognizer's: no detector here yet. + assert!(!model.is_cached()); + std::fs::write(dir.path().join("det.onnx"), b"weights").unwrap(); + assert!(model.is_cached()); } } diff --git a/server-rs/crates/lrg-ml/src/lib.rs b/server-rs/crates/lrg-ml/src/lib.rs index 0f2b56a5..504aab65 100644 --- a/server-rs/crates/lrg-ml/src/lib.rs +++ b/server-rs/crates/lrg-ml/src/lib.rs @@ -138,6 +138,7 @@ pub(crate) fn apply_kleidiai_policy( } pub mod aesthetic; +pub mod auraface; pub mod bioclip; pub mod bioclip_pre; pub mod clip_iqa; diff --git a/server-rs/crates/lrg-ml/src/model_paths.rs b/server-rs/crates/lrg-ml/src/model_paths.rs index 9bbdabd1..61b832bd 100644 --- a/server-rs/crates/lrg-ml/src/model_paths.rs +++ b/server-rs/crates/lrg-ml/src/model_paths.rs @@ -136,7 +136,7 @@ pub fn resolve_bioclip() -> BioclipModelPaths { } } -/// Where the YuNet detector and FaceNet recognizer live. +/// Where the YuNet detector and the FaceNet and AuraFace recognizers live. /// /// Same per-artifact env-var convention and same cache directory as SigLIP2 /// and BioCLIP. These used to resolve out of `INSIGHTFACE_ROOT` @@ -151,5 +151,17 @@ pub fn resolve_face() -> FaceModelPaths { FaceModelPaths { det_onnx: env_or_default("LRG_YUNET_ONNX", dir.join("yunet_face_detection.onnx")), rec_onnx: env_or_default("LRG_FACENET_ONNX", dir.join("facenet_vggface2.onnx")), + auraface_onnx: env_or_default("LRG_AURAFACE_ONNX", dir.join("auraface_glintr100.onnx")), } } + +/// Where the backend remembers which face recognizer the user chose +/// (`facenet` or `auraface`, see [`crate::faces::Recognizer`]). A plain text +/// file next to the models, since the choice belongs to the models on this +/// computer rather than to one catalog. `LRG_FACE_RECOGNIZER_FILE` moves it. +pub fn face_recognizer_setting() -> PathBuf { + env_or_default( + "LRG_FACE_RECOGNIZER_FILE", + default_models_dir().join("face_recognizer.txt"), + ) +} diff --git a/server-rs/crates/lrg-ml/tests/face_goldens.rs b/server-rs/crates/lrg-ml/tests/face_goldens.rs index 0512af71..79f73793 100644 --- a/server-rs/crates/lrg-ml/tests/face_goldens.rs +++ b/server-rs/crates/lrg-ml/tests/face_goldens.rs @@ -1,4 +1,5 @@ -//! End-to-end golden verification of the YuNet + FaceNet face pipeline +//! End-to-end golden verification of the YuNet + FaceNet face pipeline (and, +//! when its model and reference are present, the AuraFace one) //! against the reference implementations — OpenCV's `FaceDetectorYN` and //! `facenet-pytorch`'s Inception-ResNet-v1 — as captured by //! `testdata/make_face_goldens.py` on a real photo (skimage's astronaut @@ -14,7 +15,7 @@ //! `LRG_REQUIRE_GOLDENS=face`, making a broken download a test failure //! rather than a silent pass. -use lrg_ml::faces::{FaceModel, FaceModelPaths}; +use lrg_ml::faces::{FaceModel, FaceModelPaths, Recognizer}; mod common; @@ -54,6 +55,7 @@ fn detects_and_embeds_real_face_matching_reference_models() { let model = FaceModel::new(FaceModelPaths { det_onnx: paths.det_onnx, rec_onnx: paths.rec_onnx, + auraface_onnx: paths.auraface_onnx, }); let golden_path = concat!( @@ -123,3 +125,73 @@ fn detects_and_embeds_real_face_matching_reference_models() { ); } } + +/// The AuraFace chain (YuNet landmarks, ArcFace warp, `auraface::to_blob`, +/// the network) against InsightFace's own embedding of the same face, made by +/// `scripts/auraface_reference.py`. Skips until both the model and that +/// reference exist; `LRG_REQUIRE_GOLDENS=auraface` makes the skip a failure. +#[test] +fn auraface_matches_insightface_reference() { + let paths = lrg_ml::model_paths::resolve_face(); + let golden_path = concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../testdata/face_goldens.json" + ); + let goldens: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap(); + // No reference recorded yet is not a missing download, so it skips even + // under `LRG_REQUIRE_GOLDENS=all`: there is nothing to compare against + // until `scripts/auraface_reference.py` has been run and its output + // committed. + let Some(reference) = goldens["astronaut_512"]["faces"][0]["auraface_embedding"].as_array() + else { + eprintln!( + "skipping AuraFace golden: no `auraface_embedding` in face_goldens.json yet \ + (run scripts/auraface_reference.py)" + ); + return; + }; + if !common::assets_ready( + "auraface", + paths.ready_for(Recognizer::AuraFace), + &format!( + "needs {} and {}", + paths.det_onnx.display(), + paths.auraface_onnx.display() + ), + ) { + return; + } + let expected: Vec = reference + .iter() + .map(|v| v.as_f64().unwrap() as f32) + .collect(); + + let model = FaceModel::new(FaceModelPaths { + det_onnx: paths.det_onnx, + rec_onnx: paths.rec_onnx, + auraface_onnx: paths.auraface_onnx, + }); + model.set_recognizer(Recognizer::AuraFace).unwrap(); + + let img_path = concat!(env!("CARGO_MANIFEST_DIR"), "/../../testdata/astronaut.jpg"); + let decoded = image::load_from_memory(&std::fs::read(img_path).unwrap()) + .unwrap() + .to_rgb8(); + let (width, height) = (decoded.width() as usize, decoded.height() as usize); + let (faces, model_id) = model + .detect_faces_with_model( + &decoded.into_raw(), + width, + height, + &lrg_imaging::cull_config::FaceMetricsConfig::defaults(), + lrg_ml::faces::FacePass::Full, + ) + .unwrap(); + assert_eq!(model_id, lrg_ml::faces::AURAFACE_MODEL_ID); + assert_eq!(faces.len(), 1, "face count mismatch"); + assert_eq!(expected.len(), 512, "reference embedding must be 512-d"); + let cos = cosine(&faces[0].embedding, &expected); + println!("auraface embedding_cosine={cos:.4}"); + assert!(cos > 0.99, "AuraFace embedding cosine {cos} too low"); +} diff --git a/server-rs/scripts/auraface_reference.py b/server-rs/scripts/auraface_reference.py new file mode 100644 index 00000000..eccce094 --- /dev/null +++ b/server-rs/scripts/auraface_reference.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.10,<3.13" +# dependencies = [ +# "huggingface_hub>=0.24", +# "insightface==0.7.3", +# "numpy<2.0.0", +# "onnxruntime>=1.17.0", +# "opencv-python-headless>=4.8", +# ] +# /// +"""Reference embedding for the AuraFace recognizer, made by InsightFace itself. + +The backend runs AuraFace (``fal/AuraFace-v1``, ``glintr100.onnx``) through its +own Rust preprocessing (``lrg-ml/src/auraface.rs``: the ArcFace 5-point warp, +RGB, ``(v - 127.5) / 127.5``). This script computes the embedding of the face +in ``testdata/astronaut.jpg`` the way AuraFace is published to be used — +InsightFace's ``norm_crop`` and ``ArcFaceONNX.get_feat`` — from the same five +landmarks the FaceNet golden already stores, and writes it into +``testdata/face_goldens.json`` as ``auraface_embedding``. The golden test +``auraface_matches_insightface_reference`` then checks the Rust chain against +it (cosine > 0.99). + +Run:: + + uv run --no-project server-rs/scripts/auraface_reference.py --revision main + +and pin the printed commit as ``AURAFACE_REVISION`` in +``lrg-api/src/routes/face_recognizer.rs`` once the test passes. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + +import cv2 +import numpy as np +from huggingface_hub import HfApi, hf_hub_download +from insightface.model_zoo.arcface_onnx import ArcFaceONNX +from insightface.utils import face_align + +REPO = "fal/AuraFace-v1" +FILE = "glintr100.onnx" +ROOT = Path(__file__).resolve().parents[1] +GOLDENS = ROOT / "testdata" / "face_goldens.json" +IMAGE = ROOT / "testdata" / "astronaut.jpg" + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--revision", default="main") + parser.add_argument("--output", type=Path, help="also copy the ONNX file here") + args = parser.parse_args() + + commit = HfApi().model_info(REPO, revision=args.revision).sha + path = Path(hf_hub_download(REPO, FILE, revision=commit)) + data = path.read_bytes() + print(f"{REPO}@{commit} {FILE}: {len(data)} bytes, sha256 {hashlib.sha256(data).hexdigest()}") + if args.output: + args.output.write_bytes(data) + + model = ArcFaceONNX(model_file=str(path)) + model.prepare(ctx_id=-1) + print(f"InsightFace picked input_mean={model.input_mean} input_std={model.input_std}") + + goldens = json.loads(GOLDENS.read_text()) + face = goldens["astronaut_512"]["faces"][0] + kps = np.array(face["kps"], dtype=np.float32) + img_bgr = cv2.imread(str(IMAGE)) + aligned = face_align.norm_crop(img_bgr, landmark=kps, image_size=112) + emb = model.get_feat(aligned).flatten() + emb = emb / np.linalg.norm(emb) + + face["auraface_embedding"] = [float(v) for v in emb] + goldens["astronaut_512"]["auraface_source"] = { + "repo": REPO, + "revision": commit, + "file": FILE, + "sha256": hashlib.sha256(data).hexdigest(), + "input_mean": float(model.input_mean), + "input_std": float(model.input_std), + } + GOLDENS.write_text(json.dumps(goldens)) + print(f"wrote auraface_embedding ({emb.shape[0]}-d) to {GOLDENS}") + + +if __name__ == "__main__": + main() From caf262ca8654a742627ba37c976734e6d0cfbbcf Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 04:32:08 +0000 Subject: [PATCH 04/10] Drop Lightroom previews (requestJpegThumbnail) again: export only requestJpegThumbnail delivers a few previews and then stops answering, and what it delivers comes in whatever size is cached: embeddings and aesthetic scores made from those drift from the ones made from exports. Indexing and Add Missing Search Previews now always export, in batches. Removes LightroomPreviews.lua, the Plug-in Manager option, Util.jpegDimensions and Util.previewFallbackWarning; the docs explain why previews are not used. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HaMMvqfgsKcoNWQaKy3c9C --- .luacheckrc | 1 - docs/wiki/Dev-Backend-API.md | 2 +- docs/wiki/Dev-Plugin-README.md | 8 +- docs/wiki/Help-Advanced-Search.md | 4 +- docs/wiki/Help-Analyze-and-Index.md | 3 +- docs/wiki/Help-Performance-Tips.md | 32 ++--- .../LrGeniusAI.lrdevplugin/APISearchIndex.lua | 59 +-------- plugin/LrGeniusAI.lrdevplugin/Init.lua | 5 - .../LightroomPreviews.lua | 124 ------------------ .../PluginInfoDialogSections.lua | 26 ---- .../TaskFillSearchPreviews.lua | 92 +++---------- plugin/LrGeniusAI.lrdevplugin/Util.lua | 78 ----------- plugin/README.md | 8 +- plugin/spec/util_export_and_faces_spec.lua | 44 +++++++ plugin/spec/util_preview_render_spec.lua | 105 --------------- 15 files changed, 92 insertions(+), 499 deletions(-) delete mode 100644 plugin/LrGeniusAI.lrdevplugin/LightroomPreviews.lua create mode 100644 plugin/spec/util_export_and_faces_spec.lua delete mode 100644 plugin/spec/util_preview_render_spec.lua diff --git a/.luacheckrc b/.luacheckrc index 3ce3a6f1..5c00913c 100644 --- a/.luacheckrc +++ b/.luacheckrc @@ -22,7 +22,6 @@ globals = { "UpdateCheck", "Defaults", "KeywordConfigProvider", - "LightroomPreviews", "LocalModelCatalog", "MetadataManager", "OnboardingWizard", diff --git a/docs/wiki/Dev-Backend-API.md b/docs/wiki/Dev-Backend-API.md index b03f6307..e26fe427 100644 --- a/docs/wiki/Dev-Backend-API.md +++ b/docs/wiki/Dev-Backend-API.md @@ -344,7 +344,7 @@ No database bound → the empty answer. Costs a metadata scan (like ### `POST /v1/photos/thumbnail` Stores the search-page preview of an already indexed photo, without indexing it again. Multipart: `photo_id` and one `image` part — any JPEG/PNG; the plug-in -sends Lightroom's cached preview or a small export. The backend scales it down +sends a small export (512 px). The backend scales it down to the size indexing stores (256 px long edge, quality 75) and writes it where `GET /v1/photos/thumbnail` reads it, replacing any earlier one. Nothing else about the photo changes. diff --git a/docs/wiki/Dev-Plugin-README.md b/docs/wiki/Dev-Plugin-README.md index 34511e96..057b743b 100644 --- a/docs/wiki/Dev-Plugin-README.md +++ b/docs/wiki/Dev-Plugin-README.md @@ -43,8 +43,8 @@ The plugin is designed to work with local and cloud providers, while keeping Lig - Every processed photo also gets a small preview (256 px, ~13 KB, stored by the backend beside the index) for the browser search page. Photos indexed before previews were stored get theirs from *Add Missing Search Previews...* (`TaskFillSearchPreviews.lua`): it asks the backend which - indexed photos lack one, takes Lightroom's cached preview where it is big enough - (`LightroomPreviews.lua`) and a small export otherwise, and touches nothing else + indexed photos lack one, exports small JPEGs of those in batches, and touches nothing else. + (Not `requestJpegThumbnail`: Lightroom delivers a few previews and then stops answering.) ### Advanced Search @@ -321,9 +321,7 @@ In the plugin settings dialog you can configure: The section reports why it is unavailable when the host cannot use it — a source build without the `llamacpp` feature, or a missing MLX helper. - Export size and quality used for AI processing, photos per export session (indexing exports - in batches of that size, `Util.exportChunkSize`, so Lightroom frees its memory between them), - and whether indexing may use Lightroom's cached previews instead of exporting - (`LightroomPreviews.lua`; photos without a large-enough preview are exported) + in batches of that size, `Util.exportChunkSize`, so Lightroom frees its memory between them) - Prompt presets — a preset is the *system* prompt: which expert is looking at the photo, which vocabulary they use, how specific they may be, and what they must never invent. **Default** (the analytical voice) and **Family & diff --git a/docs/wiki/Help-Advanced-Search.md b/docs/wiki/Help-Advanced-Search.md index edef34ca..1d432cc0 100644 --- a/docs/wiki/Help-Advanced-Search.md +++ b/docs/wiki/Help-Advanced-Search.md @@ -167,8 +167,8 @@ photos, beside the search index. Photos indexed before this version have none yet: they show *No preview yet*, and the page says how many. To add them, run **Library ▸ Plug-in Extras ▸ Add Missing Search Previews...** — for the whole catalog, or click **Show in Lightroom** first and choose *Selected photos*. It -only adds the previews (from Lightroom's own previews where it can, from a small -export otherwise); nothing is analyzed again. +only adds the previews, from a small export of each photo; nothing is analyzed +again. A preview is deleted with its photo's index data (the index cleanup, or removing the photo from the index), and is not part of a database backup — diff --git a/docs/wiki/Help-Analyze-and-Index.md b/docs/wiki/Help-Analyze-and-Index.md index f38056b8..13bb1eff 100644 --- a/docs/wiki/Help-Analyze-and-Index.md +++ b/docs/wiki/Help-Analyze-and-Index.md @@ -110,8 +110,7 @@ same. Photos indexed before previews existed get theirs from **Library ▸ Plug-in Extras ▸ Add Missing Search Previews...**: choose selected photos, the current view or the whole catalog, and it adds a preview to every indexed photo that has -none — from Lightroom's own previews where they are big enough, from a small -export otherwise — without analyzing anything again. (A run of Analyze & Index +none, from a small export of each photo, without analyzing anything again. (A run of Analyze & Index that processes a photo adds one too, but *New or unprocessed photos* leaves these photos out, because nothing a search, a cull or a face lookup needs is missing.) diff --git a/docs/wiki/Help-Performance-Tips.md b/docs/wiki/Help-Performance-Tips.md index eac93351..ff1b8c99 100644 --- a/docs/wiki/Help-Performance-Tips.md +++ b/docs/wiki/Help-Performance-Tips.md @@ -107,21 +107,15 @@ batch, so a run over thousands of photos no longer makes it swell until the computer slows to a crawl. Lower it if Lightroom still grows on your computer; raise it on a machine with plenty of memory for slightly faster runs. -### Use Lightroom's previews where they are large enough - -With **Use Lightroom's previews where they are large enough (faster)** on, -indexing takes Lightroom's cached preview of a photo instead of exporting it, -whenever that preview is at least three quarters of the export size (768 px -for the default 1024 px). Photos without such a preview are exported as usual, -and the run's completion dialog says how many that were. Previews show your -develop edits, so crops and adjustments still count — with one caveat: a -preview Lightroom has not re-rendered since your last edit shows the photo as -it was, and the aesthetic score is made from what the backend sees. - -To get the most out of it, select the photos and run *Library ▸ Previews ▸ -Build Standard-Sized Previews* first, with *Standard Preview Size* in -*Catalog Settings ▸ File Handling* at least as large as the export size. The -option is off by default and has no effect when *Submit original files* is on. +### Why exports and not Lightroom's previews + +Lightroom's cached previews (`requestJpegThumbnail` in its plug-in SDK) would +skip the export, but they are not reliable for indexing: Lightroom delivers a +few and then stops answering, and the ones it delivers come in whatever size +happens to be cached. Embeddings and aesthetic scores made from pictures of +varying size drift apart from the ones made from exports, which makes search +and culling less consistent. Every photo is therefore exported at the size set +above; batching is what makes that fast. --- @@ -199,13 +193,13 @@ parallelism in a batched run is entirely on the backend's side, inside one request: it is free to use several cores, while the plugin still waits for a single answer. -On macOS, when you're *not* using "submit originals" or Lightroom's -previews, the plugin pipelines export and upload — the next batch's JPEG +On macOS, when you're *not* using "submit originals," the plugin pipelines +export and upload — the next batch's JPEG export overlaps with the previous photos' backend round-trips, so you're not paying export time and network time back-to-back. On Windows the exports of a batch are still rendered together, in one export session, and then sent one -by one; there the "submit originals" fast path (§4), Lightroom's previews -(§3) or a smaller export size buy you more. +by one; there the "submit originals" fast path (§4) or a smaller export size +(§3) buys you more. --- diff --git a/plugin/LrGeniusAI.lrdevplugin/APISearchIndex.lua b/plugin/LrGeniusAI.lrdevplugin/APISearchIndex.lua index 4dd29238..bbf6e493 100644 --- a/plugin/LrGeniusAI.lrdevplugin/APISearchIndex.lua +++ b/plugin/LrGeniusAI.lrdevplugin/APISearchIndex.lua @@ -308,10 +308,6 @@ local EXPORT_SETTINGS = { LR_embeddedMetadataOption = "all", } ---- How long indexing waits for Lightroom's cached preview of one photo ---- before it exports the photo instead. -local PREVIEW_TIMEOUT_SECONDS = 10 - --- The export settings for one session: the defaults above with the current --- size and quality preferences (which can change after this file loaded), --- the temp folder as destination, and `overrides` on top. A fresh table each @@ -2586,21 +2582,12 @@ function SearchIndexAPI.analyzeAndIndexSelectedPhotos(selectedPhotos, progressSc -- Opt-in fast path: submit the original file instead of exporting a JPEG. -- Local backend gets just the path; remote backends get the raw upload. local useOriginals = (prefs and prefs.indexSubmitOriginals == true) - -- Opt-in: take Lightroom's cached preview instead of exporting, when it is - -- big enough (LightroomPreviews). Only meaningful in export mode. - local usePreviews = (not useOriginals) and prefs ~= nil and prefs.indexUseLightroomPreviews == true - local exportLongEdge = tonumber(prefs and prefs.exportSize) or 1024 - -- A preview a quarter smaller than the export would have been is still - -- plenty for the models (SigLIP2 and the aesthetic scorer look at far - -- less); anything smaller is exported. - local minPreviewEdge = math.floor(exportLongEdge * 0.75) local chunkSize = Util.exportChunkSize(prefs and prefs.exportChunkSize) - local previewStats = { used = 0, exported = 0 } local errorMessages = {} local warningsList = {} - if MAC_ENV and numPhotos > 1 and not useOriginals and not usePreviews then + if MAC_ENV and numPhotos > 1 and not useOriginals then -- Pipelined export/index path (macOS only). A producer drives the export -- sessions and pushes rendered JPEG paths into a queue; a consumer POSTs -- each photo to the backend as soon as its export completes, so the next @@ -2998,34 +2985,11 @@ function SearchIndexAPI.analyzeAndIndexSelectedPhotos(selectedPhotos, progressSc end end - if not success and usePreviews then - local previewPath, why = LightroomPreviews.previewFile( - photo, - exportLongEdge, - minPreviewEdge, - PREVIEW_TIMEOUT_SECONDS - ) - if previewPath then - log:trace("Using Lightroom's preview for " .. filename) - previewStats.used = previewStats.used + 1 - success, indexResponse = - SearchIndexAPI.analyzeAndIndexPhoto(photoId, previewPath, photoOptions) - LrTasks.pcall(function() - LrFileUtils.delete(previewPath) - end) - else - log:trace( - "No usable Lightroom preview for " .. filename .. " (" .. tostring(why) .. ")" - ) - previewStats.exported = previewStats.exported + 1 - end - end - if not success then - -- Pure export mode renders the next photos with this one, - -- in one session (see prefillExports). In the other modes - -- an export is the exception, so it stays a single one. - if not useOriginals and not usePreviews and exportMemo[photo] == nil then + -- Export mode renders the next photos with this one, in one + -- session (see prefillExports). With originals an export is + -- the exception (a fallback), so it stays a single one. + if not useOriginals and exportMemo[photo] == nil then prefillExports(photo) end local exportedPhotoPath = exportMemo[photo] @@ -3144,19 +3108,6 @@ function SearchIndexAPI.analyzeAndIndexSelectedPhotos(selectedPhotos, progressSc return "canceled", stats.processed, stats.failed, processedPhotos end - -- Not a failure — those photos were exported instead — but the user chose - -- previews to make the run faster, and this is why it was not. - if usePreviews and previewStats.exported > 0 then - table.insert( - warningsList, - Util.previewFallbackWarning( - previewStats.exported, - previewStats.used + previewStats.exported, - minPreviewEdge - ) - ) - end - local status, combinedError, combinedWarnings = SearchIndexAPI.summarizeRun(stats, errorMessages, warningsList) return status, stats.processed, stats.failed, processedPhotos, combinedError, combinedWarnings end diff --git a/plugin/LrGeniusAI.lrdevplugin/Init.lua b/plugin/LrGeniusAI.lrdevplugin/Init.lua index bff773ea..54466028 100644 --- a/plugin/LrGeniusAI.lrdevplugin/Init.lua +++ b/plugin/LrGeniusAI.lrdevplugin/Init.lua @@ -40,7 +40,6 @@ require("PromptConfigProvider") require("UpdateCheck") require("ErrorHandler") require("APISearchIndex") -require("LightroomPreviews") require("IndexCleanup") require("SpeciesLinks") require("PhotoSelector") @@ -138,10 +137,6 @@ if _G.prefs.indexSubmitOriginals == nil then _G.prefs.indexSubmitOriginals = false end -if _G.prefs.indexUseLightroomPreviews == nil then - _G.prefs.indexUseLightroomPreviews = false -end - if _G.prefs.exportSize == nil then _G.prefs.exportSize = Defaults.defaultExportSize end diff --git a/plugin/LrGeniusAI.lrdevplugin/LightroomPreviews.lua b/plugin/LrGeniusAI.lrdevplugin/LightroomPreviews.lua deleted file mode 100644 index 8a1f20b0..00000000 --- a/plugin/LrGeniusAI.lrdevplugin/LightroomPreviews.lua +++ /dev/null @@ -1,124 +0,0 @@ ---[[ - LightroomPreviews.lua - - Rendered JPEGs of photos without an export: Lightroom's own previews, - through `photo:requestJpegThumbnail`. - - An export renders the photo from scratch, which is what makes indexing a - large selection slow and what lets Lightroom's memory climb over a long - run. A preview is already rendered — with the photo's current develop - settings — whenever Lightroom has one cached, so handing it over costs - almost nothing. Two things make it unreliable, and both are handled here - rather than by every caller: - - * Lightroom answers with whatever it has cached, which can be far smaller - than what was asked for (a 160 px grid thumbnail when only that was ever - built). The size is read from the JPEG itself (`Util.jpegDimensions`) - and the caller decides what is big enough; anything smaller is the - caller's cue to export instead. - * The callback may never come (a photo whose preview cannot be built), or - come more than once (a small preview first, a better one after). The - wait has a deadline, and a better answer arriving shortly after a first - one is still taken. - - Previews can lag behind the latest edit until Lightroom re-renders them; - for a search preview that is fine, for an aesthetic score it is the - trade-off the "Use Lightroom previews" setting names. -]] - -LightroomPreviews = {} - ---- How long to keep waiting for a better answer once a first one arrived. -local SETTLE_SECONDS = 0.4 - ---- --- Asks Lightroom for a rendered JPEG of `photo`, at most `longEdge` px on its --- long edge. --- @param photo LrPhoto --- @param longEdge number The size wanted. --- @param timeoutSeconds number|nil How long to wait for Lightroom (default 10). --- @return string|nil jpegData, number longEdgeReceived (0 when none), --- string|nil reason Why there is no data, for the log. --- -function LightroomPreviews.requestJpeg(photo, longEdge, timeoutSeconds) - local deadline = LrDate.currentTime() + (timeoutSeconds or 10) - local best, bestEdge, failure = nil, 0, nil - local firstAt = nil - - local ok, request = LrTasks.pcall(function() - return photo:requestJpegThumbnail(longEdge, longEdge, function(data, err) - if type(data) == "string" and #data > 0 then - local w, h = Util.jpegDimensions(data) - local edge = math.max(w or 0, h or 0) - if edge > bestEdge then - best, bestEdge = data, edge - end - firstAt = firstAt or LrDate.currentTime() - else - failure = err or "Lightroom returned no preview" - end - end) - end) - if not ok then - return nil, 0, "requestJpegThumbnail failed: " .. tostring(request) - end - - -- `request` must stay referenced until the callback has run: Lightroom - -- cancels a request whose handle is garbage-collected. - while LrDate.currentTime() < deadline do - if bestEdge >= longEdge then - break - end - if firstAt and LrDate.currentTime() - firstAt >= SETTLE_SECONDS then - break - end - if failure and not best then - break - end - LrTasks.sleep(0.05) - end - request = nil -- luacheck: ignore 311 - - if best == nil then - return nil, 0, failure or "Lightroom did not answer in time" - end - return best, bestEdge, nil -end - ---- --- Writes JPEG bytes to a new file in the temp folder. --- @return string|nil path, string|nil error --- -function LightroomPreviews.writeTemp(data, nameHint) - local name = "lrgenius-preview-" - .. tostring(nameHint or "photo"):gsub("[^%w%-_]", "_") - .. "-" - .. tostring(math.floor(LrDate.currentTime() * 1000)) - .. ".jpg" - local path = LrPathUtils.child(LrPathUtils.getStandardFilePath("temp"), name) - local fh, err = io.open(path, "wb") - if not fh then - return nil, "could not write " .. path .. ": " .. tostring(err) - end - fh:write(data) - fh:close() - return path, nil -end - ---- --- A rendered JPEG of `photo` on disk, from its Lightroom preview when that is --- at least `minLongEdge` px, else nil (the caller exports instead). --- @return string|nil path, string|nil reason Why the preview was not used. --- -function LightroomPreviews.previewFile(photo, wantedLongEdge, minLongEdge, timeoutSeconds) - local data, edge, reason = LightroomPreviews.requestJpeg(photo, wantedLongEdge, timeoutSeconds) - if data == nil then - return nil, reason - end - if edge < minLongEdge then - return nil, "the cached preview is " .. tostring(edge) .. " px, " .. tostring(minLongEdge) .. " px needed" - end - return LightroomPreviews.writeTemp(data, photo.localIdentifier) -end - -return LightroomPreviews diff --git a/plugin/LrGeniusAI.lrdevplugin/PluginInfoDialogSections.lua b/plugin/LrGeniusAI.lrdevplugin/PluginInfoDialogSections.lua index 974dd34b..34b09199 100644 --- a/plugin/LrGeniusAI.lrdevplugin/PluginInfoDialogSections.lua +++ b/plugin/LrGeniusAI.lrdevplugin/PluginInfoDialogSections.lua @@ -75,7 +75,6 @@ function PluginInfoDialogSections.startDialog(propertyTable) propertyTable.exportSize = prefs.exportSize propertyTable.exportQuality = prefs.exportQuality propertyTable.indexSubmitOriginals = (prefs.indexSubmitOriginals == true) - propertyTable.indexUseLightroomPreviews = (prefs.indexUseLightroomPreviews == true) propertyTable.exportChunkSize = tostring(Util.exportChunkSize(prefs.exportChunkSize)) propertyTable.promptTitles = Util.promptMenuItems(prefs.prompts, Defaults.defaultPromptName) @@ -1556,30 +1555,6 @@ function PluginInfoDialogSections.sectionsForTopOfDialog(f, propertyTable) enabled = false, }), }), - f:row({ - fill_horizontal = 1, - f:checkbox({ - value = bind("indexUseLightroomPreviews"), - title = "Use Lightroom's previews where they are large enough (faster)", - enabled = LrView.bind({ - key = "indexSubmitOriginals", - transform = function(v) - return v ~= true - end, - }), - }), - }), - f:row({ - fill_horizontal = 1, - f:static_text({ - title = "Takes Lightroom's cached preview instead of exporting when it is at least three\n" - .. "quarters of the export size; other photos are exported as usual. Previews show your\n" - .. "edits, but one Lightroom has not re-rendered since the last edit shows the photo as\n" - .. "it was then — which matters for the aesthetic score. Build Standard-Sized Previews\n" - .. "(Library > Previews) first to get the most out of this.", - enabled = false, - }), - }), }), }, { @@ -2004,7 +1979,6 @@ function PluginInfoDialogSections.endDialog(propertyTable) prefs.exportSize = propertyTable.exportSize prefs.exportQuality = propertyTable.exportQuality prefs.indexSubmitOriginals = (propertyTable.indexSubmitOriginals == true) - prefs.indexUseLightroomPreviews = (propertyTable.indexUseLightroomPreviews == true) prefs.exportChunkSize = Util.exportChunkSize(propertyTable.exportChunkSize) prefs.prompts = propertyTable.prompts diff --git a/plugin/LrGeniusAI.lrdevplugin/TaskFillSearchPreviews.lua b/plugin/LrGeniusAI.lrdevplugin/TaskFillSearchPreviews.lua index d43cd181..46e2f115 100644 --- a/plugin/LrGeniusAI.lrdevplugin/TaskFillSearchPreviews.lua +++ b/plugin/LrGeniusAI.lrdevplugin/TaskFillSearchPreviews.lua @@ -4,28 +4,20 @@ without indexing them again. The backend says which photos of the chosen scope are indexed but have no - preview (/v1/photos/thumbnails/missing). For each of those this task takes - Lightroom's own cached preview when it is big enough (LightroomPreviews), - and exports a small JPEG — a few dozen photos per export session — for the - rest. Each picture goes to /v1/photos/thumbnail, which scales it to the - preview size. Nothing else about a photo is touched. + preview (/v1/photos/thumbnails/missing). Those are exported as small JPEGs, + a few dozen photos per export session, and each goes to + /v1/photos/thumbnail, which scales it to the preview size. Nothing else + about a photo is touched. + + Export, not `photo:requestJpegThumbnail`: Lightroom answers that a few + times and then stops delivering, so a run over a catalog ends up exporting + anyway, after a timeout per photo. ]] ---- Long edge asked of Lightroom and of the fallback export. The backend ---- stores 256 px, so this leaves room for its own downscale. +--- Long edge of the export. The backend stores 256 px, so this leaves room +--- for its own downscale. local RENDER_LONG_EDGE = 512 ---- Smallest cached preview that is still used: the stored size itself. -local MIN_PREVIEW_LONG_EDGE = 256 - ---- How long to wait for Lightroom's answer per photo before exporting it. -local PREVIEW_TIMEOUT_SECONDS = 5 - ---- Misses in a row after which Lightroom's previews are not asked any more: ---- a catalog whose previews were never built would otherwise cost the full ---- timeout per photo before each export. -local MAX_PREVIEW_MISSES_IN_A_ROW = 10 - --- Photo ids per question to the backend, so a whole catalog does not go out --- as one request body. local LOOKUP_CHUNK = 2000 @@ -42,8 +34,8 @@ local function chooseScope(context) spacing = f:control_spacing(), f:static_text({ title = "Photos indexed before LrGeniusAI stored previews have no picture on the\n" - .. "browser search page. This adds one to each of them, from Lightroom's\n" - .. "previews where possible. Nothing else about the photos changes.", + .. "browser search page. This exports a small picture of each of them and\n" + .. "adds it. Nothing else about the photos changes.", }), f:row({ f:static_text({ title = "Photos:" }), @@ -168,21 +160,13 @@ local function run(context) local total = #missing local added, failures = 0, {} - local fromPreview, exported = 0, 0 - local needExport = {} - - local function upload(photoId, path, how) + local function upload(photoId, path) local ok, err = SearchIndexAPI.uploadPhotoThumbnail(photoId, path) LrTasks.pcall(function() LrFileUtils.delete(path) end) if ok then added = added + 1 - if how == "preview" then - fromPreview = fromPreview + 1 - else - exported = exported + 1 - end else local photo = byId[photoId] local name = photo and photo:getFormattedMetadata("fileName") or photoId @@ -197,43 +181,14 @@ local function run(context) scope:setCaption(caption .. " (" .. done .. " of " .. total .. ")") end - -- Pass 1: Lightroom's cached previews. - local missesInARow = 0 - for _, photoId in ipairs(missing) do - if scope:isCanceled() then - break - end - if missesInARow >= MAX_PREVIEW_MISSES_IN_A_ROW then - table.insert(needExport, photoId) - else - local photo = byId[photoId] - local path, reason = - LightroomPreviews.previewFile(photo, RENDER_LONG_EDGE, MIN_PREVIEW_LONG_EDGE, PREVIEW_TIMEOUT_SECONDS) - if path then - missesInARow = 0 - upload(photoId, path, "preview") - step("Adding previews") - else - missesInARow = missesInARow + 1 - log:trace( - "No usable Lightroom preview for " .. photoId .. " (" .. tostring(reason) .. "); exporting it" - ) - table.insert(needExport, photoId) - if missesInARow == MAX_PREVIEW_MISSES_IN_A_ROW then - log:info("Search previews: no usable Lightroom preview for 10 photos in a row; exporting the rest") - end - end - end - end - - -- Pass 2: small exports for the rest, a few dozen per session. + -- Small exports, a few dozen per session. local chunkSize = Util.exportChunkSize(prefs.exportChunkSize) local index = 1 - while index <= #needExport and not scope:isCanceled() do + while index <= #missing and not scope:isCanceled() do local chunkIds, chunkPhotos = {}, {} - for i = index, math.min(index + chunkSize - 1, #needExport) do - table.insert(chunkIds, needExport[i]) - table.insert(chunkPhotos, byId[needExport[i]]) + for i = index, math.min(index + chunkSize - 1, #missing) do + table.insert(chunkIds, missing[i]) + table.insert(chunkPhotos, byId[missing[i]]) end index = index + #chunkIds local paths, exportFailures = SearchIndexAPI.exportPhotosForIndexing(chunkPhotos, { @@ -245,7 +200,7 @@ local function run(context) for i, photoId in ipairs(chunkIds) do local photo = chunkPhotos[i] if paths[photo] then - upload(photoId, paths[photo], "export") + upload(photoId, paths[photo]) elseif not scope:isCanceled() then table.insert( failures, @@ -264,14 +219,7 @@ local function run(context) local canceled = scope:isCanceled() scope:done() - local summary = added - .. " of " - .. total - .. " photos got a preview (" - .. fromPreview - .. " from Lightroom's previews, " - .. exported - .. " exported)." + local summary = added .. " of " .. total .. " photos got a preview." if canceled then summary = "Stopped. " .. summary end diff --git a/plugin/LrGeniusAI.lrdevplugin/Util.lua b/plugin/LrGeniusAI.lrdevplugin/Util.lua index 5ab3e594..3182bf69 100644 --- a/plugin/LrGeniusAI.lrdevplugin/Util.lua +++ b/plugin/LrGeniusAI.lrdevplugin/Util.lua @@ -3363,65 +3363,6 @@ function Util.cullFlagSummaryLines(counts) return lines end ---- --- Width and height of a JPEG, read from its frame header (SOFn) without --- decoding it. Used to tell whether a preview Lightroom handed back is big --- enough to stand in for an export: `requestJpegThumbnail` answers with --- whatever it has cached, which can be far smaller than what was asked for. --- @param data string The JPEG's bytes. --- @return number|nil width, number|nil height nil, nil for anything that is not --- a JPEG or has no frame header before the image data. --- -function Util.jpegDimensions(data) - if type(data) ~= "string" or #data < 4 then - return nil, nil - end - if data:byte(1) ~= 0xFF or data:byte(2) ~= 0xD8 then - return nil, nil - end - local i = 3 - local n = #data - while i + 3 <= n do - if data:byte(i) ~= 0xFF then - return nil, nil - end - -- Any number of 0xFF fill bytes may precede a marker. - while i <= n and data:byte(i) == 0xFF do - i = i + 1 - end - if i > n then - return nil, nil - end - local marker = data:byte(i) - i = i + 1 - if marker == 0xD9 or marker == 0xDA then - -- End of image, or start of scan: no frame header came first. - return nil, nil - end - -- Markers without a length: TEM and the restart markers. - if not (marker == 0x01 or (marker >= 0xD0 and marker <= 0xD7)) then - if i + 1 > n then - return nil, nil - end - local length = data:byte(i) * 256 + data:byte(i + 1) - -- SOF0..SOF15, except DHT (C4), JPG (C8) and DAC (CC). - if marker >= 0xC0 and marker <= 0xCF and marker ~= 0xC4 and marker ~= 0xC8 and marker ~= 0xCC then - if i + 6 > n then - return nil, nil - end - local height = data:byte(i + 3) * 256 + data:byte(i + 4) - local width = data:byte(i + 5) * 256 + data:byte(i + 6) - return width, height - end - if length < 2 then - return nil, nil - end - i = i + length - end - end - return nil, nil -end - --- Photos per export session when indexing exports JPEGs. Util.DEFAULT_EXPORT_CHUNK_SIZE = 25 @@ -3452,25 +3393,6 @@ function Util.exportChunkSize(configured) return n end ---- --- The run-end note for indexing with "Use Lightroom previews" on, when some --- photos had no cached preview big enough and were exported instead. --- @param exported number Photos exported instead. --- @param total number Photos that went through the preview check. --- @param minEdge number The smallest long edge a preview needed. --- @return string --- -function Util.previewFallbackWarning(exported, total, minEdge) - return tostring(exported) - .. " of " - .. tostring(total) - .. " photos had no Lightroom preview of at least " - .. tostring(minEdge) - .. " px and were exported instead, which is slower. To use previews for all of them next time, " - .. "select the photos and run Library > Previews > Build Standard-Sized Previews first " - .. "(with the standard preview size in Catalog Settings at least as large)." -end - --- -- One paragraph for the Plug-in Manager about the face recognizer, from -- GET /v1/faces/recognizer. diff --git a/plugin/README.md b/plugin/README.md index da0b3c39..022c2e82 100644 --- a/plugin/README.md +++ b/plugin/README.md @@ -39,8 +39,8 @@ The plugin is designed to work with local and cloud providers, while keeping Lig - Every processed photo also gets a small preview (256 px, ~13 KB, stored by the backend beside the index) for the browser search page. Photos indexed before previews were stored get theirs from *Add Missing Search Previews...* (`TaskFillSearchPreviews.lua`): it asks the backend which - indexed photos lack one, takes Lightroom's cached preview where it is big enough - (`LightroomPreviews.lua`) and a small export otherwise, and touches nothing else + indexed photos lack one, exports small JPEGs of those in batches, and touches nothing else. + (Not `requestJpegThumbnail`: Lightroom delivers a few previews and then stops answering.) ### Advanced Search @@ -317,9 +317,7 @@ In the plugin settings dialog you can configure: The section reports why it is unavailable when the host cannot use it — a source build without the `llamacpp` feature, or a missing MLX helper. - Export size and quality used for AI processing, photos per export session (indexing exports - in batches of that size, `Util.exportChunkSize`, so Lightroom frees its memory between them), - and whether indexing may use Lightroom's cached previews instead of exporting - (`LightroomPreviews.lua`; photos without a large-enough preview are exported) + in batches of that size, `Util.exportChunkSize`, so Lightroom frees its memory between them) - Prompt presets — a preset is the *system* prompt: which expert is looking at the photo, which vocabulary they use, how specific they may be, and what they must never invent. **Default** (the analytical voice) and **Family & diff --git a/plugin/spec/util_export_and_faces_spec.lua b/plugin/spec/util_export_and_faces_spec.lua new file mode 100644 index 00000000..a374081f --- /dev/null +++ b/plugin/spec/util_export_and_faces_spec.lua @@ -0,0 +1,44 @@ +-- Pure helpers behind indexing's batched exports and the Plug-in Manager's +-- face recognition box. + +local Util = require("Util") + +describe("Util.exportChunkSize", function() + it("defaults when nothing usable is saved", function() + assert.are.equal(Util.DEFAULT_EXPORT_CHUNK_SIZE, Util.exportChunkSize(nil)) + assert.are.equal(Util.DEFAULT_EXPORT_CHUNK_SIZE, Util.exportChunkSize("lots")) + assert.are.equal(Util.DEFAULT_EXPORT_CHUNK_SIZE, Util.exportChunkSize(0 / 0)) + end) + + it("keeps a saved size within bounds", function() + assert.are.equal(40, Util.exportChunkSize("40")) + assert.are.equal(7, Util.exportChunkSize(7.9)) + assert.are.equal(1, Util.exportChunkSize(-3)) + assert.are.equal(200, Util.exportChunkSize(5000)) + end) +end) + +describe("Util.faceRecognizerSummary", function() + local state = { + active = "auraface", + recognizers = { + { id = "facenet", name = "FaceNet", description = "Included." }, + { id = "auraface", name = "AuraFace", description = "More accurate." }, + }, + warnings = {}, + } + + it("names the recognizer in use", function() + assert.are.equal("In use: AuraFace. More accurate.", Util.faceRecognizerSummary(state)) + end) + + it("adds the backend's first warning", function() + state.warnings = { "AuraFace was chosen, but its model file is missing." } + assert.truthy(Util.faceRecognizerSummary(state):find("model file is missing", 1, true)) + end) + + it("says so when the answer is unusable", function() + assert.truthy(Util.faceRecognizerSummary(nil):find("Unknown", 1, true)) + assert.truthy(Util.faceRecognizerSummary({ active = "x", recognizers = {} }):find("Unknown", 1, true)) + end) +end) diff --git a/plugin/spec/util_preview_render_spec.lua b/plugin/spec/util_preview_render_spec.lua deleted file mode 100644 index be1e3d4e..00000000 --- a/plugin/spec/util_preview_render_spec.lua +++ /dev/null @@ -1,105 +0,0 @@ --- Util.jpegDimensions: reads a JPEG's size from its frame header, which is how --- a Lightroom preview is checked against the size an export would have had. - -local Util = require("Util") - -local function u16(n) - return string.char(math.floor(n / 256), n % 256) -end - --- SOI, an APP0 segment, then a frame header of the given marker and size. -local function jpeg(marker, width, height) - local app0 = "\255\224" .. u16(16) .. "JFIF\0" .. string.rep("\0", 9) - local sof = "\255" .. string.char(marker) .. u16(17) .. "\8" .. u16(height) .. u16(width) .. string.rep("\0", 10) - return "\255\216" .. app0 .. sof .. "\255\218" -end - -describe("Util.jpegDimensions", function() - it("reads a baseline frame header after other segments", function() - local w, h = Util.jpegDimensions(jpeg(0xC0, 1024, 683)) - assert.are.equal(1024, w) - assert.are.equal(683, h) - end) - - it("reads a progressive frame header", function() - local w, h = Util.jpegDimensions(jpeg(0xC2, 640, 960)) - assert.are.equal(640, w) - assert.are.equal(960, h) - end) - - it("skips a Huffman table, which shares the SOF marker range", function() - local dht = "\255\196" .. u16(5) .. "\0\0\0" - local data = "\255\216" .. dht .. jpeg(0xC0, 300, 200):sub(3) - local w, h = Util.jpegDimensions(data) - assert.are.equal(300, w) - assert.are.equal(200, h) - end) - - it("reads a real file", function() - local fh = assert(io.open("server-rs/testdata/astronaut.jpg", "rb")) - local data = fh:read("*a") - fh:close() - local w, h = Util.jpegDimensions(data) - assert.are.equal(512, w) - assert.are.equal(512, h) - end) - - it("answers nil for anything that is not a usable JPEG", function() - assert.is_nil(Util.jpegDimensions(nil)) - assert.is_nil(Util.jpegDimensions("")) - assert.is_nil(Util.jpegDimensions("\137PNG\r\n\26\n")) - -- Truncated inside the frame header. - assert.is_nil(Util.jpegDimensions(jpeg(0xC0, 10, 10):sub(1, 26))) - -- Image data before any frame header. - assert.is_nil(Util.jpegDimensions("\255\216\255\218\0\0")) - end) -end) - -describe("Util.exportChunkSize", function() - it("defaults when nothing usable is saved", function() - assert.are.equal(Util.DEFAULT_EXPORT_CHUNK_SIZE, Util.exportChunkSize(nil)) - assert.are.equal(Util.DEFAULT_EXPORT_CHUNK_SIZE, Util.exportChunkSize("lots")) - assert.are.equal(Util.DEFAULT_EXPORT_CHUNK_SIZE, Util.exportChunkSize(0 / 0)) - end) - - it("keeps a saved size within bounds", function() - assert.are.equal(40, Util.exportChunkSize("40")) - assert.are.equal(7, Util.exportChunkSize(7.9)) - assert.are.equal(1, Util.exportChunkSize(-3)) - assert.are.equal(200, Util.exportChunkSize(5000)) - end) -end) - -describe("Util.previewFallbackWarning", function() - it("says how many were exported and what makes the next run use previews", function() - local text = Util.previewFallbackWarning(3, 40, 768) - assert.truthy(text:find("3 of 40 photos", 1, true)) - assert.truthy(text:find("768 px", 1, true)) - assert.truthy(text:find("Build Standard-Sized Previews", 1, true)) - end) -end) - -describe("Util.faceRecognizerSummary", function() - local state = { - active = "auraface", - recognizers = { - { id = "facenet", name = "FaceNet", description = "Included." }, - { id = "auraface", name = "AuraFace", description = "More accurate." }, - }, - warnings = {}, - } - - it("names the recognizer in use", function() - assert.are.equal("In use: AuraFace. More accurate.", Util.faceRecognizerSummary(state)) - end) - - it("adds the backend's first warning", function() - state.warnings = { "AuraFace was chosen, but its model file is missing." } - assert.truthy(Util.faceRecognizerSummary(state):find("model file is missing", 1, true)) - end) - - it("says so when the answer is unusable", function() - assert.truthy(Util.faceRecognizerSummary(nil):find("Unknown", 1, true)) - assert.truthy(Util.faceRecognizerSummary({ active = "x", recognizers = {} }):find("Unknown", 1, true)) - end) -end) From 1ae4a82b67f74085b3772391f05e8ee8fa0a4f1a Mon Sep 17 00:00:00 2001 From: Bastian Machek <16717398+bmachek@users.noreply.github.com> Date: Fri, 30 Oct 2026 05:34:43 +0100 Subject: [PATCH 05/10] refactor: rearrange menu items --- plugin/LrGeniusAI.lrdevplugin/Info.lua | 36 +++++++++----------------- 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/plugin/LrGeniusAI.lrdevplugin/Info.lua b/plugin/LrGeniusAI.lrdevplugin/Info.lua index 1647adf0..2f643094 100644 --- a/plugin/LrGeniusAI.lrdevplugin/Info.lua +++ b/plugin/LrGeniusAI.lrdevplugin/Info.lua @@ -50,10 +50,6 @@ return { title = LOC("$$$/LrGeniusAI/Menu/RetrieveMetadata=Retrieve Metadata from Backend..."), file = "TaskRetrieveMetadata.lua", }, - { - title = "Add Missing Search Previews...", - file = "TaskFillSearchPreviews.lua", - }, { title = LOC("$$$/LrGeniusAI/Menu/People=People..."), file = "TaskPeople.lua", @@ -66,14 +62,6 @@ return { title = LOC("$$$/LrGeniusAI/Menu/FindSimilarFaces=Find Similar Faces..."), file = "TaskFindSimilarFaces.lua", }, - { - title = LOC("$$$/LrGeniusAI/Training/MenuItem=Save Edits as AI Training Examples..."), - file = "TaskTrainFromEdits.lua", - }, - { - title = LOC("$$$/LrGeniusAI/Menu/DeduplicateKeywords=Deduplicate Keyword Synonyms..."), - file = "TaskDeduplicateKeywords.lua", - }, }, LrExportMenuItems = { @@ -105,10 +93,6 @@ return { title = LOC("$$$/LrGeniusAI/Menu/RetrieveMetadata=Retrieve Metadata from Backend..."), file = "TaskRetrieveMetadata.lua", }, - { - title = "Add Missing Search Previews...", - file = "TaskFillSearchPreviews.lua", - }, { title = LOC("$$$/LrGeniusAI/Menu/People=People..."), file = "TaskPeople.lua", @@ -121,14 +105,6 @@ return { title = LOC("$$$/LrGeniusAI/Menu/FindSimilarFaces=Find Similar Faces..."), file = "TaskFindSimilarFaces.lua", }, - { - title = LOC("$$$/LrGeniusAI/Training/MenuItem=Save Edits as AI Training Examples..."), - file = "TaskTrainFromEdits.lua", - }, - { - title = LOC("$$$/LrGeniusAI/Menu/DeduplicateKeywords=Deduplicate Keyword Synonyms..."), - file = "TaskDeduplicateKeywords.lua", - }, }, LrHelpMenuItems = { @@ -144,6 +120,18 @@ return { title = LOC("$$$/LrGeniusAI/Menu/ExportCullFixture=Export Culling Fixture..."), file = "TaskExportCullFixture.lua", }, + { + title = "Add Missing Search Previews...", + file = "TaskFillSearchPreviews.lua", + }, + { + title = LOC("$$$/LrGeniusAI/Training/MenuItem=Save Edits as AI Training Examples..."), + file = "TaskTrainFromEdits.lua", + }, + { + title = LOC("$$$/LrGeniusAI/Menu/DeduplicateKeywords=Deduplicate Keyword Synonyms..."), + file = "TaskDeduplicateKeywords.lua", + }, }, LrShutdownApp = "ShutdownApp.lua", From 7574d296aba3daa3c987e3d5d066591bae15ea21 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 04:47:41 +0000 Subject: [PATCH 06/10] Export: pin sRGB and no enlarging; point docs to the Help menu Indexing exports now state the colour space (sRGB) and never enlarge a photo smaller than the export size, so two exports of the same photo always match instead of depending on Lightroom's defaults. The menu rearrangement moved Add Missing Search Previews, Save Edits as AI Training Examples and Deduplicate Keyword Synonyms to the Help menu; the wiki, README, search page banner, the no_thumbnail error and the AI Edit hint now say so. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HaMMvqfgsKcoNWQaKy3c9C --- README.md | 4 ++-- docs/wiki/Dev-Backend-API.md | 2 +- docs/wiki/Dev-Project-README.md | 4 ++-- docs/wiki/FAQ.md | 2 +- docs/wiki/Help-Advanced-Search.md | 2 +- docs/wiki/Help-Analyze-and-Index.md | 4 ++-- docs/wiki/Help-Keyword-Dedup-and-Declutter.md | 2 +- docs/wiki/Help-Performance-Tips.md | 4 ++++ docs/wiki/Help-Train-From-Edits.md | 2 +- docs/wiki/Plugin-Guide.md | 9 +++++++-- plugin/LrGeniusAI.lrdevplugin/APISearchIndex.lua | 9 +++++++++ plugin/LrGeniusAI.lrdevplugin/TaskAiEditPhotos.lua | 2 +- server-rs/crates/lrg-api/src/routes/index.rs | 2 +- server-rs/crates/lrg-api/src/ui/search.html | 4 ++-- 14 files changed, 35 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index b62ec4ba..7ffd53b9 100644 --- a/README.md +++ b/README.md @@ -64,8 +64,8 @@ Whether you prefer running local models to ensure maximum privacy or want to lev - **Cull Similar Photos...** *(beta)* — burst grouping and auto-ranking - **People...** — face clusters and named person collections - **Find Similar Images...** — find near-duplicates or visually similar photos - - **Deduplicate Keyword Synonyms...** — clean up synonym sprawl in your catalog -6. For AI Edit, first teach it your style: edit a few photos by hand and run **Save Edits as AI Training Examples...**. AI Edit builds every recipe from those examples and needs at least five of them — it calls no LLM. Keep **Review each proposed edit before applying it** enabled while you validate the results. + - **Deduplicate Keyword Synonyms...** — clean up synonym sprawl in your catalog (in the **Help** menu) +6. For AI Edit, first teach it your style: edit a few photos by hand and run **Save Edits as AI Training Examples...** (in the **Help** menu). AI Edit builds every recipe from those examples and needs at least five of them — it calls no LLM. Keep **Review each proposed edit before applying it** enabled while you validate the results. *For comprehensive details, model setup guides, and tips, please visit [lrgenius.com/help](http://lrgenius.com/help/).* diff --git a/docs/wiki/Dev-Backend-API.md b/docs/wiki/Dev-Backend-API.md index e26fe427..ccf1dafb 100644 --- a/docs/wiki/Dev-Backend-API.md +++ b/docs/wiki/Dev-Backend-API.md @@ -326,7 +326,7 @@ them again. ### `POST /v1/photos/thumbnails/missing` Which **indexed** photos have no search-page preview — what the plug-in's -*Library ▸ Plug-in Extras ▸ Add Missing Search Previews* asks before it renders +*Help ▸ Add Missing Search Previews* asks before it renders anything. Body (JSON, every field optional): | Field | Meaning | diff --git a/docs/wiki/Dev-Project-README.md b/docs/wiki/Dev-Project-README.md index 5f26e6c6..c0b3f1e4 100644 --- a/docs/wiki/Dev-Project-README.md +++ b/docs/wiki/Dev-Project-README.md @@ -68,8 +68,8 @@ Whether you prefer running local models to ensure maximum privacy or want to lev - **Cull Similar Photos...** *(beta)* — burst grouping and auto-ranking - **People...** — face clusters and named person collections - **Find Similar Images...** — find near-duplicates or visually similar photos - - **Deduplicate Keyword Synonyms...** — clean up synonym sprawl in your catalog -6. For AI Edit, first teach it your style: edit a few photos by hand and run **Save Edits as AI Training Examples...**. AI Edit builds every recipe from those examples and needs at least five of them — it calls no LLM. Keep **Review each proposed edit before applying it** enabled while you validate the results. + - **Deduplicate Keyword Synonyms...** — clean up synonym sprawl in your catalog (in the **Help** menu) +6. For AI Edit, first teach it your style: edit a few photos by hand and run **Save Edits as AI Training Examples...** (in the **Help** menu). AI Edit builds every recipe from those examples and needs at least five of them — it calls no LLM. Keep **Review each proposed edit before applying it** enabled while you validate the results. *For comprehensive details, model setup guides, and tips, please visit [lrgenius.com/help](http://lrgenius.com/help/).* diff --git a/docs/wiki/FAQ.md b/docs/wiki/FAQ.md index 3ab387d0..e21c74ab 100644 --- a/docs/wiki/FAQ.md +++ b/docs/wiki/FAQ.md @@ -182,7 +182,7 @@ None. AI Edit does not call a language model at all — it interpolates the deve ### AI Edit says my style profile is not ready -It needs at least five saved training examples before it can produce anything. Edit some photos the way you like them and run **Save Edits as AI Training Examples** (*Library → Plug-in Extras*). See [Help: Train from Edits](Help-Train-From-Edits). +It needs at least five saved training examples before it can produce anything. Edit some photos the way you like them and run **Save Edits as AI Training Examples** (*Help* menu). See [Help: Train from Edits](Help-Train-From-Edits). ### I want to review edits before they are applied diff --git a/docs/wiki/Help-Advanced-Search.md b/docs/wiki/Help-Advanced-Search.md index 1d432cc0..59e69215 100644 --- a/docs/wiki/Help-Advanced-Search.md +++ b/docs/wiki/Help-Advanced-Search.md @@ -165,7 +165,7 @@ Index Photos** stores for each photo it processes, whatever you ask it to do; they take about 13 KB a photo on the backend's disk, roughly 130 MB per 10,000 photos, beside the search index. Photos indexed before this version have none yet: they show *No preview yet*, and the page says how many. To add them, run -**Library ▸ Plug-in Extras ▸ Add Missing Search Previews...** — for the whole +**Help ▸ Add Missing Search Previews...** — for the whole catalog, or click **Show in Lightroom** first and choose *Selected photos*. It only adds the previews, from a small export of each photo; nothing is analyzed again. diff --git a/docs/wiki/Help-Analyze-and-Index.md b/docs/wiki/Help-Analyze-and-Index.md index 13bb1eff..1d37bcb6 100644 --- a/docs/wiki/Help-Analyze-and-Index.md +++ b/docs/wiki/Help-Analyze-and-Index.md @@ -107,8 +107,8 @@ photos with existing data* skips everything else for a photo. If one cannot be saved, the completion dialog lists it as a warning; the photo is indexed all the same. -Photos indexed before previews existed get theirs from **Library ▸ Plug-in -Extras ▸ Add Missing Search Previews...**: choose selected photos, the current +Photos indexed before previews existed get theirs from **Help ▸ Add Missing +Search Previews...**: choose selected photos, the current view or the whole catalog, and it adds a preview to every indexed photo that has none, from a small export of each photo, without analyzing anything again. (A run of Analyze & Index that processes a photo adds one too, but *New or unprocessed photos* leaves diff --git a/docs/wiki/Help-Keyword-Dedup-and-Declutter.md b/docs/wiki/Help-Keyword-Dedup-and-Declutter.md index f0036456..3542a400 100644 --- a/docs/wiki/Help-Keyword-Dedup-and-Declutter.md +++ b/docs/wiki/Help-Keyword-Dedup-and-Declutter.md @@ -17,7 +17,7 @@ This is a **catalog-modifying operation**. Back up your catalog before running i ### How to run it -`Library -> Plug-in Extras -> Deduplicate Keyword Synonyms...` +`Help -> Deduplicate Keyword Synonyms...` The workflow has five steps: diff --git a/docs/wiki/Help-Performance-Tips.md b/docs/wiki/Help-Performance-Tips.md index ff1b8c99..de9902eb 100644 --- a/docs/wiki/Help-Performance-Tips.md +++ b/docs/wiki/Help-Performance-Tips.md @@ -89,6 +89,10 @@ Manager → LrGeniusAI* control that export: 1024, 2048, 3072, 4096. - **Export JPEG quality in percent** — default **60%**. +Everything else about the export is fixed so that two exports of the same +photo always come out the same: **sRGB**, no output sharpening, and photos +smaller than the export size are **not enlarged**. + Larger/higher-quality exports mean bigger uploads (cloud) and more pixels for a local model to decode (local) — both cost time, and for cloud providers also cost more tokens/money. **1024px / 60%** is a reasonable default for diff --git a/docs/wiki/Help-Train-From-Edits.md b/docs/wiki/Help-Train-From-Edits.md index 2a56c3de..31634592 100644 --- a/docs/wiki/Help-Train-From-Edits.md +++ b/docs/wiki/Help-Train-From-Edits.md @@ -4,7 +4,7 @@ The **Save Edits as AI Training Examples** workflow lets you teach LrGeniusAI yo ## How to use it -`Library → Plug-in Extras → Save Edits as AI Training Examples...` +`Help → Save Edits as AI Training Examples...` 1. Edit a set of photos in Lightroom's Develop module the way you like them. 2. Select those photos in the Library grid. diff --git a/docs/wiki/Plugin-Guide.md b/docs/wiki/Plugin-Guide.md index 1be4bc59..e6fb484e 100644 --- a/docs/wiki/Plugin-Guide.md +++ b/docs/wiki/Plugin-Guide.md @@ -8,7 +8,7 @@ For detailed technical usage of the plugin component, please view the [`plugin/R ## All menu items -All actions are available under `Library → Plug-in Extras`: +The everyday actions are under `Library → Plug-in Extras`: | Menu item | What it does | |---|---| @@ -17,11 +17,16 @@ All actions are available under `Library → Plug-in Extras`: | Advanced Search... | Semantic free-text photo search | | Cull Similar Photos... | Burst grouping, ranking, Picks/Alternates/Rejects | | Retrieve Metadata from Backend... | Pull AI-generated metadata back into Lightroom | -| Add Missing Search Previews... | Give photos indexed before previews existed their picture on the browser search page, without re-indexing | | Import Metadata from Catalog... | Sync existing Lightroom metadata into backend | | People... | Browse face clusters, assign names, open person collections | | Find Similar Faces... | Find photos with the same face as a selected photo | | Find Similar Images... | Find near-duplicates or visually similar photos | + +Occasional and maintenance actions are in the `Help` menu: + +| Menu item | What it does | +|---|---| +| Add Missing Search Previews... | Give photos indexed before previews existed their picture on the browser search page, without re-indexing | | Save Edits as AI Training Examples... | Feed your own edits to the AI as style reference | | Deduplicate Keyword Synonyms... | Find and merge near-duplicate keywords in your catalog | diff --git a/plugin/LrGeniusAI.lrdevplugin/APISearchIndex.lua b/plugin/LrGeniusAI.lrdevplugin/APISearchIndex.lua index bbf6e493..75a41d74 100644 --- a/plugin/LrGeniusAI.lrdevplugin/APISearchIndex.lua +++ b/plugin/LrGeniusAI.lrdevplugin/APISearchIndex.lua @@ -291,10 +291,19 @@ local ENDPOINTS = { AGENT_COMMANDS = "/v1/agent/commands", -- suffix //result, //progress } +-- Every setting that changes the pixels is stated rather than left to +-- Lightroom's defaults: embeddings, aesthetic scores and faces are compared +-- across runs, so two exports of the same photo have to come out the same. local EXPORT_SETTINGS = { LR_export_destinationType = "specificFolder", LR_export_useSubfolder = false, LR_format = "JPEG", + -- The models expect sRGB, and a profile Lightroom picks by default could + -- change between versions or presets. + LR_export_colorSpace = "sRGB", + -- A photo smaller than the export size stays as it is: upscaling adds no + -- detail, only interpolation the models would read as softness. + LR_size_doNotEnlarge = true, LR_jpeg_quality = tonumber(prefs.exportQuality) or 60, LR_minimizeEmbeddedMetadata = false, LR_outputSharpeningOn = false, diff --git a/plugin/LrGeniusAI.lrdevplugin/TaskAiEditPhotos.lua b/plugin/LrGeniusAI.lrdevplugin/TaskAiEditPhotos.lua index ddce1951..9ba2f28b 100644 --- a/plugin/LrGeniusAI.lrdevplugin/TaskAiEditPhotos.lua +++ b/plugin/LrGeniusAI.lrdevplugin/TaskAiEditPhotos.lua @@ -329,7 +329,7 @@ LrTasks.startAsyncTask(function() LrDialogs.message( LOC("$$$/LrGeniusAI/TaskAiEditPhotos/NoStyleProfileTitle=Style profile not ready"), LOC( - "$$$/LrGeniusAI/TaskAiEditPhotos/NoStyleProfile=AI Edit builds every recipe from your own saved edits, and it needs at least ^1 training examples to do that. You currently have ^2.\n\nEdit a handful of photos the way you like them, then run Library → Plug-in Extras → Save Edits as AI Training Examples and start AI Edit again.", + "$$$/LrGeniusAI/TaskAiEditPhotos/NoStyleProfile=AI Edit builds every recipe from your own saved edits, and it needs at least ^1 training examples to do that. You currently have ^2.\n\nEdit a handful of photos the way you like them, then run Help → Save Edits as AI Training Examples and start AI Edit again.", tostring(MIN_TRAINING_EXAMPLES), tostring(trainingCount) ), diff --git a/server-rs/crates/lrg-api/src/routes/index.rs b/server-rs/crates/lrg-api/src/routes/index.rs index e452c519..3b3b910e 100644 --- a/server-rs/crates/lrg-api/src/routes/index.rs +++ b/server-rs/crates/lrg-api/src/routes/index.rs @@ -590,7 +590,7 @@ async fn get_thumbnail( None => ( StatusCode::NOT_FOUND, Json(json!({ - "error": "No preview is stored for this photo. Run Library > Plug-in Extras > Add Missing Search Previews to add one.", + "error": "No preview is stored for this photo. Run Help > Add Missing Search Previews in Lightroom to add one.", "error_code": "no_thumbnail", })), ) diff --git a/server-rs/crates/lrg-api/src/ui/search.html b/server-rs/crates/lrg-api/src/ui/search.html index 8c5d7e89..2f3370ad 100644 --- a/server-rs/crates/lrg-api/src/ui/search.html +++ b/server-rs/crates/lrg-api/src/ui/search.html @@ -712,8 +712,8 @@

Search

banner( missing + (missing === 1 ? " of these photos has" : " of these photos have") + - " no preview yet: indexed before LrGeniusAI stored previews. To add them, run Library ▸ " + - "Plug-in Extras ▸ Add Missing Search Previews in Lightroom — for the whole catalog, or click " + + " no preview yet: indexed before LrGeniusAI stored previews. To add them, run Help ▸ " + + "Add Missing Search Previews in Lightroom — for the whole catalog, or click " + "“Show in Lightroom” first and choose “Selected photos”. It only adds the previews; nothing " + "else is analyzed again.", "note", From 08000d7a10642f31b922c5ec13de484364164a53 Mon Sep 17 00:00:00 2001 From: Bastian Machek <16717398+bmachek@users.noreply.github.com> Date: Fri, 30 Oct 2026 07:22:46 +0100 Subject: [PATCH 07/10] AuraFace: record InsightFace's reference embedding, pin the model revision scripts/auraface_reference.py, run against fal/AuraFace-v1 at af6d057c9b0ec4071d4c49c80e3539258798b609 (glintr100.onnx, sha256 a7933ea5330113b01c9b60351d8f4c33003f145d8470ac5f0e52ee2effe25c60, the same file the backend's download fetches), wrote auraface_embedding into face_goldens.json. InsightFace picked input_mean/std 127.5, as auraface.rs assumes, and the Rust chain matches the reference at cosine 0.9988 (auraface_matches_insightface_reference, needs > 0.99), so the golden test no longer skips once the model is downloaded. The download is pinned to that commit instead of `main`, so a later push to the Hugging Face repository cannot change the model under a verified chain. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HaMMvtFGGZAyc9nbCZiG6T --- docs/wiki/Dev-Server-README.md | 9 ++++++--- server-rs/README.md | 9 ++++++--- server-rs/crates/lrg-api/src/routes/face_recognizer.rs | 10 ++++++---- server-rs/testdata/face_goldens.json | 2 +- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/docs/wiki/Dev-Server-README.md b/docs/wiki/Dev-Server-README.md index 687c5cb5..1a4dee9b 100644 --- a/docs/wiki/Dev-Server-README.md +++ b/docs/wiki/Dev-Server-README.md @@ -427,11 +427,14 @@ and `POST /v1/faces/recognizer` switches. The choice is saved in `~/.cache/lrgenius/models/face_recognizer.txt` (or `LRG_FACE_RECOGNIZER_FILE`) and restored at start. It runs on the same YuNet landmarks and the same 112x112 ArcFace crop InsightFace makes, RGB, `(v - 127.5) / 127.5` -(`lrg-ml/src/auraface.rs`). To check that chain numerically, write -InsightFace's own embedding into the golden file and run the golden test: +(`lrg-ml/src/auraface.rs`). The download is pinned to the Hugging Face commit +(`AURAFACE_REVISION` in `lrg-api/src/routes/face_recognizer.rs`) whose model +InsightFace's own embedding in `testdata/face_goldens.json` was made from; the +golden test checks the chain against it (cosine 0.9988, needs > 0.99). To move +the pin, record the reference for the new commit and run the test again: ```bash -uv run --no-project server-rs/scripts/auraface_reference.py --output ~/.cache/lrgenius/models/auraface_glintr100.onnx +uv run --no-project server-rs/scripts/auraface_reference.py --revision --output ~/.cache/lrgenius/models/auraface_glintr100.onnx cargo test -p lrg-ml --test face_goldens -- auraface # LRG_REQUIRE_GOLDENS=auraface to fail instead of skip ``` diff --git a/server-rs/README.md b/server-rs/README.md index 50912e37..f4be4da9 100644 --- a/server-rs/README.md +++ b/server-rs/README.md @@ -423,11 +423,14 @@ and `POST /v1/faces/recognizer` switches. The choice is saved in `~/.cache/lrgenius/models/face_recognizer.txt` (or `LRG_FACE_RECOGNIZER_FILE`) and restored at start. It runs on the same YuNet landmarks and the same 112x112 ArcFace crop InsightFace makes, RGB, `(v - 127.5) / 127.5` -(`lrg-ml/src/auraface.rs`). To check that chain numerically, write -InsightFace's own embedding into the golden file and run the golden test: +(`lrg-ml/src/auraface.rs`). The download is pinned to the Hugging Face commit +(`AURAFACE_REVISION` in `lrg-api/src/routes/face_recognizer.rs`) whose model +InsightFace's own embedding in `testdata/face_goldens.json` was made from; the +golden test checks the chain against it (cosine 0.9988, needs > 0.99). To move +the pin, record the reference for the new commit and run the test again: ```bash -uv run --no-project server-rs/scripts/auraface_reference.py --output ~/.cache/lrgenius/models/auraface_glintr100.onnx +uv run --no-project server-rs/scripts/auraface_reference.py --revision --output ~/.cache/lrgenius/models/auraface_glintr100.onnx cargo test -p lrg-ml --test face_goldens -- auraface # LRG_REQUIRE_GOLDENS=auraface to fail instead of skip ``` diff --git a/server-rs/crates/lrg-api/src/routes/face_recognizer.rs b/server-rs/crates/lrg-api/src/routes/face_recognizer.rs index 512fcc1f..4165564d 100644 --- a/server-rs/crates/lrg-api/src/routes/face_recognizer.rs +++ b/server-rs/crates/lrg-api/src/routes/face_recognizer.rs @@ -31,10 +31,12 @@ const DOWNLOAD_KEY: &str = "face_recognizer"; /// Where AuraFace is published, Apache-2.0. const AURAFACE_REPO: &str = "fal/AuraFace-v1"; -/// The repository revision downloaded. `main` until the exact commit has been -/// verified against `scripts/auraface_reference.py`'s reference embedding; -/// pin it then, like YuNet's commit in `export_face_models.py`. -const AURAFACE_REVISION: &str = "main"; +/// The repository revision downloaded: the commit whose `glintr100.onnx` +/// (sha256 `a7933ea5…`) `scripts/auraface_reference.py` made the golden +/// reference embedding from, which the Rust chain matches (cosine 0.9988). +/// Pinned like YuNet's commit in `export_face_models.py`; moving it means +/// running the script for the new commit and the golden test again. +const AURAFACE_REVISION: &str = "af6d057c9b0ec4071d4c49c80e3539258798b609"; /// The recognition network in that repository (the others are InsightFace's /// detector and attribute models, which this pipeline does not use). const AURAFACE_FILE: &str = "glintr100.onnx"; diff --git a/server-rs/testdata/face_goldens.json b/server-rs/testdata/face_goldens.json index e903228c..cc1e9918 100644 --- a/server-rs/testdata/face_goldens.json +++ b/server-rs/testdata/face_goldens.json @@ -1 +1 @@ -{"astronaut_512": {"width": 512, "height": 512, "faces": [{"bbox": [179.13382568359376, 64.11212768554688, 269.59995727539064, 173.82073364257815], "kps": [[201.74453125, 104.982177734375], [245.293115234375, 105.34576416015625], [220.66279296875, 126.8361083984375], [203.65838623046875, 144.840771484375], [241.990625, 144.978173828125]], "det_score": 0.9431958198547363, "embedding": [0.013513335958123207, 0.03552423417568207, 0.05688447505235672, -0.003538884222507477, 0.02187528647482395, 0.0170944444835186, -0.05028330907225609, -0.020455366000533104, 0.039024800062179565, -0.03140786290168762, -0.034710802137851715, 0.004504606127738953, 0.021206719800829887, 0.07965287566184998, -0.009218870662152767, 0.014309441670775414, 0.08001935482025146, 0.12310704588890076, -0.008425913751125336, 0.023850679397583008, 0.05434480682015419, -0.04251399636268616, 0.019260529428720474, -0.0796394795179367, -0.04599601402878761, 0.01090801227837801, 0.036534059792757034, 0.10105296969413757, 0.0471428744494915, -0.008032332174479961, -0.004963373299688101, -0.06321012228727341, 0.001574227586388588, 0.012196275405585766, 0.023937180638313293, -0.0654715895652771, -0.013859586790204048, -0.01314372755587101, -0.014256936497986317, -0.028910299763083458, 0.005670937709510326, -0.03313359245657921, -0.03921167925000191, 0.028397666290402412, -0.0006350941839627922, -0.03402511775493622, -0.0021566262003034353, -0.03216635808348656, -0.04647371172904968, -0.036270055919885635, -0.09944566339254379, -0.004059445112943649, -0.015893708914518356, -0.048420052975416183, -0.04049826040863991, -0.06699362397193909, 7.786802598275244e-05, -0.040069978684186935, 0.03016781434416771, 0.04346166551113129, -0.006618059240281582, 0.0019008716335520148, 0.005818247329443693, 0.07368450611829758, -0.06130365654826164, -0.013340781442821026, 0.064145527780056, 0.04583815485239029, -0.03073897957801819, -0.03853690251708031, -0.012799565680325031, -0.058059561997652054, 0.06257904320955276, -0.0021041252184659243, -0.009523759596049786, -0.04205476865172386, 0.014514791779220104, 0.04201289266347885, -0.009640481323003769, -0.09532398730516434, -0.025162260979413986, -0.05480686202645302, 0.064853236079216, 0.04239547997713089, -0.07504183053970337, -0.00984091218560934, 0.0012075365521013737, 0.01484911609441042, -0.06894576549530029, -0.0001060811264324002, -0.03270347788929939, -0.0033645490184426308, -0.03434508666396141, 0.03429720923304558, 0.03971198946237564, 0.034459587186574936, -0.00634327158331871, -0.001879255403764546, -0.0366867296397686, -0.06267329305410385, 0.004698866978287697, 0.04788306728005409, -0.04991818964481354, 0.09232640266418457, -0.043738044798374176, -0.029016369953751564, 0.05856643617153168, 0.005075464956462383, -0.028227636590600014, 0.0540909618139267, -0.05452285334467888, -0.030619081109762192, -0.021506289020180702, 0.056030984967947006, -0.0033803230617195368, -0.047483984380960464, -0.03949197009205818, 0.062422975897789, 0.09065616875886917, -0.007638793904334307, 0.11554799973964691, 0.005865904968231916, -0.022464942187070847, -0.04997226595878601, -0.06903783977031708, 0.004956497810781002, -0.020672239363193512, 0.06734756380319595, -0.04037332534790039, -0.015917344018816948, -0.02903899736702442, 0.0695568397641182, -0.054893989115953445, -0.014048532582819462, 0.04485481232404709, 2.8090773412259296e-05, -0.009698979556560516, -0.08744174242019653, -0.06342674791812897, -0.003987821284681559, 0.0020103470887988806, -0.005338377319276333, -0.034117769449949265, -0.0201374851167202, -0.0022641406394541264, -0.06522877514362335, 0.002895863726735115, -0.08289957046508789, -0.00018432991055306047, -0.02961805835366249, 0.012410813011229038, 0.020212450996041298, 0.043496184051036835, 0.007806194014847279, 0.027856402099132538, -0.042755402624607086, -0.0629059448838234, -0.09362766146659851, -0.03411004692316055, -0.0019542600493878126, -0.04547019302845001, 0.09279822558164597, 0.01595512218773365, 0.073454849421978, 0.010674583725631237, 0.053088147193193436, 0.030728422105312347, -0.006268845405429602, -0.038154806941747665, 0.037238504737615585, 0.00657521840184927, 0.08429296314716339, -0.02138616144657135, -0.0054775262251496315, -0.0061383494175970554, -0.06239357218146324, -0.0217136200517416, 0.020265910774469376, 0.05558445304632187, 0.008317962288856506, 0.041851840913295746, 0.079482801258564, -0.03177275508642197, 0.022517766803503036, 0.012374709360301495, -0.02041841484606266, -0.08637694269418716, -0.03718717768788338, 0.08144901692867279, -0.03984607011079788, 0.03456014767289162, 0.01090577058494091, -0.07327531278133392, 0.027857055887579918, 0.014863698743283749, -0.019636178389191628, -0.12194974720478058, 0.03254003822803497, -0.02066223695874214, 0.03301236778497696, 0.05668412894010544, -0.008310968987643719, -0.04803905636072159, 0.049886304885149, -0.07669530063867569, 0.016248919069767, -0.07079877704381943, 0.014871063642203808, -0.0016133871395140886, -0.03471954166889191, 0.0689745619893074, -0.005593284033238888, 0.0013272984651848674, 0.0682763084769249, 0.03460555523633957, -0.068846195936203, -0.03545607626438141, 0.06510081142187119, -0.014944888651371002, 0.047962747514247894, 0.03945469856262207, 0.007993528619408607, -0.06502973288297653, -0.04954289644956589, -0.03448652848601341, 0.07605063170194626, 0.07064887881278992, -0.01840902492403984, -0.053421296179294586, -0.016993271186947823, -0.021873509511351585, -0.04822858050465584, 0.05935445427894592, -0.02505362592637539, -0.007795183919370174, 0.014592938125133514, 0.033059753477573395, -0.0384046696126461, 0.05274626240134239, 0.04485143721103668, -0.011236390098929405, 0.015228701755404472, 0.06673561036586761, -0.009807245805859566, -0.06605672836303711, 0.007128911558538675, 0.058747950941324234, 0.03623116761445999, -0.02068769372999668, 0.021895311772823334, -0.04574386775493622, -0.07888875901699066, 0.03716074675321579, -0.01292780227959156, -0.0506756454706192, 0.05770420283079147, 0.0791463702917099, 0.0315258763730526, -0.030738787725567818, -0.008314014412462711, -0.04067361727356911, 0.03553410992026329, -0.041514329612255096, -0.004513721913099289, 0.0026798867620527744, 0.0021248317789286375, 0.036588042974472046, -0.05030660703778267, 0.04895492643117905, 0.01702815666794777, 0.06177413836121559, -0.0531771257519722, 0.14458437263965607, -0.014505586586892605, 0.06699799001216888, 0.007921341806650162, -0.007899932563304901, 0.013478344306349754, 0.016841445118188858, -0.037101730704307556, -0.020296702161431313, -0.018203889951109886, 0.011302824132144451, 0.05995794013142586, -0.02333962917327881, 0.005980135872960091, 0.0012829732149839401, 0.003176286583766341, 0.05249307304620743, 0.02088780514895916, 0.0004660444683395326, -0.07463273406028748, -0.04935610666871071, -0.059951331466436386, 0.002659172285348177, -0.05773184821009636, 0.03082268498837948, 0.023267054930329323, 0.029713815078139305, -0.07082369178533554, -0.014689600095152855, -0.1154676079750061, -0.02439752034842968, 0.026169711723923683, -0.04399487003684044, 0.0563223697245121, -0.0010905432282015681, -0.04110633581876755, 0.004796264227479696, 0.007699532900005579, -0.012949839234352112, 0.031916022300720215, -0.028448093682527542, 0.004061872139573097, -0.035414356738328934, -0.0002821986563503742, -0.012017499655485153, 0.01821109838783741, 0.08835014700889587, 0.009840051643550396, -0.03154147416353226, -0.05408887565135956, 0.0033085590694099665, -0.008449471555650234, -0.010239382274448872, -0.04287136346101761, 0.029836509376764297, 0.042008209973573685, 0.04273183271288872, -0.03652992472052574, -0.020420966669917107, -0.07054346054792404, -0.020503902807831764, -0.061635617166757584, 0.0022443775087594986, 0.009013262577354908, 0.08362516760826111, -0.07620283216238022, -0.06025237217545509, -0.019406570121645927, -0.07664510607719421, -0.0038493177853524685, 0.025932347401976585, 0.0026112094055861235, -0.03527161106467247, 0.0797271654009819, -0.03176652640104294, 0.02738024666905403, -0.05965788662433624, 0.038413744419813156, 0.02902727574110031, -0.008306275121867657, 0.06418056786060333, 0.07139807939529419, 0.021623721346259117, -0.0725100114941597, -0.013597621582448483, 0.09489242732524872, 0.0021902918815612793, 0.030901115387678146, -0.030589327216148376, 0.018432799726724625, 0.018367454409599304, -0.023047106340527534, -0.034717120230197906, -0.019083602353930473, -0.03868735209107399, 0.01504641491919756, 0.040343787521123886, -0.001903304597362876, 5.903674264118308e-06, 0.012333793565630913, 0.029453566297888756, 0.026231318712234497, 0.018436554819345474, 0.020453665405511856, 0.07839369773864746, 0.07723172754049301, -0.04396659508347511, 0.028396710753440857, -0.00031788903288543224, 0.005847201216965914, 0.01983710564672947, -0.029132289811968803, 0.017187463119626045, 0.0010830587707459927, -0.029637612402439117, -0.0009637774201110005, 0.039146799594163895, -0.03852805867791176, -0.07248535007238388, 0.08601894229650497, -0.01364500354975462, 0.05485472455620766, -0.01557887438684702, 0.05817742273211479, -0.029398150742053986, 0.04611032083630562, 0.011346949264407158, -0.006507873069494963, 0.030659399926662445, -0.015012620948255062, -0.020676348358392715, 0.024143796414136887, 0.061287034302949905, -0.004231231287121773, -0.006039237603545189, -0.0030562167521566153, 0.008498754352331161, 0.043625373393297195, -0.030704230070114136, 0.052916716784238815, 0.06857573986053467, 0.1293746531009674, 0.04934697598218918, -0.037569817155599594, 0.02747318148612976, -0.038188088685274124, 0.03968730568885803, -0.05692492052912712, 0.021270377561450005, 0.03455786779522896, 0.033183079212903976, -0.04004547744989395, -0.033581480383872986, 0.023548904806375504, -0.05959898233413696, -0.05429805442690849, -0.09259957820177078, 0.020014530047774315, -0.025657203048467636, -0.031953807920217514, -0.04648406058549881, 0.041608188301324844, 0.057309553027153015, 0.08706308901309967, -0.01753014698624611, 0.022337261587381363, 0.11236553639173508, -0.013517208397388458, 0.02346067503094673, 0.07652797549962997, -0.011292807757854462, -0.05146007239818573, -0.013494198210537434, 0.017786167562007904, -0.008526128716766834, -0.058424316346645355, 0.01420593447983265, 0.06946490705013275, -0.05233963206410408, 0.025444619357585907, -0.015221566893160343, 0.06055983528494835, 0.004903245251625776, 0.004390350077301264, -0.04201102629303932, 0.020836681127548218, -0.004000192508101463, 0.026532670482993126, 0.05536966770887375, -0.017368722707033157, -0.04076547920703888, -0.04764010012149811, 0.08628389239311218, 0.05564761906862259, -0.0431993305683136, -0.07242660224437714, 0.060236282646656036, 0.08113879710435867, 0.03543011099100113, -0.05227779969573021, 0.04203886538743973, 0.01357585284858942, 0.06472063809633255, 0.04309173673391342, 0.08852197974920273, 0.01882972940802574, -0.028108973056077957, 0.035874202847480774, 0.07864027470350266, -0.001931990496814251, -0.0521618016064167, 0.009736504405736923, -0.07050395756959915, 0.05462994799017906, 0.017030352726578712, -0.04582693800330162, 0.013289899565279484, 0.03034176304936409, -0.07412642240524292, 0.017280444502830505, -0.07020281255245209, -0.010828903876245022, 0.013319304212927818, -0.04619235917925835, -0.04388128221035004, 0.036927368491888046, -0.021930893883109093, 0.03889084607362747, -0.031834475696086884, -0.02293967641890049, 0.0048389132134616375, -0.04287312552332878, -0.007713694125413895, -0.0095838513225317, 0.020498378202319145, -0.01259702444076538, -0.052085030823946, 0.014438627287745476, 0.008889690972864628, 0.02098825015127659]}]}} \ No newline at end of file +{"astronaut_512": {"width": 512, "height": 512, "faces": [{"bbox": [179.13382568359376, 64.11212768554688, 269.59995727539064, 173.82073364257815], "kps": [[201.74453125, 104.982177734375], [245.293115234375, 105.34576416015625], [220.66279296875, 126.8361083984375], [203.65838623046875, 144.840771484375], [241.990625, 144.978173828125]], "det_score": 0.9431958198547363, "embedding": [0.013513335958123207, 0.03552423417568207, 0.05688447505235672, -0.003538884222507477, 0.02187528647482395, 0.0170944444835186, -0.05028330907225609, -0.020455366000533104, 0.039024800062179565, -0.03140786290168762, -0.034710802137851715, 0.004504606127738953, 0.021206719800829887, 0.07965287566184998, -0.009218870662152767, 0.014309441670775414, 0.08001935482025146, 0.12310704588890076, -0.008425913751125336, 0.023850679397583008, 0.05434480682015419, -0.04251399636268616, 0.019260529428720474, -0.0796394795179367, -0.04599601402878761, 0.01090801227837801, 0.036534059792757034, 0.10105296969413757, 0.0471428744494915, -0.008032332174479961, -0.004963373299688101, -0.06321012228727341, 0.001574227586388588, 0.012196275405585766, 0.023937180638313293, -0.0654715895652771, -0.013859586790204048, -0.01314372755587101, -0.014256936497986317, -0.028910299763083458, 0.005670937709510326, -0.03313359245657921, -0.03921167925000191, 0.028397666290402412, -0.0006350941839627922, -0.03402511775493622, -0.0021566262003034353, -0.03216635808348656, -0.04647371172904968, -0.036270055919885635, -0.09944566339254379, -0.004059445112943649, -0.015893708914518356, -0.048420052975416183, -0.04049826040863991, -0.06699362397193909, 7.786802598275244e-05, -0.040069978684186935, 0.03016781434416771, 0.04346166551113129, -0.006618059240281582, 0.0019008716335520148, 0.005818247329443693, 0.07368450611829758, -0.06130365654826164, -0.013340781442821026, 0.064145527780056, 0.04583815485239029, -0.03073897957801819, -0.03853690251708031, -0.012799565680325031, -0.058059561997652054, 0.06257904320955276, -0.0021041252184659243, -0.009523759596049786, -0.04205476865172386, 0.014514791779220104, 0.04201289266347885, -0.009640481323003769, -0.09532398730516434, -0.025162260979413986, -0.05480686202645302, 0.064853236079216, 0.04239547997713089, -0.07504183053970337, -0.00984091218560934, 0.0012075365521013737, 0.01484911609441042, -0.06894576549530029, -0.0001060811264324002, -0.03270347788929939, -0.0033645490184426308, -0.03434508666396141, 0.03429720923304558, 0.03971198946237564, 0.034459587186574936, -0.00634327158331871, -0.001879255403764546, -0.0366867296397686, -0.06267329305410385, 0.004698866978287697, 0.04788306728005409, -0.04991818964481354, 0.09232640266418457, -0.043738044798374176, -0.029016369953751564, 0.05856643617153168, 0.005075464956462383, -0.028227636590600014, 0.0540909618139267, -0.05452285334467888, -0.030619081109762192, -0.021506289020180702, 0.056030984967947006, -0.0033803230617195368, -0.047483984380960464, -0.03949197009205818, 0.062422975897789, 0.09065616875886917, -0.007638793904334307, 0.11554799973964691, 0.005865904968231916, -0.022464942187070847, -0.04997226595878601, -0.06903783977031708, 0.004956497810781002, -0.020672239363193512, 0.06734756380319595, -0.04037332534790039, -0.015917344018816948, -0.02903899736702442, 0.0695568397641182, -0.054893989115953445, -0.014048532582819462, 0.04485481232404709, 2.8090773412259296e-05, -0.009698979556560516, -0.08744174242019653, -0.06342674791812897, -0.003987821284681559, 0.0020103470887988806, -0.005338377319276333, -0.034117769449949265, -0.0201374851167202, -0.0022641406394541264, -0.06522877514362335, 0.002895863726735115, -0.08289957046508789, -0.00018432991055306047, -0.02961805835366249, 0.012410813011229038, 0.020212450996041298, 0.043496184051036835, 0.007806194014847279, 0.027856402099132538, -0.042755402624607086, -0.0629059448838234, -0.09362766146659851, -0.03411004692316055, -0.0019542600493878126, -0.04547019302845001, 0.09279822558164597, 0.01595512218773365, 0.073454849421978, 0.010674583725631237, 0.053088147193193436, 0.030728422105312347, -0.006268845405429602, -0.038154806941747665, 0.037238504737615585, 0.00657521840184927, 0.08429296314716339, -0.02138616144657135, -0.0054775262251496315, -0.0061383494175970554, -0.06239357218146324, -0.0217136200517416, 0.020265910774469376, 0.05558445304632187, 0.008317962288856506, 0.041851840913295746, 0.079482801258564, -0.03177275508642197, 0.022517766803503036, 0.012374709360301495, -0.02041841484606266, -0.08637694269418716, -0.03718717768788338, 0.08144901692867279, -0.03984607011079788, 0.03456014767289162, 0.01090577058494091, -0.07327531278133392, 0.027857055887579918, 0.014863698743283749, -0.019636178389191628, -0.12194974720478058, 0.03254003822803497, -0.02066223695874214, 0.03301236778497696, 0.05668412894010544, -0.008310968987643719, -0.04803905636072159, 0.049886304885149, -0.07669530063867569, 0.016248919069767, -0.07079877704381943, 0.014871063642203808, -0.0016133871395140886, -0.03471954166889191, 0.0689745619893074, -0.005593284033238888, 0.0013272984651848674, 0.0682763084769249, 0.03460555523633957, -0.068846195936203, -0.03545607626438141, 0.06510081142187119, -0.014944888651371002, 0.047962747514247894, 0.03945469856262207, 0.007993528619408607, -0.06502973288297653, -0.04954289644956589, -0.03448652848601341, 0.07605063170194626, 0.07064887881278992, -0.01840902492403984, -0.053421296179294586, -0.016993271186947823, -0.021873509511351585, -0.04822858050465584, 0.05935445427894592, -0.02505362592637539, -0.007795183919370174, 0.014592938125133514, 0.033059753477573395, -0.0384046696126461, 0.05274626240134239, 0.04485143721103668, -0.011236390098929405, 0.015228701755404472, 0.06673561036586761, -0.009807245805859566, -0.06605672836303711, 0.007128911558538675, 0.058747950941324234, 0.03623116761445999, -0.02068769372999668, 0.021895311772823334, -0.04574386775493622, -0.07888875901699066, 0.03716074675321579, -0.01292780227959156, -0.0506756454706192, 0.05770420283079147, 0.0791463702917099, 0.0315258763730526, -0.030738787725567818, -0.008314014412462711, -0.04067361727356911, 0.03553410992026329, -0.041514329612255096, -0.004513721913099289, 0.0026798867620527744, 0.0021248317789286375, 0.036588042974472046, -0.05030660703778267, 0.04895492643117905, 0.01702815666794777, 0.06177413836121559, -0.0531771257519722, 0.14458437263965607, -0.014505586586892605, 0.06699799001216888, 0.007921341806650162, -0.007899932563304901, 0.013478344306349754, 0.016841445118188858, -0.037101730704307556, -0.020296702161431313, -0.018203889951109886, 0.011302824132144451, 0.05995794013142586, -0.02333962917327881, 0.005980135872960091, 0.0012829732149839401, 0.003176286583766341, 0.05249307304620743, 0.02088780514895916, 0.0004660444683395326, -0.07463273406028748, -0.04935610666871071, -0.059951331466436386, 0.002659172285348177, -0.05773184821009636, 0.03082268498837948, 0.023267054930329323, 0.029713815078139305, -0.07082369178533554, -0.014689600095152855, -0.1154676079750061, -0.02439752034842968, 0.026169711723923683, -0.04399487003684044, 0.0563223697245121, -0.0010905432282015681, -0.04110633581876755, 0.004796264227479696, 0.007699532900005579, -0.012949839234352112, 0.031916022300720215, -0.028448093682527542, 0.004061872139573097, -0.035414356738328934, -0.0002821986563503742, -0.012017499655485153, 0.01821109838783741, 0.08835014700889587, 0.009840051643550396, -0.03154147416353226, -0.05408887565135956, 0.0033085590694099665, -0.008449471555650234, -0.010239382274448872, -0.04287136346101761, 0.029836509376764297, 0.042008209973573685, 0.04273183271288872, -0.03652992472052574, -0.020420966669917107, -0.07054346054792404, -0.020503902807831764, -0.061635617166757584, 0.0022443775087594986, 0.009013262577354908, 0.08362516760826111, -0.07620283216238022, -0.06025237217545509, -0.019406570121645927, -0.07664510607719421, -0.0038493177853524685, 0.025932347401976585, 0.0026112094055861235, -0.03527161106467247, 0.0797271654009819, -0.03176652640104294, 0.02738024666905403, -0.05965788662433624, 0.038413744419813156, 0.02902727574110031, -0.008306275121867657, 0.06418056786060333, 0.07139807939529419, 0.021623721346259117, -0.0725100114941597, -0.013597621582448483, 0.09489242732524872, 0.0021902918815612793, 0.030901115387678146, -0.030589327216148376, 0.018432799726724625, 0.018367454409599304, -0.023047106340527534, -0.034717120230197906, -0.019083602353930473, -0.03868735209107399, 0.01504641491919756, 0.040343787521123886, -0.001903304597362876, 5.903674264118308e-06, 0.012333793565630913, 0.029453566297888756, 0.026231318712234497, 0.018436554819345474, 0.020453665405511856, 0.07839369773864746, 0.07723172754049301, -0.04396659508347511, 0.028396710753440857, -0.00031788903288543224, 0.005847201216965914, 0.01983710564672947, -0.029132289811968803, 0.017187463119626045, 0.0010830587707459927, -0.029637612402439117, -0.0009637774201110005, 0.039146799594163895, -0.03852805867791176, -0.07248535007238388, 0.08601894229650497, -0.01364500354975462, 0.05485472455620766, -0.01557887438684702, 0.05817742273211479, -0.029398150742053986, 0.04611032083630562, 0.011346949264407158, -0.006507873069494963, 0.030659399926662445, -0.015012620948255062, -0.020676348358392715, 0.024143796414136887, 0.061287034302949905, -0.004231231287121773, -0.006039237603545189, -0.0030562167521566153, 0.008498754352331161, 0.043625373393297195, -0.030704230070114136, 0.052916716784238815, 0.06857573986053467, 0.1293746531009674, 0.04934697598218918, -0.037569817155599594, 0.02747318148612976, -0.038188088685274124, 0.03968730568885803, -0.05692492052912712, 0.021270377561450005, 0.03455786779522896, 0.033183079212903976, -0.04004547744989395, -0.033581480383872986, 0.023548904806375504, -0.05959898233413696, -0.05429805442690849, -0.09259957820177078, 0.020014530047774315, -0.025657203048467636, -0.031953807920217514, -0.04648406058549881, 0.041608188301324844, 0.057309553027153015, 0.08706308901309967, -0.01753014698624611, 0.022337261587381363, 0.11236553639173508, -0.013517208397388458, 0.02346067503094673, 0.07652797549962997, -0.011292807757854462, -0.05146007239818573, -0.013494198210537434, 0.017786167562007904, -0.008526128716766834, -0.058424316346645355, 0.01420593447983265, 0.06946490705013275, -0.05233963206410408, 0.025444619357585907, -0.015221566893160343, 0.06055983528494835, 0.004903245251625776, 0.004390350077301264, -0.04201102629303932, 0.020836681127548218, -0.004000192508101463, 0.026532670482993126, 0.05536966770887375, -0.017368722707033157, -0.04076547920703888, -0.04764010012149811, 0.08628389239311218, 0.05564761906862259, -0.0431993305683136, -0.07242660224437714, 0.060236282646656036, 0.08113879710435867, 0.03543011099100113, -0.05227779969573021, 0.04203886538743973, 0.01357585284858942, 0.06472063809633255, 0.04309173673391342, 0.08852197974920273, 0.01882972940802574, -0.028108973056077957, 0.035874202847480774, 0.07864027470350266, -0.001931990496814251, -0.0521618016064167, 0.009736504405736923, -0.07050395756959915, 0.05462994799017906, 0.017030352726578712, -0.04582693800330162, 0.013289899565279484, 0.03034176304936409, -0.07412642240524292, 0.017280444502830505, -0.07020281255245209, -0.010828903876245022, 0.013319304212927818, -0.04619235917925835, -0.04388128221035004, 0.036927368491888046, -0.021930893883109093, 0.03889084607362747, -0.031834475696086884, -0.02293967641890049, 0.0048389132134616375, -0.04287312552332878, -0.007713694125413895, -0.0095838513225317, 0.020498378202319145, -0.01259702444076538, -0.052085030823946, 0.014438627287745476, 0.008889690972864628, 0.02098825015127659], "auraface_embedding": [-0.03799444064497948, 0.0836239755153656, -0.11143390089273453, -0.054017599672079086, 0.03098551370203495, 0.04076646268367767, -0.05022979527711868, 0.04118705168366432, 0.06365291029214859, 0.03241406008601189, -0.015343987382948399, -0.04981659725308418, 0.04081951081752777, 0.04862617701292038, -0.10553012788295746, 0.07953647524118423, -0.00537462905049324, 0.07112342864274979, 0.09354852885007858, -0.0443839356303215, -0.02066071890294552, -0.035662587732076645, 0.03751859813928604, -0.08743070065975189, -0.09550055861473083, -0.0700145959854126, 0.042402975261211395, 0.032154425978660583, -0.0368865430355072, 0.03835882991552353, 0.0289042629301548, 0.04454656317830086, 0.028613779693841934, 0.041155748069286346, -0.0045319669879972935, 0.007583947852253914, 0.060862667858600616, -0.02828039601445198, 0.038836345076560974, -0.006382460240274668, 0.05543737858533859, 0.06125757843255997, -0.04295719042420387, -0.007744070142507553, -0.0037796329706907272, 0.0714200884103775, 0.02716321125626564, -0.05825706571340561, -0.013859725557267666, 0.03097839094698429, -0.028572704643011093, -0.04134959354996681, -0.05345277488231659, -0.02118479274213314, 0.0027805487625300884, -0.05321525037288666, 0.032584164291620255, -0.009118779562413692, 0.09253240376710892, 0.008217032067477703, -0.05232881382107735, -0.051997728645801544, -0.007401136681437492, 0.06745750457048416, 0.023795146495103836, 0.03489379957318306, -0.01418302021920681, -0.0789034366607666, 0.1038179099559784, -0.030028603971004486, 0.029894646257162094, -0.05078820511698723, 0.0613156333565712, 0.029495147988200188, -0.025454195216298103, -0.00615087291225791, -0.0465446412563324, -0.0007060030475258827, -0.047622378915548325, 0.010464430786669254, -0.07267770916223526, 0.05920804664492607, 0.051306623965501785, -0.07208184897899628, 0.013576134108006954, 0.03608925640583038, 0.049090418964624405, -0.018632112070918083, 0.020517440512776375, -0.008740591816604137, -0.009743706323206425, 0.046514127403497696, 0.08821699768304825, 0.0483834408223629, 0.010411491617560387, -0.0949162095785141, -0.02464103512465954, -0.009647427126765251, -0.02160990983247757, 0.040879275649785995, -0.08201175183057785, -0.06739991158246994, 0.051367223262786865, -0.006888366304337978, -0.004681554157286882, -0.01569312810897827, -0.02062935009598732, -0.032410819083452225, 0.003958406392484903, -0.11198306083679199, 0.05295116454362869, -0.0018534671980887651, -0.0787169486284256, 0.05482657626271248, -0.08936027437448502, -0.05030450224876404, -0.033505991101264954, 0.06337697803974152, 0.05790233612060547, -0.029965180903673172, 0.013101769611239433, 0.030788064002990723, 0.055239737033843994, -0.034957628697156906, 0.026556136086583138, -0.014096718281507492, 0.011915571056306362, 0.05769060552120209, 0.021129904314875603, 0.04322769120335579, -0.11464216560125351, -0.05863502249121666, 0.039635296911001205, 0.025906013324856758, -0.10007030516862869, -0.01589466631412506, -0.04728575050830841, 0.03614956513047218, -0.06296242028474808, -0.021842345595359802, -0.0026417516637593508, 0.014944618567824364, -0.05159686505794525, -0.0027121580205857754, 0.013417060486972332, 0.03917611390352249, -0.12344778329133987, -0.04579126089811325, -0.029093563556671143, -0.022945165634155273, -0.009073389694094658, 0.013252667151391506, 0.015233276411890984, -0.0027986476197838783, 0.029529530555009842, 0.04555033892393112, 0.03477170690894127, -0.026000548154115677, 0.006219295784831047, 0.019825050607323647, -0.05507836118340492, 0.018951166421175003, -0.032616518437862396, -0.05282427743077278, -0.0524224154651165, -0.06560493260622025, 0.014054221101105213, 0.039421383291482925, -0.04303412511944771, -0.05939439311623573, 0.02685636654496193, -2.6786243324750103e-05, 0.030823156237602234, -0.022359302267432213, 0.11715640872716904, -0.0019038215978071094, -0.08416631817817688, -0.03820035606622696, 0.0007208166061900556, -0.030885983258485794, -0.006933772470802069, -0.0006920642917975783, 0.06642378121614456, 0.05501927062869072, 0.01858764886856079, -0.01196350622922182, -0.06482455134391785, -0.03902658075094223, -0.031661782413721085, 0.0726037472486496, -0.023077068850398064, -0.008052438497543335, -0.029322946444153786, -0.05178716033697128, -0.0008550470229238272, 0.014908403158187866, 1.4245580359784071e-06, -0.01575285568833351, 0.07125503569841385, 0.03447376564145088, 0.008526821620762348, -0.00560714490711689, -0.1067676991224289, 0.0013467423850670457, 0.026573546230793, 0.022827396169304848, 0.013021005317568779, -0.037964288145303726, 0.03925706446170807, -0.011994477361440659, -0.013316849246621132, 0.02804226242005825, 0.022290926426649094, 0.012604302726686, -0.054302461445331573, -0.06346619874238968, -0.05918024480342865, 0.04738897830247879, 0.050266116857528687, -0.06281650811433792, 0.013954922556877136, -0.009139049798250198, -0.013742309994995594, 0.027138041332364082, -0.024892643094062805, -0.023915521800518036, -0.009652889333665371, 0.009330973960459232, -0.06199246644973755, -0.06738169491291046, -0.0353936068713665, -0.002843203954398632, -0.04153160750865936, -0.008156832307577133, -0.011813672259449959, -0.007997721433639526, -0.08949943631887436, -0.04193497076630592, -0.013308310881257057, 0.0005426111165434122, -0.024733031168580055, 0.013836954720318317, 0.009338975884020329, 0.0323452353477478, -0.0133870430290699, 0.08078236132860184, -0.07132363319396973, 0.004890226759016514, -0.0020329037215560675, 0.01591130718588829, -0.043777644634246826, 0.0018298657378181815, -0.01385588850826025, 0.07291083037853241, -0.018895097076892853, 0.009081635624170303, -0.01969165913760662, -0.012506842613220215, 0.05924268811941147, 0.0008666045614518225, -0.04612020030617714, -0.018780402839183807, -0.011895311996340752, 0.12212128192186356, 0.029945559799671173, 0.03922213986515999, -0.08218931406736374, 0.009205227717757225, -0.06993961334228516, 0.015222804620862007, 0.009902921505272388, 0.026396619156003, 0.04138693958520889, 0.007424382958561182, -0.035389188677072525, -0.05428456887602806, -0.025213224813342094, -0.013524350710213184, 0.01653178408741951, -0.041606057435274124, 0.0056521096266806126, 0.008940482512116432, -0.018934326246380806, 0.004250342957675457, 0.06369178742170334, -0.03141530603170395, 0.02591557428240776, 0.07621489465236664, -0.017314886674284935, -0.0008383752428926528, 0.04816051200032234, 0.0157842505723238, -0.022349132224917412, 0.03001946210861206, 0.016812734305858612, 0.016992317512631416, -0.001936833024956286, 0.03954596444964409, 0.006103330757468939, -0.03633322939276695, 0.05314920097589493, -0.016529198735952377, -0.08701246231794357, -0.03815971687436104, 0.03316405415534973, -0.0011148484190925956, 0.019263846799731255, 0.07994415611028671, 0.014869393780827522, 0.0038127368316054344, 0.006147252395749092, 0.032673195004463196, 0.00419669970870018, -0.0014728241367265582, 0.03456913307309151, 0.09903787076473236, -0.016177061945199966, 0.0299637783318758, 0.045361313968896866, -0.05263521522283554, -0.037395477294921875, 0.007446906063705683, -0.024184120818972588, -0.040794700384140015, 0.07887280732393265, 0.001869737752713263, 0.03804241865873337, 0.07737036794424057, -0.09265606850385666, 0.004087749868631363, -0.019762229174375534, 0.027955977246165276, 0.08155251294374466, -0.00142304424662143, 0.03542250394821167, 0.05644936114549637, 0.06979646533727646, 0.04298216104507446, -0.04376509413123131, 0.013895308598876, 0.02385002374649048, 0.049873124808073044, -0.027473989874124527, 0.06613638252019882, -0.04393010959029198, -0.03198802471160889, -0.02572411485016346, -0.06973157823085785, 0.04775651916861534, -0.008210291154682636, -0.024038737639784813, -0.02084561064839363, 0.0002145085745723918, -0.0041586970910429955, -0.053414568305015564, -0.03297661244869232, -0.03431420773267746, -0.04217250645160675, -0.08342160284519196, 0.02130257524549961, -0.020695440471172333, 0.008040585555136204, -0.016200874000787735, 0.047639720141887665, 0.019424162805080414, -0.02876928448677063, -0.006686609238386154, 0.002683590864762664, 0.07059793174266815, 0.07582128793001175, -0.04518020153045654, -0.03930132836103439, -0.07125194370746613, 0.025594666600227356, 0.06447367370128632, 0.043777938932180405, -0.010831610299646854, -0.010270509868860245, -0.015646200627088547, -0.022046618163585663, 0.03745895251631737, 0.017380688339471817, 0.010684183798730373, 0.016353292390704155, -0.012536582536995411, -0.0006659181090071797, 0.03250446170568466, -0.025556722655892372, -0.019341178238391876, 0.04249022528529167, 0.03478110954165459, -0.10642334818840027, 0.07015302032232285, 0.012445829808712006, -0.004153423011302948, 0.029118971899151802, -0.03855689615011215, -0.09369663149118423, -0.018668850883841515, 0.005986872594803572, -0.03350198641419411, -0.08368265628814697, 0.03680216148495674, -0.017794273793697357, -0.03203713521361351, 0.00967615656554699, 0.06263182312250137, 0.03432411327958107, -0.03313856199383736, -0.031206293031573296, 0.000695848953910172, -0.006141439080238342, -0.003342160489410162, -0.011482102796435356, 0.019210435450077057, 0.011402229778468609, 0.033946145325899124, 0.006251658778637648, -0.028655489906668663, 0.004792502615600824, -0.056461308151483536, 0.0019056639866903424, 0.04924926534295082, -0.037842247635126114, -0.005083304364234209, -0.04141578450798988, 0.05401838198304176, 0.03240339085459709, -0.03899182751774788, -0.007905610837042332, 0.06860347092151642, 0.06404329091310501, 0.010388294234871864, -0.005057428497821093, 0.012449443340301514, 0.023372001945972443, 0.017629629001021385, -0.04993236064910889, 0.03388028219342232, -0.09372373670339584, 0.06629173457622528, 0.010677753947675228, -0.002351866103708744, -0.08307833969593048, 0.01971364952623844, 0.02485574409365654, 0.07540301233530045, -0.03315254673361778, 0.07740803807973862, -0.08545646071434021, -0.01963018998503685, -0.03480420261621475, 0.09101227670907974, 0.023538414388895035, 0.07348760962486267, 0.011347927153110504, 0.04410087317228317, -0.007272419985383749, -0.055769309401512146, 0.06237493455410004, -0.06642663478851318, 0.04667466878890991, 0.11089182645082474, 0.07423635572195053, 0.0021169844549149275, -0.0023581315763294697, 0.013804005458950996, -0.029064832255244255, 0.017731735482811928, 3.7448902730830014e-05, 0.0033728783018887043, -0.017733333632349968, 0.002331110881641507, 0.02517792023718357, 0.01695212349295616, 0.0591556690633297, 0.0989610105752945, 0.05231719836592674, 0.02285662665963173, 0.004013587720692158, 0.01733327843248844, -0.04384347423911095, -0.04013308510184288, 0.02239014394581318, 0.06313587725162506, -0.0784158781170845, 0.05067882314324379, 0.014497012831270695, -0.013364923186600208, 0.0314195342361927, -0.008608970791101456, 0.011616305448114872, 0.0480954572558403, -0.0335468091070652, 0.04741618037223816, 0.022557822987437248, -0.014819160103797913, -0.057447366416454315, -0.06196126714348793, 0.0004988081054762006, 0.030097689479589462, -0.02044272981584072, 0.04561210051178932, 0.06942591071128845, 0.04074731469154358, 0.008185670711100101, -0.019061679020524025, 0.0034824323374778032, -0.027719849720597267, 0.03931022062897682, 0.011561905965209007, 0.04012657701969147]}], "auraface_source": {"repo": "fal/AuraFace-v1", "revision": "af6d057c9b0ec4071d4c49c80e3539258798b609", "file": "glintr100.onnx", "sha256": "a7933ea5330113b01c9b60351d8f4c33003f145d8470ac5f0e52ee2effe25c60", "input_mean": 127.5, "input_std": 127.5}}} \ No newline at end of file From a383823c0e96de44e0aefdc0d1374b9ccad31d55 Mon Sep 17 00:00:00 2001 From: Bastian Machek <16717398+bmachek@users.noreply.github.com> Date: Fri, 30 Oct 2026 07:22:59 +0100 Subject: [PATCH 08/10] Export: pass the JPEG quality on Lightroom's 0-1 scale LR_jpeg_quality runs from 0 to 1, and Lightroom takes anything above 1 as full quality. The "Export JPEG quality in percent" preference went through unchanged, so every indexing and AI export was written at 100%: measured in Lightroom 15.6 on a 2048 px raw export, 1.44 MB at "50" against 160 KB at 0.5. The Add Missing Search Previews task passed 80 the same way. Util.jpegQualityFromPercent converts the preference (clamped to 1-100, 60 when unset, as before), exportSettingsFor applies it per session, and the preview task passes 0.8. Exports get several times smaller; embeddings shift slightly the next time a photo is indexed, which Help-Performance-Tips now says, together with the real defaults (3072 px, 50%). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HaMMvtFGGZAyc9nbCZiG6T --- docs/wiki/Help-Performance-Tips.md | 7 +++++-- .../LrGeniusAI.lrdevplugin/APISearchIndex.lua | 5 +++-- .../TaskFillSearchPreviews.lua | 2 +- plugin/LrGeniusAI.lrdevplugin/Util.lua | 20 +++++++++++++++++++ plugin/spec/util_export_and_faces_spec.lua | 20 +++++++++++++++++++ 5 files changed, 49 insertions(+), 5 deletions(-) diff --git a/docs/wiki/Help-Performance-Tips.md b/docs/wiki/Help-Performance-Tips.md index de9902eb..cf1b7668 100644 --- a/docs/wiki/Help-Performance-Tips.md +++ b/docs/wiki/Help-Performance-Tips.md @@ -85,9 +85,12 @@ Every photo sent to a vision LLM is first exported to a temporary JPEG (unless you use the originals fast path below). Two settings in *Plug-in Manager → LrGeniusAI* control that export: -- **Export size in pixels (long edge)** — default **1024px**. Options: 512, +- **Export size in pixels (long edge)** — default **3072px**. Options: 512, 1024, 2048, 3072, 4096. -- **Export JPEG quality in percent** — default **60%**. +- **Export JPEG quality in percent** — default **50%**. Plug-ins before this + version exported at 100% whatever this said, so the temporary JPEGs are now + several times smaller; embeddings shift slightly the next time a photo is + indexed. Everything else about the export is fixed so that two exports of the same photo always come out the same: **sRGB**, no output sharpening, and photos diff --git a/plugin/LrGeniusAI.lrdevplugin/APISearchIndex.lua b/plugin/LrGeniusAI.lrdevplugin/APISearchIndex.lua index 75a41d74..5f147bc0 100644 --- a/plugin/LrGeniusAI.lrdevplugin/APISearchIndex.lua +++ b/plugin/LrGeniusAI.lrdevplugin/APISearchIndex.lua @@ -304,7 +304,7 @@ local EXPORT_SETTINGS = { -- A photo smaller than the export size stays as it is: upscaling adds no -- detail, only interpolation the models would read as softness. LR_size_doNotEnlarge = true, - LR_jpeg_quality = tonumber(prefs.exportQuality) or 60, + -- LR_jpeg_quality comes from the preference, per session: exportSettingsFor. LR_minimizeEmbeddedMetadata = false, LR_outputSharpeningOn = false, LR_size_doConstrain = true, @@ -326,7 +326,8 @@ local function exportSettingsFor(overrides) for k, v in pairs(EXPORT_SETTINGS) do settings[k] = v end - settings.LR_jpeg_quality = tonumber(prefs.exportQuality) or 60 + -- 0 to 1, not percent: anything above 1 exports at full quality. + settings.LR_jpeg_quality = Util.jpegQualityFromPercent(prefs.exportQuality) settings.LR_size_maxHeight = tonumber(prefs.exportSize) or 1024 settings.LR_export_destinationPathPrefix = LrPathUtils.getStandardFilePath("temp") for k, v in pairs(overrides or {}) do diff --git a/plugin/LrGeniusAI.lrdevplugin/TaskFillSearchPreviews.lua b/plugin/LrGeniusAI.lrdevplugin/TaskFillSearchPreviews.lua index 46e2f115..91b509a8 100644 --- a/plugin/LrGeniusAI.lrdevplugin/TaskFillSearchPreviews.lua +++ b/plugin/LrGeniusAI.lrdevplugin/TaskFillSearchPreviews.lua @@ -193,7 +193,7 @@ local function run(context) index = index + #chunkIds local paths, exportFailures = SearchIndexAPI.exportPhotosForIndexing(chunkPhotos, { LR_size_maxHeight = RENDER_LONG_EDGE, - LR_jpeg_quality = 80, + LR_jpeg_quality = 0.8, -- 0 to 1, not percent }, function() return scope:isCanceled() end) diff --git a/plugin/LrGeniusAI.lrdevplugin/Util.lua b/plugin/LrGeniusAI.lrdevplugin/Util.lua index 3182bf69..662e2d07 100644 --- a/plugin/LrGeniusAI.lrdevplugin/Util.lua +++ b/plugin/LrGeniusAI.lrdevplugin/Util.lua @@ -3363,6 +3363,26 @@ function Util.cullFlagSummaryLines(counts) return lines end +--- +-- Lightroom's `LR_jpeg_quality` for a quality in percent (the Plug-in +-- Manager's "Export JPEG quality in percent"). +-- +-- The export setting runs from 0 to 1. Lightroom takes anything above 1 as +-- full quality, so passing the percent straight through exported every photo +-- at 100%: about nine times the bytes of 50%, for nothing the models can see. +-- @param percent any The saved preference (prefs.exportQuality), if any. +-- @param default number|nil Percent to use when `percent` is not a number +-- (60 when left out). +-- @return number Between 0.01 and 1. +-- +function Util.jpegQualityFromPercent(percent, default) + local n = tonumber(percent) + if n == nil or n ~= n then + n = tonumber(default) or 60 + end + return math.max(1, math.min(100, n)) / 100 +end + --- Photos per export session when indexing exports JPEGs. Util.DEFAULT_EXPORT_CHUNK_SIZE = 25 diff --git a/plugin/spec/util_export_and_faces_spec.lua b/plugin/spec/util_export_and_faces_spec.lua index a374081f..8abbf882 100644 --- a/plugin/spec/util_export_and_faces_spec.lua +++ b/plugin/spec/util_export_and_faces_spec.lua @@ -18,6 +18,26 @@ describe("Util.exportChunkSize", function() end) end) +describe("Util.jpegQualityFromPercent", function() + it("turns the percent into Lightroom's 0-1 scale", function() + assert.are.equal(0.5, Util.jpegQualityFromPercent(50)) + assert.are.equal(0.8, Util.jpegQualityFromPercent("80")) + assert.are.equal(1, Util.jpegQualityFromPercent(100)) + end) + + it("keeps the result within Lightroom's range", function() + assert.are.equal(0.01, Util.jpegQualityFromPercent(0)) + assert.are.equal(0.01, Util.jpegQualityFromPercent(-20)) + assert.are.equal(1, Util.jpegQualityFromPercent(250)) + end) + + it("uses the default when nothing usable is saved", function() + assert.are.equal(0.6, Util.jpegQualityFromPercent(nil)) + assert.are.equal(0.5, Util.jpegQualityFromPercent("high", 50)) + assert.are.equal(0.5, Util.jpegQualityFromPercent(0 / 0, 50)) + end) +end) + describe("Util.faceRecognizerSummary", function() local state = { active = "auraface", From bedb2d6a776f9ce182c2f96375da00538e56a5a7 Mon Sep 17 00:00:00 2001 From: Bastian Machek <16717398+bmachek@users.noreply.github.com> Date: Fri, 30 Oct 2026 07:23:07 +0100 Subject: [PATCH 09/10] Face search: say when faces of the other recognizer were left out After a switch between FaceNet and AuraFace, /v1/faces/search skips the rows the other recognizer embedded, as it should, but said nothing: right after a switch it answered "ok" with no results, which reads as "nobody matches". It now counts the rows it left out and says so in `warnings`, with what to run to bring them back (none searched at all, or N of M). Find Similar Faces read only the old single `warning` field. It now goes through Util.responseWarnings, shows the warnings in place of the generic "no similar faces" text when nothing was found, and as a warning dialog next to a result otherwise. Checked in Lightroom 15.6 against the running backend: with AuraFace active and only FaceNet rows stored, 0 results and the "None of the 135 faces" warning; with FaceNet, 20 results and no warning. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HaMMvtFGGZAyc9nbCZiG6T --- docs/wiki/Dev-Backend-API.md | 4 +- docs/wiki/Help-People-Faces.md | 3 +- .../TaskFindSimilarFaces.lua | 27 +++---- .../lrg-api/src/routes/face_recognizer.rs | 2 +- server-rs/crates/lrg-api/src/routes/faces.rs | 71 ++++++++++++++++++- 5 files changed, 89 insertions(+), 18 deletions(-) diff --git a/docs/wiki/Dev-Backend-API.md b/docs/wiki/Dev-Backend-API.md index ccf1dafb..d9728818 100644 --- a/docs/wiki/Dev-Backend-API.md +++ b/docs/wiki/Dev-Backend-API.md @@ -731,7 +731,9 @@ Detects faces in an uploaded image and stores their embeddings. Finds photos containing faces similar to those in a reference image. Only stored faces from the recognizer that embedded the query (`face_model`) are ranked: after a FaceNet/AuraFace switch the other rows are vectors of another -space until their photos are re-indexed. +space until their photos are re-indexed. How many were left out comes back in +`warnings`, so a search right after a switch (no results at all) says why +instead of looking like nobody matched. ### `GET /v1/faces/recognizer` Which network makes face identity embeddings (`lrg_ml::faces::Recognizer`): diff --git a/docs/wiki/Help-People-Faces.md b/docs/wiki/Help-People-Faces.md index 24209639..33581224 100644 --- a/docs/wiki/Help-People-Faces.md +++ b/docs/wiki/Help-People-Faces.md @@ -215,7 +215,8 @@ The two recognizers describe faces in different ways that cannot be compared, so after a switch: - Faces found with the other recognizer are left **unassigned** by **Cluster - faces** and are not offered by **Find Similar Faces**. + faces** and are not offered by **Find Similar Faces**, which says how many it + left out. - Their photos count as unprocessed again, so **Analyze & Index Photos** with *New or unprocessed photos* and *Enable face detection* recognises them anew. - **Names you gave are kept.** The face *detector* is the same for both, so a diff --git a/plugin/LrGeniusAI.lrdevplugin/TaskFindSimilarFaces.lua b/plugin/LrGeniusAI.lrdevplugin/TaskFindSimilarFaces.lua index a244c65b..042347e9 100644 --- a/plugin/LrGeniusAI.lrdevplugin/TaskFindSimilarFaces.lua +++ b/plugin/LrGeniusAI.lrdevplugin/TaskFindSimilarFaces.lua @@ -297,14 +297,11 @@ LrTasks.startAsyncTask(function() ) table.insert(resolutionWarnings, "One of the faces could not be looked up, so it is not named.") end - if qResp and qResp.warning then + for _, warning in ipairs(Util.responseWarnings(qResp)) do log:warn( - "queryFacesByImage warning during face resolution (idx " - .. tostring(i - 1) - .. "): " - .. tostring(qResp.warning) + "queryFacesByImage warning during face resolution (idx " .. tostring(i - 1) .. "): " .. warning ) - table.insert(resolutionWarnings, tostring(qResp.warning)) + table.insert(resolutionWarnings, warning) end local qResults = (qResp and qResp.results) and qResp.results or {} if qResults[1] and qResults[1].person_id then @@ -331,21 +328,25 @@ LrTasks.startAsyncTask(function() ErrorHandler.handleError(LOC("$$$/LrGeniusAI/FindSimilarFaces/QueryError=Search failed"), err) return end - if queryResp and queryResp.warning then - LrDialogs.message(LOC("$$$/LrGeniusAI/common/BackendWarning=Backend Warning"), queryResp.warning, "warning") - end + -- Faces left out of the search (found with the other face recognizer) + -- explain an empty or short result, so they are shown either way. + local queryWarnings = Util.formatWarningList(Util.responseWarnings(queryResp)) local results = (queryResp and queryResp.results) and queryResp.results or {} if #results == 0 then LrDialogs.message( LOC("$$$/LrGeniusAI/FindSimilarFaces/NoResultsTitle=No similar faces"), - LOC( - "$$$/LrGeniusAI/FindSimilarFaces/NoResultsMessage=No similar faces found in the index. Run 'Analyze & Index' with face detection enabled." - ), - "info" + queryWarnings + or LOC( + "$$$/LrGeniusAI/FindSimilarFaces/NoResultsMessage=No similar faces found in the index. Run 'Analyze & Index' with face detection enabled." + ), + queryWarnings and "warning" or "info" ) return end + if queryWarnings then + LrDialogs.message(LOC("$$$/LrGeniusAI/common/BackendWarning=Backend Warning"), queryWarnings, "warning") + end -- 4) Get cluster: prefer person_id from first result, then get all photos for that person progressScope:setCaption("Collecting the matching photos...") diff --git a/server-rs/crates/lrg-api/src/routes/face_recognizer.rs b/server-rs/crates/lrg-api/src/routes/face_recognizer.rs index 4165564d..b6d5cc31 100644 --- a/server-rs/crates/lrg-api/src/routes/face_recognizer.rs +++ b/server-rs/crates/lrg-api/src/routes/face_recognizer.rs @@ -131,7 +131,7 @@ fn describe(state: &AppState) -> Value { }) } -fn name_of(r: Recognizer) -> &'static str { +pub(crate) fn name_of(r: Recognizer) -> &'static str { match r { Recognizer::FaceNet => "FaceNet", Recognizer::AuraFace => "AuraFace", diff --git a/server-rs/crates/lrg-api/src/routes/faces.rs b/server-rs/crates/lrg-api/src/routes/faces.rs index 7631cc1f..450e5fe8 100644 --- a/server-rs/crates/lrg-api/src/routes/faces.rs +++ b/server-rs/crates/lrg-api/src/routes/faces.rs @@ -175,9 +175,11 @@ async fn query(State(state): State>, body: Option>) -> // FaceNet and AuraFace the older rows are 512-d unit vectors of another // space, and a distance to them ranks nonsense among real matches. They // come back once their photos are indexed again. + let same_model = + |r: &StoreRecord| r.metadata.get("face_model").and_then(Value::as_str) == Some(face_model); let mut scored: Vec<(f64, &StoreRecord)> = records .iter() - .filter(|r| r.metadata.get("face_model").and_then(Value::as_str) == Some(face_model)) + .filter(|r| same_model(r)) .filter_map(|r| { let v = r.vector.as_ref()?; Some((cosine_distance(query_emb, v), r)) @@ -185,6 +187,21 @@ async fn query(State(state): State>, body: Option>) -> .collect(); scored.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap()); scored.truncate(n_results); + // Left out silently, a switch looks like a search that finds nobody. + let set_aside = records.iter().filter(|r| !same_model(r)).count(); + let warnings: Vec = set_aside_warning( + set_aside, + records.len(), + crate::routes::face_recognizer::name_of(state.face.recognizer()), + ) + .into_iter() + .collect(); + if !warnings.is_empty() { + log::warn!( + "Face search: {set_aside} of {} faces set aside", + records.len() + ); + } let results: Vec = scored .into_iter() @@ -205,7 +222,29 @@ async fn query(State(state): State>, body: Option>) -> }) }) .collect(); - Json(json!({"status": "ok", "results": results})).into_response() + Json(json!({"status": "ok", "results": results, "warnings": warnings})).into_response() +} + +/// What to tell the user when `set_aside` of the `total` stored faces were +/// embedded by another recognizer than `active` (the one in use) and so left +/// out of a face search; `None` when every face was searched. +fn set_aside_warning(set_aside: usize, total: usize, active: &str) -> Option { + if set_aside == 0 { + return None; + } + let reindex = "They come back once their photos are indexed again with face detection \ + on: run Analyze & Index Photos with \"New or unprocessed photos\"."; + Some(if set_aside >= total { + format!( + "None of the {total} faces in the index was found with {active}, the face \ + recognizer in use, so this search could not compare any of them. {reindex}" + ) + } else { + format!( + "{set_aside} of the {total} faces in the index were found with another face \ + recognizer than {active} and were left out of this search. {reindex}" + ) + }) } fn cosine_distance(a: &[f32], b: &[f32]) -> f64 { @@ -1512,3 +1551,31 @@ async fn merge_persons(State(state): State>, body: Option Date: Fri, 30 Oct 2026 20:07:22 +0100 Subject: [PATCH 10/10] Store: split large upserts to stay under Lance's merge memory pool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Once the `id` index covers a table, LanceDB runs merge_insert as a hash join that loads its whole input into a fixed 150 MB pool it cannot spill out of. Face clustering writes every face back in one upsert, and at about 2.5 KB a face a catalog with ~60,000 faces failed with "Resources exhausted … HashJoinInput". Store::upsert now builds its input into batches of at most 32 MB of row data (UPSERT_CHUNK_BYTES) and runs one merge_insert per batch. Each is a commit of its own, so a failure partway leaves the earlier chunks written. Raising the pool through LANCE_MEM_POOL_SIZE would only move the limit. Tests cover the split (updated and new rows, one commit per chunk) and an oversized record; a 70,000-face reproduction is #[ignore]d. The server-rs README's memory section notes the write ceiling. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HaMMvsPvtNfysffwnVi9vb --- server-rs/README.md | 6 + server-rs/crates/lrg-store/src/lib.rs | 163 ++++++++++++++++++++++++-- 2 files changed, 156 insertions(+), 13 deletions(-) diff --git a/server-rs/README.md b/server-rs/README.md index f4be4da9..a3f53a6c 100644 --- a/server-rs/README.md +++ b/server-rs/README.md @@ -662,6 +662,12 @@ in MiB: | `GENIUSAI_LANCE_INDEX_CACHE_MB` | 128 | | `GENIUSAI_LANCE_METADATA_CACHE_MB` | 128 | +Writes have a ceiling of their own. Once the `id` index covers a table, +LanceDB's `merge_insert` loads its whole input into a fixed 150 MB pool it +cannot spill out of, so `Store::upsert` splits anything over 32 MB +(`UPSERT_CHUNK_BYTES`) into several commits. Face clustering, which writes +every face back at once, is the caller that reaches it: about 60,000 faces. + Indexing reads and normalises several photos at once, which is the other place a batch can grow its peak: that many originals — 25-50 MB apiece for raw — are resident before any of them has been shrunk to a normalised JPEG. diff --git a/server-rs/crates/lrg-store/src/lib.rs b/server-rs/crates/lrg-store/src/lib.rs index 23df5d60..606425c6 100644 --- a/server-rs/crates/lrg-store/src/lib.rs +++ b/server-rs/crates/lrg-store/src/lib.rs @@ -80,6 +80,20 @@ const RETIRED_VERTEX_TABLE: &str = "image_embeddings_vertex"; /// GET_IDS_CHUNK_SIZE rationale (bounded query size on large catalogs). const GET_IDS_CHUNK_SIZE: usize = 1000; +/// Most bytes of row data (ids, vectors, metadata JSON) that one +/// `merge_insert` in [`Store::upsert`] takes in. +/// +/// Once the `id` index covers a table, Lance runs `merge_insert` as a hash +/// join that first loads the entire input into memory, inside a fixed 150 MB +/// pool it cannot spill out of (lance-datafusion's +/// `DEFAULT_LANCE_MEM_POOL_SIZE_PER_PARTITION`). Face clustering writes every +/// face back in one `upsert`, and at about 2.5 KB a face (the 512-d vector is +/// 2 KB of it) a catalog with ~60,000 faces failed with "Resources exhausted +/// … HashJoinInput". Raising the pool through `LANCE_MEM_POOL_SIZE` would only +/// move that limit, so inputs are split instead, well under the pool because +/// the rest of the plan draws from it too. +const UPSERT_CHUNK_BYTES: usize = 32 * 1024 * 1024; + /// `LIKE` terms per `DELETE` in [`Store::delete_by_id_prefixes`]. Each term is /// a pattern match, not an index lookup, so this stays well under /// `GET_IDS_CHUNK_SIZE`. @@ -310,10 +324,25 @@ fn id_prefix_predicate(prefix: &str) -> String { } fn records_to_batch(table: &str, records: &[StoreRecord]) -> Result { + // An unlimited budget never closes a batch early, so this is the only one. + Ok(records_to_batches(table, records, usize::MAX)?.swap_remove(0)) +} + +/// Builds `records` into batches of about `max_bytes` of row data each. A +/// batch is closed once it reaches the budget, so a record bigger than the +/// budget still gets a batch of its own. Always returns at least one batch. +fn records_to_batches( + table: &str, + records: &[StoreRecord], + max_bytes: usize, +) -> Result> { let dim = table_dim(table)?; + let vector_bytes = dim as usize * std::mem::size_of::(); let mut ids = StringBuilder::new(); let mut vectors = FixedSizeListBuilder::new(Float32Builder::new(), dim); let mut metadatas = StringBuilder::new(); + let mut batches = Vec::new(); + let mut rows = 0usize; for record in records { ids.append_value(&record.id); @@ -336,8 +365,27 @@ fn records_to_batch(table: &str, records: &[StoreRecord]) -> Result } } metadatas.append_value(Value::Object(record.metadata.clone()).to_string()); + rows += 1; + + let bytes = ids.values_slice().len() + metadatas.values_slice().len() + rows * vector_bytes; + if bytes >= max_bytes { + batches.push(finish_batch(dim, &mut ids, &mut vectors, &mut metadatas)?); + rows = 0; + } } + if rows > 0 || batches.is_empty() { + batches.push(finish_batch(dim, &mut ids, &mut vectors, &mut metadatas)?); + } + Ok(batches) +} +/// Turns what the builders hold into a batch and leaves them empty for the next. +fn finish_batch( + dim: i32, + ids: &mut StringBuilder, + vectors: &mut FixedSizeListBuilder, + metadatas: &mut StringBuilder, +) -> Result { Ok(RecordBatch::try_new( table_schema(dim), vec![ @@ -555,24 +603,40 @@ impl Store { } /// Insert-or-replace by id (Chroma add/update semantics). + /// + /// A large `records` is written in several `merge_insert`s (see + /// [`UPSERT_CHUNK_BYTES`]), each a commit of its own: when one fails, the + /// chunks before it have already been written. pub async fn upsert(&self, table: &str, records: &[StoreRecord]) -> Result<()> { + self.upsert_in_chunks(table, records, UPSERT_CHUNK_BYTES) + .await + } + + /// `upsert` with an explicit chunk budget, so tests can split a handful + /// of rows instead of building 32 MB of them. + async fn upsert_in_chunks( + &self, + table: &str, + records: &[StoreRecord], + max_bytes: usize, + ) -> Result<()> { if records.is_empty() { return Ok(()); } - let batch = records_to_batch(table, records)?; - let schema = batch.schema(); - let reader = RecordBatchIterator::new([Ok(batch)], schema); let tbl = self.table(table).await?; - let mut builder = tbl.merge_insert(&["id"]); - builder - .when_matched_update_all(None) - .when_not_matched_insert_all(); - builder - .execute( - Box::new(reader) as Box - ) - .await?; - self.write_ops.fetch_add(1, Ordering::Relaxed); + for batch in records_to_batches(table, records, max_bytes)? { + let schema = batch.schema(); + let reader = RecordBatchIterator::new([Ok(batch)], schema); + let mut builder = tbl.merge_insert(&["id"]); + builder + .when_matched_update_all(None) + .when_not_matched_insert_all(); + builder + .execute(Box::new(reader) + as Box) + .await?; + self.write_ops.fetch_add(1, Ordering::Relaxed); + } Ok(()) } @@ -1299,4 +1363,77 @@ mod tests { ); assert!(got[0].vector.is_some(), "the embedding has to come along"); } + + fn face_rows(n: usize, person: &str) -> Vec { + (0..n) + .map(|i| { + record( + &format!("photo-{i:06}_face_0"), + Some(vec![0.01; 512]), + json!({"person_id": person, "padding": "x".repeat(300)}), + ) + }) + .collect() + } + + #[tokio::test] + async fn upsert_splits_large_input_and_writes_every_row() { + let dir = tempfile::tempdir().unwrap(); + let store = Store::open(dir.path()).await.unwrap(); + store + .append(FACE_TABLE, &face_rows(10, "person_1")) + .await + .unwrap(); + store.optimize_all().await.unwrap(); + + // Rows 0-9 exist and are updated, 10-24 are new. A 2.3 KB face + // against a 5 KB budget closes a batch every third row. + let rows = face_rows(25, "person_2"); + assert_eq!( + records_to_batches(FACE_TABLE, &rows, 5_000).unwrap().len(), + 9 + ); + store + .upsert_in_chunks(FACE_TABLE, &rows, 5_000) + .await + .unwrap(); + + assert_eq!(store.pending_write_ops(), 9, "one commit per chunk"); + let all = store.scan_all(FACE_TABLE).await.unwrap(); + assert_eq!(all.len(), 25); + assert!(all + .iter() + .all(|r| r.metadata.get("person_id") == Some(&json!("person_2")))); + } + + #[test] + fn records_to_batches_gives_an_oversized_record_its_own_batch() { + let rows = face_rows(3, "person_1"); + let batches = records_to_batches(FACE_TABLE, &rows, 1).unwrap(); + assert_eq!( + batches + .iter() + .map(RecordBatch::num_rows) + .collect::>(), + vec![1, 1, 1] + ); + // An empty input still yields the (empty) batch `append` relies on. + assert_eq!(records_to_batches(FACE_TABLE, &[], 1).unwrap().len(), 1); + } + + /// The failure this guards against: clustering writes every face back in + /// one `upsert`, and once the `id` index covers the table Lance's merge + /// loaded all of it into a 150 MB pool. 70,000 faces asked for 172 MB. + #[tokio::test] + #[ignore = "builds ~170 MB of faces; run with --ignored"] + async fn upsert_of_every_face_in_a_large_catalog_fits_lances_memory_pool() { + let dir = tempfile::tempdir().unwrap(); + let store = Store::open(dir.path()).await.unwrap(); + let rows = face_rows(70_000, "person_unassigned"); + store.append(FACE_TABLE, &rows).await.unwrap(); + // Brings the rows under the `id` index, which is what routes + // `merge_insert` through the capped hash join. + store.optimize_all().await.unwrap(); + store.upsert(FACE_TABLE, &rows).await.unwrap(); + } }