diff --git a/.ci-vendor/.gitkeep b/.ci-vendor/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..4ac2e587 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,53 @@ +# Docker build context: only what a clean clone contains. +# Host build output and vendored libraries are host/macOS artifacts by definition (they are +# git-ignored), and must never reach an image (see docs/implementation/README.md §3). +.git +.gitignore +.dockerignore +.build/ +.swiftpm/ +build.noindex/ +Vendor/ +.dev/ +.sign-id +.DS_Store + +# The desktop shell's build output and staging (the server image builds no Tauri shell) +shell/src-tauri/target/ +shell/src-tauri/gen/ +shell/src-tauri/helper/ +shell/src-tauri/THIRD-PARTY-NOTICES.txt + +# Python environments and caches +worker/.venv/ +**/__pycache__/ +*.pyc + +# Server-side runtime state and test media (never part of an image): the corpus script puts +# pictures, videos, clips and indexes under media/, and they are mounted at run time. +server/data/ +server/tests/corpus/media/ +server/tests/corpus/media/videos/ +server/tests/corpus/media/clips/ +server/tests/corpus2/media/ +server/tests/corpus/proof/ +**/.pytest_cache/ +**/.env +.playwright-mcp/ + +# Large documentation media +docs/*.gif + +# Compose bind-mount folders: runtime media/index/model data is never build context +deploy/data/ + +# Local agent working notes: git-ignored, and git-ignored files are not build context +# (the builder's `COPY . .` must see only what a clean clone contains) +AGENTS.md +.agent-journal.md + +# The Lit + Vite frontend: node_modules and a host-built dist never reach an image — +# the frontend-builder stage runs npm ci + build inside the build and the server stages +# COPY --from it (the host dist may be stale; a host build must never ship silently). +server/ui/dist/ +server/ui/node_modules/ diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml new file mode 100644 index 00000000..ad76e6f3 --- /dev/null +++ b/.github/workflows/server.yml @@ -0,0 +1,496 @@ +# The server's CI: build the engine from source in the same multi-stage Dockerfile the +# project ships, run its `swift test` inside the builder, check the CLI runs headless in the +# runtime image, then run the adapter's contract tests against that exact payload, corpus and +# model. Needs no GUI, no GPU, no Rust, no Node, no Tauri — a clean clone is the whole input. +# Also by hand: `gh workflow run server.yml`. +name: Server + +on: + pull_request: + paths: + - "server/**" + - "deploy/**" + - "docs/implementation/**" + - ".github/workflows/server.yml" + - ".dockerignore" + - ".ci-vendor/**" + - "Package.swift" + - "Sources/**" + - "scripts/**" + push: + branches: [main, "feat/**"] + paths: + - "server/**" + - "deploy/**" + - "docs/implementation/**" + - ".github/workflows/server.yml" + - ".dockerignore" + - ".ci-vendor/**" + # The published images embed the engine: a Sources/Package.swift/scripts change that + # passes ci.yml must republish them here, or ghcr silently serves a stale engine. + - "Package.swift" + - "Sources/**" + - "scripts/**" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: server-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +# YAML anchor (a parser feature, honored by Actions): the vendor seeding is byte-identical +# for the two jobs — one recipe, two runners, no copy-paste drift between them. +X-vendor-step: &vendor + name: Vendored libraries, ffmpeg, llama.cpp (in the builder's base image, cached — ci.yml's own scripts) + run: | + mkdir -p Vendor .ci-vendor + docker run --rm -v "$PWD:/src" -v "$PWD/Vendor:/src/Vendor" -w /src ubuntu:22.04 \ + bash -c 'set -eux; \ + apt-get update -q; \ + DEBIAN_FRONTEND=noninteractive apt-get install -y -q --no-install-recommends \ + ca-certificates curl git gnupg xz-utils unzip python3 binutils build-essential \ + cmake ninja-build meson nasm pkg-config file zlib1g-dev libcurl4-openssl-dev \ + libxml2-dev libedit2 libsqlite3-0 libz3-dev libpython3-dev tzdata yasm; \ + ./scripts/fetch-vendor.sh; \ + rm -f Vendor/llama-build/CMakeCache.txt; \ + tries=0; until ./scripts/build-ffmpeg.sh; do tries=$((tries+1)); [ "$tries" -ge 3 ] && exit 1; \ + echo "build-ffmpeg: attempt $tries failed; retrying in 30 s"; sleep 30; done; \ + tries=0; until ./scripts/build-llama.sh; do tries=$((tries+1)); [ "$tries" -ge 3 ] && exit 1; \ + echo "build-llama: attempt $tries failed; retrying in 30 s"; sleep 30; done' + cp -a Vendor/. .ci-vendor/ + +jobs: + server: + runs-on: ubuntu-24.04 + # The builder compiles ffmpeg + llama.cpp + the Swift package from source (the project's + # own scripts, as ci.yml runs them). 90 min was measured too tight mid-proof; 180 covers + # a cold cache with room, and a hung job dies on its own well before the 6 h ceiling. + timeout-minutes: 180 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + # The engine owns the model pins (Sources/DigUpKit/ModelFiles.swift: revision, file + # names and both sha256s, verified at runtime by ModelFiles itself). This step keeps + # the CI download pinned to the engine's own words — it runs FIRST, before any + # compile: when upstream bumps a model, the PR fails in seconds, not after an hour + # of build minutes (its comment always promised "fails first"; placement makes it + # true). It now catches the revision and the file names too: a revision bump with + # unchanged hashes moves the download URL silently, and a renamed file moves it + # loudly — both are red here, not in a download failure mid-suite. + - name: Model pins follow the engine (drift guard) + run: | + engine=$(grep -oE 'sha256: "[0-9a-f]{64}"' Sources/DigUpKit/ModelFiles.swift | grep -oE '[0-9a-f]{64}' | sort) + ci=$(printf '%s\n%s\n' \ + 2188ac1deca4b77dffefd603c2776a9d76d9d74ec01841392982ebb840b09135 \ + c4a8a52691ecef40618438928bdf9e68379b854e24166f292592353db0aab64f | sort) + if [ "$engine" != "$ci" ]; then + echo "::error::ModelFiles.swift pins differ from server.yml — update the download step's pins in the same PR" + echo "engine: $engine"; echo "ci: $ci"; exit 1 + fi + rev=$(sed -nE 's/.*revision = "([0-9a-f]{40})".*/\1/p' Sources/DigUpKit/ModelFiles.swift) + if [ "$rev" != "bfcd298762cc34d0357ece5ebdd31791a3a374d8" ]; then + echo "::error::ModelFiles.revision moved (engine: $rev) — update this step and the download step together" + exit 1 + fi + grep -q '"embeddinggemma-2-Q8_0.gguf"' Sources/DigUpKit/ModelFiles.swift && + grep -q '"mmproj-embeddinggemma-2-Q8_0.gguf"' Sources/DigUpKit/ModelFiles.swift || { + echo "::error::ModelFiles.swift file names changed — update the download step in the same PR"; exit 1; } + + - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: Vendor + key: server-vendor-linux-${{ hashFiles('scripts/fetch-vendor.sh', 'scripts/build-ffmpeg.sh', 'scripts/build-llama.sh', 'scripts/package-linux.sh') }} + # The engine chain (vendor libraries + ffmpeg + llama.cpp) builds IN THE BUILDER'S OWN + # BASE IMAGE, not on the runner host. Measured 2026-11-04 (first fork CI run): + # host-built Vendor on the ubuntu-24.04 runner (glibc 2.39) seeded into the ubuntu:22.04 + # engine-builder (glibc 2.35) lands on its own stamps, skips the rebuild — and swift + # test dies in 2 minutes: the host-built libllama/libpdfium cannot load against the + # container's older glibc. Building in the base image keeps the cached artifacts + # ABI-matched to where they are consumed. The result is cached host-side and passed + # into the Docker build context as `.ci-vendor/`; the Dockerfile seeds its cache mounts + # from it, the stamped scripts (the same ones ci.yml runs — stamped, idempotent) land + # on their stamps and skip the rebuild. Hosted runners start BuildKit cache mounts + # cold every job, so without the seed a 15-minute ffmpeg compile lands on every PR. + # The runner host needs no toolchain for this (curl/git/python3 ship on it — ffmpeg + # does NOT, see the corpus step; the container installs the builder's build packages + # itself). Bounded retries around the + # build scripts: they survive a network hiccup and cost nothing on healthy networks. + - <<: *vendor + + - name: Model files, pinned by Sources/DigUpKit/ModelFiles.swift + run: | + mkdir -p "$RUNNER_TEMP/models" + rev=bfcd298762cc34d0357ece5ebdd31791a3a374d8 + base=https://huggingface.co/ggml-org/embeddinggemma-2-GGUF/resolve/$rev + curl -fsSL -o "$RUNNER_TEMP/models/embeddinggemma-2-Q8_0.gguf" \ + "$base/embeddinggemma-2-Q8_0.gguf" + curl -fsSL -o "$RUNNER_TEMP/models/mmproj-embeddinggemma-2-Q8_0.gguf" \ + "$base/mmproj-embeddinggemma-2-Q8_0.gguf" + echo "2188ac1deca4b77dffefd603c2776a9d76d9d74ec01841392982ebb840b09135 $RUNNER_TEMP/models/embeddinggemma-2-Q8_0.gguf" > "$RUNNER_TEMP/pins.txt" + echo "c4a8a52691ecef40618438928bdf9e68379b854e24166f292592353db0aab64f $RUNNER_TEMP/models/mmproj-embeddinggemma-2-Q8_0.gguf" >> "$RUNNER_TEMP/pins.txt" + sha256sum -c "$RUNNER_TEMP/pins.txt" + + # The builder stage is upstream's own build, in the container the packages are built + # in: vendor libraries verified against committed pins, llama.cpp b11461, `swift + # build -c release`, `swift test` — everything in the image build's log below. + - name: Build the engine (and run swift test) in the builder image + run: docker build --target engine-builder -f server/docker/Dockerfile -t digup-builder:ci . + + - name: The runtime image runs the CLI headless + run: | + docker build --target engine-runtime -f server/docker/Dockerfile -t digup-engine:ci . + docker run --rm digup-engine:ci digup --help + docker run --rm digup-engine:ci digup --help | sed -n 2p + + # The corpus fixture is fully checkout-deterministic (12 photos + 2 videos in git, + # manifest-pinned); only the synthetic clip needs ffmpeg. The hosted runner image ships + # NO ffmpeg (measured 2026-11-05, run #4: the runner's "Tools" inventory has MediaInfo + # and no ffmpeg; the apt list jumps ftp→g++), and build.sh probes candidates by + # capability — the engine's Vendor ffmpeg (decode-only, restored into the workspace by + # the step-4 cache) is probed out, so the runner needs a real one: apt's ffmpeg has + # libx264 + the mp4 muxer and passes the probe. + - name: Corpus (pinned photos are in git; the videos, clip, and manifest checks run here) + run: | + sudo apt-get update -q + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -q --no-install-recommends ffmpeg + server/tests/corpus/build.sh + + # The clip is the suite's video fixture; a runner whose ffmpeg lacks libx264 silently + # unbuilds it, ~20 tests skip, and the suite greys out green. It must exist, plainly. + - name: The corpus clip exists (the video fixtures are not skippable) + run: test -f server/tests/corpus/media/clips/scenes.mp4 + + - name: Build the test image (engine payload + adapter) + run: docker build --target server-test -f server/docker/Dockerfile -t digup-server-test:ci . + + # The contract suite against the real CLI: search/index ops and reply fields verbatim, + # the engine's honest "no good matches", idle-exit restart, single-writer lock, reload, + # the `/api/v1` surface (health/status/search/estimate/index/jobs/embed_ahead/media + # over the real adapter — the lifespan spawns the helpers as children of the test + # process, a live parent), the index-job contract (409 while a job runs, the engine's + # counts, interrupted recovery from the job log, stop-mid-job resume, log rotation + # with history, env knobs), the startup prewarm and the ghost-row rules (walked-root + # ghosts retire on plain sync; out-of-scope ghosts survive it and retire under prune), + # and the UI surface (the built Lit + Vite app, served same-origin by FastAPI from + # the dist the frontend-builder stage compiles — Node runs in the build, never at + # runtime; server-issued media ids → previews/text, 404s for unissued ids). + # Zero-skip gate: the suite skips its own module when /models, /corpus-root or the + # clip are missing (conftest) — "57 passed, 20 skipped" would be green-by-construction + # and a lie. In CI every test must RUN: any skip is red, loudly. + - name: Contract tests (every test runs — skips are red in CI) + run: | + mkdir -p "$RUNNER_TEMP/index" + # tini is the image's ENTRYPOINT (the ParentWatch rule: the test process must be a + # child, not PID 1), so the pytest command is given in full — args alone would + # replace the CMD and the container would try to exec `-rs`. + docker run --rm \ + -v "$PWD/server/tests/corpus/media:/corpus:ro" \ + -v "$PWD/server/tests/corpus:/corpus-root:ro" \ + -v "$RUNNER_TEMP/models:/models:ro" \ + -v "$RUNNER_TEMP/index:/index" \ + digup-server-test:ci /srv/server/.venv/bin/pytest -rs 2>&1 | tee "$RUNNER_TEMP/suite.log" + if grep -qE '^(SKIPPED|tests/.*SKIPPED)' "$RUNNER_TEMP/suite.log"; then + echo "::error::a test SKIPPED in CI — the suite self-skipping is a green-by-construction lie" + grep -E 'SKIPPED' "$RUNNER_TEMP/suite.log" | head -20 + exit 1 + fi + grep -qE '[0-9]+ passed' "$RUNNER_TEMP/suite.log" + + - name: The headless proof (index + search + eval, the engine's own words) + run: | + # The engine image runs as its non-root user (USER digup, uid 990); dirs under + # $RUNNER_TEMP are runner-owned (mode 775) and dirs docker auto-creates are + # root-owned — uid 990 can write NEITHER. Measured on run #5 (2026-11-05): the + # proof died in 58 s with every gate red (tee into unwritable /out/proof, index + # unwritable), while the suite passed writing the same shapes as root (its test + # image has no USER directive). CI scratch dirs get 777 by design: containers + # create files there under uids the host does not have. The proof also gets its + # OWN index dir — hermetic: the suite's /index holds root-owned rows from test + # runs, the engine must start from an empty index, not inherit fixtures. + mkdir -p "$RUNNER_TEMP/proof-index" "$RUNNER_TEMP/proof" + chmod 777 "$RUNNER_TEMP/proof-index" "$RUNNER_TEMP/proof" + docker run --rm \ + -v "$PWD:/src:ro" \ + -v "$PWD/server/tests/corpus/media:/corpus:ro" \ + -v "$RUNNER_TEMP/models:/models:ro" \ + -v "$RUNNER_TEMP/proof-index:/index" \ + -v "$RUNNER_TEMP/proof:/out/proof" \ + -e OUT=/out/proof \ + digup-engine:ci bash /src/server/tests/proof.sh + # proof.sh counts failures (exit != 0) and gates the eval ALL row (R@1 ≥ 0.60, + # R@5 ≥ 0.85) — the numbers the docs quote must be green here, not cited there. + + # The shipping image: tini PID 1 → uvicorn → helpers (the ParentWatch chain), + # non-root, HEALTHCHECK = the API's own /health, the UI served at /. SIGTERM must + # end it *stopped, not killed*: the lifespan must complete (it marks a running job + # interrupted and quit()s every helper) inside the stop grace — exit 143 (a completed + # SIGTERM stop) is the graceful code here; 137 means SIGKILL, i.e. the shutdown hung. + # Every operator-facing compose shape, schema-checked on every run: the CPU default, + # the build-GPU override, both pull-and-run (ghcr) merges, the LAN-bind merges, and + # the Nextcloud shared-stack shape (standalone + LAN-merged, the remote-app case) — + # the `!reset` of the build block, the `ports`-list merge (loopback stays, LAN + # binding appends) and the required-env forcing (`:?`) are merge features, so their + # shapes belong pinned in CI, not in prose. LAN/Nextcloud shapes use RFC 5737 + # TEST-NET addresses and synthetic data paths: documentation values, nobody's. + - name: The Compose files are valid (every merge shape, schema check) + run: | + cd deploy + docker compose config --quiet + docker compose -f docker-compose.yml -f docker-compose.nvidia.yml --profile nvidia config --quiet + docker compose -f docker-compose.yml -f docker-compose.ghcr.yml config --quiet + docker compose -f docker-compose.yml -f docker-compose.ghcr.yml -f docker-compose.ghcr.nvidia.yml --profile nvidia config --quiet + DIGUP_LAN_BIND=192.0.2.1 docker compose -f docker-compose.yml -f docker-compose.lan.yml config --quiet + DIGUP_APP_TARGET_NAME=nextcloud DIGUP_APP_TARGET_HOST_DIR=/srv/app-data docker compose -f docker-compose.yml -f docker-compose.app-targets.yml config --quiet + DIGUP_APP_TARGET_NAME=nextcloud DIGUP_APP_TARGET_HOST_DIR=/srv/app-data docker compose -f docker-compose.yml -f docker-compose.app-targets.yml -f docker-compose.nvidia.yml --profile nvidia config --quiet + DIGUP_LAN_BIND=192.0.2.1 docker compose -f docker-compose.yml -f docker-compose.lan.yml -f docker-compose.nvidia.yml --profile nvidia config --quiet + DIGUP_LAN_BIND=192.0.2.1 docker compose -f docker-compose.yml -f docker-compose.ghcr.yml -f docker-compose.lan.yml config --quiet + DIGUP_NEXTCLOUD_DATA=/srv/nextcloud/data DIGUP_NEXTCLOUD_ADMIN_USER=ci-admin \ + DIGUP_NEXTCLOUD_ADMIN_PASSWORD=ci-secret DIGUP_NEXTCLOUD_INSTANCE_ID=octciexampleid \ + DIGUP_NEXTCLOUD_READ_TOKEN=ci-read-token DIGUP_SERVER_ADMIN_TOKEN=ci-admin-token \ + DIGUP_NEXTCLOUD_EXTSTORAGE=/srv/nextcloud/extstorage \ + docker compose -f docker-compose.nextcloud.yml config --quiet + DIGUP_NEXTCLOUD_DATA=/srv/nextcloud/data DIGUP_NEXTCLOUD_ADMIN_USER=ci-admin \ + DIGUP_NEXTCLOUD_ADMIN_PASSWORD=ci-secret DIGUP_NEXTCLOUD_INSTANCE_ID=octciexampleid \ + DIGUP_NEXTCLOUD_READ_TOKEN=ci-read-token DIGUP_SERVER_ADMIN_TOKEN=ci-admin-token \ + DIGUP_LAN_BIND=192.0.2.1 docker compose -f docker-compose.nextcloud.yml -f docker-compose.lan.yml config --quiet + # …and the LAN overlay of the NEXTCLOUD web port itself (docker-compose. + # nextcloud.lan.yml: the app reaches the API inside the compose network, so + # only the nextcloud service gains the LAN binding; trusted domains are an + # env/operator concern, documented in the overlay header). + DIGUP_NEXTCLOUD_DATA=/srv/nextcloud/data DIGUP_NEXTCLOUD_ADMIN_USER=ci-admin \ + DIGUP_NEXTCLOUD_ADMIN_PASSWORD=ci-secret DIGUP_NEXTCLOUD_INSTANCE_ID=octciexampleid \ + DIGUP_NEXTCLOUD_READ_TOKEN=ci-read-token DIGUP_SERVER_ADMIN_TOKEN=ci-admin-token \ + DIGUP_NEXTCLOUD_PUBLISH_PORT=8081 DIGUP_LAN_BIND=192.0.2.1 \ + docker compose -f docker-compose.nextcloud.yml -f docker-compose.nextcloud.lan.yml config --quiet + # The WebDAV shape (§11.7): per-user davfs2 mounts as the media tree, env-only — + # the instance id is NOT required (appdata is not in any user's dav view, the + # composed exclusion is a no-op), the index layout is its own folder, and the + # Nextcloud service keeps its own data volume untouched. + DIGUP_NEXTCLOUD_DATA=/srv/nextcloud/data DIGUP_NEXTCLOUD_ADMIN_USER=ci-admin \ + DIGUP_NEXTCLOUD_ADMIN_PASSWORD=ci-secret \ + DIGUP_NEXTCLOUD_READ_TOKEN=ci-read-token DIGUP_SERVER_ADMIN_TOKEN=ci-admin-token \ + DIGUP_SERVER_ROOTS=/media/dav/ci-alice:/media/dav/ci-bob DIGUP_MEDIA_TARGET=/media/dav DIGUP_MEDIA_BIND=/media/dav \ + DIGUP_NEXTCLOUD_INDEX_DIR=./data/nextcloud/index-dav \ + docker compose -f docker-compose.nextcloud.yml config --quiet + # …and the same shape WITH the dav mount container (the sudo-free shape the + # header documents): profile + propagation + mount env, all schema-checked. + DIGUP_NEXTCLOUD_DATA=/srv/nextcloud/data DIGUP_NEXTCLOUD_ADMIN_USER=ci-admin \ + DIGUP_NEXTCLOUD_ADMIN_PASSWORD=ci-secret \ + DIGUP_NEXTCLOUD_READ_TOKEN=ci-read-token DIGUP_SERVER_ADMIN_TOKEN=ci-admin-token \ + DIGUP_SERVER_ROOTS=/media/dav/ci-alice:/media/dav/ci-bob DIGUP_MEDIA_TARGET=/media/dav DIGUP_MEDIA_BIND=/media/dav \ + DIGUP_MEDIA_BIND_OPTS=ro,rslave DAV_HOST_DIR=/media/dav \ + DIGUP_NEXTCLOUD_INDEX_DIR=./data/nextcloud/index-dav \ + NEXTCLOUD_URL=http://nextcloud DAV_USERS="ci-alice ci-bob" DAV_APPPW="ci-alice:x ci-bob:y" \ + docker compose -f docker-compose.nextcloud.yml --profile dav config --quiet + # The all-in-one EXAMPLE (docker-compose.all-in-one.yml: Nextcloud + server + + # CUDA build + LAN binds + named app target + dav, one file): pinned in CI like + # every other shape, with the full env chain the example's header prints. + DIGUP_LAN_BIND=192.0.2.1 DIGUP_NEXTCLOUD_DATA=/srv/nextcloud/data DIGUP_NEXTCLOUD_ADMIN_USER=ci-admin \ + DIGUP_NEXTCLOUD_ADMIN_PASSWORD=ci-secret DIGUP_NEXTCLOUD_INSTANCE_ID=octciexampleid \ + DIGUP_NEXTCLOUD_READ_TOKEN=ci-read-token DIGUP_SERVER_ADMIN_TOKEN=ci-admin-token \ + DIGUP_APP_TARGET_NAME=nextcloud DIGUP_APP_TARGET_HOST_DIR=/srv/app-data \ + DIGUP_SERVER_ROOTS=/media/dav/ci-alice:/media/dav/ci-bob DIGUP_MEDIA_TARGET=/media/dav DIGUP_MEDIA_BIND=/media/dav \ + DIGUP_MEDIA_BIND_OPTS=ro,rslave DAV_HOST_DIR=/media/dav \ + NEXTCLOUD_URL=http://nextcloud DAV_USERS="ci-alice ci-bob" DAV_APPPW="ci-alice:x ci-bob:y" \ + docker compose -f docker-compose.all-in-one.yml --profile nvidia --profile dav config --quiet + + - name: The server runtime image boots, answers, serves the UI, stops cleanly + run: | + docker build --target server-runtime -f server/docker/Dockerfile -t digup-server:ci . + mkdir -p "$RUNNER_TEMP/data/index" "$RUNNER_TEMP/data/config" + # 777 as in the proof step: the server image is non-root (uid 990) and must own + # its writer lock + job log under /data/index — a runner-owned 775 dir denies it. + chmod 777 "$RUNNER_TEMP/data/index" "$RUNNER_TEMP/data/config" + docker run -d --name server-ci \ + -p 127.0.0.1:8787:8787 \ + -v "$RUNNER_TEMP/data/index:/data/index" \ + -v "$RUNNER_TEMP/data/config:/data/config" \ + -v "$RUNNER_TEMP/models:/data/models" \ + -v "$PWD/server/tests/corpus/media/photos:/media/photos:ro" \ + digup-server:ci + code=0 + for i in $(seq 1 30); do + # The image's OWN healthcheck script — the exact file Docker runs, proven here + # against the live API (its exit 0 is the health Docker reports). + if docker exec server-ci python /opt/digup/healthcheck.py; then code=200; break; fi + sleep 2 + done + test "$code" = "200" + docker exec server-ci python -c 'import urllib.request; html = urllib.request.urlopen("http://127.0.0.1:8787/", timeout=3).read().decode(); assert "DigUp" in html and "app.js" in html' + docker stop --time 15 server-ci + exit_code=$(docker inspect server-ci --format '{{.State.ExitCode}}') + test "$exit_code" != "137" # 137 = SIGKILL: the lifespan shutdown hung + docker logs server-ci 2>&1 | grep -q "Application shutdown complete" + docker rm server-ci + + - name: Proof logs (the run's own output, for the PR log) + if: always() + run: | + shopt -s nullglob + files=("$RUNNER_TEMP"/proof/*.log "$RUNNER_TEMP"/suite.log) + if [ ${#files[@]} -eq 0 ]; then echo "no proof/suite logs to show"; exit 0; fi + for f in "${files[@]}"; do echo "::group::${f##*/}"; cat "$f"; echo "::endgroup::"; done + + - name: Upload proof and suite logs + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: server-proof-logs-${{ github.run_number }} + path: | + ${{ runner.temp }}/proof + ${{ runner.temp }}/suite.log + if-no-files-found: warn + + # Pull-and-run publishes here (server/README.md "Pull the published image"): pushes to + # main and to the PR branch push the shipping images to ghcr.io, and + # deploy/docker-compose.ghcr*.yml pull them, so an operator needs no toolchain — set + # the DIGUP_* env vars, `docker compose up -d`, done. Tags: main → :latest and + # :latest-cuda; a feature branch → : and :-cuda (lowercased, + # slashes → dashes — the PR's own image is :feat-server-cuda on :feat-server). + # publish never runs on a pull_request event: that context's token is scoped to the + # BASE repo's registry, and this is the head owner's registry. x86_64 only, by rule: + # cross-emulation builds are forbidden (README §11) and arm64 hosts build the image + # natively under the same rule. The publish runs only after the full `server` job is + # green — the image that reaches ghcr passed the contract suite, the headless proof + # and the clean-stop gate minutes earlier. + publish: + needs: server + if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-24.04 + # The CUDA flavor compiles llama.cpp with GGML_CUDA for seven archs on a cold gha + # cache; 180 mirrors the server job's measured ceiling and dies on its own before + # GitHub's 6 h one. + timeout-minutes: 180 + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + # The same Vendor cache the server job just filled (same key): the Dockerfile seeds + # its cache mounts from .ci-vendor/, the stamped scripts land on their stamps and + # skip the rebuild. Built in the builder's own base image for ABI reasons (see the + # server job's comment — measured on the fork's first run 2026-11-04). The CUDA + # flavor re-pays only the CUDA compile (the llama stamp embeds $FLAVOR, so a + # cpu-seeded Vendor forces exactly the GPU recompile, not a reuse), and the gha + # build cache below keeps it warm across runs. + - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: Vendor + key: server-vendor-linux-${{ hashFiles('scripts/fetch-vendor.sh', 'scripts/build-ffmpeg.sh', 'scripts/build-llama.sh', 'scripts/package-linux.sh') }} + - <<: *vendor + + # ghcr.io forces the whole path lowercase: Ocean572/DigUp publishes as + # ghcr.io/ocean572/digup-server. Derived from the repo owner so case never hard-codes. + - name: Image name (ghcr paths are lowercase — derive from the repo) + run: echo "IMAGE=ghcr.io/$(echo '${{ github.repository_owner }}' | tr '[:upper:]' '[:lower:]')/digup-server" >> "$GITHUB_ENV" + + # main is :latest / :latest-cuda; a feature branch is : / :-cuda — + # the PR branch's own build, pullable with DIGUP_IMAGE_TAG=. The branch + # name is what a push event carries; PR numbers are not derivable from a push, so + # tags are branch-named, honestly. + - name: Image tags (main → latest; other refs → the branch name) + run: | + if [ "$GITHUB_REF" = refs/heads/main ]; then + tag=latest + else + tag=$(echo "$GITHUB_REF_NAME" | tr '[:upper:]' '[:lower:]' | tr / -) + fi + echo "TAG_CPU=$tag" >> "$GITHUB_ENV" + echo "TAG_CUDA=$([ "$tag" = latest ] && echo latest-cuda || echo "$tag-cuda")" >> "$GITHUB_ENV" + + - uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4.7.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + + # The CPU image — the target server-runtime the stop-gate step above booted, + # tagged :latest on main, : on a feature branch. + - uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: . + file: server/docker/Dockerfile + target: server-runtime + push: true + tags: ${{ env.IMAGE }}:${{ env.TAG_CPU }} + platforms: linux/amd64 + cache-from: type=gha + cache-to: type=gha,mode=max + + # The CUDA image — the shape docker-compose.ghcr.nvidia.yml pulls (:latest-cuda + # on main, :-cuda on the PR branch). Fat arch list + # (61 Pascal … 90 Hopper; CUDA 12.4.1 compiles 61 as deprecated — CUDA 13 drops it, + # re-evaluate the base when the line moves); CMake lists are semicolon-separated and + # the Dockerfile's engine-builder normalizes this comma string (run #6, 2026-11-05: + # raw commas = ONE invalid arch token, configure died in seconds). Its build gate is the Dockerfile's own: + # server-runtime-cuda runs `ldd libggml-cuda.so` and fails the build on any + # "not found" (measured 2026-11-04, Gate B), so a broken CUDA image never publishes. + # Separate gha cache scope: the CPU and CUDA layer sets must not evict each other. + - uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: . + file: server/docker/Dockerfile + target: server-runtime-cuda + push: true + tags: ${{ env.IMAGE }}:${{ env.TAG_CUDA }} + platforms: linux/amd64 + build-args: | + BASE_IMAGE=nvidia/cuda:12.4.1-devel-ubuntu22.04 + CUDA_RUNTIME_IMAGE=nvidia/cuda:12.4.1-runtime-ubuntu22.04 + CUDA_ARCHS=61,70,75,80,86,89,90 + cache-from: type=gha,scope=cuda + cache-to: type=gha,mode=max,scope=cuda + + # Version hygiene: every push leaves the previous one untagged, and ghcr's free + # tier is metered by stored GB — a CUDA image is ~2 GB, so untagged leftovers get + # deleted right here. Keep rule: anything still carrying a tag survives — the + # floating tags this job writes (latest, latest-cuda, branch names) are the only + # tags it ever creates, so the registry ends up holding exactly the live builds. + - name: Delete untagged image versions (registry hygiene) + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + python3 - <<'PY' + import json, os, urllib.request, urllib.error + + owner, pkg, tok = os.environ["GITHUB_REPOSITORY"].split("/")[0], "digup-server", os.environ["GITHUB_TOKEN"] + api = "https://api.github.com" + + def req(method, url, body=None): + data = json.dumps(body).encode() if body is not None else None + r = urllib.request.Request(url, data=data, method=method, headers={ + "Authorization": "Bearer " + tok, + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + }) + with urllib.request.urlopen(r) as resp: + raw = resp.read() + return json.loads(raw) if raw else {} + + # Best-effort hygiene: publish has already succeeded at this point, so cleanup + # problems get a loud ::warning and exit 0 — never a red run, never silence. + # Two measured fixes (run #7, 2026-11-05): the heredoc body must sit at the SAME + # column as the delimiters (a deeper indent = IndentationError, exit 1 in 1 s), + # and deleting versions is POST .../versions/delete since GitHub's 2022 verb + # migration — the old DELETE answers 404. + try: + versions, page = [], 1 + while True: + batch = req("GET", f"{api}/users/{owner}/packages/container/{pkg}/versions?per_page=50&page={page}") + versions.extend(batch) + if len(batch) < 50: + break + page += 1 + doomed = [{"id": v["id"]} for v in versions if not v.get("tags")] + for i in range(0, len(doomed), 20): # the API deletes in batches of <=20 + req("POST", f"{api}/user/packages/container/{pkg}/versions/delete", {"versions": doomed[i:i + 20]}) + print(f"versions: {len(versions)} total, {len(versions) - len(doomed)} tagged kept, {len(doomed)} untagged deleted") + except urllib.error.HTTPError as exc: + print(f"::warning::registry hygiene skipped: HTTP {exc.code} {exc.reason} — {exc.read().decode()[:200]}") + except Exception as exc: + print(f"::warning::registry hygiene skipped: {exc}") + PY diff --git a/.gitignore b/.gitignore index e21ff61c..e4ed03af 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ build.noindex/ # Python: the reference worker's environment worker/.venv/ +server/.venv/ __pycache__/ *.pyc @@ -26,3 +27,36 @@ shell/src-tauri/gen/ # What scripts/package-*.sh gather for the packages shell/src-tauri/helper/ shell/src-tauri/THIRD-PARTY-NOTICES.txt +server/tests/corpus/media/videos/ +server/tests/corpus/media/clips/ +server/tests/corpus/media/index/ +server/tests/corpus/media/models/ +server/data/ +.pytest_cache/ + +# Compose bind-mount folders (runtime state, never in git) +deploy/data/ +server/tests/corpus/proof/ +server/tests/corpus2/media/ +.playwright-mcp/ + +# CI vendor seeding (server.yml): the host cache copied into the build context. Its +# .gitkeep is tracked (the Dockerfile's COPY needs the folder); a copied Vendor never is. +.ci-vendor/* +!.ci-vendor/.gitkeep + +# Local agent working notes (see the file itself): a dev aid, not part of the PR +AGENTS.md +.agent-journal.md + +# ghcr pull-and-run scratch stack (compose state, index, job logs) — same category as deploy/data +deploy/data-ghcr/ + +# stage-13 app-targets demo scratch stack (compose state, index, job logs) +deploy/data-demo/ + +# The Lit + Vite frontend (user brief 2026-11-08): npm install and the build output are +# host artifacts — dist is produced in the Dockerfile's frontend-builder stage and served +# by FastAPI; the host copy is only for `npm run dev`/local preview. +server/ui/dist/ +server/ui/node_modules/ diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 00000000..e3f4c278 --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,172 @@ +# DigUp server — Docker Compose deployment + +The shipped compose app in one page: what each file is for, every env var it reads, +the launch chains, and the API surface. The API contract itself is +[`server/docs/API.md`](../server/docs/API.md) — this page never restates schemas, only +points at them. Live measured chronicle: [`server/IMPLEMENTATION_STATUS.md`](../server/IMPLEMENTATION_STATUS.md). + +--- + +## 60-second standalone + +```bash +cd deploy +mkdir -p data/{index,models,config,media} && sudo chown -R 990:990 data # uid 990 = the image's user +docker compose up -d --build # CPU build, default chain +# → http://127.0.0.1:8787/ (UI + /api/v1) +``` + +Open the page, paste a key if the server has tokens (the gear shows it keylessly on a +tokenless box), press **Estimate** then **Index now**. Indexing never auto-starts. + +Three measured rules before anything else: + +- **The `chown 990` is not optional.** Docker creates bind dirs root-owned on first up; + the server then dies on the writer lock (`EACCES`). Measured, not folklore. +- **Index rows carry the roots they were crawled from.** Moving a mount to a new path + makes answers thin + ghost rows; cure with `POST /api/v1/index {"prune":true}`. +- **Exactly one server process**: media ids and the job runner live inside one uvicorn. + `--workers >1`, replicas, or a LB in front of two containers 404s media ids. + Concurrency *within* the process is fine. + +## Which compose file for which shape + +| File | Shape | +|---|---| +| `docker-compose.yml` | standalone server, CPU build — the whole product on its own | +| `docker-compose.ghcr.yml` | skip the build, pull the published `ghcr.io` image | +| `docker-compose.nvidia.yml` | NVIDIA box: CUDA build target + GPU device reservation (`--profile nvidia`) | +| `docker-compose.lan.yml` | LAN bind for the server port (`DIGUP_LAN_BIND`, opt-in by design) | +| `docker-compose.app-targets.yml` | add one named index target for an application's own data | +| `docker-compose.nextcloud.yml` | full Nextcloud + server stack, self-contained (dav service behind `--profile dav`) | +| `docker-compose.nextcloud.lan.yml` | LAN bind for the Nextcloud web port | +| `docker-compose.all-in-one.yml` | **example**: all of the above merged into one file (peel instructions in its header) | + +Overlay files merge onto the base — the chain order is the merge order, and every +shape below is `docker compose -f …[-f …] [--profile …] up -d`. CI schema-validates +each of these chains (see `.github/workflows/server.yml`), so the lines printed here +are exactly the CI-checked ones. + +**Recreation rule (measured, it costs hours if ignored):** every `up -d` on an existing +stack repeats the FULL file chain AND every env var used at first boot. A launch with +defaults silently remounts media one level short — the index goes out of scope, the API +answers 0 hits, health stays green. + +## Launch chains (example values only) + +```bash +# standalone, CPU +docker compose up -d --build + +# published image, no build step (x86_64) +DIGUP_IMAGE_TAG=latest \ +docker compose -f docker-compose.yml -f docker-compose.ghcr.yml up -d + +# NVIDIA box (toolkit installed; match CUDA_BASE to `nvidia-smi`'s CUDA Version, +# CUDA_ARCHS a PLAIN number: 61 Pascal, 89 Ada — apt cmake 3.22.1 rejects `6.1`/native) +CUDA_BASE=nvidia/cuda:12.4.1-devel-ubuntu22.04 CUDA_ARCHS=89 \ +docker compose -f docker-compose.yml -f docker-compose.nvidia.yml --profile nvidia up -d --build + +# home LAN (no auth — home-LAN trust model, see the overlay's header) +DIGUP_LAN_BIND=192.0.2.1 \ +docker compose -f docker-compose.yml -f docker-compose.lan.yml up -d + +# a named index for an application's data, beside the media index +DIGUP_APP_TARGET_NAME=photos DIGUP_APP_TARGET_HOST_DIR=/srv/photos-data \ +docker compose -f docker-compose.yml -f docker-compose.app-targets.yml up -d --build + +# Nextcloud + server, one stack (data shape: Nextcloud's own data dir is the media tree) +DIGUP_NEXTCLOUD_DATA=$PWD/data/nextcloud/html \ +DIGUP_NEXTCLOUD_ADMIN_USER=admin DIGUP_NEXTCLOUD_ADMIN_PASSWORD='pick-a-password' \ +DIGUP_NEXTCLOUD_INSTANCE_ID= \ +DIGUP_NEXTCLOUD_READ_TOKEN= \ +DIGUP_SERVER_ADMIN_TOKEN= \ +docker compose -f docker-compose.nextcloud.yml up -d +# → Nextcloud http://127.0.0.1:8081/ · server http://127.0.0.1:8787/ +# Add --profile dav + DAV_USERS/DAV_APPPW/NEXTCLOUD_URL for the remote-WebDAV data shape +# (per-user roots, rslave bind — the dav container mounts, the server reads, no restart +# for (re)mounts). The Nextcloud app itself is its own repository: Ocean572/digup_nextcloud. + +# everything at once (example, one file) +docker compose -f docker-compose.all-in-one.yml --profile nvidia --profile dav up -d --build +``` + +## Env var reference (compose layer) + +| Var | Effect | +|---|---| +| `DIGUP_INDEX_DIR` `DIGUP_MODELS_DIR` `DIGUP_CONFIG_DIR` `DIGUP_MEDIA_DIR` | move the bind targets off `deploy/data/` (defaults are inside the tree) | +| `DIGUP_PUBLISH_PORT` | host port for the server (container binds 8787) | +| `DIGUP_LAN_BIND` | the host's LAN address; adds the LAN port mapping (opt-in files) | +| `DIGUP_IMAGE_TAG` / `DIGUP_REGISTRY` | registry image selection (`ghcr` overlay) | +| `DIGUP_SWIFT_TESTS` | build-arg knob: `0` skips the engine-test stage on a builder whose env can't run them (CI keeps `1`) | +| `CUDA_BASE` `CUDA_RUNTIME_IMAGE` `CUDA_ARCHS` | CUDA build flavor — base matched to the host driver, plain arch numbers (see the GPU section) | +| `DIGUP_NVIDIA_GPU` | pin one card by UUID on multi-GPU hosts (`NVIDIA_VISIBLE_DEVICES` passthrough) | +| `DIGUP_SERVER_ROOTS` | colon-separated roots the searcher answers for (default `/media/photos`; per-user list on the WebDAV shape: `/media/dav/:…`) | +| `DIGUP_SERVER_TOKEN` | single-token deployment (read scope; admin token supersedes) | +| `DIGUP_SERVER_READ_TOKENS` | named read entries `name:token` (comma-separated; `app:` binds a named index) | +| `DIGUP_SERVER_ADMIN_TOKEN` | admin scope: index, prune, estimate, targets, tokens | +| `DIGUP_SERVER_APP_TARGETS` | named targets `name=roots:index` (the app-targets overlay composes it) | +| `DIGUP_SERVER_ALLOWED_HOSTS` | Host guard additions for proxy/LAN names (empty = accept-any) | +| `DIGUP_SERVER_ALLOWED_ORIGINS` | CORS allowlist for browser widgets (see API.md §CORS) | +| `DIGUP_SERVER_ESTIMATE_TIMEOUT_S` | estimate walk ceiling (default 120 s; huge trees need more, e.g. 900) | +| `DIGUP_SERVER_JOB_TIMEOUT_S` | index job ceiling; on expiry the job records *interrupted* with its counters and resumes | +| `DIGUP_SERVER_PREWARM` `DIGUP_SERVER_BACKFILL` `DIGUP_SERVER_ENCODER_IDLE` `DIGUP_SERVER_STATUS_TTL_S` `DIGUP_SERVER_JOB_LOG_SIZE` `DIGUP_SERVER_EXTRA_EXCLUDE` `DIGUP_SERVER_EXCLUDE_EXT` | tuning, defaults sane — API.md + the Dockerfile document each | +| `DIGUP_MEDIA_BIND` `DIGUP_MEDIA_TARGET` `DIGUP_MEDIA_BIND_OPTS` `DIGUP_MEDIA_GID` | the media bind: host dir, mount point, `ro` (+`,rslave` on WebDAV), group grant via `group_add` | +| `DIGUP_NEXTCLOUD_*` `DAV_*` `NEXTCLOUD_URL` | the Nextcloud stack + dav container — each namespaced var is commented where it is used in `docker-compose.nextcloud.yml` | + +## Semantics an operator must know (measured, and honest) + +- **`/health` answers 503 while warming** (~20–30 s prewarm at boot, and during a + searcher respawn). The container is healthy; the API is honest. The healthcheck counts + 503 as alive. +- **`POST /index` answering 409 = a job is running.** One writer flock per index (per + named target its own). Never "fix" a 409; ask later. +- **Stop semantics:** `docker stop` with the compose `stop_grace_period: 90s` → exit + **143**, clean shutdown, the running job recorded *interrupted* with its counters, the + next job resumes. Docker's default 10 s grace → SIGKILL 137; log replay recovers + *interrupted* at next boot — never success. +- **The server never invents numbers**: progress, z, timings, previews all come from the + engine. Fewer strong hits on tiny corpora is the engine's z-gate, not a server filter. +- **GPU is configuration, not patching**: the CUDA image is the same Dockerfile, a + different target + bases. The engine reports `device` + cosine agreement; the proof of + use is `server/tests/gpu-handoff.sh` sampling `nvidia-smi` **during a real index job** + (full procedure: `docs/implementation/gpu-handoff.md`). `DIGUP_DEVICE=cpu` is the CPU + fallback — CPU winning the measurement is a result, report the numbers. + +## API surface (one table — `server/docs/API.md` is the contract) + +| Endpoint | Scope | Notes | +|---|---|---| +| `GET /api/v1/health` | none | 503 while warming (by design) | +| `GET /api/v1/status` | read | engine counts verbatim: files, searchable, vectors, kinds, roots | +| `POST /api/v1/search` `{q, limit, kinds, code}` | read | engine answer: hits, z, strong, ms, meaning; `kinds` = media-type filter | +| `POST /api/v1/embed_ahead` | read | seeds a preview embed (UI latency path) | +| `GET /api/v1/media/{id}/preview?pixels=` | read | core/preview shape for photos | +| `GET /api/v1/media/{id}/text` | read | extracted text page(s) | +| `GET /api/v1/media/{id}/file` | read | streamed original, HTTP range requests (206) | +| `POST /api/v1/estimate` | admin | file count + cost forecast (ceiling `DIGUP_SERVER_ESTIMATE_TIMEOUT_S`) | +| `POST /api/v1/index` `{prune?}` | admin | 202 queued · **409** busy; `{"prune": true}` drops rows outside the roots | +| `GET /api/v1/jobs` · `GET /api/v1/jobs/{id}` | read | the engine's own progress records (live + history) | +| `GET /api/v1/targets` · `POST /api/v1/targets` | admin | named indexes; POST mints the target's private key | +| `GET /api/v1/tokens` · `POST /api/v1/tokens` | admin | minted-key list / rotation (keyless read for the gear on the box) | + +Bearer tokens ride `authorization: Bearer `. The single-token shape +(`DIGUP_SERVER_TOKEN`) is read-scope everywhere the admin token is not set; the named +map and minted keys bind reads to their own index — API.md §Authentication + +§Named indexes state every rule, and the UI's gear (Settings → API key, Copy + +Regenerate) shows the box's own key without ever asking the browser to hold an admin one. + +## Where the rest lives + +- **UI/CLI/API contracts:** `server/README.md` (layout, contract suite, dev run) and + `server/docs/API.md` (schemas, media ids, limits, errors, versioning, CORS, named + indexes, deploy-shape promises). +- **GPU handoff:** `docs/implementation/gpu-handoff.md` + executable + `server/tests/gpu-handoff.sh` (Gate A `--cpu-only`, Gate B with `CUDA_BASE=…`). +- **Nextcloud integration design + measured pilot gates:** + `docs/implementation/stage-12-integrations.md`; the app: + `github.com/Ocean572/digup_nextcloud` (its README carries its install steps, including + the `allow_local_servers` core knob — SSRF protection stays on). +- **Images:** `ghcr.io/ocean572/digup-server` `:latest` / `:latest-cuda` (x86_64; every + other architecture builds natively — never copy a binary across architectures). diff --git a/deploy/dav/Dockerfile b/deploy/dav/Dockerfile new file mode 100644 index 00000000..f2e1d9d6 --- /dev/null +++ b/deploy/dav/Dockerfile @@ -0,0 +1,36 @@ +# The dav mount helper (stage-12 §11.7, the WebDAV shape): davfs2 inside a container, +# so `docker compose --profile dav` sets the WebDAV mounts up behind the scenes — no +# host mount step, no fstab, no host secrets file, no sudo. +# +# privileged + /dev/fuse: davfs2 is a FUSE filesystem driver, and a FUSE mount inside a +# container needs the device and SYS_ADMIN (the standard fuse-in-container shape). The +# container's whole job is mounting the listed dav views at /mnt/dav/ READ-ONLY with +# mount propagation; it serves nothing, binds nothing else. +FROM debian:bookworm-slim + +# This box's LAN resolver hiccups for tens of seconds at times (AGENTS.md, measured): +# the bounded retry loop is the standing shape for every network step in this repo. +RUN for i in 1 2 3 4 5 6; do \ + DEBIAN_FRONTEND=noninteractive apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + davfs2 tini ca-certificates && break; \ + echo "apt failed (resolver hiccup?), retry $i"; sleep 10; \ + done && rm -rf /var/lib/apt/lists/* + +# davfs2 resolves the mounted tree's owner via getpwuid/getgrgid (measured 2026-11-07: +# "can't read user data base" when the uid= owner has no passwd entry in the mounting +# environment). The tree's owner is the server container's user — 990 — so the image +# carries that identity. (Host-mount shape: the HOST needs the same entry — documented +# in nextcloud-webdav.sh.) +RUN groupadd -g 990 digup && useradd -r -u 990 -g 990 -M -d /nonexistent digup + +# read-only indexer's davfs2 shape: locks are pointless (nothing writes), dir cache 30 s. +# buf_size 64: the measured 1.7.x read breakage on modern kernels ("reading directory: +# Invalid argument" = Launchpad #2129131, fixed upstream by raising the read buffer; +# this box's kernel 7.0 hits it with the 16 KiB default). +RUN printf 'use_locks 0\ndir_refresh 30\nbuf_size 64\n' >> /etc/davfs2/davfs2.conf + +COPY entrypoint.sh /entrypoint.sh +RUN chmod 755 /entrypoint.sh + +ENTRYPOINT ["/usr/bin/tini", "--", "/entrypoint.sh"] diff --git a/deploy/dav/entrypoint.sh b/deploy/dav/entrypoint.sh new file mode 100644 index 00000000..c7ce1438 --- /dev/null +++ b/deploy/dav/entrypoint.sh @@ -0,0 +1,67 @@ +#!/bin/sh +# dav mount entrypoint (the dav/Dockerfile image): keep every listed user's dav view +# mounted READ-ONLY at /mnt/dav//, self-healing. Compose env (docker-compose.nextcloud.yml): +# +# NEXTCLOUD_URL the Nextcloud base URL (http://nextcloud inside the compose network) +# DAV_USERS space-separated user ids +# DAV_APPPW space-separated "user:app-password" pairs (occ user:add-app-password ) +# DAV_UID/DAV_GID owner of the mounted tree (default 990 = the server container's user) +# DAV_CA optional: PEM text of a trusted self-signed server certificate (https) +# +# Credentials go to the container's /etc/davfs2/secrets (root, 600) — they ride the +# compose env chain, never a host file. Graceful stop unmounts everything (trap). +set -eu + +BASE=/mnt/dav +UID_M="${DAV_UID:-990}" +GID_M="${DAV_GID:-990}" +SECRETS=/etc/davfs2/secrets + +[ -n "${NEXTCLOUD_URL:-}" ] || { echo "FAIL NEXTCLOUD_URL is required"; exit 1; } +[ -n "${DAV_USERS:-}" ] || { echo "FAIL DAV_USERS is required"; exit 1; } +[ -n "${DAV_APPPW:-}" ] || { echo "FAIL DAV_APPPW is required (user:app-password pairs)"; exit 1; } + +url_of() { echo "${NEXTCLOUD_URL%/}/remote.php/dav/files/$1/"; } +pw_of() { for p in $DAV_APPPW; do [ "${p%%:*}" = "$1" ] && { echo "${p#*:}"; return; }; done; } + +cleanup() { + for u in $DAV_USERS; do umount "$BASE/$u" 2>/dev/null || true; done + echo "OK all dav views unmounted (graceful stop)" + exit 0 +} +trap cleanup TERM INT + +# credentials: rebuild the secrets file from the pairs on every boot (600, root) +tmp=$(mktemp) +for u in $DAV_USERS; do + pw=$(pw_of "$u" || true) + [ -n "$pw" ] || { echo "FAIL no app password pair for user '$u' in DAV_APPPW"; exit 1; } + echo "$(url_of "$u") $u $pw" >>"$tmp" +done +mkdir -p /etc/davfs2/certs +cat "$tmp" >"$SECRETS"; rm -f "$tmp"; chmod 600 "$SECRETS" + +# self-signed https: the operator pastes the server cert (PEM) — davfs2's own trust knob +if [ -n "${DAV_CA:-}" ]; then + printf '%s\n' "$DAV_CA" >/etc/davfs2/certs/trusted.pem + chmod 600 /etc/davfs2/certs/trusted.pem + printf '%s\n' "trust_server_cert trusted.pem" >>/etc/davfs2/davfs2.conf +fi + +mkdir -p "$BASE" +echo "dav mount helper up: users='$DAV_USERS' base=$BASE owner=$UID_M:$GID_M ro" + +while :; do + for u in $DAV_USERS; do + d="$BASE/$u" + mkdir -p "$d" + mountpoint -q "$d" && continue + if mount -t davfs "$(url_of "$u")" "$d" \ + -o "ro,uid=$UID_M,gid=$GID_M,dir_mode=0750,file_mode=0640,noninteractive"; then + echo "OK mounted $u ($(url_of "$u") -> $d, ro, uid $UID_M)" + else + echo "WARN mount failed for $u — retrying in 30 s (Nextcloud reachable? app password?)" + fi + done + sleep 30 +done diff --git a/deploy/docker-compose.all-in-one.yml b/deploy/docker-compose.all-in-one.yml new file mode 100644 index 00000000..7afa0ce7 --- /dev/null +++ b/deploy/docker-compose.all-in-one.yml @@ -0,0 +1,169 @@ +# All-in-one EXAMPLE: one file, the whole product. This merges the shipped chain +# docker-compose.nextcloud.yml + docker-compose.nvidia.yml + docker-compose.lan.yml +# + docker-compose.nextcloud.lan.yml + docker-compose.app-targets.yml +# into a single file: Nextcloud + the DigUp server (CUDA build) + LAN bindings on both +# web ports + an application's named index target + the WebDAV mount container. +# Every service definition matches the shipped file it derives from; the env names are +# unchanged. docs/implementation/stage-12-integrations.md is the design; the shipped +# chain stays canonical (this example mirrors it, it does not replace it). +# +# cd deploy +# mkdir -p data/nextcloud/{index,models,config,apps,html,server-config,dav} \ +# && sudo chown -R 33:33 data/nextcloud +# DIGUP_LAN_BIND=192.0.2.1 \ # the host's own LAN address (example; ip -4 route get 8.8.8.8) +# DIGUP_NEXTCLOUD_DATA=$PWD/data/nextcloud/html \ +# DIGUP_NEXTCLOUD_ADMIN_USER=admin \ +# DIGUP_NEXTCLOUD_ADMIN_PASSWORD='pick-a-password' \ +# DIGUP_NEXTCLOUD_INSTANCE_ID= \ +# DIGUP_NEXTCLOUD_READ_TOKEN= \ +# DIGUP_SERVER_ADMIN_TOKEN= \ +# DIGUP_APP_TARGET_HOST_DIR=/path/to/an/application/data \ +# CUDA_ARCHS=61 \ # plain card number: 61 = Pascal sm_61, 89 = Ada sm_89 +# docker compose -f docker-compose.all-in-one.yml --profile nvidia --profile dav up -d --build +# → Nextcloud on http://:8081/ (and 127.0.0.1:8081), DigUp on http://:8787/ (and 127.0.0.1:8787). +# +# Peeling the layers (the same env names as the shipped chain): +# - CPU-only: delete the server's `profiles: ["nvidia"]` line, its build `target:` + +# the three CUDA `args:` lines, `NVIDIA_VISIBLE_DEVICES`, and the whole +# `deploy:` block — then plain `docker compose -f … up -d --build`. +# - Loopback: delete the two `${DIGUP_LAN_BIND…}` port lines. +# - Data shape (the Nextcloud data dir IS the media tree): delete the dav service; +# leave DIGUP_MEDIA_BIND unset (the server binds the data dir read-only). +# - WebDAV shape (files of a REMOTE Nextcloud): keep dav, and add +# NEXTCLOUD_URL=http://nextcloud DAV_USERS="alice bob" +# DAV_APPPW="alice: bob:" \ +# DIGUP_MEDIA_BIND=$PWD/data/nextcloud/html DIGUP_MEDIA_TARGET=/media/dav \ +# DIGUP_MEDIA_BIND_OPTS=ro,rslave \ +# DIGUP_SERVER_ROOTS=/media/dav/alice:/media/dav/bob +# — roots are the per-user colon list (the engine walk does not cross +# mount points, measured); trusted_domains below already carries the +# `nextcloud` service name the dav container dials (400 without it). +# - No app target: delete the two DIGUP_APP_TARGET volumes and the +# DIGUP_SERVER_APP_TARGETS line. +# +# The GPU stays honest: this file makes the card AVAILABLE, never claims it is USED — +# the engine measures (`speed --model` device + cosine agreement) and the utilization +# gate is server/tests/gpu-handoff.sh. Match CUDA_BASE/CUDA_RUNTIME_IMAGE to the host +# driver's `nvidia-smi` "CUDA Version" (defaults: 12.4.1; Pascal with an 11.8 driver → +# nvidia/cuda:11.8.0-devel-ubuntu22.04 + …-runtime-ubuntu22.04). +# The server has no auth: the LAN bindings ride the home-LAN trust model, as documented +# in docker-compose.lan.yml. Nextcloud keeps its own login in front of :8081. + +name: digup-server-all + +services: + server: + profiles: ["nvidia"] # the CUDA shape of the server: delete this line for CPU-only + build: + context: .. + dockerfile: server/docker/Dockerfile + target: server-runtime-cuda # CPU-only: target: server-runtime and drop the args block + args: + BASE_IMAGE: ${CUDA_BASE:-nvidia/cuda:12.4.1-devel-ubuntu22.04} + CUDA_RUNTIME_IMAGE: ${CUDA_RUNTIME_IMAGE:-nvidia/cuda:12.4.1-runtime-ubuntu22.04} + CUDA_ARCHS: ${CUDA_ARCHS:-native} + image: digup-server:local + restart: unless-stopped + stop_grace_period: 90s + group_add: + - "${DIGUP_MEDIA_GID:-33}" # the media tree's group (the server's uid is 990) + ports: + - "127.0.0.1:${DIGUP_PUBLISH_PORT:-8787}:8787" + - "${DIGUP_LAN_BIND:?the LAN bind is opt-in in the shipped chain; delete this port line for a loopback-only stack}:${DIGUP_PUBLISH_PORT:-8787}:8787" + depends_on: + - nextcloud + environment: + NVIDIA_VISIBLE_DEVICES: ${DIGUP_NVIDIA_GPU:-all} # pin one card by UUID on multi-GPU hosts + DIGUP_SERVER_INDEX: /data/index + DIGUP_SERVER_ROOTS: ${DIGUP_SERVER_ROOTS:-/media/photos} + DIGUP_SERVER_MODELS: /data/models + DIGUP_SERVER_CONFIG: /data/config + # Stage 13 named indexes (the app-targets layer): one named target with its OWN + # index folder beside the media index — the app's key mints and answers that + # target alone. name=roots:index, roots colon-separated. + DIGUP_SERVER_APP_TARGETS: ${DIGUP_APP_TARGET_NAME:-app}=/media/app:/data/index-${DIGUP_APP_TARGET_NAME:-app} + # The Nextcloud junk exclusion composed from the instance id (inert on the WebDAV + # shape — appdata is in no user's dav view; the engine's exact-path rule no-ops it). + DIGUP_SERVER_EXCLUDE: ${DIGUP_MEDIA_TARGET:-/media/photos}/appdata_${DIGUP_NEXTCLOUD_INSTANCE_ID:-}${DIGUP_SERVER_EXTRA_EXCLUDE:+:}${DIGUP_SERVER_EXTRA_EXCLUDE:-} + DIGUP_SERVER_EXCLUDE_EXT: ${DIGUP_SERVER_EXCLUDE_EXT:-} + DIGUP_SERVER_READ_TOKENS: nextcloud:${DIGUP_NEXTCLOUD_READ_TOKEN:?a secret for the Nextcloud app — read scope, i.e. search media health} + DIGUP_SERVER_ADMIN_TOKEN: ${DIGUP_SERVER_ADMIN_TOKEN:?a secret for the operator — index jobs and estimates} + DIGUP_SERVER_ALLOWED_HOSTS: ${DIGUP_SERVER_ALLOWED_HOSTS:-server,127.0.0.1,localhost} + DIGUP_SERVER_ENCODER_IDLE: ${DIGUP_SERVER_ENCODER_IDLE:-300} + DIGUP_SERVER_PREWARM: ${DIGUP_SERVER_PREWARM:-1} + DIGUP_SERVER_BACKFILL: ${DIGUP_SERVER_BACKFILL:-1} + DIGUP_SERVER_JOB_LOG_SIZE: ${DIGUP_SERVER_JOB_LOG_SIZE:-1048576} + DIGUP_SERVER_ESTIMATE_TIMEOUT_S: ${DIGUP_SERVER_ESTIMATE_TIMEOUT_S:-120} + DIGUP_SERVER_JOB_TIMEOUT_S: ${DIGUP_SERVER_JOB_TIMEOUT_S:-3600} + volumes: + - ${DIGUP_NEXTCLOUD_INDEX_DIR:-./data/nextcloud/index}:/data/index + - ${DIGUP_NEXTCLOUD_MODELS_DIR:-./data/nextcloud/models}:/data/models + - ${DIGUP_NEXTCLOUD_SERVER_CONFIG_DIR:-./data/nextcloud/server-config}:/data/config + # The application's own data folder (app-targets layer), read-only, and its OWN + # index folder — recreation never destroys it; its writer lock is its own, so its + # crawl never queues behind the media crawl. + - ${DIGUP_APP_TARGET_HOST_DIR:?the application's data folder (read-only mount); delete this volume + DIGUP_SERVER_APP_TARGETS for no app target}:/media/app:ro + - ${DIGUP_APP_TARGET_INDEX_HOST_DIR:-./data/index-${DIGUP_APP_TARGET_NAME:-app}}:/data/index-${DIGUP_APP_TARGET_NAME:-app} + # The media tree the server reads, READ-ONLY: the Nextcloud data dir by default + # (data shape); the WebDAV shape binds the dav-mount parent with rslave so the + # dav container's mounts propagate live (no server restart for (re)mounts). + - ${DIGUP_MEDIA_BIND:-${DIGUP_NEXTCLOUD_DATA}}:${DIGUP_MEDIA_TARGET:-/media/photos}:${DIGUP_MEDIA_BIND_OPTS:-ro} + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [cuda] + + dav: + # The mount helper of the WebDAV shape (§11.7): davfs2 in a container, mounts every + # user's dav view at /mnt/dav/ and propagates them into the host directory the + # server binds. Opt-in: --profile dav. Env names/defaults exactly as shipped. + profiles: ["dav"] + build: + context: ./dav + dockerfile: Dockerfile + image: digup-davfs:local + restart: unless-stopped + stop_grace_period: 30s + privileged: true # FUSE mount + propagation (SYS_ADMIN); mounts only, serves nothing + devices: + - /dev/fuse:/dev/fuse + environment: + # NOT :?-forced on purpose: compose interpolates profile-off services too, and the + # data-shape runs must not need dav variables. The entrypoint is the loud gate. + NEXTCLOUD_URL: "${NEXTCLOUD_URL:-}" # in this stack: http://nextcloud + DAV_USERS: "${DAV_USERS:-}" # space-separated user ids + DAV_APPPW: "${DAV_APPPW:-}" # space-separated user:app-password pairs + DAV_UID: ${DAV_UID:-990} # the tree's owner: the server container's user + DAV_GID: ${DAV_GID:-990} + volumes: + - ${DAV_HOST_DIR:-${DIGUP_MEDIA_BIND:-${DIGUP_NEXTCLOUD_DATA}}}:/mnt/dav:shared + + nextcloud: + image: nextcloud:${DIGUP_NEXTCLOUD_TAG:-28-apache} + restart: unless-stopped + group_add: + - "${DIGUP_NEXTCLOUD_EXTSTORAGE_GID:-1000}" # the extstorage tree's group for the web process (uid 33) + ports: + - "127.0.0.1:${DIGUP_NEXTCLOUD_PUBLISH_PORT:-8081}:80" + - "${DIGUP_LAN_BIND:?opt-in by design — the host's own LAN address; Nextcloud keeps its own login in front}:${DIGUP_NEXTCLOUD_PUBLISH_PORT:-8081}:80" + environment: + NEXTCLOUD_ADMIN_USER: ${DIGUP_NEXTCLOUD_ADMIN_USER:?the first run installs headlessly — name the admin account} + NEXTCLOUD_ADMIN_PASSWORD: ${DIGUP_NEXTCLOUD_ADMIN_PASSWORD:?the first run installs headlessly — set the admin password} + # The `nextcloud` entry is the compose service name the dav container dials — the + # WebDAV requests carry that Host and a missing entry answers 400 (measured). The + # LAN name/IP is added here too when DIGUP_LAN_BIND is set (trusted domains are + # applied by the image ONLY at first install; after that occ owns them — see the + # stage-12 LAN note). + NEXTCLOUD_TRUSTED_DOMAINS: ${DIGUP_NEXTCLOUD_TRUSTED_DOMAINS:-127.0.0.1 localhost nextcloud} + APACHE_DISABLE_REWRITE_IP: "1" + volumes: + - ${DIGUP_NEXTCLOUD_DATA:?the Nextcloud data directory (its files live under /files) — required}:/var/www/html/data + - ${DIGUP_NEXTCLOUD_CONFIG_DIR:-./data/nextcloud/config}:/var/www/html/config + - ${DIGUP_NEXTCLOUD_APPS_DIR:-./data/nextcloud/apps}:/var/www/html/custom_apps + # External Storage bind (a server-side deployment REFERS to an existing tree at + # /extstorage read-only — the DigUp server binds the SAME host folder). + - ${DIGUP_NEXTCLOUD_EXTSTORAGE:-./data/nextcloud/extstorage}:/extstorage:ro + - ${DIGUP_NEXTCLOUD_EXTSTORAGE_HOOK:-./nextcloud-hooks/extstorage-group.sh}:/docker-entrypoint-hooks.d/before-starting/extstorage-group.sh:ro diff --git a/deploy/docker-compose.app-targets.yml b/deploy/docker-compose.app-targets.yml new file mode 100644 index 00000000..75a5e5a0 --- /dev/null +++ b/deploy/docker-compose.app-targets.yml @@ -0,0 +1,55 @@ +# Stage 13 opt-in application-targets fragment: the standalone server also indexes and +# answers for an APPLICATION's own folder, in its OWN index, next to (never inside) its +# media index. docs/API.md "Named indexes" is the contract; targets.py documents the +# registry. The default deployment (plain docker-compose.yml) is byte-identical without +# this file — the feature engages only when you include -f this-file. +# +# Bring-up (full chain rule: all files, all boot env, never defaults): +# +# DIGUP_APP_TARGET_NAME=nextcloud \ +# DIGUP_APP_TARGET_HOST_DIR=/path/to/the/application/data \ +# docker compose -f docker-compose.yml -f docker-compose.app-targets.yml up -d +# +# (DIGUP_APP_TARGET_INDEX_HOST_DIR names the host folder for the target's index; the +# default sits beside the media index, ./data/index-.) +# +# Then the flow, end to end: +# 1. The server mints the target's application key at first boot (nothing to type — +# mint_tokens; stored 0600 in the config folder, recreation-safe) and the admin key. +# Minting turns auth on, so the admin key is bootstrapped from the file on the +# operator's own mount: `cat data/config/tokens.json`. +# 2. Open the UI → gear → Settings → API key: the minted keys render as rows (the +# admin key first, then each app key named by its target) — Copy takes the app's +# key to the clipboard; Regenerate (confirm) mints fresh values and the old ones +# die immediately: every application that has a key pasted loses access until the +# new key is pasted into it (this browser adopts its new admin key itself). +# The main page needs NO key: in the minted shape the main index's reads — the +# search box, previews, status, the gear's index rows — answer openly; the keys +# guard index jobs, estimates, the keys surface, and each app's own index. +# (Index now/Estimate in this browser answer 401 until it carries the admin key — +# `cat data/config/tokens.json` and set it in localStorage as `digupBearer`, or +# Regenerate to adopt a fresh one.) +# 3. Paste it into the application's DigUp bridge settings (Nextcloud app: server URL + +# this key). The bridge's searches, status, and "index now" all answer for the +# target's OWN index — the media index is untouched, and a second application gets +# its own target the same way (DIGUP_SERVER_APP_TARGETS lists them all, ';'-separated, +# and wins over the persisted /targets.json per name). +# Stage 14: applications can also REGISTER AT RUNTIME with the admin key — +# `POST /api/v1/targets {"name": …, "roots": […]}` mints their private key, gives +# them their own index folder, and persists the registration (docs/API.md). +# 4. The target's index folder rides its own bind mount (below), so indexing survives +# recreation exactly like the media index's; its writer lock, jobs log, and searcher +# are per target — a crawl here never queues behind the media crawl. +services: + server: + volumes: + # The application's data folder, mounted read-only: the server indexes it, the + # engine never writes to it (README: indexing reads media, writes only the index). + - ${DIGUP_APP_TARGET_HOST_DIR:?set it to the application's data folder (read-only mount)}:/media/app:ro + # The target's OWN index folder — a bind of its own, beside the media index's, + # so recreation never destroys it (README §5, same rule as every index). + - ${DIGUP_APP_TARGET_INDEX_HOST_DIR:-./data/index-${DIGUP_APP_TARGET_NAME:-app}}:/data/index-${DIGUP_APP_TARGET_NAME:-app} + environment: + # name=roots:index — roots colon-separated (as in DIGUP_SERVER_ROOTS), the index + # the LAST folder, under /data (a bind mount, so it survives recreation). + DIGUP_SERVER_APP_TARGETS: ${DIGUP_APP_TARGET_NAME:-app}=/media/app:/data/index-${DIGUP_APP_TARGET_NAME:-app} diff --git a/deploy/docker-compose.ghcr.nvidia.yml b/deploy/docker-compose.ghcr.nvidia.yml new file mode 100644 index 00000000..45f739e1 --- /dev/null +++ b/deploy/docker-compose.ghcr.nvidia.yml @@ -0,0 +1,28 @@ +# Pull-and-run GPU shape: the published CUDA image, no local CUDA build. +# +# docker compose -f docker-compose.yml -f docker-compose.ghcr.yml \ +# -f docker-compose.ghcr.nvidia.yml --profile nvidia up -d +# +# Host prerequisites are docker-compose.nvidia.yml's, unchanged: the NVIDIA Container +# Toolkit, and on multi-GPU hosts DIGUP_NVIDIA_GPU pins the card (a UUID from +# `nvidia-smi -L`). The engine, not this file, decides whether the card is USED +# (`digup speed --model` measures it; the proof of use is gpu-handoff.sh's utilization +# gate) — the image shape is the same server-runtime-cuda target the build path ships. +# +# Tag knob mirrors docker-compose.ghcr.yml, with the -cuda suffix appended by the +# literal below: DIGUP_IMAGE_TAG=latest → :latest-cuda; DIGUP_IMAGE_TAG=feat-server → +# :feat-server-cuda. Requires docker-compose.ghcr.yml in the -f chain (it carries the +# `!reset` that removes the build block; this file alone keeps the base's build shape). +services: + server: + profiles: ["nvidia"] + image: ghcr.io/ocean572/digup-server:${DIGUP_IMAGE_TAG:-latest}-cuda + environment: + NVIDIA_VISIBLE_DEVICES: ${DIGUP_NVIDIA_GPU:-all} + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [cuda] diff --git a/deploy/docker-compose.ghcr.yml b/deploy/docker-compose.ghcr.yml new file mode 100644 index 00000000..c14fa9e1 --- /dev/null +++ b/deploy/docker-compose.ghcr.yml @@ -0,0 +1,23 @@ +# Pull-and-run override (server/README.md "Pull the published image"): use the ghcr.io +# image instead of building from source. +# +# docker compose -f docker-compose.yml -f docker-compose.ghcr.yml up -d +# +# The Server workflow publishes these images — x86_64 only, by rule: cross-emulation +# builds are forbidden (docs/implementation/README.md §11), so arm64 hosts build the +# image natively under that same rule. The tag knob picks the build: +# +# DIGUP_IMAGE_TAG=latest → main's published image (default) +# DIGUP_IMAGE_TAG=feat-server → the PR branch's image (branch name, lowercased, +# slashes → dashes; pr--style tags do not exist, +# a push event carries the branch, not the PR number) +# +# The package is public when the repository is public (this repo is) — no login needed. +# If a package ever goes private, `docker login ghcr.io` with a PAT (read:packages) +# first. All the DIGUP_* env, knobs and volumes come from docker-compose.yml unchanged: +# this file only swaps where the image comes from. `!reset` nulls the base file's +# `build:` block — a service with only `image:` is pulled, never built. +services: + server: + build: !reset null + image: ghcr.io/ocean572/digup-server:${DIGUP_IMAGE_TAG:-latest} diff --git a/deploy/docker-compose.lan.yml b/deploy/docker-compose.lan.yml new file mode 100644 index 00000000..140b63f6 --- /dev/null +++ b/deploy/docker-compose.lan.yml @@ -0,0 +1,23 @@ +# ── Opt-in LAN exposure ────────────────────────────────────────────────────────────── +# Appends a SECOND host binding for the API: other machines on your network reach +# http://:8787/ (the base file's 127.0.0.1 binding stays in place, so local use +# never breaks). The address is REQUIRED, set per run: +# +# DIGUP_LAN_BIND=192.168.1.20 \ +# docker compose -f docker-compose.yml -f docker-compose.lan.yml up -d +# +# READ THIS BEFORE USING IT: the server has NO authentication (server/README §7 — +# localhost-only by design). With this file every device on your network can read the +# WHOLE indexed library through the UI and the API — searches, previews, paths. Bind a +# single LAN address (the host's own, from `ip -4 route get 8.8.8.8`); do NOT put a +# wildcard there, and do not use this on a network you do not own — on untrusted +# networks the §7 shape is an authenticating reverse proxy in front of the loopback +# binding, not this file. Home LAN use is exactly what this file is for. +# +# DIGUP_PUBLISH_PORT keeps working (the LAN binding follows the same host port). +# Compose note: `ports` lists MERGE across files, so the loopback mapping from the +# base file stays active — this file adds, it never replaces. +services: + server: + ports: + - "${DIGUP_LAN_BIND:?this file is opt-in by design — set DIGUP_LAN_BIND to the host's own LAN address (e.g. 192.168.1.20); the server has no auth, see the header comment}:${DIGUP_PUBLISH_PORT:-8787}:8787" diff --git a/deploy/docker-compose.nextcloud.lan.yml b/deploy/docker-compose.nextcloud.lan.yml new file mode 100644 index 00000000..38827179 --- /dev/null +++ b/deploy/docker-compose.nextcloud.lan.yml @@ -0,0 +1,26 @@ +# ── Opt-in LAN exposure for the Nextcloud + DigUp stack ────────────────────────────── +# Appends a SECOND host binding for the NEXTCLOUD web port (the DigUp API stays +# loopback-only: the app reaches it server-to-server inside the compose network). +# The base file's 127.0.0.1 binding stays in place (compose merges `ports` lists), so +# local use never breaks. The address is REQUIRED, set per run: +# +# DIGUP_LAN_BIND=192.0.2.10 DIGUP_NEXTCLOUD_TRUSTED_DOMAINS="127.0.0.1 localhost \ +# nextcloud nc.example.lan" \ +# docker compose -f docker-compose.nextcloud.yml -f docker-compose.nextcloud.lan.yml \ +# --profile dav up -d +# +# The hostname your LAN DNS resolves to (here: nc.example.lan) must be in +# DIGUP_NEXTCLOUD_TRUSTED_DOMAINS — Nextcloud answers untrusted Host headers with 400 +# "Remote Address (…): Not Trusted" (measured). Wildcards are refused by core. +# +# Same honesty note as docker-compose.lan.yml: this exposes Nextcloud's OWN login — +# every device on the network can reach the Nextcloud login page (and through the app, +# the searches of the users who log in there). Home LAN use is what this file is for; +# on a network you do not own, put TLS + hardening in front instead. +# +# DIGUP_NEXTCLOUD_PUBLISH_PORT moves the port; the LAN binding follows it (the +# container always binds 80 internally). +services: + nextcloud: + ports: + - "${DIGUP_LAN_BIND:?opt-in by design — the host's own LAN address (ip -4 route get 8.8.8.8); Nextcloud keeps its own login in front, see the header}:${DIGUP_NEXTCLOUD_PUBLISH_PORT:-8081}:80" diff --git a/deploy/docker-compose.nextcloud.yml b/deploy/docker-compose.nextcloud.yml new file mode 100644 index 00000000..63a5fc59 --- /dev/null +++ b/deploy/docker-compose.nextcloud.yml @@ -0,0 +1,212 @@ +# Nextcloud + DigUp server, one stack (docs/implementation/stage-12-integrations.md). +# +# cd deploy +# mkdir -p data/nextcloud/{index,models,config,apps,html} && sudo chown -R 33:33 data/nextcloud +# DIGUP_NEXTCLOUD_DATA=$PWD/data/nextcloud/html \ +# DIGUP_NEXTCLOUD_ADMIN_USER=admin \ +# DIGUP_NEXTCLOUD_ADMIN_PASSWORD='pick-a-password' \ +# DIGUP_NEXTCLOUD_INSTANCE_ID= \ +# DIGUP_NEXTCLOUD_READ_TOKEN= \ +# DIGUP_SERVER_ADMIN_TOKEN= \ +# docker compose -f docker-compose.nextcloud.yml up -d +# → Nextcloud on http://127.0.0.1:8081/, DigUp on http://127.0.0.1:8787/ +# +# The design (stage-12 §11.1): Nextcloud owns its files, users, ACLs and previews; the +# DigUp server reads the same data directory, mounted READ-ONLY at /media/photos, and the +# DigUp Nextcloud app (its own repository) queries the API server-to-server inside this +# compose network with the read token. Zero extra containers for the integration: the app +# runs inside Nextcloud, the search runs on this machine's GPU/CPU — the Nextcloud box +# never needs the card (the remote shape: DIGUP_LAN_BIND on the AI server + the app's +# settings pointing at its URL; the data folder reaches the AI server as an NFS/SMB +# read-only mount, stage-12 §11.5). +# +# Why uid/gid: the official image writes the data dir as www-data (uid/gid 33), so this +# file defaults DIGUP_MEDIA_GID to 33 — the server (uid 990) joins group 33 and reads the +# tree the scanner owns. The chown 33:33 above is what makes the first boot work (the same +# measured EACCES class as the base file's chown 990, in reverse). +# +# The exclusions (stage-12 §11.3): a Nextcloud data dir is not only files — previews, +# thumbnails, versions and temporaries live in appdata_/, and indexing those +# copies is pure pollution of the index. The engine skips folders by EXACT PATH, so the +# appdata exclusion is composed from the deployment's own instance id: +# +# docker compose -f docker-compose.nextcloud.yml exec nextcloud \ +# php occ config:system:get instanceid +# +# …and CACHEDIR.TAG-tagged folders (preview/, thumbnails/, versions/, cache/ inside appdata +# are exactly those) are auto-skipped by the engine's cache-folder rule even before the +# appdata exclusion applies — one gate, measured: estimate counts the users' files only, +# the junk copies not one (the pilot numbers are in IMPLEMENTATION_STATUS.md). Folders per +# deployment go appended via DIGUP_SERVER_EXTRA_EXCLUDE (colon-separated, the engine's +# pathsep list). +# +# The token map: this stack registers Nextcloud under the name `nextcloud` in +# DIGUP_SERVER_READ_TOKENS — the app gets THAT token and nothing else. Several Nextcloud +# instances may share one server (mount each data dir at its own prefix, register one map +# entry each): revoking one instance touches no other, and the server log names the +# principal on every guarded request (security.py PrincipalLog). +# +# Indexing never starts by itself (base file, §5): index through the DigUp UI with the +# admin token, from the server machine. Root-scope rule (base file header): this index +# folder belongs to this root layout; a moved mount needs one prune job. +# +# The WebDAV shape (stage-12 §11.7 — the per-user davfs2 deployment): instead of the raw +# data dir, the server's roots are ONE FOLDER OF PER-USER WebDAV MOUNTS — each user's dav +# view mounted READ-ONLY at // — so every file under // is, by +# construction, in that user's own view (own files + shares; groupfolders and received +# external shares included). The mount is the setup: with `--profile dav` this stack runs +# a `dav` mount container (davfs2 + FUSE + mount propagation, deploy/dav/) that mounts +# every user itself — no host mount step, no fstab, no host secrets file, no sudo: +# +# NEXTCLOUD_URL=http://nextcloud (the compose-network name; add `nextcloud` to +# DIGUP_NEXTCLOUD_TRUSTED_DOMAINS) +# DAV_USERS="alice bob carol" +# DAV_APPPW="alice: bob: carol:" (occ user:add-app-password ) +# DIGUP_SERVER_ROOTS=/media/dav/alice:/media/dav/bob:/media/dav/carol \ +# — ONE root per mounted user, colon-separated. Measured 2026-11-07: the engine's +# walk does not cross mount points, so a walk rooted AT each dav view works, while +# one walk through the shared parent sees zero subtrees. §11.5's multi-root list +# is exactly this shape. +# DIGUP_MEDIA_TARGET=/media/dav \ +# DIGUP_MEDIA_BIND=$PWD/data/ncdav DIGUP_MEDIA_BIND_OPTS=ro,rslave \ +# DAV_HOST_DIR=$PWD/data/ncdav \ +# DIGUP_NEXTCLOUD_INDEX_DIR=./data/nextcloud/index-dav (index folder per root layout!) +# (no DIGUP_NEXTCLOUD_INSTANCE_ID needed — appdata is not in any dav view) +# docker compose -f docker-compose.nextcloud.yml --profile dav up -d --build +# +# …with the app's settings in `webdav` mapping mode and the same prefix (/media/dav). +# The server's media bind rides the propagation (rslave): the server SEES mounts appear +# and vanish LIVE — stop/start the dav container is the whole remount procedure, no +# server restart. The dav container is privileged (FUSE needs SYS_ADMIN + /dev/fuse, +# the standard fuse-in-container shape); its whole job is these ro mounts, it serves +# nothing and binds nothing else. Ghost-row rule unchanged: this root layout owns its +# own index folder. Accepted costs (measured numbers in IMPLEMENTATION_STATUS.md): +# shares duplicate in the index (one row per viewer-mount), davfs2 crawl latency, app +# passwords per user. Wins: zero Nextcloud-host setup (its built-in WebDAV endpoint IS +# the share), server-side-encryption-safe (dav serves plaintext), per-user ACL comes +# from the mount, read-only enforced at the mount itself. +# +# Nextcloud's SSRF firewall: PHP requests to private addresses (this compose network) are +# blocked by Nextcloud's firewall by default — the app README documents the config.php +# line `'allow_local_servers' => true`; the config volume below keeps it persistent. +# +# LAN exposure of Nextcloud's web port (opt-in): `docker-compose.nextcloud.lan.yml` +# appends a LAN host binding for the nextcloud service (the API stays loopback: the app +# reaches it inside the compose network). The hostname your LAN DNS resolves to must be +# in DIGUP_NEXTCLOUD_TRUSTED_DOMAINS — the official image applies that env only at FIRST +# INSTALL (measured 2026-11-07), so on an installed stack add it with +# `php occ config:system:set trusted_domains --value=`. + +name: digup-server-nextcloud + +services: + dav: + # The mount helper of the WebDAV shape (header + §11.7): davfs2 in a container, + # mounts every user's dav view at /mnt/dav/ and propagates them to the host + # directory the server binds. Opt-in: --profile dav. + profiles: ["dav"] + build: + context: ./dav + dockerfile: Dockerfile + image: digup-davfs:local + restart: unless-stopped + stop_grace_period: 30s + privileged: true # FUSE mount + propagation (SYS_ADMIN); mounts only, serves nothing + devices: + - /dev/fuse:/dev/fuse + environment: + # NOT `:?`-forced on purpose: compose interpolates profile-off services too, and + # the data-shape runs must not need dav variables. The loud gate is the entrypoint + # itself: with the profile ON these are required, and it fails fast naming each. + NEXTCLOUD_URL: "${NEXTCLOUD_URL:-}" # in this stack: http://nextcloud + DAV_USERS: "${DAV_USERS:-}" # space-separated user ids + DAV_APPPW: "${DAV_APPPW:-}" # space-separated user:app-password pairs + DAV_UID: ${DAV_UID:-990} # the tree's owner: the server container's user + DAV_GID: ${DAV_GID:-990} + volumes: + # Default = exactly the host directory the server binds (same chain, one less env + # var to keep in sync): dav mounts INTO what the server reads. + - ${DAV_HOST_DIR:-${DIGUP_MEDIA_BIND:-${DIGUP_NEXTCLOUD_DATA}}}:/mnt/dav:shared + + nextcloud: + image: nextcloud:${DIGUP_NEXTCLOUD_TAG:-28-apache} + restart: unless-stopped + # The external-storage bind (the /extstorage volume below) is readable by the web + # server process (uid 33) only if that process also holds the host tree's GROUP — + # the same group_add mechanism the server service uses for its media bind. Default + # 1000 = the host's first human gid (what a user-owned data tree on a data disk + # carries); set DIGUP_NEXTCLOUD_EXTSTORAGE_GID to the tree's gid. + group_add: + - "${DIGUP_NEXTCLOUD_EXTSTORAGE_GID:-1000}" + ports: + - "127.0.0.1:${DIGUP_NEXTCLOUD_PUBLISH_PORT:-8081}:80" + environment: + NEXTCLOUD_ADMIN_USER: ${DIGUP_NEXTCLOUD_ADMIN_USER:?the first run installs headlessly — name the admin account} + NEXTCLOUD_ADMIN_PASSWORD: ${DIGUP_NEXTCLOUD_ADMIN_PASSWORD:?the first run installs headlessly — set the admin password} + NEXTCLOUD_TRUSTED_DOMAINS: ${DIGUP_NEXTCLOUD_TRUSTED_DOMAINS:-127.0.0.1 localhost} + APACHE_DISABLE_REWRITE_IP: "1" + volumes: + - ${DIGUP_NEXTCLOUD_DATA:?the Nextcloud data directory (its files live under /files) — required}:/var/www/html/data + - ${DIGUP_NEXTCLOUD_CONFIG_DIR:-./data/nextcloud/config}:/var/www/html/config + - ${DIGUP_NEXTCLOUD_APPS_DIR:-./data/nextcloud/apps}:/var/www/html/custom_apps + # Read-only host folder for Nextcloud's External Storage app (the local backend's + # datadir points inside /extstorage): this is how a server-side deployment lets + # Nextcloud REFERENCE an existing tree (e.g. a data dir on another disk) without + # moving it — the mount is read-only in Nextcloud's own configuration too + # (files_external readonly), and the DigUp server binds the SAME host folder. + - ${DIGUP_NEXTCLOUD_EXTSTORAGE:-./data/nextcloud/extstorage}:/extstorage:ro + # Official-image before-starting hook (see the script): grants www-data the + # tree's group inside the CONTAINER, so the apache worker can read the bind. + - ${DIGUP_NEXTCLOUD_EXTSTORAGE_HOOK:-./nextcloud-hooks/extstorage-group.sh}:/docker-entrypoint-hooks.d/before-starting/extstorage-group.sh:ro + + server: + build: + context: .. + dockerfile: server/docker/Dockerfile + target: server-runtime + image: digup-server:local + restart: unless-stopped + stop_grace_period: 90s + group_add: + - "${DIGUP_MEDIA_GID:-33}" # the data dir's group: the official image writes it as www-data + ports: + - "127.0.0.1:${DIGUP_PUBLISH_PORT:-8787}:8787" + depends_on: + - nextcloud + environment: + DIGUP_SERVER_INDEX: /data/index + DIGUP_SERVER_ROOTS: ${DIGUP_SERVER_ROOTS:-/media/photos} + DIGUP_SERVER_MODELS: /data/models + DIGUP_SERVER_CONFIG: /data/config + # The Nextcloud junk exclusion, composed from this deployment's instance id — and + # nothing about this file indexes copies of the users' previews. The path follows + # DIGUP_MEDIA_TARGET so the exclusion always names the mounted tree. In the WebDAV + # shape (root layout `//…`, §11.7) appdata is NOT in any user's dav + # view, so the exclusion is inert there and the instance id may stay unset (the + # composed path points at a folder that does not exist: the engine skips nothing, + # no-op by the engine's exact-path rule). + DIGUP_SERVER_EXCLUDE: ${DIGUP_MEDIA_TARGET:-/media/photos}/appdata_${DIGUP_NEXTCLOUD_INSTANCE_ID:-}${DIGUP_SERVER_EXTRA_EXCLUDE:+:}${DIGUP_SERVER_EXTRA_EXCLUDE:-} + DIGUP_SERVER_EXCLUDE_EXT: ${DIGUP_SERVER_EXCLUDE_EXT:-} + # The token map (§11.2): Nextcloud's app carries the read-scope entry named + # `nextcloud`; the operator keeps the admin token. The guard is on because a + # token-guarded deployment is exactly what names its hosts. + DIGUP_SERVER_READ_TOKENS: nextcloud:${DIGUP_NEXTCLOUD_READ_TOKEN:?a secret for the Nextcloud app — read scope, i.e. search media health} + DIGUP_SERVER_ADMIN_TOKEN: ${DIGUP_SERVER_ADMIN_TOKEN:?a secret for the operator — index jobs and estimates} + DIGUP_SERVER_ALLOWED_HOSTS: ${DIGUP_SERVER_ALLOWED_HOSTS:-server,127.0.0.1,localhost} + DIGUP_SERVER_ENCODER_IDLE: ${DIGUP_SERVER_ENCODER_IDLE:-300} + DIGUP_SERVER_PREWARM: ${DIGUP_SERVER_PREWARM:-1} + DIGUP_SERVER_BACKFILL: ${DIGUP_SERVER_BACKFILL:-1} + DIGUP_SERVER_JOB_LOG_SIZE: ${DIGUP_SERVER_JOB_LOG_SIZE:-1048576} + DIGUP_SERVER_ESTIMATE_TIMEOUT_S: ${DIGUP_SERVER_ESTIMATE_TIMEOUT_S:-120} + DIGUP_SERVER_JOB_TIMEOUT_S: ${DIGUP_SERVER_JOB_TIMEOUT_S:-3600} + volumes: + - ${DIGUP_NEXTCLOUD_INDEX_DIR:-./data/nextcloud/index}:/data/index + - ${DIGUP_NEXTCLOUD_MODELS_DIR:-./data/nextcloud/models}:/data/models + - ${DIGUP_NEXTCLOUD_SERVER_CONFIG_DIR:-./data/nextcloud/server-config}:/data/config + # The media tree the server reads, READ-ONLY: the Nextcloud data dir by default + # (data shape), any folder by env — the WebDAV shape binds its dav-mount parent + # (see the header). The target must match DIGUP_SERVER_ROOTS and the app's prefix. + # DIGUP_MEDIA_BIND_OPTS: the bind suffix — `ro` default; the WebDAV shape adds + # `,rslave` so the dav container's mounts ride the propagation and the server + # sees remounts live (no server restart for (re)mounts). + - ${DIGUP_MEDIA_BIND:-${DIGUP_NEXTCLOUD_DATA}}:${DIGUP_MEDIA_TARGET:-/media/photos}:${DIGUP_MEDIA_BIND_OPTS:-ro} diff --git a/deploy/docker-compose.nvidia.yml b/deploy/docker-compose.nvidia.yml new file mode 100644 index 00000000..d160d296 --- /dev/null +++ b/deploy/docker-compose.nvidia.yml @@ -0,0 +1,54 @@ +# Stage 9 opt-in NVIDIA override (docs/implementation/gpu-handoff.md §Gate B, item 4). +# The CPU-only docker-compose.yml stays the default: this file only takes effect with +# +# docker compose -f docker-compose.yml -f docker-compose.nvidia.yml --profile nvidia up -d --build +# +# Prerequisite: the NVIDIA Container Toolkit on the host (its test is in gpu-handoff.md). +# `libcuda.so.1` and the driver's CUDA userland come from the HOST through the toolkit — +# never baked into an image. The image carries the CUDA backend only when it was built +# from a matching CUDA base (see the Dockerfile's server-runtime-cuda target): +# +# DOCKER_BUILDKIT=1 docker build --target server-runtime-cuda \ +# --build-arg BASE_IMAGE=nvidia/cuda:.X.0-devel-ubuntu22.04 \ +# --build-arg CUDA_RUNTIME_IMAGE=nvidia/cuda:.X.0-runtime-ubuntu22.04 \ +# -f server/docker/Dockerfile -t digup-server:local .. +# +# (X matches the host driver's `nvidia-smi` "CUDA Version"; scripts/build-llama.sh finds +# nvcc itself; pass `--build-arg CUDA_ARCHS=` for the card — the builder's +# apt cmake 3.22.1 rejects dotted `6.1` and `native` alike (measured 2026-11-04): use +# `61` for Pascal sm_61, `89` for Ada sm_89. Fat lists (`61,70,75,80,86,89,90`) build a +# many-card binary; the Dockerfile normalizes the separators to CMake's semicolons.) +# +# This file makes the device AVAILABLE. It never claims the device is USED: the engine +# measures (`digup speed --model` keeps `device` in costs.json, and DIGUP_DEVICE=cpu is +# the CPU fallback). The proof of use is the utilization gate in gpu-handoff.sh — +# nvidia-smi sampled DURING a real index job, not a device that is merely visible. +# +# Multi-GPU hosts: DIGUP_NVIDIA_GPU pins the card (a UUID from `nvidia-smi -L`, any value +# NVIDIA_VISIBLE_DEVICES accepts) — the explicit env below outranks the reservation's +# device selection, so the container sees exactly that card. Default: all (a single-GPU +# box needs nothing set). +services: + server: + profiles: ["nvidia"] + # The CUDA shape of the image: same Dockerfile, the server-runtime-cuda target, the + # matching devel + runtime bases, the card's CUDA_ARCHS — a plain number for one card + # (apt cmake 3.22.1, see the comment above) or a fat list like `61,70,89` for many; + # the Dockerfile normalizes the separators. Compose merges this over the base file's + # CPU build, so `--profile nvidia … up -d --build` produces the CUDA image on the box + # that has it. + build: + target: server-runtime-cuda + args: + BASE_IMAGE: ${CUDA_BASE:-nvidia/cuda:12.4.1-devel-ubuntu22.04} + CUDA_RUNTIME_IMAGE: ${CUDA_RUNTIME_IMAGE:-nvidia/cuda:12.4.1-runtime-ubuntu22.04} + CUDA_ARCHS: ${CUDA_ARCHS:-native} + environment: + NVIDIA_VISIBLE_DEVICES: ${DIGUP_NVIDIA_GPU:-all} + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [cuda] diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml new file mode 100644 index 00000000..0f1f7bb4 --- /dev/null +++ b/deploy/docker-compose.yml @@ -0,0 +1,101 @@ +# The DigUp server, CPU-only first release (docs/implementation/README.md §3, §5, §7, §11). +# +# docker compose up -d --build → http://127.0.0.1:8787/ (UI + /api/v1) +# +# One service. The image is built from the repo's multi-stage Dockerfile on this machine's +# own architecture (never under emulation, §11). Bind mounts keep the index, model, config +# and job log OUTSIDE the image: container recreation never destroys them (§5). Media is +# mounted read-only; the server never modifies originals and never takes paths from +# clients (§7). Indexing never starts by itself: "Index now" in the UI, or POST /index. +# +# Paths come from the environment so this file is portable; defaults sit beside it under +# deploy/data/. On a production box, point the DIGUP_* variables at the real folders and +# make them owned by uid 990 (the image's `digup` user) so the bind mounts are writable. +# +# GPU is an override for the box that has the card, built and measured there (§11): +# docker-compose.nvidia.yml arrives at Stage 9, not as a promise here. +# +# Root scope (engine semantics, pinned in the Stage-7 proof + tests): index rows carry +# the absolute paths of the roots they were crawled from, and the searcher answers only +# for files under the roots it was started with. Change where the media mounts live +# (DIGUP_MEDIA_DIR's path), then run one job with prune — "Index now" + "prune missing", +# or POST /index {"prune":true} — to re-scope the index and retire the ghost rows of +# the old paths (rows outside the current roots keep counting in files/vectors and +# dilute z for live hits). Give each root layout its OWN index folder: sharing one +# index folder across two root scopes is how ghost rows get made. The engine's design, +# not a bug: hits are always real files under the configured roots. + +# Compose project name, explicit: this stack is `digup-server`, never the folder's +# default — other stacks living under folders named `deploy` keep their own project. + +name: digup-server + +services: + server: + build: + context: .. + dockerfile: server/docker/Dockerfile + target: server-runtime + args: + # The engine's `swift test` gate runs inside the builder stage by default (CI + # builds run it too). A box whose builder environment is measured to fail an + # engine-side test (docs/implementation notes; the server's own gate is the + # contract suite) builds with DIGUP_SWIFT_TESTS=0 — a documented escape hatch, + # not a default: unset means the gate runs. + SWIFT_TESTS: ${DIGUP_SWIFT_TESTS:-1} + image: digup-server:local + # A job mid-embed (a photo takes ~10–40 s of CPU) needs room for the engine's safe + # point: with the Docker default 10 s the stop lands as SIGKILL and the job recovers + # as *interrupted* at the next boot (honest, and resume works — measured Stage 8). + # 90 s covers one file's embed, so `docker stop` ends the job truthfully *inside* + # the grace, with its partial counts: 143 + "Application shutdown complete". + stop_grace_period: 90s + # Survive a daemon restart (a reboot, `docker pull` upgrading, a crash): come back + # with Docker, and after a manual `docker stop`, stay down — the honest semantics of + # `unless-stopped` for a single-user service. + restart: unless-stopped + # Media readability on a real Linux host: the bind is :ro (the server never touches + # originals), and a tree whose folders are owner/group only (a Nextcloud data dir, + # drwxr-x---) is readable by the image's uid 990 only when the container also holds + # that tree's group. Default 1000 = the host's first human uid; set DIGUP_MEDIA_GID + # to the tree's gid, or clear it with group_add: []. + group_add: + - "${DIGUP_MEDIA_GID:-1000}" + ports: + # localhost-only; LAN exposure needs auth first (§7). DIGUP_PUBLISH_PORT moves the + # host side when 8787 is taken; the container keeps binding 8787. + - "127.0.0.1:${DIGUP_PUBLISH_PORT:-8787}:8787" + environment: + DIGUP_SERVER_INDEX: /data/index + DIGUP_SERVER_ROOTS: ${DIGUP_SERVER_ROOTS:-/media/photos} + DIGUP_SERVER_MODELS: /data/models + DIGUP_SERVER_CONFIG: /data/config + DIGUP_SERVER_EXCLUDE: ${DIGUP_SERVER_EXCLUDE:-} + DIGUP_SERVER_EXCLUDE_EXT: ${DIGUP_SERVER_EXCLUDE_EXT:-} + DIGUP_SERVER_ENCODER_IDLE: ${DIGUP_SERVER_ENCODER_IDLE:-300} + DIGUP_SERVER_PREWARM: ${DIGUP_SERVER_PREWARM:-1} + DIGUP_SERVER_BACKFILL: ${DIGUP_SERVER_BACKFILL:-1} + DIGUP_SERVER_JOB_LOG_SIZE: ${DIGUP_SERVER_JOB_LOG_SIZE:-1048576} + # Estimate's ceiling: 120 s fits corpora; a full crawl of a large media tree on a + # spinning disk needs minutes (measured 2026-11-04: 187k files on a 2 TB HDD). + DIGUP_SERVER_ESTIMATE_TIMEOUT_S: ${DIGUP_SERVER_ESTIMATE_TIMEOUT_S:-120} + # A job's ceiling: 3600 s fits corpora; a real tree's first pass outlives an hour + # (measured 2026-11-04: interrupted with partial counters at 19,007/131,972 — the + # next job resumes). The engine keeps walking; this is only the runner's patience. + DIGUP_SERVER_JOB_TIMEOUT_S: ${DIGUP_SERVER_JOB_TIMEOUT_S:-3600} + # Off by default, like the Compose file itself: host validation and bearer tokens guard + # a Docker-network / reverse-proxy / LAN deployment (docs/API.md "Authentication"). A + # localhost-only box leaves all three unset. + DIGUP_SERVER_ALLOWED_HOSTS: ${DIGUP_SERVER_ALLOWED_HOSTS:-} + DIGUP_SERVER_TOKEN: ${DIGUP_SERVER_TOKEN:-} + DIGUP_SERVER_ADMIN_TOKEN: ${DIGUP_SERVER_ADMIN_TOKEN:-} + # The standalone box mints its own API key at boot (gear → API key: Copy the key; + # applications bring it to POST /api/v1/targets and get their own private key and + # their own index — docs/API.md "Named indexes"). Main-page reads stay key-free; + # the minted key guards index/estimate/keys. Unset keeps a plain tokenless server. + DIGUP_SERVER_MINT_KEYS: ${DIGUP_SERVER_MINT_KEYS:-} + volumes: + - ${DIGUP_INDEX_DIR:-./data/index}:/data/index + - ${DIGUP_MODELS_DIR:-./data/models}:/data/models + - ${DIGUP_CONFIG_DIR:-./data/config}:/data/config + - ${DIGUP_MEDIA_DIR:-./data/media/photos}:/media/photos:ro diff --git a/deploy/nextcloud-hooks/extstorage-group.sh b/deploy/nextcloud-hooks/extstorage-group.sh new file mode 100755 index 00000000..f7a73c07 --- /dev/null +++ b/deploy/nextcloud-hooks/extstorage-group.sh @@ -0,0 +1,12 @@ +#!/bin/sh +# Container-local fix for the extstorage bind: apache workers run as www-data and +# Apache calls initgroups() for the configured User, so the web process reads a +# group-restricted host tree only if www-data holds that group in the container's +# /etc/group. Create the group with the tree's gid (DIGUP_NEXTCLOUD_EXTSTORAGE_GID, +# default 1000) and add www-data to it. Nothing on the host tree changes. +gid="${DIGUP_NEXTCLOUD_EXTSTORAGE_GID:-1000}" +if ! getent group "$gid" >/dev/null; then + groupadd -g "$gid" extstorage +fi +usermod -aG extstorage www-data 2>/dev/null || true +exit 0 diff --git a/deploy/nextcloud-webdav.sh b/deploy/nextcloud-webdav.sh new file mode 100755 index 00000000..2c143c1b --- /dev/null +++ b/deploy/nextcloud-webdav.sh @@ -0,0 +1,169 @@ +#!/bin/sh +# nextcloud-webdav.sh — per-user WebDAV mounts for the DigUp server +# (docs/implementation/stage-12-integrations.md §11.7, the WebDAV deployment shape). +# +# WHAT THIS DOES: mounts each Nextcloud user's own WebDAV view +# (/remote.php/dav/files//) READ-ONLY on the DigUp server machine, one +# mount per user under //. Every file under // is, by +# construction, a file THAT user can see in Nextcloud (own files + shares + groupfolders) +# — the mount is the ACL. The DigUp server indexes the mount tree read-only; the +# digup_nextcloud app must be set to mapping mode "webdav" with this same base path. +# +# RUN AS ROOT on the DIGUP SERVER machine (the machine with the GPU — not the Nextcloud +# machine; Nextcloud's built-in WebDAV endpoint is the share, it needs no setup at all). +# +# SETUP (once per machine): +# sudo apt install -y davfs2 +# # davfs2 resolves the mounted tree's owner via getpwuid (measured 2026-11-07: +# # "can't read user data base" when DAV_UID has no passwd entry on THIS machine): +# grep -q '^digup:' /etc/passwd || sudo useradd -r -u 990 -M -d /nonexistent digup +# PER USER: an app password (Nextcloud → Settings → Security → Create new app +# password, or on the Nextcloud machine: occ user:add-app-password ). +# +# NOTE: the SHIPPED shape is the compose `dav` mount container (docker-compose. +# nextcloud.yml, --profile dav, stage-12 §11.7) — zero host steps. This script is the +# host-mount ALTERNATIVE for deployments that mount at the OS level (fstab/automount). +# +# NEXTCLOUD_URL=https://nc.example.lan DAV_USERS="alice bob carol" \ +# DAV_APPPW_ALICE=... DAV_APPPW_BOB=... DAV_APPPW_CAROL=... \ +# sudo -E ./nextcloud-webdav.sh mount +# +# -e NEXTCLOUD_URL the Nextcloud base URL (https://… normally; http:// on a pilot) +# -e DAV_USERS space-separated Nextcloud user ids to mount +# -e DAV_APPPW_ that user's app password (one env per user) +# -e DAV_BASE mount root (default /media/dav) +# -e DAV_UID -e DAV_GID (default 990 990 — the server container's user) +# +# Mounts run as davfs2's "daemon" (user=daemon): davfs2 keys its per-mount cache + +# credentials by mount URL + run uid, and a service mount under http needs a system +# account (measured: a plain login uid dies "can't read user data base" unless its +# davfs2 config home exists). The per-user identity lives in the mount URL/path. +# If you mount through this script as a NON-root user, also mark your login user +# "system" in /etc/davfs2/davfs2.conf (davfs2.conf(5)). +# +# Then in deploy/: point the server stack at the mounts (compose header in +# docker-compose.nextcloud.yml, the WebDAV shape) with +# DIGUP_SERVER_ROOTS=/:/:… (one root PER mounted +# user — measured 2026-11-07: the engine's walk does not cross mount points, a walk +# rooted AT each dav view works, one walk through sees zero subtrees) +# DIGUP_MEDIA_TARGET= +# DIGUP_MEDIA_BIND= DIGUP_NEXTCLOUD_INDEX_DIR= +# and set the app's settings (Administration settings → DigUp Search) to prefix +# and mapping mode "WebDAV mounts". Then RESTART the server container +# (a FUSE mount made after the container started is invisible to its bind) and index. +# +# Subcommands: +# mount (default) create mount dirs, register credentials, mount every user +# check per user: mounted? readable? file count visible +# umount unmount every user under DAV_BASE (then the server sees 0 files) +# fstab print the fstab lines that make the mounts boot-automounted +# (_netdev,x-systemd.automount) — paste-ready, append to /etc/fstab +# +# Credentials land in root-owned /etc/davfs2/secrets (chmod 600), keyed by mount URL — +# davfs2's own mechanism. Idempotent: re-running updates entries, skips live mounts. +# The mounts are read-only at the mount itself (ro): this machine cannot write through +# them, so the Nextcloud files are safe from this box by construction, not by convention. + +set -eu + +BASE="${DAV_BASE:-/media/dav}" +UID_RUN="${DAV_UID:-990}" +GID_RUN="${DAV_GID:-990}" +SECRETS=/etc/davfs2/secrets +CMD="${1:-mount}" + +die() { echo "FAIL: $*" >&2; exit 1; } + +[ "$(id -u)" -eq 0 ] || die "run this as root: sudo -E $0 $CMD" + +need_env() { [ -n "${NEXTCLOUD_URL:-}" ] || die "NEXTCLOUD_URL is required (e.g. https://nc.example.lan)"; [ "${DAV_USERS:+set}" = set ] || die "DAV_USERS is required (space-separated user ids)"; } +url_of() { echo "${NEXTCLOUD_URL%/}/remote.php/dav/files/$1/"; } + +check_davfs2() { + command -v mount.davfs >/dev/null 2>&1 \ + || die "davfs2 is not installed — sudo apt install -y davfs2" +} + +secret_set() { # $1 url, $2 user, $3 password: replace/append the URL's line, keep 600 + mkdir -p /etc/davfs2 + touch "$SECRETS"; chmod 600 "$SECRETS" + tmp=$(mktemp) + grep -v "^$1[[:space:]]" "$SECRETS" >"$tmp" || true + echo "$1 $2 $3" >>"$tmp" + cat "$tmp" >"$SECRETS"; rm -f "$tmp" + chmod 600 "$SECRETS" +} + +do_mount() { + check_davfs2 + mkdir -p "$BASE" + for u in $DAV_USERS; do + apppw_var="DAV_APPPW_$(printf '%s' "$u" | tr 'abcdefghijklmnopqrstuvwxyz' 'ABCDEFGHIJKLMNOPQRSTUVWXYZ')" + apppw=$(printenv "$apppw_var" || true) + [ -n "$apppw" ] || die "no app password for user '$u' — export $apppw_var= (create one: occ user:add-app-password $u)" + url=$(url_of "$u") + dir="$BASE/$u" + mkdir -p "$dir" + secret_set "$url" "$u" "$apppw" + if mountpoint -q "$dir"; then + echo "OK $u already mounted at $dir" + continue + fi + # ro = the Nextcloud files are un-writable through this mount; the uid/gid/ + # mode options put the tree in the server container's hands (uid 990) and + # nobody else's (0640/0750); noninteractive keeps the script CI-friendly. + # user=daemon: davfs2 keys its cache/secrets per Mount-point + Run-uid, not + # per login user — the per-user identity lives in the mount URL/path, and + # "daemon" is the account davfs2 accepts for a server-daemon mount (a plain + # service uid with no home dies in its user-data-base preflight; a REAL user + # is required over https, which this box's app passwords don't need here). + mount -t davfs "$url" "$dir" \ + -o ro,user=daemon,uid="$UID_RUN",gid="$GID_RUN",dir_mode=0750,file_mode=0640,noninteractive + echo "OK mounted $u ($url -> $dir, ro, uid $UID_RUN)" + done + echo "NOTE restart the server container after (re)mounting: docker compose restart server" + echo "NOTE app: prefix $BASE + mapping mode WebDAV; then index (POST /index, admin token)." +} + +do_check() { + for u in $DAV_USERS; do + dir="$BASE/$u" + if ! mountpoint -q "$dir"; then + echo "FAIL $u: not mounted at $dir" + continue + fi + n=$(find "$dir" -type f 2>/dev/null | wc -l) + if [ "$n" -eq 0 ]; then + echo "WARN $u: mounted, 0 files visible (app password stale? view empty?)" + else + echo "OK $u: mounted, $n files visible" + fi + done +} + +do_umount() { + for u in $DAV_USERS; do + if mountpoint -q "$BASE/$u"; then + umount "$BASE/$u" && echo "OK unmounted $u" + else + echo "OK $u was not mounted" + fi + done + echo "NOTE the server now sees 0 files under $BASE — Test connection says so;" + echo "NOTE remount + 'docker compose restart server' to recover." +} + +do_fstab() { + echo "# Paste into /etc/fstab (append-only). Auto-mounts on first access after boot." + for u in $DAV_USERS; do + echo "$(url_of "$u") $BASE/$u davfs ro,_netdev,x-systemd.automount,uid=$UID_RUN,gid=$GID_RUN,dir_mode=0750,file_mode=0640 0 0" + done +} + +case "$CMD" in + mount) need_env; do_mount ;; + check) need_env; do_check ;; + umount) need_env; do_umount ;; + fstab) need_env; do_fstab ;; + *) die "unknown subcommand '$CMD' (mount|check|umount|fstab)" ;; +esac diff --git a/docs/implementation/README.md b/docs/implementation/README.md new file mode 100644 index 00000000..100247d3 --- /dev/null +++ b/docs/implementation/README.md @@ -0,0 +1,298 @@ +# DigUp Server — implementation instructions (binding) + +These are the rules this project follows. They are not suggestions. When a stage's work +violates one of them, the stage is not done. + +Companion documents: + +- [reconnaissance.md](reconnaissance.md) — what the existing repository is, verified by reading + it and probing the environment. Start there. +- [review-round.md](review-round.md) — the adversarial review before the PR: every finding that + was real, its fix, the test that pins it, and what was deliberately left alone with its reason. +- [stage-12-integrations.md](stage-12-integrations.md) — the integration contract for serving + applications *of* the library (Nextcloud first): the shared-stack shape, the named-token map, + the data-dir exclusion gate, the CORS allowlist and its guard exception, the embed widget, + the multi-instance honesty. Its §11.x numbers are quoted in code comments. +- `api.md`, `deployment.md`, `operations.md` — written as those parts are implemented. + +--- + +## 1. The engine is DigUp's. It is not rewritten. + +The original Swift engine remains responsible for reading media, extracting images / frames / +audio / text, generating embeddings, storing them, searching, and ranking. The server is a +wrapper around it. + +Do **not**: + +- replace EmbeddingGemma 2 or llama.cpp; +- write a second embedding engine (in Python or anything else); +- change preprocessing, vector dimensions, similarity math, video frame sampling, audio + segmentation, ranking behaviour, or the SQLite index schema; +- build a parallel/incompatible vector store; +- invent similarity scores, progress percentages, timestamps, or metadata the engine did not + produce. + +`worker/digup_worker.py` in this repo is the mlx-vlm **reference** runtime used by the evals on +Apple Silicon. It is **not** a server engine and must not be used by the server. + +## 2. DigUp is a native Swift application + +This is the single most important fact for packaging, and the one most often mistaken: + +- The search engine is Swift (`Sources/DigUpKit`, `Sources/LlamaRuntime`), built into the + executable product `digup` (`Package.swift`). llama.cpp is a C/C++ library that Swift calls. +- The Linux and Windows applications are **not** the engine. They are a Tauri shell (Rust + + web UI) that starts the Swift `digup` binary as helper processes and talks to it over + JSON-lines. `shell/src-tauri/src/helper.rs`, first sentence: *"DigUp's helper processes, + which do all the work."* +- Therefore: a headless DigUp server **is** a Swift binary at its core. The Python layer is + the HTTP/JSON wrapper; the web UI is a browser client. Both are thin. +- Swift is required **at build time only**. A runtime image needs no Swift compiler and no + Swift toolchain: `scripts/package-linux.sh` links the CLI with + `--static-swift-stdlib` (Swift 6.4's static stdlib needs its archives re-listed at the end of + the link line — copy that list from the script), and the shipped payload is the binary plus + native `.so` files with `rpath = $ORIGIN`. + +## 3. Linux Docker build (mandatory shape) + +Use a **multi-stage** build for the server image: + +1. **Builder stage** (one architecture, pinned versions, pinned commits — see reconnaissance + §9 for the exact pins): + - official Swift Linux toolchain (`swift-6.4.0-RELEASE`, download.swift.org, GPG-verified, + as `.github/workflows/packages.yml` does); + - C/C++ toolchain, CMake, Ninja, Python 3, curl, git, patch; + - the repository's own scripts: `scripts/fetch-vendor.sh` (SQLite/FTS5, stb, miniz, PDFium) + and `scripts/build-llama.sh` (llama.cpp `b11461`, patched, shared libraries with + `GGML_BACKEND_DL` + `GGML_CPU_ALL_VARIANTS`); + - `swift build -c release --product digup --static-swift-stdlib …` (the exact flags live in + `scripts/package-linux.sh`); + - run the test suite in this stage (`swift test`), never in the runtime image. +2. **Runtime stage**: a slim Linux base with only + `digup`, llama.cpp's shared libraries (CPU variants; Vulkan/CUDA only when that backend is + in use), `libpdfium.so`, `ffmpeg`, `ffprobe`, the runtime libraries they need + (`libgomp1`, `libstdc++6`, `libcurl4`; `libvulkan1` only for the Vulkan override), the + FastAPI application, the built web frontend, and Python. + - non-root user, no desktop environment, no Tauri, no WebKit/GTK, no X11, no Wayland, no + systemd; + - a Docker `HEALTHCHECK`; + - correct SIGTERM handling: the API process owns its child processes and stops them cleanly. + +Rules: + +- **Compile for the image's architecture.** An `x86_64` image is built on/before an + `x86_64` host. An `arm64` image is built on `arm64`. Never ship a binary of another + architecture, and never rely on emulation for indexing/search work. +- **Do not install the Swift compiler in the runtime image.** +- **Do not use macOS binaries, macOS frameworks, or a macOS-built `digup`** in any image. + Image work happens in Linux containers. +- **Verify the CLI inside the final runtime image** as part of the build/acceptance + procedure, headlessly, at minimum: + + ```sh + digup --help + digup status --index "$DIGUP_INDEX" + digup search "wakeboarding" --index "$DIGUP_INDEX" -n 5 + ``` + + If the container cannot run the engine's own CLI, the image is not finished. +- **CPU first.** Get CPU inference indexed and searching correctly before adding any NVIDIA / + Vulkan dependency. GPU support arrives as a separate Compose override + (`deploy/docker-compose.nvidia.yml`), is opt-in, and the server reports the backend it + actually measured (`digup speed --model`), never the backend it hopes for. +- Cross-check the self-built image against the upstream release payload + (`DigUp__amd64.deb`, `/usr/lib/DigUp/helper/`) when comparing search results, so a + build problem is distinguishable from an engine problem. Upstream artifacts are a + **reference**, not the shipped build. + +## 4. Engine interface + +`digup` already exposes persistent, documented, JSON-lines services, and they are the +interface for the server adapter (reconnaissance §5): + +- `digup search-service … --index DIR` — `search`, `status`, `summaries`, `text`, + `render`, `reload`, `folders`, `prewarm`, `quit`; keeps the index open and the query encoder + warm; real timings; real index counts; renders preview JPEGs (photos, PDF pages, video + frames at a second). +- `digup index-service … --index DIR` — commands `sync`, `sync code`, `stop`, + `backfill on|off`; emits `scanning`, `sync` (added/changed/moved/removed/unchanged), `file` + (done/total), `rest`, `summary`, `waiting`, `idle`, `error`, `exit`. Real progress counts. +- `digup status` / `digup index` / `digup search --json` / `digup estimate --json` / + `digup download-model` — one-shot commands, useful for diagnostics. +- The adapter exposes `health() / index() / search() / status() / close()` and nothing about + DigUp internals leaks above it. Subprocess calls use argument arrays, never shell strings. + +## 5. Persistence layout + +Everything mutable lives on named volumes/bind mounts outside the image: + +``` +/data/ index/ index.sqlite, code.sqlite, previews/, costs.json, worker logs + models/ embeddinggemma-2-Q8_0.gguf, mmproj-embeddinggemma-2-Q8_0.gguf + config/ server settings, job state (jobs.jsonl) +/media/photos/ read-only mount of the original media +``` + +- The single-writer lock is a file **beside the index** (`/.digup-server.writer.lock`, + `server/digup_server/engine/index.py`), not a separate folder: it travels with the volume + that holds the index it protects, and a moved index folder carries its lock with it. + +- `--index /data/index` is always explicit. `/data/models` is published as `$DIGUP_MODELS`. +- Media roots are mounted read-only, validated against an allowlist, and are never chosen by + the browser. Original files are never modified. +- Container recreation never destroys the index, model, config, or logs. +- Indexing never starts automatically at container start. +- One indexing job at a time, protected by an OS-level `flock` beside the index file (above), + plus the engine's SQLite locking. An interrupted job is recovered as *interrupted*, never + as success. + +## 6. API surface (`/api/v1`) + +`GET /health`, `GET /status`, `POST /search`, `POST /index`, `GET /jobs/{id}`, +`POST /embed_ahead`, `GET /media/{id}/preview`, `GET /media/{id}/text`, +`GET /media/{id}/file`. Pydantic models, OpenAPI documentation, versioned from the start. + +The plan's `GET /media/{id}/open` shipped as **`/file`** (the UI's Open + Download), and +the plan's `GET /settings` + `PATCH /settings` shipped as **no settings endpoints**: the +settings the UI offers (appearance, limit, kinds, code mode, prune) are browser-side +state and request parameters, and the server-side knobs (encoder idle, prewarm, backfill, +job-log size, roots) are container configuration in the environment — `server/.env.example` +— not a remote-writeable surface. A single-user localhost server writes its own config +through Compose, not through HTTP (§7: no settings endpoints to protect). + +- Search passes DigUp's ranking through untouched; media filters map onto DigUp's kinds + (`image`/`screenshot` → photo, `video`, `audio`, `pdf`, `doc`). +- Video hits keep DigUp's seconds (`loc`, `loc_end`) exactly as the engine produced them. +- Errors are reported as the engine reported them. +- Media endpoints accept only ids from the index, never paths from the client; they resolve + inside configured roots and refuse symlink traversal. + As shipped: `/file`'s type, size and span come from **one open** (`os.fstat` + + `stat.S_ISREG`) so a replacement mid-request can't make the headers lie about the bytes; + root confinement is `Path.is_relative_to`, not a prefix string (which matched + `/media/photos_evil` as a child of `/media/photos`); ranged spans are capped + (8 MiB) and **streamed** in chunks — `Range: bytes=0-` is what every media element + sends first, and an uncapped open range buffered the whole file; every `/file` answer + carries `x-content-type-options: nosniff` as the belt behind magic-bytes typing. + +## 7. Security and privacy + +- binds `127.0.0.1:8787` by default; LAN exposure only after authentication is implemented + and enabled; +- no plaintext passwords and no settings endpoints to protect (§6): the server's own + configuration is container configuration, and there are no uploads, no remote writes to + settings, and no auth state to leak. CORS is same-origin by construction — the UI is + served by the API, so no cross-origin allowance is granted at all; +- no shell interpolation of user input, no arbitrary path or URL acceptance, no cloud embedding + APIs, no uploads of media anywhere; +- filenames and metadata are escaped in the UI; +- the model is local: documented download path plus support for an already-cached model + directory (`$HF_HUB_CACHE`, `digup download-model --to /data/models`). + +## 8. Order of work + +Stage 1 reconnaissance → 2 headless proof of the engine → 3 engine adapter + tests → +4 API (health/status/search) → 5 index jobs, locking, recovery → 6 web application → +7 Docker (CPU-only) → 8 persistence and reliability → 9 GPU → 10 PhotoPrism → 11 polish. + +At the end of every stage: run tests, list files changed, give exact commands, give real +results, document open issues. **A stage that fails does not proceed by substituting mocks.** + +## 9. Git safety + +Preserve the repository: no `git reset --hard`, no `git clean -fd`, no overwriting user +changes, no remote changes, no force-push, no commits or pushes unless explicitly asked. +Server code stays separate from upstream engine code. + +## 10. Built to be merged upstream (sustainability) + +This server is intended as a **pull request to `ARahim3/DigUp`**, and as the basis of a public +page. Build it so a maintainer can adopt it, and so it survives the project's next 20 releases. + +1. **Additive only.** `Sources/**`, `Package.swift`, `scripts/package-*.sh`, `shell/`, + `website/` are not edited to make the server convenient. New code lives in + `server/` (FastAPI app + engine adapter + tests + Docker) and `docs/implementation/`, with a + dedicated `.github/workflows/server.yml`. If the engine must expose something the protocol + lacks, that is a separate minimal change to the documented JSON protocol — backward + compatible with the desktop shell, tested, and justified on its own. +2. **Speak the published protocol.** Only `search-service` / `index-service` / one-shot CLI + operations that the shipping shell already uses. Contract tests in `server/tests/` pin the + ops and reply fields. No private fields, no side channels, no parallel store, no re-ranking. +3. **Follow the house style.** Pinned versions with committed hashes (as + `scripts/vendor-checksums.json`), nothing vendored into git, dense comments that explain + *why*, MIT licensing and `scripts/notices.py` notices respected, `.env.example` for + configuration, no secrets, no large binaries. Python dependencies via `uv` + (`pyproject.toml` + `uv.lock`) — the repository's existing convention in `worker/`. +4. **One frontend, not two.** The repository's UI is dependency-free vanilla HTML/CSS/JS with + no build step (`shell/ui`, ~5 000 lines, `frontendDist: "../ui"`). The server UI is built in + that same style talking to `/api/v1` with `fetch()`. Adding React + TypeScript + Vite for the + server alone would leave the project maintaining two UI paradigms: not the route. (Structure + the server UI so a shared layer with `shell/ui` stays possible later.) + + > **Amended 2026-11-08 by explicit user brief** (server/ui/AGENTS.md is the architecture + > manual): the SERVER UI migrated to **Lit + JavaScript ES modules + Vite** — no TypeScript, + > no other framework, no router/state library. The rule's intent survives intact and is + > now pinned by tests: one shell/ui stays vanilla and untouched; the server UI is the + > project's one Lit surface, same-origin to `/api/v1`, loads no external URLs, keeps the + > visual identity and the API contracts verbatim, and Node exists at BUILD time only + > (the Dockerfile's `frontend-builder` stage; the runtime images ship static files). + > "Two UI paradigms" is the accepted trade, scoped by the brief; the shared layer the old + > text hoped for (format.js, media templates, tokens) is factored so it stays possible. +5. **Reviewable from a clean clone, headless.** One command sequence in CI builds the CLI, runs + `swift test`, verifies `digup --help` inside the runtime image, indexes a committed tiny + corpus, asserts semantic ranking (`wakeboarding`, `hiking`), and runs the API tests. It needs + no GUI, no GPU, no Rust, no Node, no Tauri. + + > Amended 2026-11-08 with §10.4: the review path gained **Node at Docker build time** + > (the frontend-builder stage runs `npm ci`, vitest, and `vite build` inside the build; + > the published image and every runtime stage stay Node-free — measured: the runtime + > stage COPYs the static dist and nothing else). CI stays headless. +6. **Stable public surface.** API versioned at `/api/v1`; engine version, vector source, llama.cpp + build tag and device backend published in `/health`; `IMPLEMENTATION_STATUS.md` kept current; + server state confined to `/data` volumes; the engine's own files (`costs.json`, `speed.json`) + are read, not rewritten, by the server. + +## 11. What is universal in Docker — and what is not + +Docker is universal; **images are not**. This section settles where things get built, because +it is the question that decides every later deployment. + +| universal (one copy, runs anywhere) | architecture-specific (built per machine class) | +|---|---| +| `Dockerfile`, Compose files, scripts, config, docs | every compiled artifact: the `digup` binary, llama.cpp `.so` files, `ffmpeg`, the images containing them | +| Python server code, adapter, tests | the GPU backends: Vulkan/CUDA need the real device and driver present | +| the browser UI (`fetch()` to `/api/v1`) | | + +So: + +- An `arm64` image runs on `arm64` machines; an `x86_64` image runs on `x86_64` machines. The + same *Dockerfile* builds the correct image on each — that is what "universal" means here. +- `docker pull` gives every machine "the same" image only when a **multi-architecture manifest + list** is published (each machine is handed the variant built for its own architecture, each + built natively). That is a CI feature for later (`docker buildx --platform …`), not a license + to copy a binary across architectures. +- Running an `x86_64` container on Apple Silicon is QEMU **emulation**: fine for a shell + script, useless for a C++/Swift compile (10–50× slower) and for embedding work. Committed + rule: **never ship an image that was built under emulation.** + +Decision (2026-11-02), given a dev Mac (Apple Silicon, no CUDA, no Vulkan device) and a +production box (x86_64 + RTX 4070): + +1. **This Mac builds and proves the arm64 images natively** — Stages 1–2, all adapter/API/UI + development, the corpus proof, the API tests, the CI-equivalent run. +2. **The x86_64 production image is built natively on the production server** from the same + `server/docker/Dockerfile`, on that machine, where the RTX 4070 is: the Vulkan/CUDA + override (`deploy/docker-compose.nvidia.yml`) is built and measured there, by + `digup speed --model`, nowhere else. +3. Later, a GitHub Actions `x86_64` job can build and publish the production image to a + registry so the server only pulls and runs Compose — same Dockerfile, same rules. Nothing + in the repository depends on which of 2 or 3 is used. + +## 12. Definition of done (first release) + +`docker compose up -d --build` starts the app → the user opens the web UI, configures a media +folder, starts indexing, watches honest status, searches `wakeboarding` and `hiking`, sees real +DigUp results (photos and videos, video hits keeping their timestamps), the index and model +survive container recreation, the HTTP API works without the browser, media stays read-only, +no cloud AI service is involved. Progress and limits are recorded in +`IMPLEMENTATION_STATUS.md`. diff --git a/docs/implementation/gpu-handoff.md b/docs/implementation/gpu-handoff.md new file mode 100644 index 00000000..58b0e97e --- /dev/null +++ b/docs/implementation/gpu-handoff.md @@ -0,0 +1,138 @@ +# Stage 9 handoff — run it on an x86_64 host with an NVIDIA GPU + +This file exists because the development machine (an arm64 Mac) has **no CUDA device and +no Vulkan device**, and building an x86_64 image under emulation is forbidden by +[README §11](README.md) ("never ship an image that was built under emulation"). So the GPU work +is *authored* here and *measured* there. Nothing in this repository claims GPU acceleration was +verified — the CPU-only path is what the contract suite, the CI workflow and +`IMPLEMENTATION_STATUS.md` measure. + +Run [`../../server/tests/gpu-handoff.sh`](../../server/tests/gpu-handoff.sh) on the box. It +verifies the CPU baseline first (the thing the PR promises), then the GPU path, and writes one +log with a PASS/FAIL line per gate. Its gates count failures and exit non-zero: a script that +prints a nice sentence and exits 0 while the GPU was never used is exactly the artifact this +project refuses. + +## Host prerequisites (check them, do not assume them) + +```sh +uname -m # x86_64, and the build happens here — not copied from the Mac +docker --version && docker compose version +nvidia-smi # the driver, on the host +docker run --rm --gpus all nvidia/cuda:12.4.1-base-ubuntu22.04 nvidia-smi # NVIDIA Container Toolkit +``` + +The last line is the toolkit test. If it fails, install/configure the NVIDIA Container Toolkit +before anything else — `libcuda.so.1` and the driver's CUDA userland come from the host, and an +image cannot supply them. + +## What the engine already does (reuse it, do not rebuild it) + +`scripts/build-llama.sh` picks the backend by what the machine has: **CUDA when `nvcc` / +`CUDACXX` is on PATH** (`CUDA_ARCHS`, default `native`), else **Vulkan when `glslc` is present**, +else CPU — and on Linux/Windows the ggml backends are shared libraries loaded at run time +(`GGML_BACKEND_DL`). `LlamaEmbedder.loadBackends()` looks in `$DIGUP_BACKENDS`, then beside the +binary, then ggml's own search; the `engine-payload` stage copies **every** +`Vendor/llama/lib/*.so*` with `rpath=$ORIGIN`, so a CUDA-enabled builder automatically ships +`libggml-cuda.so` beside `/opt/digup/digup` — no Dockerfile change. + +Device choice is **measured, never assumed**: `digup speed --model` times GPU against processor, +keeps `device` in `costs.json` beside the index, and accepts the GPU only when its vectors agree +with the processor's (cosine ≥ 0.999 — Mesa's `lavapipe` software "GPU" made 0.94–0.97 and is +rejected by this rule). The helper that loses gets `DIGUP_DEVICE=cpu`, and `DIGUP_DEVICE=cpu` +also *unloads* the GPU backend (Vulkan's setup costs seconds). `Sources/digup/ModelSpeed.swift` +and `Sources/LlamaRuntime/LlamaEmbedder.swift` are the code to read first. + +## Gate A — the CPU baseline, on this box (this part must pass) + +```sh +server/tests/gpu-handoff.sh --cpu-only +``` + +The gates, in order, each a PASS/FAIL line in the log: build the corpus fixtures +(`server/tests/corpus/build.sh`, `server/tests/corpus2/build.sh`); create +`$HOME/digup-data/{index,models,config}` owned by uid/gid 990; fetch the two pinned model files — +the URLs carry the pinned revision from `Sources/DigUpKit/ModelFiles.swift`, and the hashes must +equal the pins duplicated in `.github/workflows/server.yml`: + +```sh +curl -fL -o "$HOME/digup-data/models/embeddinggemma-2-Q8_0.gguf" \ + https://huggingface.co/ggml-org/embeddinggemma-2-GGUF/resolve/bfcd298762cc34d0357ece5ebdd31791a3a374d8/embeddinggemma-2-Q8_0.gguf +curl -fL -o "$HOME/digup-data/models/mmproj-embeddinggemma-2-Q8_0.gguf" \ + https://huggingface.co/ggml-org/embeddinggemma-2-GGUF/resolve/bfcd298762cc34d0357ece5ebdd31791a3a374d8/mmproj-embeddinggemma-2-Q8_0.gguf +sha256sum "$HOME/digup-data/models"/*.gguf +``` + +Then build the `server-test` target **natively**; run the contract suite with the four required +mounts (corpus, corpus-root, models, index) via `/srv/server/.venv/bin/pytest -rs` — every test +RUNS and any SKIPPED line is a FAIL; then `docker compose up -d --build`, the healthcheck, a real +search, and a `docker stop` that must end the container **not** 137, with +`Application shutdown complete` last in the log. The script echoes every command it runs; read it +for the exact invocations. + +Expected on x86_64: the same suite counts as CI (all green, zero skips), `/health` answering +(the engine's fingerprints, `build_tag` b11461), a search with real `z`, a 206 on a `Range` +request. If Gate A does not pass, the GPU work is not the problem — fix this first. + +## Gate B — the GPU build, measured where the card is + +1. **Build the builder stage with a CUDA toolchain.** The repo's builder stage is Ubuntu 22.04 + without `nvcc`; the CUDA build is a base-image decision, not a code change: + `docker build --build-arg BASE_IMAGE=nvidia/cuda:12.4.1-devel-ubuntu22.04 --target engine-builder …` + (pick the CUDA version that matches the host driver; check `nvidia-smi`'s "CUDA Version"). + `scripts/build-llama.sh` finds `nvcc` by itself and builds the CUDA backend; + `CUDA_ARCHS=…` pins the architecture. Pass the card's **plain number** (measured 2026-11-04: + the builder's cmake is Ubuntu 22.04's apt 3.22.1, which rejects dotted `6.1` **and** `native` + alike — `native` needs cmake ≥ 3.24 *and* a GPU visible at build time, which the builder + container never has). `61` = Pascal sm_61 (GTX 1070), `89` = Ada sm_89. +2. **Check the payload carries the backend and nothing duplicated:** + `docker run --rm digup-payload:x86 sh -c 'ls -l /opt/digup | grep -i cuda; ldd /opt/digup/libggml-cuda.so'` + — the CUDA runtime libs it asks for (cudart/cublas family) must resolve: ship exactly those + from a CUDA *runtime* base, never the toolkit, never a second copy of `libggml*`. + `libcuda.so.1` comes from the host driver through the toolkit — it must **not** be baked in. +3. **Prove it runs on the GPU, not "sees" a GPU.** The engine's own log line + `backends: CPU, CUDA` (from `LlamaEmbedder.loadBackends`) is necessary, not sufficient: + * `digup speed --model --json --index ` reports the device it measured and the + agreement check; the record's `device` is what every helper then uses. + * Sample the card **while indexing**: `nvidia-smi --query-gpu=utilization.gpu,memory.used \ + --format=csv,interval=1` — utilization must move during a real index job. A container that + only *detects* the device and embeds on CPU is a failure of this gate, and it is the easy + false positive. Sampling cadence is measured (2026-11-04, GTX 1070): a small-corpus GPU job + is ~150 ms kernel bursts separated by P8 idle — a 2 s-interval sampler sees 0–12% on a job + that is provably on the GPU (model resident in the card's VRAM, engine costs at the GPU's + image rate), while a 0.15 s sampler across the job window sees peak 94% and 39% busy time. + Judge the busy-time fraction and VRAM residency, not only the peak: a CPU-embedded job is + flat — zero busy samples, no model VRAM on the card. (`gpu-handoff.sh` samples the job at + 0.15 s and passes on peak > 20% or ≥ 10% busy with the model on the card.) + * Compare `speed --model` times against the CPU-only build on the same machine, same corpus, + same `DIGUP_THREADS`. Faster *and* agreeing vectors is the pass; the engine keeps CPU when + CPU wins (that is not a failure, it is the engine's measurement — report the numbers). +4. **The opt-in Compose shape** (to be added on this box, in `deploy/`, keeping + `docker-compose.yml` CPU-only as the default): an override with the `nvidia` profile and the + device request — + ```yaml + services: + server: + profiles: ["nvidia"] + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [cuda] + ``` + run with `docker compose -f docker-compose.yml -f docker-compose.nvidia.yml --profile nvidia up -d`, + documented with the toolkit prerequisite, the `DIGUP_DEVICE=cpu` CPU fallback, and startup + diagnostics that print the backend that actually ran. Validate the config with + `docker compose config --quiet` **with and without** the profile. +5. Vulkan (the driver-free-ish path) is the same story with `glslc` + a Vulkan ICD and + `--device /dev/dri`; measure it the same way. Keep one GPU path, not two, in the shipped + override. + +## Report back (what goes into `IMPLEMENTATION_STATUS.md`) + +`uname -a` + driver version + `nvidia-smi -L`; the CUDA base image and `CUDA_ARCHS` used; +`ldd` of the CUDA backend; the `backends:` line; `speed --model --json` for CPU and GPU with +their `device` and agreement values; `nvidia-smi` utilization sampled during a job; the suite +counts (with zero skips) on x86_64; and the honest statement of what was **not** measured. diff --git a/docs/implementation/reconnaissance.md b/docs/implementation/reconnaissance.md new file mode 100644 index 00000000..cecee738 --- /dev/null +++ b/docs/implementation/reconnaissance.md @@ -0,0 +1,418 @@ +# Stage 1 — reconnaissance + +Date: 2026-11-02. Everything below was verified by reading the repository and probing this +machine, not by reading the README alone. Commands are listed in §14. + +## 1. Repository identity + +| | | +|---|---| +| Working directory | `~/Projects/digup` | +| Remote | `origin https://github.com/ARahim3/DigUp.git` (fetch and push) — **a clone of upstream, not a fork** | +| Branch | `main`, tracking `origin/main`, in sync (`## main...origin/main`) | +| HEAD | `dc2651a Merge pull request #142 from Nagi-Inaba/fix/ime-composition-search` | +| Local changes | **none** (`git status --short` is empty) | +| License | MIT | +| Latest release | `v0.7.0-beta.3` (Mac `.dmg` releases stop at `v0.6.2`; `v0.7.0-beta.3` ships Windows + Linux) | +| Package manifest | `swift-tools-version: 6.2`, single Swift package | + +So: this working copy is pristine upstream code. Anything the server adds must stay additive. + +## 2. Where the engine lives (the packaging fact that matters) + +- `Package.swift` products: `.executable(name: "digup")` from the Swift targets `digup` + + `DigUpKit` + `LlamaRuntime`, plus `.library(name: "DigUpKit")`. On macOS only, also + `DigUpApp` (SwiftUI menubar app, Sparkle updates). +- `Sources/DigUpKit` — "the engine: crawler, extractors, index store, the embedder interface + and worker client, search. No UI." (its own comment in `Package.swift`). +- `Sources/LlamaRuntime` — EmbeddingGemma 2 on llama.cpp (`LlamaEmbedder`, `LlamaModels`). +- `Sources/CLlama` (macOS, static + Metal), `Sources/CLlamaShared` (Linux, shared libs + + `$ORIGIN` rpath), `Sources/CLlamaWindows` (Windows). +- The **Linux/Windows application is a shell, not the engine**: `shell/src-tauri` is Rust/Tauri + and `shell/ui` is dependency-free vanilla JS. `shell/src-tauri/src/helper.rs` line 1: + *"DigUp's helper processes, which do all the work: `digup search-service` answers the panel + … and `digup index-service` keeps the index current."* Every feature of the Linux app is the + Swift CLI answering JSON lines. +- Consequence for us: a headless server = the Swift `digup` binary + a thin HTTP layer. Swift + is needed **only to build**. `scripts/package-linux.sh` proves it: it builds + `swift build -c release --product digup --static-swift-stdlib` (with Swift's runtime archives + re-listed after the link) so "no Swift libraries ship". + +`worker/digup_worker.py` is the **mlx-vlm reference runtime** for evals (`WorkerConfig.python`, +`$DIGUP_WORKER_DIR`). mlx-vlm needs Apple Silicon. It is not a server engine. + +## 3. Layout + +``` +Package.swift one Swift package, per-platform targets +Sources/ + DigUpKit/ engine: Crawler, Indexer, IndexStore, Database, Searcher, SearchText, + Estimator, Rules, FolderSelection, OfficeText, MeasuredCosts, + ModelFiles, PreviewImages, Vectors, Worker, Platform/ (per-OS readers) + LlamaRuntime/ LlamaEmbedder.swift, LlamaModels.swift, ImageSizing.swift + digup/ main.swift, IndexService.swift, SearchService.swift, WorkerService.swift, + DownloadModel.swift, ModelSpeed.swift, MeasuredCosts…, Watchdog, + ParentWatch, SystemLoad + DigUpApp/ macOS SwiftUI app (not built on Linux) + CLlama*/ CPDFium/ C module maps for llama.cpp and PDFium +Tests/ DigUpKitTests (crawler/index/search/readers, Pictures/, Videos/), + LlamaRuntimeTests (ReferenceVectorTests + stored reference vectors) +scripts/ fetch-vendor.sh, build-llama.sh, build-ffmpeg.sh, package-linux.sh, + package-windows.sh, check-linux-package.sh, make-testbed.sh, + testbed/ (media/doc/lang/long generators), dev-run.sh, release.sh, + notices.py, appcast.py, vendor-checksums.json +evals/ synthetic.json, multilingual.json, code.json (ground-truth queries) +shell/ ui/ (vanilla JS: panel, window, settings, setup) + src-tauri/ (Rust) +website/ the product site +docs/ evals/ updates/ Resources/ .github/workflows/ +``` + +`.gitignore` keeps `.build/`, `.swiftpm/`, `build.noindex/`, `Vendor/`, `.dev/`, +`shell/src-tauri/target/`, `shell/src-tauri/helper/` out of git. Vendor dependencies are never +committed — they are fetched at pinned versions. + +## 4. The CLI (from `Sources/digup/main.swift`, verbatim usage) + +``` +digup estimate ... [--json] what would be indexed, how long the first pass takes +digup index ... [--limit N] sync + embed what's new, then the rest of long files +digup search ... [--kind K,K] [-n 10] [--json] +digup status what's in the index +digup speed [--model [--if-needed] [--when-quiet S] [--tries N]] [--json] +digup eval [-n 10] recall of known answers +digup index-service ... [--backfill] [--prune] [--memory-limit MB] [--time-limit S] +digup worker [--text-only] the model as a process (JSON lines) +digup search-service ... [--encoder-idle S] the searcher as a process (JSON lines) +digup download-model [--to DIR] the model files (JSON lines) + + --index DIR index folder (default: AppPaths.defaultIndex) + --code the code index (code.sqlite) + --kind K,K screenshot, image, pdf, doc, audio, video + --exclude DIR / --exclude-ext EXT / --include-code / --include-datasets + --reindex drop vectors and embed everything again + --zgate 2.0 --word-weight 2.0 --shot-tokens 280 --code-stretch 1800 + --limit N -n N --new-limit --linger --encoder-idle --tries --when-quiet +``` + +Verified behaviours relevant to the server: + +- `search --json` prints an array of hit objects built by `hitFields()`: + `path, kind, segment, segment_id, score, modified, size, info`, plus optional + `z, cosine, keyword_rank, loc, loc_end, focus, excerpt`. Scores are the engine's own; + `loc`/`loc_end` are the page number, the second of a video/audio moment, the line range, + the cell range. There is **no** percentage similarity to fake. +- `status` prints index path, vector and segment counts, per-kind per-state counts, long files + left, the index's `fingerprint` ("built with:"), `llama.cpp , vectors `, + and the index size. +- The model source and the index must agree: `queryVectors()` refuses to search when the index's + `vectorSource` differs from the encoder's (`index vectors come from … but the query encoder is …`), + and `SearchService` falls back to keyword-only with a `warning` in the reply. +- Non-macOS specifics: `main.swift` forces `LlamaEmbedder.cpuOnly = true` for `search-service` + (a query is a few words: the CPU encodes faster than a cold GPU; indexing keeps the GPU), and + uses `_exit()` for the services because glibc's `exit()` deadlocks on the stdin reader thread. +- `ParentWatch.start()` runs for `index-service`, `worker`, `search-service`, `download-model`: + the helper exits when its parent dies. Supervision consequence: the API process must be the + long-lived parent of the services. +- `MeasuredCosts.load(from: index).speed?.device == "cpu"` forces CPU for every command except + `speed` — i.e. the device choice is persisted beside the index by `digup speed --model`. + `DIGUP_DEVICE=cpu` forces CPU. `LlamaEmbedder.hasGPU` detects an actual GPU device. + +## 5. Persistent services: the engine interface we want (Approach B exists, and is public) + +`digup search-service` (`Sources/digup/SearchService.swift`) — one JSON request per line on +stdin, one reply per line on stdout carrying the request's `id`; lines without an `id` are +events. Ops (documented in the source header): + +| op | reply | +|---|---| +| `search` (`q, code, kinds[], limit, meaning, prefix_last`) | `hits[]` (as `search --json`, plus `segment_id, modified, size, info`), `strong`, `meaning`, `ms:{embed,search}`, optional `warning` | +| `embed_ahead` (`q, code`) | `{ok:true}` — pre-embeds and caches a partial query | +| `text` (`segment`) | `{text}` — the passage/page/frame text a hit points at | +| `render` (`path, kind:pdf\|video\|image, loc, pixels`) | `{image: }` — a JPEG preview/thumbnail, made beside requests, cached in `/previews` | +| `reload` | `{ok, vectors, code_vectors}` — reopens the index after indexing ran | +| `summaries` (`code`) | per-root `{path, missing, unreadable, files, indexed, pending, failed, skipped, unfinished, kinds, skipped_reasons}` | +| `folders` (`roots, skipped, types, code_roots, code_skipped`) | `{ok}` — changes scope at runtime | +| `prewarm` | `{ok, encoder, vectors}` — starts encoder + loads vectors | +| `status` | `{encoder: stopped\|starting\|ready\|unavailable, vectors, code_vectors, index, files, searchable, pending, unfinished}` | +| `quit` | `{ok}` then exit | + +Notes: search is scoped to the positional roots (and `--exclude`/`--exclude-ext`) and drops hits +whose file no longer exists. `--encoder-idle S` (default 300 s) releases the model and vectors +after idle; after `2 × S` the process exits itself (`event: exit, reason: idle`) — the adapter +must restart it. `search-service` prints `{event: ready, index}` at start and +`{event: encoder, state: ready, seconds, source}` when the encoder is up. + +`digup index-service` (`Sources/digup/IndexService.swift`) — stdin commands `sync`, +`sync code`, `stop`, `backfill on`, `backfill off` (EOF = stop). Events: + +`missing` (folders gone/unreadable) · `pruned` · `rescan` · `stopped` (watchdog) · `scanning` +(checked, root) · `sync` (`new, added, changed, moved, removed, unchanged, ms, started, left_out`) · +`file` (`done, total, path, kind, state, seconds, note, new`) · `summary` (states, ms) · +`rest` (long files: done/total) · `waiting` (for model) · `idle` · `error` (with `fatal`) · +`exit` · `reindex` (model changed). Counts (`files, searchable, pending, vectors, unfinished`) +ride on the count events, `code_`-prefixed for the code index. Progress is real file counts, +never an invented percentage. A missing/unreadable root is skipped, never treated as "all +deleted". Limits: `--memory-limit`, `--time-limit` (Watchdog), `--new-limit` (default 50), +`--linger` (default 60 s). + +`digup worker` (`Sources/digup/WorkerService.swift` + `DigUpKit/Worker.swift`) — the JSON-lines +embedding protocol (`text`, `image`, `audio`, `quit`; base64 float vectors), i.e. the same +protocol the Python reference worker speaks. Useful for testing, not needed by the server. + +## 6. Index storage + +- `IndexStore.mainFile = "index.sqlite"`, `IndexStore.codeFile = "code.sqlite"`, in the `--index` + folder; plus `previews/` (rendered JPEGs, cleared at each `search-service` start), + `costs.json` (`MeasuredCosts`: per-machine costs and the chosen `device`), `speed.json`, + `worker.log` / `worker-query.log`, a `Watchdog` note file. +- Default index folder when `--index` is absent: macOS `~/Library/Application Support/DigUp`, + Linux `$XDG_DATA_HOME/digup` else `~/.local/share/digup` (`AppPaths` in + `Sources/DigUpKit/Platform/Portable.swift`), Windows `%LOCALAPPDATA%\DigUp`. **The server + always passes `--index` explicitly.** +- SQLite: opened read-write, `PRAGMA journal_mode = WAL`, `synchronous = NORMAL`, + `foreign_keys = ON`, `busy_timeout = 5000` (`Database.swift`); vendored SQLite **with FTS5** + (`SQLITE_ENABLE_FTS5`, `Package.swift`) for the exact-word index. Vendor SQLite is used + because Linux's may lack FTS5/headers. +- Fingerprints: `WorkerInfo.vectorSource = "@ "`; the index + stores `fingerprint` and refuses cross-source searches. `LlamaEmbedder.vectorVersion = 1`, + `buildTag = LLAMA_BUILD_TAG` (written by `scripts/build-llama.sh` into `llama-build.h`). +- File states used in counts: `done`, `partial`, `pending`, `failed`, and skipped-with-reason + (`no text`, `silent or shorter than 2 s`, …). + +## 7. Model storage + +`Sources/DigUpKit/ModelFiles.swift` pins exactly two files from +`ggml-org/embeddinggemma-2-GGUF` at revision `bfcd298762cc34d0357ece5ebdd31791a3a374d8`: + +| file | bytes | sha256 | +|---|---|---| +| `embeddinggemma-2-Q8_0.gguf` (text, queries) | 309,855,456 | `2188ac1deca4b77dffefd603c2776a9d76d9d74ec01841392982ebb840b09135` | +| `mmproj-embeddinggemma-2-Q8_0.gguf` (image + audio encoders) | 554,821,024 | `c4a8a52691ecef40618438928bdf9e68379b854e24166f292592353db0aab64f` | + +865 MB total, Q8_0 only (never F16 — the activations overflow float16). Lookup order +(`LlamaModels.locate`): **`$DIGUP_MODELS` — and if set, only there** → `AppPaths` models folder +(Linux: `~/.local/share/digup/Models`) → the Hugging Face cache snapshot +(`$HF_HUB_CACHE` / `$HF_HOME/hub` / `~/.cache/huggingface/hub/models--ggml-org--embeddinggemma-2-GGUF/snapshots/`). +`digup download-model [--to DIR]` fetches with resume and checksum verification, printing JSON +lines. `HF_HUB_OFFLINE=1` once files are cached (what `WorkerConfig.python` does). + +## 8. Media support (Linux vs macOS) + +Engine kinds: `screenshot, image, pdf, doc, audio, video, code`. + +| kind | Linux reader | notes | +|---|---|---| +| images | stb (`CStb`) | JPEG/PNG/WebP/HEIC-decodable set; README: RAW photos are Mac-only | +| screenshots | stb | **no in-image OCR on Linux** (Apple Vision): found by what they show, not by the text in them | +| PDF | PDFium (`CPDFium`) | text + page renders, scans included; up to 5,000 pages, first 30 in the first pass | +| docs | `OfficeText` + miniz (`CMiniz`) | txt, md, typ, rtf, docx, odt, html, xlsx, ods, pptx, odp; legacy `.doc` is Mac-only | +| audio | ffmpeg/ffprobe + model | 30 s windows, overlapping; hits carry the second | +| video | ffmpeg/ffprobe + model | frames per shot / every few seconds + soundtrack; hits carry the second | +| code | `CodeExtractor` | separate `code.sqlite`, off unless enabled | + +README FAQ confirms the Linux gaps: screenshot text, `.doc`, RAW, SVG are macOS-only. The +server's UI must therefore not promise audio/document/OCR features it cannot verify. + +## 9. Build system, pinned versions + +`scripts/fetch-vendor.sh` (Linux/Windows only) → `Vendor/`: SQLite amalgamation +`2026/sqlite-amalgamation-3540000.zip` (3.54.0), stb @ `2c980bb59875b0d32144a71867fbdebb2f77cd20`, +miniz `3.1.2`, PDFium `chromium/8086` prebuilt (`pdfium-linux-x64`, `pdfium-linux-arm64` for +aarch64). All downloads are verified against **committed pins** in +`scripts/vendor-checksums.json`; writes `Vendor/VENDOR` stamp. Needs curl, tar, python3. + +`scripts/build-llama.sh` → `Vendor/llama/{lib,include}` + `Vendor/llama.cpp` + `Vendor/llama-build`: +llama.cpp tag `b11461` at commit `4d756bc72bf00a4aacf410ae15a2d315f3db400d`, plus +`scripts/llama.cpp-no-logits.patch`. Linux/Windows: `BUILD_SHARED_LIBS=ON`, +`GGML_BACKEND_DL=ON`, `GGML_CPU_ALL_VARIANTS=ON`, `GGML_NATIVE=OFF`, `LLAMA_BUILD_MTMD=ON`, +tools/server/examples/tests/app/ui off, `LLAMA_CURL=OFF`. GPU choice: CUDA when `nvcc` +(`CUDACXX`, `CUDA_ARCHS` default `native`), else Vulkan when `glslc` is on PATH, else CPU. +arm64 needs a compiler that accepts `-march=armv9.2-a+sme` (script probes `cc`, `gcc-14`, +`gcc-15`, `clang`); it hard-fails otherwise. macOS: static libs with Metal +(`GGML_METAL_EMBED_LIBRARY=ON`). + +`.github/workflows/packages.yml` (Linux job) is the authoritative dependency list: builds inside +`container: ubuntu:22.04` on `ubuntu-24.04` runners so the output needs **glibc ≤ 2.35**; +apt packages `ca-certificates curl git gnupg xz-utils unzip python3 binutils build-essential cmake +ninja-build meson nasm pkg-config patchelf file zlib1g-dev libcurl4-openssl-dev libxml2-dev +libedit2 libsqlite3-0 libz3-dev libpython3-dev tzdata … libvulkan1`; Swift +`6.4.0-RELEASE` from download.swift.org verified by GPG signature; Rust (rustup, minimal) and +Node v24.20.0 (for Tauri's CLI — **only the shell needs them, not the engine**); Vulkan SDK +`1.4.363.0` for the Vulkan backend's shader compiler. Then +`scripts/fetch-vendor.sh && scripts/build-llama.sh && scripts/package-linux.sh`. +`scripts/check-linux-package.sh` is the upstream "installed package actually launches" check. + +`scripts/package-linux.sh` shows the exact release build of the CLI +(`--static-swift-stdlib` + the trailing `-Xlinker -l…` list) and the runtime payload: `digup`, +`libpdfium.so`, `ffmpeg`, `ffprobe`, llama's `.so` files, all `patchelf --set-rpath '$ORIGIN'`. + +## 10. Environment probe (this machine) + +| probe | result | +|---|---| +| `uname -m` | `arm64` (Apple Silicon) | +| `sw_vers` | macOS 26.5.1 | +| CPUs / RAM | 8 physical, 8 589 934 592 B (8 GB) | +| disk | 386 GB volume, **41 GB available** | +| Docker | client 28.3.0, Docker Desktop 4.43.1, engine linux/**arm64**, 7 CPUs, 6.77 GiB, root `/var/lib/docker` | +| `docker compose` | v2.38.1-desktop.1 | +| Xcode | **16.4**; `swift --version` = Apple Swift **6.1.2** | +| CMake / uv / ninja (host) | none installed | +| `swift-tools-version` needed | **6.2** (and the README asks Swift 6.4 / Xcode 26 for non-Mac builds; Xcode 26 for the app's macros) | + +Consequence: **the host cannot build this package as installed** (Xcode 16.4's 6.1.2 < +tools-version 6.2), and the host is arm64 macOS so its binaries are excluded from images by the +rules in [README.md](README.md). Engine build and proof must happen in Linux containers +(arm64 here) and on the eventual x86_64 server. + +Toolchain availability (HTTP probes, 2026-11-02): + +| artifact | result | +|---|---| +| `swift-6.4.0-RELEASE-ubuntu22.04.tar.gz` | **200**, 1 123 431 313 B | +| `swift-6.4.0-RELEASE-ubuntu22.04-aarch64.tar.gz` | **exists** (206 on a ranged probe) | +| `swift-6.4.0-RELEASE-ubuntu24.04_arm64.tar.gz` | 404 | +| `swift-6.2.0-RELEASE-ubuntu24.04_arm64.tar.gz` | 404 | + +⇒ arm64 Linux builds use **ubuntu:22.04 + the `-aarch64` Swift tarball** (which also matches +upstream's glibc-2.35 target); x86_64 builds use ubuntu:22.04 + the plain tarball. No arm64 +Ubuntu 24.04 Swift release exists. + +## 11. Upstream release artifacts (the runtime payload, verified by extraction) + +Downloaded `DigUp_0.7.0-beta.3_amd64.deb` (41,565,124 B) and listed its payload: +`/usr/bin/digup-shell` (7.7 MB, Tauri shell), `/usr/lib/DigUp/THIRD-PARTY-NOTICES.txt`, and +`/usr/lib/DigUp/helper/`: + +``` +digup 81,410,360 libllama.so.0 4,825,512 libmtmd.so.0 1,896,696 +ffmpeg 12,277,760 libggml.so.0 55,184 libggml-base.so.0 971,368 +ffprobe 12,064,704 libggml-vulkan.so 46,010,608 +libpdfium.so 7,982,296 libggml-cpu-{x64,sse42,sandybridge,ivybridge,haswell,alderlake, + icelake,cascadelake,cooperlake,zen4,sapphirerapids,cannonlake,piledriver}.so +``` + +Control file: `Depends: libcurl4t64 | libcurl4, libgomp1, libayatana-appindicator3-1, +libwebkit2gtk-4.1-0, libgtk-3-0`, `Recommends: libvulkan1`, `Installed-Size: 191747` KB. +GTK/WebKit/appindicator are the shell's; the headless payload needs `libcurl4`, `libgomp1` +(+ `libstdc++6`, and `libvulkan1` only if Vulkan is used). + +Points this proves: (1) an x86_64 `digup` with static Swift stdlib runs with **no Swift +toolchain installed**; (2) llama.cpp ships CPU dispatch variants + a Vulkan backend and **no +CUDA** in upstream's Linux releases (CI has no `nvcc`); (3) the AppImage (146 MB, amd64) is the +self-contained variant. These artifacts are for **comparison only** — the server image is built +from source per [README.md](README.md). + +## 12. Existing UI (reuse candidate) + +`shell/ui`: `panel.js` 581, `window.js` 1212, `settings.js` 553, `setup.js` 719, +`results.js` 269, `shortcut.js` 55 (+ `panel.css` 402, `window.css` 667, `settings.css`, +`setup.css`, 4 HTML files) = ~5 000 lines, **no framework, no build step** +(`tauri.conf.json: "frontendDist": "../ui"`, `withGlobalTauri: true`). All engine access goes +through Tauri `invoke()` (22 call sites in `window.js`). The project's house style is +deliberately dependency-free. The server UI follows that style (§13): HTML + CSS + vanilla JS +talking to `/api/v1` with `fetch()`, in the same visual language. Adopting React/TypeScript/Vite +for the server only would leave the repository with two frontends in two paradigms — poor for +upstream maintenance, so it is not the route. + +## 13. Sustainability / upstream-contribution constraints + +This server is intended as a **pull request to ARahim3/DigUp** and as the basis of a public +page. Rules that follow from that: + +1. **Additive only.** Engine, manifest, packaging scripts, shell, and website stay untouched. + New code lives in `server/` (API + adapter + tests + Docker) with its own README, and + `docs/implementation/` for documentation, plus a dedicated + `.github/workflows/server.yml`. No edits to `Sources/**` for server convenience; if the + engine genuinely must expose something new, it is a separate, minimal, tested, + backward-compatible change to the JSON protocol — never a private field. +2. **Speak the documented protocol only.** `search-service` and `index-service` are the same + protocol the shipping desktop shell uses, so upstream keeps them working. Contract tests in + `server/tests/` assert the ops/fields above; a protocol doc mirrors the source header + comments. Nothing is invented, nothing is cached in a parallel store. +3. **Upstream conventions.** Pinned versions with pinned hashes (like `vendor-checksums.json`), + no vendored blobs in git, `uv` for Python (repo already uses `uv` + `pyproject.toml` + + `uv.lock` in `worker/`), dependency-light frontend, dense "why" comments in the house style, + MIT license and `scripts/notices.py` notices respected, `.env.example` for configuration, + no secrets committed. +4. **CI-able and headless.** Everything a reviewer needs runs from a clean clone in containers: + build → `swift test` → `digup --help` → index a generated corpus → semantic search assertions + → API tests → Docker acceptance. The workflow must not need a GUI, a GPU, the desktop shell, + Rust, Node, or Tauri. +5. **Long-lived compatibility surface.** Version the HTTP API (`/api/v1`), publish the engine + version/vector source in `/health`, keep one `CHANGELOG`-style status document + (`IMPLEMENTATION_STATUS.md`), and never break the API's published contract. +6. **Small blast radius.** Server runtime state lives in `/data` volumes; server settings are + server-only; the engine's own settings files (`costs.json`, `speed.json`) are read, not + rewritten, by the server. + +## 14. Reproduce these findings + +```sh +pwd; git status --short --branch; git remote -v; git log -1 --oneline; git branch -a +ls -la; sed -n 1,257p README.md; cat Package.swift +sed -n 1,240p Sources/DigUpKit/Worker.swift +sed -n 1,385p Sources/digup/SearchService.swift +sed -n 1,404p Sources/digup/IndexService.swift +sed -n 1,180p Sources/digup/main.swift && sed -n 300,760p Sources/digup/main.swift +grep -n "mainFile\|busy_timeout\|journal_mode" Sources/DigUpKit/IndexStore.swift Sources/DigUpKit/Database.swift +sed -n 1,80p Sources/DigUpKit/ModelFiles.swift +sed -n 1,136p scripts/build-llama.sh; sed -n 1,85p scripts/fetch-vendor.sh; cat scripts/package-linux.sh +sed -n 1,120p .github/workflows/packages.yml +head -30 scripts/make-testbed.sh; head -20 evals/synthetic.json +wc -l shell/ui/*.js; grep -c "invoke(" shell/ui/*.js +# upstream artifact inspection (reference only) +curl -L -o DigUp.deb https://github.com/ARahim3/DigUp/releases/download/v0.7.0-beta.3/DigUp_0.7.0-beta.3_amd64.deb +ar x DigUp.deb && python3 -c "import tarfile;[print(m.size,m.name) for m in tarfile.open('data.tar.xz').getmembers()]" +# environment +uname -m; sw_vers; sysctl -n hw.ncpu hw.memsize; df -h /; docker version; docker compose version +xcode-select -p; xcodebuild -version; swift --version +curl -sIL https://download.swift.org/swift-6.4.0-release/ubuntu2204/swift-6.4.0-RELEASE/swift-6.4.0-RELEASE-ubuntu22.04.tar.gz +curl -s -o /dev/null -w "%{http_code}" -L -r 0-1 https://download.swift.org/swift-6.4.0-release/ubuntu2204-aarch64/swift-6.4.0-RELEASE/swift-6.4.0-RELEASE-ubuntu22.04-aarch64.tar.gz +``` + +## 15. Test results so far, and open items + +Ran so far: repository inspection, script/protocol/CLI reading, environment probes, toolchain +availability probes, upstream artifact extraction, the corpus build, and **the first Linux build +of the engine: `swift build -c release` and `swift test` pass in the arm64 container (116 + 7 +tests, 0 failures), and the 296 MB runtime image runs `digup` headlessly**. The exact results, +commands and image layout are in [stage-1-linux-build.md](stage-1-linux-build.md). Stage 2 (the +headless index + search proof on the corpus) is running as this is written. + +Open items: + +1. ~~Build-host decision~~ — **decided**: Stages 1–2 build and prove natively in arm64 Linux + containers on this Mac; the x86_64 production image is built **natively on the x86_64 Linux + server** with the same Dockerfile (images are per-architecture; see README §11). +2. `make-testbed.sh` is macOS-specific (Xcode-run Swift generators, APFS clones, screenshot + xattrs). The Linux-side test corpus now exists: `server/tests/corpus/` (pinned Commons + pictures, pinned videos, a synthetic 3-scene clip, `corpus.json` ground truth). + **Stage 2 is complete** — headless index + search + eval in the CPU runtime image + (R@1 0.71 / R@5 0.88 over 17 labeled queries, exact video timestamps, honest + "no good matches"): [stage-2-headless-proof.md](stage-2-headless-proof.md). +3. Index locking: SQLite is WAL + `busy_timeout`, and upstream runs one `index-service` at a + time. Confirm the engine's behaviour with two concurrent writers and add an OS-level lock in + the adapter so only the server ever starts indexing. +4. `search-service` self-exits after `2 × --encoder-idle`; the adapter needs supervised restart + with a `reload` after indexing, and must never treat the idle exit as a failure. +5. `render` previews land inside `/previews` (world-readable paths): the API must read + them itself and never expose that path to clients. +6. Video search in the browser needs codecs the container doesn't control (MP4/H.264 is + generally fine in browsers; HEIC/HEVC/ProRes is not) — decide what the preview endpoint + transcodes (if anything) without touching DigUp's own renders. +7. No CUDA in upstream's Linux releases and no CUDA possible on this Mac; production GPU support + starts as Vulkan (documented path) with a measured CPU/GPU comparison by + `digup speed --model`, which also enforces vector agreement (cosine ≥ 0.999) before trusting + a GPU. +8. **`estimate --json` and ParentWatch (Stage 3, correcting Stage 2's report).** Stage 2 + recorded "`estimate --json` prints nothing on Linux (exit 0)" as an upstream finding. + It is not a bug: `estimate --json` calls `ParentWatch.start()` (`main.swift:699`), and on + Linux ParentWatch treats **parent == PID 1 as "the app is gone"** (`guard parent != 1 + else { _exit(0) }`, `ParentWatch.swift`) — the helper exits within a second. Every silent + observation came from running it as PID 1's child (`bash -c` / `proof.sh` as PID 1); with + a live parent (the pytest/uvicorn process, direct `docker run digup …`) the same binary + prints its JSON lines fine, skipped files included. Server rule, now binding: the API + process is the long-lived parent of every helper (spawn direct, never through a PID-1 + shell wrapper) — which `protocol.py` enforces — and it *can* use `estimate --json`, + tested in `server/tests/test_cli.py`. diff --git a/docs/implementation/review-round.md b/docs/implementation/review-round.md new file mode 100644 index 00000000..263404b4 --- /dev/null +++ b/docs/implementation/review-round.md @@ -0,0 +1,128 @@ +# Review round — the adversarial pass before the PR + +Three read-only adversarial reviews of the whole PR (Python backend; UI + media surface; +build / deploy / CI / docs), each looking for blockers with concrete failure scenarios, +plus the empirical checks that confirmed or killed each finding. Everything the reviews +found that was real is fixed **and pinned by a test**; what is not fixed is listed with +its reason, not left silent. + +## The suite and the harness rule this round exposed + +The first suite run after the fixes came back `7 failed, 9 passed, 59 errors` with every +helper dying `exit 0, no stderr`. Not a code bug: `docker run` makes the test process +**PID 1**, and ParentWatch treats PID 1 as "the app is gone" +(`Sources/digup/ParentWatch.swift:19`) — so every `search-service`/`index-service`/ +`estimate --json` child ended at once. The suite invocation must give the test process a +live parent. The runtime image already gets this from tini; the **test image now carries +`tini` as its ENTRYPOINT** (`server/docker/Dockerfile`), so the suite is PID-1-safe by +construction and CI needs no `--init`. Measured after the change: the full suite green +(see [IMPLEMENTATION_STATUS](../../server/IMPLEMENTATION_STATUS.md) for the run's numbers). + +Same rule in CLI form, discovered while reproducing it: `digup … --index` takes a +**folder**, not the `.sqlite` file — `IndexStore(directory:)` calls +`createDirectory(at:)` on it, so a file path answers NSCocoaError 516 "a file with the +same name already exists" (`Sources/DigUpKit/IndexStore.swift:71-72`). Documented at +`server/digup_server/engine/config.py:71-75` and in reconnaissance §15 item 8. + +## Backend — fixed + +| Finding (severity) | Concrete failure the reviewer showed | Fix + pinning test | +| --- | --- | --- | +| `shutdown()` self-deadlock (`jobs.py:136-138`, blocker) | `_mark()` called from inside `with self._lock:`, `_mark` takes the same plain Lock: a `docker stop` while a job outlived the 20 s join grace hung the lifespan forever — job never logged *interrupted*, `service.quit()` never ran, job polls consumed threadpool threads until the API stopped answering | read `_active` under the lock, mark it outside → `test_shutdown_mid_job_finishes_and_marks_the_job_interrupted` (fake `_run_one` sleeps past the grace; asserts return < 30 s, *interrupted*, `why "server stopped"`) | +| Runner thread died on a non-`RuntimeError` (blocker) | the post-job refresh raises `TimeoutError` on a 60 s prewarm — **not** a `RuntimeError`, so `except RuntimeError` missed it: worker thread gone, `_active` stuck, every later `POST /index` answered 409 naming a job that could never run, queued jobs frozen until a restart | `except Exception` + `finally` clearing `_active`/`_service` → `test_a_crashing_job_runner_leaves_indexing_alive`, `test_a_prewarm_timeout_after_done_keeps_the_job_done` | +| `_spawn_locked` leaked the child on a failed handshake | `self._proc` was assigned only after a successful `start()`; a raised handshake left the fresh child + 3 pipes + 2 reader threads unreferenced until its own idle-exit (2 × `--encoder-idle` = 600 s), and every retry spawned another | reap on any failure, map `OSError` to `CLIError` → `test_a_failed_handshake_reaps_the_child_it_spawned`, `test_a_missing_engine_binary_answers_clierror` (502 quoting the spawn, not a 500 traceback) | +| `ok:false` without an `error` field (500) | the guard demanded `"error" in reply`, so `{"ok":false,"id":4}` passed `_checked` and `stripped["hits"]` raised KeyError — the exact 500 the Stage-10 battery hit | fire on the shape, message defaults to "no reason given" → `test_a_refusal_without_an_error_field_is_still_a_refusal` | +| `render` serialized behind the query lock | the engine runs `render` **beside** its read loop on 2 slots with replies routed by id ("typing never waits on one", `SearchService.swift:52-56, 99-113`); the adapter held the serialize lock across the engine's up-to-120 s ffmpeg work, so a big-look preview froze every keystroke-search, and a 40-hit gallery serialized 40 renders | `_render_slots = Semaphore(2)` + a lock-free `_render` (lock held only for the spawn check) → `test_renders_run_beside_searching_not_ahead_of_it` (typing answers in ms beside an in-flight 1.2 s render, reply order intact) | +| Unbounded `Range: bytes=0-` | uncapped open/suffix/huge ranges read the WHOLE file into RAM (a 20 GB clip asks for a 20 GB buffer) | `MAX_RANGE_SPAN = 8 MiB` cap + chunked streaming from the span's own handle → `test_open_and_huge_ranges_are_capped_at_the_served_span`, `test_open_range_streams_the_video_rather_than_buffering_it` | +| `subprocess.TimeoutExpired` unmapped (500) | it is a `SubprocessError`, not a `TimeoutError`: a stalled `digup status` behind the writer's WAL, or an estimate crossing 120 s, died as a bare traceback | mapped in `cli.run` (covers every one-shot call) → `test_a_cli_timeout_surfaces_as_a_quoted_502_not_a_500` | +| Job-log replay trusted any parsed JSON | `record["id"]` on a line like `5` raised in `__init__`: the app refused to boot | skip non-dict/no-id lines and torn tails → `test_replay_ignores_lines_that_are_not_job_records` | +| Rotation shifted a `.1…N` chain | O(rotations) renames per rotation, and a kill mid-shift left the chain out of order: a `running` line read after its `done` reported a finished job as *interrupted* | one rename to a `time_ns` suffix, replay = sort → `test_the_job_log_rotates_and_history_survives_rotation` | +| `HitCache.get` didn't refresh (fake LRU) | eviction was insertion-order: the UI re-searches per keystroke-pause (≤40 handles each), so after ~26 searches the handles of the rows ON SCREEN were evicted — their thumbnails, preview and file 404'd with no recovery | `get()` does `move_to_end` → `test_hit_cache_is_a_true_lru_get_refreshes` | +| Two `status` polls = two subprocesses | UI polls `/status` every 15 s, HEALTHCHECK hits `/health` every 30 s, and each spawned a `digup status` doing real SQLite counting (on the writer's hot index) | TTL cache (`DIGUP_SERVER_STATUS_TTL_S`, default 15 s) for the identity fields → `test_the_cli_identity_is_served_from_a_short_cache` | +| Parallel estimates | every full-tree crawl (120 s) ran unguarded: a second tab, a retry click, a retry during a job stacked crawls | busy-lock → 409 "an estimate is already running" → `test_a_slow_estimate_answers_busy_never_parallel` | +| Pydantic dropping engine fields | `SearchReply` (extra=ignore) dropped top-level fields this model has never seen — hits already rode raw, the reply itself must too | `model_config = ConfigDict(extra="allow")` → `test_search_reply_publishes_fields_this_model_has_never_seen` | +| Media surface hardening | size/type read from `path.stat()` (a replacement mid-request makes the headers lie about the bytes); prefix-string root matching; no `nosniff` on the `text/plain` path; events deque and timed-out replies unbounded; `close()` could fail the shutdown that called it | one open + `os.fstat` + `stat.S_ISREG`, `is_relative_to`, `x-content-type-options: nosniff` (200 and 206), `_events` `maxlen=256`, drop the timed-out reply, suppress the last post-kill `wait` → `test_file_carries_nosniff` + the range/streaming tests | + +## UI — fixed + +| Finding (severity) | Concrete failure | Fix | +| --- | --- | --- | +| Space hijacked every button (blocker) | native button activation fires on **keyup**; the document-level Space handler (Quick Look) `preventDefault()`ed on keydown for any focused element except a few tags — so the gear, ✕, Index now, Estimate, "all", the Esc close, and the Open/Download links stopped activating for keyboard users while the big-look flashed open over the page. Space is the standard activation key, and the footer advertises it | guard is a `closest("button, a, [role='button'], input, select, textarea")` — activation targets keep Space; bare page areas preview. Pinned by `test_the_space_key_keeps_buttons_and_links_activated` | +| A failed job poll killed the loop (blocker) | server restart / engine-mid-respawn 503 rejected the polling promise unhandled: the loop died, the job line froze at its last state, and "indexing…" stayed in the footer forever | poll in `try/catch`: honest `job poll failed: …`, clear `state.indexing`, stop. Repeated busy-clicks poll ONE loop (`state.jobPoll` dedupe), not N → `test_the_ui_survives_a_failed_job_poll` | +| FastAPI list-`detail` rendered `[object Object]` | validation errors answer a list of dicts; `new Error(list)` is that literal on the footer | `api()` renders each `{loc, msg}`, quotes a string, falls back to statusText for a non-JSON body | +| Settings panel: aria desync + dropped focus | outside-click closed the panel without syncing the gear's `aria-expanded` (a screen reader reported an open dialog whose panel was gone) and left a keyboard user at ``; five close paths each did their own partial version | one `closeSettings()` everywhere (aria sync + focus return; opening focuses the first control) | +| Stale estimate reply | a second click during a slow crawl let the FIRST crawl's late reply write into a closed-and-reopened panel | generation token (`estimateToken`) | +| Space on the "weaker" row | the field handler opened the big look on the folded row the document handler toggles | same `toggleWeaker()` on both paths | +| Roving tabindex / listbox roles | every row had `tabIndex=0` — an arrow-key list of 40-100 rows turned every search into a tab storm; group titles claimed `role=option`; the folded weaker row (selectable by ↑↓/↩/Space) said nothing | selected 0 / rest −1, `aria-selected`, group titles `role=presentation`, fold gets `role=option` + `aria-expanded`/`aria-selected` | +| Ticked chip contrast (dark) | kind-color text on its own 14 % tint: the audio chip measured 4.3:1 — under AA | 10 % mix, measured 4.54:1 (the rest 5.4 and up) | +| `.badge.warn` lost the cascade | `.warn` and `.badge` share specificity, `.badge` came later: the estimate's "can't read it" badge rendered in muted grey, identical to the decorative badges beside it | `.badge.warn` rule after `.badge` | +| Audio control in the preview pane | audio hits live in `.audio-picture` (app.js) — the `#preview .picture audio` selector never matched, so the control rendered at its ~300 px default | selector covers both boxes | +| Mobile footer | 48-char keys line at 0.8rem with `white-space: nowrap` overflowed a 320 px viewport | `white-space: normal` under 720px | +| Query field vs API cap | the API rejects > 512 chars; the field accepted any length and the user only learned by an error strip | `maxlength="512"` | + +## Build / CI / docs — fixed + +* **CI ceiling + cache** (the suite's "tripwire" role needed it): `timeout-minutes` 90 → + 180, and the engine chain (ffmpeg, llama.cpp) is built **host-side with + `actions/cache`** — the same repo-pinned action and hash keys `ci.yml` uses — then + passed into the build context as `.ci-vendor/`; the Dockerfile seeds its BuildKit cache + mount from it, so `fetch-vendor`/`build-ffmpeg`/`build-llama` land on their own stamps + (`Vendor/VENDOR`, `ffmpeg/BUILD`, `llama/BUILD`) and skip the rebuild. Hosted runners + start their BuildKit cache mounts cold, and a 15-minute ffmpeg compile per PR was + measured overrunning the ceiling. A local build has an empty `.ci-vendor/` (only the + tracked `.gitkeep`) and pays nothing. +* **Zero-skip gate**: the suite self-skips a module when `/models`, `/corpus-root` or the + clip is missing, so `57 passed, 20 skipped` would be green-by-construction. CI fails on + **any** SKIPPED line and requires `[0-9]+ passed`. The clip (`media/clips/scenes.mp4`, + the video fixtures) is asserted to exist as its own step. +* **Drift guard runs FIRST** and now pins `ModelFiles.revision` and both file **names** + as well as the two sha256s: a revision bump with unchanged hashes moves the download + URL silently, a renamed file moves it loudly — both are red in seconds, not after an + hour of build minutes. +* **proof.sh reports failure**: it ran each gate through `cmd | tee log` and exited 0 + unconditionally, so a run where the index died and every search answered nothing was + "green" and CI cited its `EXIT=0`. It now counts failed gates (`pipefail` makes the + pipeline status real), labels the gate that failed, and gates the eval ALL row + (R@1 ≥ 0.60, R@5 ≥ 0.85 — the recorded 0.71/0.88 clear it with margin) before + exiting non-zero. +* **Runtime image honesty** (measured, not claimed): the base is Debian 13 (trixie), + glibc 2.41-12+deb13u4 — newer than the payload's build glibc 2.35 (Ubuntu 22.04), + forward never back; the healthcheck is `server/docker/healthcheck.py` and accepts + **503 as alive** (`/health` answers 503 while the searcher respawns — the engine's own + refusal; Docker must not restart a live server over it; urlopen raises `HTTPError` on + 503 so it is caught; the build asserts the script exits non-zero with nothing + listening, CI polls with it); `/data/locks` is gone (the writer lock is + `/.digup-server.writer.lock`); the inert `DIGUP_SERVER_HOST/PORT` ENV lines are + gone (uvicorn's `--host/--port` args win in a container — those envs feed + `python -m digup_server.api`); `digup --help` in the image is described as what it + is (the CLI runs headless) with §3's full acceptance — CLI answering status and + searches from a live index — proven by CI's boot step and the proof battery. +* **Compose**: `restart: unless-stopped` (back with the daemon, down after a manual + stop) and `DIGUP_PUBLISH_PORT` for the host side when 8787 is taken; localhost-only + publish unchanged; `docker compose config --quiet` is a CI step. +* **Build context**: `.dockerignore` now excludes the corpus videos/clips and corpus2 + media (the reviewer measured 45 MB of corpus2 photos riding into the build context), + `.playwright-mcp/`; `.gitignore` ignores `.ci-vendor/*` with a `!`-exception for its + tracked `.gitkeep` (the Dockerfile's `COPY` needs the folder). +* **Docs measured, not asserted**: test counts, image sizes, glibc, contrast, suite + buckets, and the quick start's `mkdir -p … && sudo chown -R 990:990 …` for real Linux + hosts (uid 990 vs Docker-created root dirs → EACCES crash loop) follow the runs that + proved them. + +## Deliberately not fixed (with reasons) + +* **No auth, localhost-only publish.** Single-user loopback by design (§7); LAN exposure + requires authentication first. Recorded in README and the compose comment. +* **`pixels` is not capped server-side.** Single-user localhost, and the engine renders on + 2 slots; a malicious client asking for a huge render costs the user their own CPU, + which they already own. +* **No `fsync` on the job log after each line.** `fsync` per line is the classic + performance victim; a crash loses at most the last record, and the SQLite index — the + real state — is durable. The job log is a *journal of intent*, replayed to mark + unfinished jobs *interrupted*: losing its tail loses a job's last note, never a lie. +* **The test image runs as root.** Its scratch indexes and host mounts are the tests' + own; the shipping image is non-root (`uid 990`), which is what a user runs. +* **The engine's own limits are left alone** (by the PR's rule): `zGate` 2.0, default + limit 40, render slots 2, idle-exit timings — the server speaks the published + protocol and shows the engine's numbers verbatim, so parity with the Mac app comes + from not adding anything. diff --git a/docs/implementation/stage-1-linux-build.md b/docs/implementation/stage-1-linux-build.md new file mode 100644 index 00000000..8b2af405 --- /dev/null +++ b/docs/implementation/stage-1-linux-build.md @@ -0,0 +1,111 @@ +# Stage 1 — the original DigUp CLI, built and tested for Linux + +Date: 2026-11-02. Built in a Linux container on this machine (arm64), from the repository's own +scripts, with the pinned versions the repository pins. Stage 2 is complete: the headless index + +semantic-search proof on the corpus is documented in [stage-2-headless-proof.md](stage-2-headless-proof.md). + +## What was built + +| part | result | +|---|---| +| Swift toolchain | `swift-6.4.0-RELEASE-ubuntu22.04-aarch64`, GPG-verified in the image build (`Good signature from "Swift 6.x Release Signing Key"`) | +| base image | `ubuntu:22.04` — the same base upstream's Linux CI builds in, so binaries ask for glibc ≤ 2.35 | +| vendor libraries | `scripts/fetch-vendor.sh`: SQLite 3.54.0 (FTS5), stb @ `2c980bb`, miniz 3.1.2, PDFium `chromium/8086` (`pdfium-linux-arm64`) — every download verified against `scripts/vendor-checksums.json` | +| ffmpeg | `scripts/build-ffmpeg.sh`: dav1d 1.5.2 + ffmpeg 9.0.2, 8.9 MB binaries (HEIC-capable, which the picture reader needs) | +| llama.cpp | `Built llama.cpp b11461 (patched, cpu, clang) into Vendor/llama` — shared libraries, `GGML_BACKEND_DL=ON`, `GGML_CPU_ALL_VARIANTS=ON`, `GGML_NATIVE=OFF`, mtmd on | +| compiler | Ubuntu 22.04's stock `cc` (gcc 12) does **not** accept `-march=armv9.2-a+sme`; `clang` (clang 14) does, so the build set `CC=clang CXX=clang++` and the arm64 CPU variants (SME included) built | +| engine build | `swift build -c release` — succeeded | +| **tests** | **`swift test` on Linux: 116 tests in 16 suites passed (`DigUpKitTests`) and 7 tests in 2 suites passed (`LlamaRuntimeTests`), 0 failures.** `vectorsMatchTheReference()` skipped itself: the model files are not in the build image by design (exactly what upstream's CI comment describes) | +| build time | 256 s for vendor + ffmpeg + llama.cpp + `swift build -c release` + `swift test` on 7 cores (BuildKit cache mounts on `Vendor/` and `.build/`) | + +## Commands + +```sh +# build (arm64, native: the image's architecture is the host's) +docker builder prune -f; docker container prune -f; docker image prune -f +DOCKER_BUILDKIT=1 docker build --platform linux/arm64 --target engine-builder \ + -f server/docker/Dockerfile -t digup-builder:local . +DOCKER_BUILDKIT=1 docker build --platform linux/arm64 --target engine-runtime \ + -f server/docker/Dockerfile -t digup-engine:local . + +# inside the build container +docker run --rm -w /src digup-builder:local bash -c \ + 'export PATH=/src/Vendor/ffmpeg/bin:$PATH; .build/release/digup --help; swift --version' +``` + +The image is a three-stage build (`engine-builder` → `engine-payload` → `engine-runtime`): +Swift, CMake, compilers and headers exist only in the builder; the runtime image carries +`digup`, llama.cpp's shared libraries, `libpdfium.so`, `ffmpeg`, `ffprobe`, `libcurl4`, +`libgomp1`, `libstdc++6` — no compiler, no Swift toolchain, no Tauri, no GTK/WebKit, no display +server, running as the `digup` user. `engine-payload` links the CLI the way +`scripts/package-linux.sh` does (`--static-swift-stdlib` plus Swift's archives re-listed at the +end of the link line) and re-points every rpath at `$ORIGIN`. + +## The runtime image, verified + +`digup-engine:local` — **296 MB**, `engine-runtime` stage, non-root user `digup` (uid 990): + +``` +$ docker run --rm digup-engine:local bash -c 'id; ls /opt/digup; digup --help; command -v ffmpeg; ffmpeg -version | head -1' +uid=990(digup) gid=990(digup) groups=990(digup) +/opt/digup: digup ffmpeg ffprobe libpdfium.so libllama.so.0 libmtmd.so.0 libggml.so.0 + libggml-base.so.0 libggml-cpu-armv8.0_1.so libggml-cpu-armv8.2_1.so + libggml-cpu-armv8.2_2.so libggml-cpu-armv8.2_3.so libggml-cpu-armv8.6_1.so + libggml-cpu-armv8.6_2.so libggml-cpu-armv9.2_1.so libggml-cpu-armv9.2_2.so +digup --help → prints the engine's own usage (the CLI runs with no Swift toolchain installed) +/opt/digup/ffmpeg → ffmpeg version 9.0.2 +``` + +Same payload shape as upstream's `.deb` helper folder (reconnaissance §11), minus Vulkan (this +is the CPU baseline), minus the shell. The apt deps are `libcurl4 libgomp1 libstdc++6` only. +A container run with `-v …/index:/index` writes `index.sqlite` to the host mount as uid 990 +(verified with a touch test), which is how the server's `/data` volume will work. + +## Model files, verified against the repository's pins + +Downloaded to `~/Projects/digup-data/models` (outside the repository) from +`ggml-org/embeddinggemma-2-GGUF` at the pinned revision +`bfcd298762cc34d0357ece5ebdd31791a3a374d8`: + +``` +embeddinggemma-2-Q8_0.gguf 309,855,456 B 2188ac1d…b09135 OK +mmproj-embeddinggemma-2-Q8_0.gguf 554,821,024 B c4a8a526…0aab64f OK +``` + +Both match the SHA-256 pins in `Sources/DigUpKit/ModelFiles.swift` (`shasum -a 256 -c`: `OK`). + +## Corpus for Stage 2 + +`server/tests/corpus/` — 12 Commons pictures committed with pinned sha256 + attribution +(licenses: CC BY-SA 4.0 / CC BY 4.0 / CC BY 2.0 / Attribution / Public domain), two real videos +fetched on demand (`wakeboarding-vienna.webm` 59 MB, `pct-hiking-southern-oregon.webm` 33 MB), +`build.sh` (fetches and verifies every pin), and `corpus.json` — the ground truth in the +`evals/` file format, covering wakeboarding, hiking, campfire, dog-on-the-beach, boat-at-sunset, +fishing-from-a-dock, Christmas dinner, the three control pictures, the two real videos, and three +timestamp assertions on a synthetic 3-scene clip (`clips/scenes.mp4@0-6`, `@6-12`, `@12-18`; +sha256 `528db370…45355b`, 766,733 B). All fetched files verified against their manifest pins. + +One discovery: **the engine's own ffmpeg (`scripts/build-ffmpeg.sh`) is decode-only** — encoders +`mjpeg`, `ppm`, `rawvideo`, `pcm_s16le`, no `lavfi` sources, no `libx264`, no `-preset`. It is a +reader, as designed. The synthetic clip is therefore generated with a full ffmpeg (apt's, in a +throwaway `ubuntu:22.04` container); `build.sh` says so when it cannot make the clip. + +## Disk + +Docker's VM disk is on the volume with 43 GB free, and a build died with +`no space left on device` before pruning. After `docker builder prune -f` + `docker container +prune -f` + `docker image prune -f` (4.0 GB reclaimed, build cache 562 MB) the build completes. +`.dockerignore` keeps host build output, `Vendor/`, the corpus media and `server/data/` out of +the build context: they are host artifacts, and the corpus media is mounted at run time. + +## Known, documented limits + +1. This build is **arm64 CPU-only** (no Vulkan, no CUDA): it proves the Linux engine, not the + production GPU path. `engine-runtime` deliberately installs no `libvulkan1`; the GPU path + arrives as the opt-in NVIDIA/Vulkan override (Stage 9), where the device is chosen by + `digup speed --model` and the agreement check (cosine ≥ 0.999), never assumed. +2. `vectorsMatchTheReference()` skips itself without the model files (upstream's own design). + Running it needs a model-mounted test run — Stage 2 does that, which is also the first check + that *this* build produces the project's reference vectors. +3. `swift test` builds a debug configuration in addition to the release build; keeping it in the + builder (never in the runtime image) is what the disk cost is about. diff --git a/docs/implementation/stage-10-ui-parity.md b/docs/implementation/stage-10-ui-parity.md new file mode 100644 index 00000000..8dd24274 --- /dev/null +++ b/docs/implementation/stage-10-ui-parity.md @@ -0,0 +1,170 @@ +# Stage 10 — the web UI reaches desktop parity (theme, filters, opening files) + +The user's brief: *"the webUI should function similar to the digup application … add option +for light and dark mode … run through the desktop application and make sure all functions +are also in the docker application … find a way to download more images for testing."* +Stage 6 shipped a minimal honest UI; this stage makes it behave like the app, adds the one +endpoint the app's "Open" needs, and grows the test material — with the desktop's own +source (`shell/ui/panel.js`, `results.js`, `settings.js`, `settings.html`, +`Sources/DigUpApp/SearchPanel.swift`) read through first, feature by feature. + +## What changed + +**`GET /api/v1/media/{media_id}/file` — the app's "Open", over HTTP** (`api.py`): + +* the same discipline as `/preview`: only a server-issued handle resolves; the path it + names must resolve **inside the server's configured roots** (the engine produced it; + the roots are the deployment's, never a client's) and must be a file that is there now + (404 for unissued ids, out-of-roots paths, and files that left the folders); +* the media type comes from the file's **magic bytes**, not its name, and code/doc content + is served `text/plain` **as an attachment** — a source file named `index.html` never + arrives as markup that could run script on the server's origin; +* media kinds (image/screenshot/video/audio/pdf) arrive `inline` so the browser opens + them; everything else `attachment` (Download keeps the name via `filename*=UTF-8''`); +* **`Accept-Ranges: bytes` + Range parsing**: `bytes=a-b`, open ranges, suffix ranges → + 206 with `Content-Range`, 416 with `bytes */size` for an unsatisfiable span. That is + what makes `
  • ` discs. Ported the vanilla rules + verbatim into the component's shadow styles (+ a row `:focus-visible` outline). +3. *Open/Download as plain links*: the pane builds `a.button-like` (lib/media.js) but + `digup-preview` never imported `controlCss` — imported; real buttons. +4. *Settings panel rework (the brief)*: the Estimate toggle moved to the RIGHT of the + Estimate button and lost its "on/off" caption; THE INDEX reads `Looking in: ` + first, then the fact badges, then the index rows; the API key section shows ONE label + with the key in a readonly text box (click selects all), `⧉ Copy` and `↻ Regenerate` + at its right (minted app keys use the same shape). Measured after the batch: unit + **52 passed / 52** (the settings test updated for the new row shape in the same + change, disclosed), build 283 ms (`app.js` 65.05 kB / 19.89 gzip), checkpoint suite + **137 passed, 1 deselected, 0 skipped, 172.83 s** — the layout pins (2 rem rows, the + minmax() key grid, `#keys-regenerate` margin-left, the estimate switch, + `overflow-x: clip`, `.set-facts` nowrap) verified present in the bundle. + Lesson recorded in `server/ui/AGENTS.md`: a component owns its shadow styles — the + migration must move EVERY rule the vanilla page had for that region; a missing import + is invisible to unit tests and shows up in the browser as an unstyled region. + +**User browser pass, round 3 (2026-11-08):** (1) every panel line is one row height (the +2rem token) and the one-line target rows are gone — the index's counts ride beside +`Looking in: ` in parentheses, abbreviated, from `/api/v1/status` verbatim (the +shell's status slice gained `files`/`searchable`); the named-app-index counts left the +panel with them (disclosed to the user; `GET /api/v1/targets` unchanged). (2) The API key +field widened (label column `minmax(4rem, 6rem)`, field `minmax(0, 1fr)`) and the actions +became icon-only `⧉` / `↻` with `aria-label` + `title`. (3) The estimate knob is centered +in the oval (`top: 50%` + `translateY(-50%)`). Measured: unit **52/52** (the settings test +updated for the new shapes in the same change), build 286 ms (`app.js` 66.20 kB / +20.18 gzip), bundle pins verified (`roots-sum`, `key-icon` ×3, knob centering, +`minmax(4rem, 6rem)` grid, `id="targets"` → 0 occurrences), checkpoint suite +**137 passed, 1 deselected, 0 skipped, 172.44 s** (two contract pins restated in the same +change, disclosed: `appkeys` + `roots-sum`; the key-grid prefix). + +**User browser pass, round 4 (2026-11-08) — Settings redesign (the user's ChatGPT + layout brief):** the gear panel's internals were reorganized in the SAME component into + four stacked sections — **Appearance · Indexing · Index status · API access** — each with + a small inline-SVG icon (palette / database / activity / key: 15 px, accent-colored, + `aria-hidden`) and a 15.2 px semibold heading, 22 px section rhythm around 1 px dividers + (`--space-md` above and below), 12 px radii on every panel box (the key rows + field get + it panel-scoped — the shared `keyRowCss` leaves the embed widget's rows as measured), + muted descriptions, restrained blue accents, no box-around-every-section. The modal frame + is untouched: same width, anchor under the gear, overlay order, the five close paths. + INDEXING: Index now + prune missing on one row; Automatic estimates on its own row, + switch right-aligned, muted description beneath; the forecast box scrolls inside its own + bounds only. INDEX STATUS is the one distinct card: folder icon + `Looking in: `, + **Total files / Searchable files** as two side-by-side stat tiles (the `/api/v1/status` + slice verbatim, `—` when no data), the capability badges wrapping naturally with semantic + tones from the engine's own strings (`vectors > 0` and `encoder ready` → green, + `starting` → amber, `unavailable` → red, `stopped` → muted). API ACCESS keeps the round-3 + shape (wide field, icon-only `⧉`/`↻`). Preserved verbatim: the store slice, all six + events, `focusFirst()`, the empty-section collapse, every control id. Measured: unit + **52/52**, build 283 ms (`app.js` 68.83 kB / 20.89 gzip), bundle bytes verified + (heading-id order, heading texts, `status-card`/`stats`/`toggle-row`, `border-top: 1px + solid var(--line)`, `flex-wrap: wrap`, `overflow-y: auto` → 0); four contract pins in + `test_media_ui.py` restated in the same change, disclosed: the section-order pin rides + the heading ids + texts (the icons now live inside the title spans), `.set-facts` + nowrap → wrap (badges wrap by brief), and the new card/stats/toggle-row/divider pins. + Checkpoint suite: image unpack 2.7 s, **137 passed, 1 deselected, 0 skipped, 183.24 s** + — the restated pins are in that green run. + +**User browser pass, round 5 (2026-11-08):** (1) The `Looking in: ` line left the + status card (the roots stay in `GET /status`; the card's two tiles carry the counts). + (2) The estimates description line left with the rename: the row is now **Estimate** + + right-aligned switch (the description span and its `.set-desc` rule are gone; the toggle + keeps its `title`). (3) The copy buttons now work where the panel is served over **http** + on the LAN — an insecure context has no `navigator.clipboard` at all: `#copy()` uses the + async clipboard when present, falls back to hidden-textarea `execCommand("copy")`, and + the button answers `✓`/`—` for a moment. New unit test covers the fallback (jsdom has no + clipboard — the exact LAN shape): unit **53/53**. The regenerate chain was proven on the + live dev stack (8787, minted shape): keyless `GET /tokens` → ticket, `POST /tokens` + (admin) → rotated 200 with a new key, `POST /tokens` with the dead key → **403** ("the + keys answer the admin token only"), keyless `GET /tokens` after rotate → the ticket is + the new key (the gear adopts it, the panel shows it, copy copies it). Disclosure: the + proof rotated the dev stack's minted keys — browsers self-heal on the next gear open. + (4) Main-page layout (the user's layout brief): the header bar bleeds to the **screen** + edges (`margin-inline: calc(50% - 50vw)` + 1.25 rem padding, `overflow-x: clip` on body + absorbing the scrollbar sliver) — DigUp left, gear right, settings panel re-anchored at + `right: 1.25rem` so its edge lines up with the gear; the footer strip bleeds the same way + — summary left, keyboard instructions **right edge**; the search input is centered on the + exact axis of the chips under it (the limit select + code box leave the row's flow above + 1000 px, riding the input's right; below it they drop back as a centered row — new 1000 px + fold, 720 px fold kept). Measured: unit 53/53, build 307 ms (`app.js` 69.24 kB / 21.18 + gzip), bundle verified (`roots-sum`/`rootsline` → 0, bleed/`right: 1.25rem`/`grid-column: + 1 / -1` present, pinned strings intact); one contract pin restated (`roots-sum` → + `Total files`, disclosed); status card/stats/toggle-row pins kept green unchanged. + Checkpoint suite: image unpack 2.3 s, **137 passed, 1 deselected, 0 skipped, 172.65 s** + — the restated pin and the 53-test frontend gate (frontend-builder stage) are in that + green run. + +**User browser pass, round 6 (2026-11-08) — the estimate flow compacted:** the estimates + toggle and the whole estimate section (forecast box + its rows) are **removed**; the + Estimate button rides the row **left of Index now** (with `prune missing`). The INDEXING + heading now carries the live state at its right edge: the true-circle spinner while the + engine works (index job **or** estimate), ✓ (green) when it finished, `!` (warn) for an + interrupted/busy job or a failed estimate (the full engine message rides the `title`), + and the brief update beside it — the job's own words verbatim, or for an estimate the + engine's seconds summed brief (`estimate ≈ 4 min 10 s · 3 folders`; disclosed: the sum is + arithmetic over the engine's `folders[].seconds`, the per-folder rows left the UI, `POST + /estimate` unchanged). Disclosed mechanics: the estimate CONTROLLER keeps its gate + + click-again-cancel + `digupEstimateOn` browser key, and the shell now forces `on` on + every Estimate click — a browser that had the old toggle switched off would otherwise be + left with a dead button (measured in the code path). Measured: unit **53/53** (the + forecast-rows test became the heading-summary test; the toggle event test is gone), + build 284 ms (`app.js` 65.30 kB / 20.27 gzip — the panel shrank 4 kB), bundle verified + (`estimate-toggle`/`estimate-switch`/`estimate-ctl`/`toggle-row` → 0, `set-head-row`/ + `set-state`/`state-done`/`state-err` present, `id="estimate"` count 0); contract pins + restated in the same change, disclosed: `id="estimate"` count 1→0, `estimate-toggle` + presence → absence, `estimate-ctl` → `not in css`, the `.estimate-switch` checked-selector + pin dropped, `toggle-row` → `set-state`. Checkpoint suite: image unpack 2.3 s, **137 + passed, 1 deselected, 0 skipped, 191.10 s** — all round-6 restatements are in that + green run. + +**User browser pass, round 7 (2026-11-08) — `all` toggle + Nextcloud app filter & + progress:** (1) Standalone: the `all` button in the kind row is a TOGGLE — all six ticked + → click clears all → click restores all; `aria-pressed` + an accent border mark the + all-on state. Engine semantics verbatim (all ticked = every kind; cleared = the engine's + answer over an empty kinds list — 0 hits is the honest filter answer). Unit gate added a + toggle test: **54/54**. (2) Nextcloud app **0.4.3**: the same six chips + the same + all-toggle under the search bar; the ticked set rides the engine's `kinds` filter + verbatim (PHP whitelists the list to the server's Kind literals — `ApiController` gained + the `kinds` param, `DigupService` forwards it, routes/shapes unchanged). The activity + spinner now stays visible for as long as the server reports a job `running`, with the + live `files_done / total (pct %)` counts beside it — the progress poll re-arms itself on + every `running` answer (user: "how many indexed out of how many remaining"). Deployed on + the pilot (`docker cp` onto the bind-mounted app dir — the install tree is www-data-owned; + `php -l` ×3 in-container clean; the registered version needed `occ upgrade` — the pilot + instance had `needsDbUpgrade: true`, which blocked `app:update`; upgrade clean, integrity + OK, maintenance off). Pilot live gate over the LAN name (session as the user the index + covers): `[image]` → 8 hits all image · `[audio]` → 3 hits · all six → mixed · + progress `ok: true` `files_done 10,254 / index_files 131,973` · served `app.js` carries + the new code (cache-bust verified). Measured on this box (ai-ubuntu-computer): unit + 54/54, build 285 ms (`app.js` 65.67 kB / 20.39 gzip). README history entry 13. + +**Cleanup audit after the redesign storm (2026-11-08, the user's "did we clean up?" + ask):** audited the whole frontend — every CSS selector in every `css`` chunk and + global file against every class/id rendered anywhere, every `lib/format.js` export, + every store slice's writers vs readers, every controller export, every shell listener. + Result: **zero dead CSS selectors, zero unused helpers, zero unused controller exports** + (the estimate controller's gate/cancel/`resetRetried` are all live code paths). + Removed the three residues the rounds left: the shell's `roots` store write (its + consumer, the panel's roots line, left in round 5 — writer without a reader), the + shell's `@digup:estimate-toggle` listener (no dispatcher since the round-6 switch + removal), and the rule-less `class="set-lines"` on the keys block. Kept by contract, + disclosed: `keys-controller`'s `GET /targets` + its `s.targets` slice (the panel no + longer renders the rows, but the endpoint literal and the reload-after-a-job behavior + are pinned by the contract suite) and the estimate controller's on/gate state (the + shell forces it on click so a browser with the old persisted toggle-off is never left + with a dead button). Measured: unit **54/54**, build 287 ms (`app.js` 65.53 kB / + 20.36 gzip), bundle strings `digup:estimate-toggle`/`set-lines`/`roots: st.roots` → 0 + occurrences. Audit recorded in `server/ui/AGENTS.md` §State. Checkpoint suite: image + unpack 2.3 s, **137 passed, 1 deselected, 0 skipped, 172.19 s** — the frontend-builder + stage ran the unit gate (54/54) on the cleaned tree on its way in. + +**Refactoring pass (2026-11-09, the user's cleanup+audit brief — full-surface audit, +then small verifiable changes):** the whole server reviewed — 2,776 Python lines +(api/security/jobs/targets + engine adapter), 2,447 UI JS, HTML, CSS, the 367-line +Dockerfile, all 9 compose files, the 507-line CI workflow. **Audit result: the +implementation was disciplined** — AST scan found zero unused imports/functions/classes +(every flagged name is decorator- or import-consumed: routes, exception handlers, +middleware); zero dead CSS selectors and zero unused UI exports; every Dockerfile stage +referenced (`engine-runtime` is CI-built, `cuda-libs` by the CUDA target); no engine +duplication (the server is passthrough — status parsing is the published CLI surface, +jobs count the engine's own events); embed widget intentionally separate (contract +pins); chronicle docs binding (§-refs in code comments) — kept. **Applied** (each a +small verifiable change): (1) removed `IndexService.sync_code` (zero refs, product and +tests) and `SearchService.folders()` + `_folders_fields` respawn replay (the server's +scope is argv at spawn — the replay existed only for a wrapper nothing called); +(2) **jobs.py counter consolidation**: the live job record already carries every +counter as the engine emits it (`_progress`), so the per-event `events` list + the +`_counts` re-scan were a second source of the same numbers — a 1.8 M-file job no +longer keeps a full copy of every event in memory; endings now read `_live_counts` +(keys the engine never published = None, same published shape; the unit pin that +called `JobRunner._counts([])` was adapted to assert the same guarantee through +`_progress` seeding + `_live_counts` — same assertions, no weakening); +(3) **api.py `make_runner`**: the JobRunner construction (config_dir, per-target log +name, reload→prewarm, ceiling, rotation) existed twice verbatim (lifespan + app +target spawn) — now one source; (4) **CI vendor-seeding consolidation**: the 18-line +vendor step was byte-identical in both jobs (server + publish) — now one YAML anchor +referenced by both (merge key; PyYAML-verified: parsed, one step per job, run scripts +md5-identical); (5) **UI export hygiene**: 11 internal-only `export` keywords demoted — +2 helpers in `media.js` + the 9 self-registering component classes (every component +file ends in its own `customElements.define`; zero importers in the app and in the +unit tests); (6) docs: `DIGUP_SERVER_STATUS_TTL_S` documented in `.env.example` + +the deploy README env table (code read it, docs never named it). **Kept by design, +disclosed**: `estimate_json/estimate_text/speed_model` (test-pinned CLI surface), +`raw()` (contract-pinned), the tokens-file merge logic, all security guards. **Gates:** +py_compile all modules + adapted test; UI unit gate 54/54 (host, and again inside the +`frontend-builder` stage at image build); vite build clean; workflow YAML parsed with +md5-identical merged vendor steps; **full contract suite twice on the refactored tree — +first on the Python batch: 137 passed, 1 deselected, 0 skipped, 195.70 s; then on the +exact final tree (image rebuilt with the UI export demotions): 137 passed, 1 deselected, +0 skipped, 173.56 s.** Changes left uncommitted per the user's rule (no commits/pushes +for this task). + +**All-in-one compose example + operator README (2026-11-09, the user's asks):** +`deploy/docker-compose.all-in-one.yml` merges the shipped chain +(`docker-compose.nextcloud.yml` + `nvidia` + `lan` + `nextcloud.lan` + `app-targets`) +into ONE file with the env names unchanged — Nextcloud + server (CUDA build target) + +LAN binds on both web ports + a named app target + the `--profile dav` mount container. +The header prints the launch chain and how to peel each layer (CPU-only = five named +line deletions; loopback = two port lines; data vs WebDAV shape; no app target). +`deploy/README.md` is the operator page the compose app needed: file map, every env var +with its effect, the launch chains (example values), the measured operator rules +(chown 990, ghost-row prune, single uvicorn, 503-warming honesty, 409 one-writer lock, +143/137 stop semantics, GPU-is-measured honesty), and the API surface table with scopes, +pointing at `server/docs/API.md` as the canonical contract (test-pinned; this page never +restates schemas). Measured on this box: host schema checks for the all-in-one — +full (`--profile nvidia --profile dav`), no-profile, dav-only — all OK; the resolved +config carries `target: server-runtime-cuda`, the nvidia device reservation, +`DIGUP_SERVER_APP_TARGETS app=/media/app:/data/index-app`, four port mappings +(8787 loopback+LAN, 8081 loopback+LAN), the ro media bind + ro app bind + shared dav +propagation. The exact CI line was run on the host (OK): the compose-schema step now +validates the merged shape with the full synthetic env chain (15th shape; RFC 5737 +example addresses and `/srv/…` paths only). Privacy grep over the three files: zero +personal hits; the app-repo URL reference follows the tracked-docs precedent. + +## 2026-11-08 — Stage 13: named indexes + minted keys — one compose app, standalone AND serving applications + +The user's ask, built end to end: the compose server keeps running standalone over its own +media index, AND acts as a server for other applications — it mints their access tokens, +and each application's calls build and search a SEPARATE index, never the media one. The +engine changes nothing: it already answers for any (roots, `--index`) pair. + +**Registry (`server/digup_server/targets.py`).** `DIGUP_SERVER_APP_TARGETS=nc=/media/nc:/idx/nc;…` +(entries `;`-separated, each `name=roots=index`, roots colon-separated) merged with a persisted +`/targets.json` (env wins per name). Validation refuses at startup with the entry +quoted: non-slug names, the reserved name `media`, relative roots/index, and a target whose +index IS the main index (separate indexes are the point). + +**Per-target supervision (`api.py`).** Every guarded endpoint addresses a target (`?target=name`; +default `media`). The main scope is the lifespan's, exactly as before. App targets spawn +**lazily** — first request addressing them pays the spawn, so registered-but-idle targets cost +zero engines: each gets its own `search-service` child, its own `JobRunner` (its own jobs log +`jobs-.jsonl`), and its own writer lock for free (the flock lives beside each index +folder — "one writer at a time" is per index, so a bridge crawl never queues behind the media +crawl). Media handles resolve their target from the path's owning roots (longest-root match), +so `/preview`, `/text`, `/file` work across indexes unchanged. `GET /api/v1/targets` lists the +registry with counts from the CLI status (no engine spawn for a listing). + +**Minted application keys (`security.mint_tokens`).** When targets exist and env left a target +unbound, the server mints `secrets.token_urlsafe(24)` keys (`app:`), plus an admin key +when the deployment has none, stored 0600 in `/tokens.json` (config folder = +recreation-safe; a recreation never silently revokes a connected app; restart-stable and +displayable from any boot). Minting fires ONLY when targets exist — a target-less standalone +deployment runs exactly as it has, unguarded. Env secrets are never stored, never echoed. +`GET /api/v1/tokens` shows minted values to the admin principal only; the UI's gear → Settings +→ Access gained **Application keys** (copy buttons) and **Indexes** sections (hidden for +non-admin keys). + +**The bearer gate binds tokens to targets.** `admin` answers every target; `read` and plain +named tokens answer the main index — their pre-stage-13 contract, unchanged. An `app:` +token answers its target and nothing else, **by default**: plain paths (`/search`, `/index`) +address its own index (the binding is what the token carries — the bridge's code path needs no +target parameter), and any explicit target it does not match answers `403`. Mutating routes +accept an app token only for its own target: the bridge queues its own crawl, never anyone +else's. Review caught and fixed: a plain-named-token 403 regression, the injected-searcher +seam the 503 contract pins (the main scope now resolves `app.state.search` at call time), the +tokens-file write before the config folder existed, the main-index clash for env entries, a +cross-target counts bleed in the single-slot status cache (now keyed per index folder), and +the tokens endpoint refusing an admin token presented on the read side +(`principal="admin (read scope)"`). + +**Surfaces:** `server/docs/API.md` "Named indexes" (+ auth bullets), `server/.env.example` +(`DIGUP_SERVER_APP_TARGETS`), `deploy/docker-compose.app-targets.yml` (opt-in `-f`; the +target's index rides its OWN bind mount, so indexing survives recreation like the media +index's), CI validates the fragment with and without the nvidia profile. +`server/tests/test_targets.py` pins the registry, the mint rules (inert/stable/0600/env- +authority), the binding table, and the flow end to end on the real engine (minted key → own +status → own crawl 202, media 403, own search, cross-target media preview, per-target busy +409, parallel crawls on two targets, tokens visibility). + +**Suite (this box, container, `-rs`, zero skips):** run 3 (new suite in): 6 failed / 123 +passed / 1 deselected / 0 skipped → review-pass fixes above; run 4: 2 failed / 127 passed — +the two were a stale binding-default expectation (superseded by design) and a render +mid-reload answering the engine's honest 503 (the pinned refusal state — the contract now +polls, like clients do, and names the last answer's body: the engine's own words are the +diagnosis); run 5: 1 failed / 128 — the preview poll's own bug (it picked `hits[0]`, a text +doc: the engine's render refuses text docs permanently, "unreadable: can't decode image" — +the contract previews the image hit); runs 7 and 8: **129 passed / 1 deselected / 0 failed / +0 skipped**, twice. + +Two more findings from the demo bring-up, both fixed in code: + +1. **Registry shape:** the parser demanded `name=roots=index` (two `=`) while the docs, + the compose fragment, and `.env.example` all shipped the house shape `name=roots:index` + (roots colon-separated, index LAST — the same shape as `DIGUP_SERVER_ROOTS`). The demo + container crashed at startup quoting its own entry — the parser moved to the documented + shape (entry → `name=`, then colon-split: ≥ 2 parts, last is the index), the tests and + docs pin it, and the env path now enforces the main-index clash too. +2. **`/api/v1/targets` counts:** the CLI status has no `files`/`searchable` keys (those are + the searcher's live fields), and a store whose writer has exited can keep its fresh rows + in the WAL pinned by the searcher's reads (measured in the suite: main db 4096 B + live + `-wal`, searcher 12 rows, a fresh CLI read 0). The listing now takes counts from a + running target's live searcher (the authority on a store it has open), from the CLI's + own table (`kinds` totals + `vectors`) for unspawned targets, `null` for an empty index + folder — no engine is ever spawned for a GET, and no number is ever invented. + +`server/docker/Dockerfile` gained `ARG SWIFT_TESTS=1` gating the engine `swift test` inside +the builder stage: CI never passes the arg (gate unchanged); local suite builds use +`--build-arg SWIFT_TESTS=0` after the measured box fact — a from-scratch engine build on +this box dies in the engine's own test `gitListingNeverRunsRepositoryFsmonitor` +(`CodeTests.swift:203`, the fsmonitor hook never runs in this box's builder), and the +server's gate is the contract suite (`swift test` belongs to ci.yml/packages.yml on their +supported environments). + +**Live demo gate (this box, compose stack project `digup-demo`, port 8792, CPU flavor, +public Wikimedia corpus photos only):** tokens file `0600` with minted admin + app key; UI +200 with both sections; unkeyed API `401`; admin answers `target media` roots `/media/photos`; +the app key answers `target nextcloud` roots `/media/app` and is `403` on `?target=media`; +its `POST /index` → `202`, a second one → `409` naming the live job; the job `done`; app +status **36 files / 36 searchable**; the media index untouched (**0 files**, 0 hits); app +search 3 hits all under `/media/app`; targets visibility (admin: both, app: only its own); +`GET /tokens` shows minted values to admin, `403` to the app key. Audit lines carry +`principal=… target=…`. Final suite run with these fixes: **129 passed / 1 deselected / +0 failed / 0 skipped in 160.04 s** (2026-11-08, this box, container, four mounts, `-rs` — +zero SKIPPED lines in the log). + +**The Nextcloud bridge (app repo `digup_nextcloud` 0.4.0).** `DigupService::serverStatus()` +(GET `/api/v1/status`) and `startIndex()` (POST `/api/v1/index`) ride the same +server-to-server path as the search — the key never reaches the browser, every number is the +server's verbatim. Admin-only routes `api#status`/`api#indexnow`; the settings form gained the +index line (target + counts) and an **Index now** button; the token field is now "Access key" +with the Application-keys page named. Deployed on the pilot (occ `installed_version 0.4.0`, +files bind-swapped, `php -l` green in the container, `node --check` green on both JS files). +Gates on this box (measured, pilot stack, session as the admin user): status → 200, +`target: media`, files **131,973**, searchable **115,098**, roots `['/media/photos']` (the +live GPU run's counts, mid-embed); index-now with the pilot's read-scope key → the server's +own words passed through verbatim ("that is this server's read-scope token; indexing and +estimating need the admin token") — the honest state until the pilot's image carries +stage-13; non-admin user → 403 on both routes. The pilot's full named-index flow (minted key +→ its own index) lands with the image rebuild after the running embed job finishes — a GPU +run is never interrupted for a redeploy. + +**Bench + pilot move onto the stage-13 CUDA image (2026-11-08, measured).** CUDA rebuild: +`server-runtime-cuda`, bases 11.8.0 devel/runtime, `CUDA_ARCHS=61` (the server-pinned +GTX 1070, sm_61), `SWIFT_TESTS=0` (the local build knob; compose's `DIGUP_SWIFT_TESTS` +added with default 1, so CI/standard flows keep the engine-test gate unchanged). Bench +`:8787` recreated on the full chain (chain files + boot env verbatim — media index +untouched): health 200; status `target media` **131,972 files / 131,758 searchable** (the +bind-mounted index survived recreation, the jobs log intact); UI 200; unkeyed API 401; +read-token status 200 / its index POST 403 (the pre-stage-13 token contract, unchanged); +`targets(admin) → [('media', 131972, 131758)]` live counts; `tokens(admin) → 200` with the +env admin flagged `configured` and its value never echoed, `apps: []` — **minting is inert +with no targets registered: no `tokens.json` in the config folder**; search 5 hits, 5 +strong, 0.751 s. Standalone behavior is unchanged, exactly as designed. The pilot's GPU +embed job hit its 10-hour ceiling (`DIGUP_SERVER_JOB_TIMEOUT_S=36000`) and recorded +`interrupted` at **118,206** files (searchable 120,693, index 753.9 MB) — the pinned stop +semantics working: the resume `POST /index` answered 202 with a fresh job continuing from +the interruption; the 10-query battery across the partial index: 10 hits / 10 strong on +every query (top z 3.67–7.33), latencies 0.43–1.63 s; the app settings line (status 200) +showed the same counts live. + +**UI rework (2026-11-08, the user's layout brief): the main page nag is gone; the gear is +Appearance · Index · Estimate · The index · API key.** The front page keeps no status +indicator (the "status unavailable: … paste your key" line is deleted — counts live in +`The index`); the gear's old Access/Application-keys/Indexes sections collapse into two: +`The index` (per-target rows with the engine's counts + roots, then the badges) and +**`API key`** — the minted keys as plain rows (`API key`, `API key — `) each with +**Copy**, a **Regenerate** button behind a confirm dialog, and the small paste field for +deployments whose keys come from the environment (env keys are never shown, and never +rotated). `POST /api/v1/tokens` (admin scope) is the rotation: minted values (admin + +app keys) get fresh `secrets.token_urlsafe(24)` strings, persisted 0600, live immediately +— old values answer 401 from that call on, which is exactly what the confirm warns about +("the applications that have these keys lose access"); env-configured keys are kept +untouched (a rotation must not silently override the deployment's own config), and a +deployment with nothing minted answers 400 honestly. The rotation swaps the bearer gate's +auth reference through the built middleware chain (the stack is built at the first +request; `Auth` stays frozen). The admin-scope check also accepts a direct admin-bearer +match: an admin-token-only deployment deliberately leaves reads open (principal `none`), +and the keys surface exists to answer the admin key's owner. The Nextcloud bridge follows +the same shape: its settings section is ONE field, labeled **API key** — paste it (app +0.4.1, deployed on the pilot: files bind-swapped, `php -l` green, occ `installed_version +0.4.1`). Suite: three new contracts pin the rotation (minted values rotate + file 0600 + +old keys die at the live auth; env keys never rotate; endpoint e2e: old key 401 → new key +200 target `nc`, non-admin POST 403, env deployment 400 with honest words), and the UI +contract asserts the new panel (sections in reading order, no main-page nag, +Regenerate + Copy wired). Measured: suite run 13 on this tree — **133 passed / 1 deselected +/ 0 failed / 0 skipped, 224.61 s**; the demo stack (:8792, rebuilt on this tree) passes the +full flow gate (minted file 0600, unkeyed 401, app key → target `nextcloud` roots +`/media/app`, 403 on `?target=media`, 202→409→done, app 36/36, media 0 untouched, targets +`media 0` + `nextcloud 36`, tokens admin-only) and the rotate gate live: `POST` 200 +`rotated: true` with fresh admin + app values shown, **old app key → 401 and old admin key +→ 401 on the same call**, new app key → 200 target `nextcloud` (files 36), 403 cross-target, +403 rotate by app key, tokens file 0600 carrying exactly the new values, media untouched +(0), app search after rotate 3 hits all under `/media/app`; served UI markup carries the +new sections (`appkeys`, `targets`, `keys-regenerate` present, main-page `statusline` +absent). The bench stack (:8787, the real tree) rebuilt on the same CUDA image (11.8.0-devel +base, `CUDA_ARCHS=61`, `SWIFT_TESTS=0`, engine layer cache hit) and recreated on the full +chain: index intact (131,972 files / 131,758 searchable / 403,831 vectors / pending 0), +contract unchanged (tokenless 401, read-token index 403, admin search 200), 10/10 queries +10 hits + 10 strong at 0.83–1.09 s (top_z 2.86–7.14), and stage 13 inert on an env-keyed +standalone: no keys file minted, `targets` shows `media 131,972`, `GET /tokens` 200 with no +value echoed, `POST /tokens` 400. + +**Round 2 (same day, the user's live test of :8787 and :8090): the minted shape opens the +main index's reads; the gear's API key section is Copy + Regenerate only.** The user's test +hit the guarded shape (`status unavailable` nag on the main page) and the gear's paste field +bothered them — the brief: the standalone app searches with no key, keys exist for the +applications. The bearer gate now distinguishes the sources: keys the deployment TYPED in its +environment keep guarding every call (test_security's shape, untouched); keys the server MINTED +guard the index operations, the keys surface, and each application's own index — the main +index's reads (status/search/status targets/embed_ahead) answer key-less on the operator's box. +The media-id side is scoped with it: a handle owned by an application's roots answers `403` to +the open principal and `200` to that application's key (preview/text/file). The gear's section +lost the paste field, Save, Clear: rows = label | value | Copy in a uniform grid (one row +height, equal padding/border, ellipsis-clipped key text, Copy carries the whole value), plus +Regenerate; the browser adopts its admin key on Regenerate. The app got the matching shape +(0.4.2): the preview pane's `display: flex` out-specified `[hidden]` — the pane, its bar +(‹ ›, ✕, Open-in-Files) and the border rendered on the empty page and ✕ "did nothing"; +`#digup-detail[hidden] { display: none; }` (the line the lightbox already had) fixes all +three complaints at once. Measured: suite run 14 — **135 passed / 1 deselected / 0 failed / +0 skipped, 164.25 s** (two new contracts: open main reads with guarded mutations/keys/app +target; open principal never previews an application's files); demo flow gate 4 on the +rebuilt minted stack (`status no key 200 target media`, `index/estimate no key 401`, +`tokens no key 403`, app-target read `401` key-less, app key `200` own target + `403` +cross-target, 202→409→done, app 36/36, media untouched) and rotate gate 2 (rotated 200, old +keys 401 on the same call, new keys answer, file 0600 carrying new values, media untouched, +search after rotate all under the app's roots); bench :8787 relaunched TOKENLESS on the new +CUDA image: "wakeboarding" un-keyed 10 hits 10 strong 0.72 s, 10/10 queries strong +0.73–0.92 s, served UI: no paste field, grid rows live, search works with no key. + +**Rounds 3–4 (the user's live test of the gear panel): one row height for the whole panel, +the API-key section collapses when there is nothing to show, and Estimate shows a spinning +wheel in the gear instead of a main-page popup.** The user's live-test report: the panel +looked uneven, the API-key section held only a lone Regenerate button, and the estimate's +main-page progress line was noise. Measured behavior now: every panel control — the +appearance select, Index now, the prune row, Estimate, Regenerate, every key/index row — is +exactly 2rem tall (`#settings select, #settings button, … { min-height: 2rem }`), titles +align to the first row, key rows are `label | value | Copy` in the grid with the key text +ellipsis-clipped (Copy carries the whole value). The section logic tells the truth per +shape: minted keys render as rows; an environment-keyed deployment renders the honest row +"Set in the server's environment" (the server never echoes typed keys — so it says so); a +tokenless deployment has NO keys (minting fires when an application registers) and the +whole section collapses — no title-only fragment. Estimate: a ⟳ beside the gear's button +spins while the engine walks (CSS keyframe spin), the main-page +"estimating — the engine is walking the folders…" popup is deleted, the forecast fills the +panel when the answer lands, and the failure line stays honest. Measured: suite runs 15 and +16 — **135 passed / 1 deselected / 0 failed / 0 skipped** (205.85 s and 164.51 s; module run +mid-round: 37 passed on targets + media_ui); served UI on both stacks (:8787 tokenless, +:8792 minted): `api-key-sec` present, `estimate-spin` present, paste field absent, grid-row +CSS + 2rem rule + spin keyframe live; "wakeboarding" un-keyed 10 hits 10 strong. + +**Stage 14 (the user's plug-in model made real at runtime): the standalone mints its API key +at boot (`DIGUP_SERVER_MINT_KEYS=1`) and applications REGISTER AT RUNTIME (`POST /api/v1/targets`).** +The box mints its admin key with no application registered — the gear's Copy row is the ticket. +An application brings it to `POST /api/v1/targets` with `{name, roots, index?}` (the startup +validator runs at runtime: slug rules, absolute paths, not-the-main-index, `media` reserved, +duplicates `409`): the server mints its PRIVATE key (returned once, persisted 0600), gives it +its own index folder (default `/index-`), persists the registration +(`targets.json` — restarts keep it), and the application's key answers only its target from +then on: own status/search/previews/crawl, `403` on the main index, `403` on registering, +and its crawl runs in the background, invisible to the main page; the open principal never +previews its files. Compose passes the knob; the env-fragment registration (stage 13) stays +the declarative shape, env wins per name. Measured (suite run 17 — **137 passed / 1 deselected / +0 failed / 0 skipped, 199.60 s**, module run 37 mid-round; live on :8787): minted file 0600, +`tokens` admin 200 key shown / key-less 403, `index` key-less 401, wakeboarding un-keyed 10 hits +10 strong; registration flow: `POST /targets` 200 with key + default index `/data/config/index-`, +app key 200 its own target + 403 media + 403 register, targets list `media 131,972` + app rows; +test registrations then wiped and the stack force-recreated to a pristine standalone (fresh mint, +un-keyed search 10/10 strong). + +**Round 7 (the settings-layout brief): the panel's grid arithmetic, measured.** The API-key +row was `9rem minmax(0,1fr) auto auto`: label (144 px) + 3 gaps (24) + two padded buttons +(Copy ≈ 70, Regenerate ≈ 138) + row padding/border (18) ≈ **394 px minimum inside a control +column of 271 px** (25 rem panel − label column) → the row sat ~120 px past the panel's right +edge, worse with the `overflow-y: auto` scrollbar. Every track is now `minmax()`-bounded +(`minmax(6rem,9rem) | minmax(0,1fr) | min-content min-content` + `min-width: 0`), the panel is +`min(27rem, 100vw − 2rem)` with `overflow-x: clip` and NO scroller, `#settings .set-sec > *` +carries `min-width: 0`, the Appearance select fills its control column, the section folds to +one column under 720 px. The Estimate control is a REAL switch (native checkbox + track/knob, +2.6rem×1.5rem, rides the 2rem row system) gating the Estimate button, its rows stacked +directly above the control line (`#estimate-ctl`). The gear ADOPTS the minted key it is shown +(the panel's own controls are admin-scope: Index now / Estimate / Regenerate work with zero +typing; a stored key the server no longer recognizes is replaced by the value shown now) and a +finished crawl re-reads the index list. The engine's own-words line (`#engine-says`) — lost by +my round-6 line-deletion — is restored (its absence threw at first paint). +Measured (suite run 19b — **137 passed / 1 deselected / 0 failed / 0 skipped, 189.48 s**; +module runs 50 green; geometry gate 34 PASS lines; round-7 layout gate 42 PASS at 1280/900/640; +live :8787 27 PASS): panel right 1196/1280, key row right 1182.2 inside the panel; control +column aligned in all five sections (878.6 = 562.6 = 25.8 px folding); select fills the column +(303.6 px wide at 1280, 580.4 at 640); toggle measures 41.6×24 px = 2.6rem×1.5rem; no inner +scrollers, no page horizontal scroll at any width; toggle flips by click and Space; Estimate +runs keyless from the gear (adoption) and renders rows in the panel, main page clean; Copy +puts the full key on the clipboard; Regenerate mints a new key (row + browser updated, old key +401); wakeboarding un-keyed 10 hits 10 strong; index/estimate un-keyed 401. + +**Round 6 (the gear as the ticket — the user's brief of 2026-11-08, "there is no api key +with a copy button").** `GET /api/v1/tokens` is **key-free in a minted deployment**: the gear +shows `API key | value | Copy` with **no key in the browser** — the same open shape as the +main page's keyless reads (new `env_secrets(auth)` in security.py draws the line: typed +secrets = guarded shape, keys never displayed, calls 401 at the gate as test_security pins; +server-MINTED keys = the ticket, displayed). An `app:` key is refused the surface (403 — +applications get their own key at registration, they never list the server's). Regenerate +moved INTO the key row's right edge (grid: label | key | Copy | Regen, `margin-left: auto`) — +no loose button under the section. The estimate forecast div moved beside the button (a plain +row column — the stray empty box near Estimate is gone); a finished crawl re-reads the index +list, so no stale target rows survive a job. Measured (suite run 18 — **137 passed / +1 deselected / 0 failed / 0 skipped, 197.37 s**; module run 50 green; live on :8787): keyless +`tokens` 200 shows the minted admin value (equals the 0600 file's value), wakeboarding +un-keyed 10 hits 10 strong, `index`/`estimate` un-keyed 401, admin `tokens` 200; app-connect +flow on the live box: register → app row shows in the keyless list (ticket model), app key on +the surface 403, app key answers its own target; test registration wiped → pristine standalone +(fresh mint keyless-shown, media 131,972 only). +## 2026-11-08 — Pilot: live progress, Access UI, the excludes fix, and the engine picks GPU + +**The excludes fix (b81e9e8).** The index job path was the gap: `estimate_argv` and +`search_argv` carried `--exclude`/`--exclude-ext`, `index_argv` did not. Measured: the +pilot's job walked **1,643,492 files** (the Nextcloud data dir's preview folders = +1,470,439 files) against a 131,973-file forecast. Engine proof that the flag works on +`index-service`: a scratch run reports `left_out: {"folders excluded folder": 1}`. +Recovery as designed: graceful stop (`Application shutdown complete`, the running job +recorded *interrupted* at files_done 34,931 / 1,643,492), the contaminated index wiped, +image rebuilt with the fix, chain re-queued. The job log lives at `/jobs.jsonl` +(jobs.py §header), so it **survives an index wipe** — boot replay restored the +interrupted record, and the newest record drives the app's progress line. The suite +asserts the job path carries the excludes (test_integrations). + +**Fresh run (measured):** estimate walk 1,762 s → **131,973 files** (106,346 image, +19,818 video, 2,868 pdf, 2,817 doc, 85 audio, 39 screenshot); forecast 8.78 h (the +engine's own `seconds` 31,614, `calibrated: false` → M4 Pro baseline). The walk total +131,973 = the excludes honored in the job. On the CPU-flavor image the default chain +measured 1.68 files/s (255→431 in 105 s) → 130,742 ÷ 1.68 ≈ 21.6 h. + +**GPU switch (config only, Stage 9 procedure):** the ghcr CUDA image predates the +excludes fix, so the CUDA image was built on the box from the current tree +(`--target server-runtime-cuda`, CUDA 11.8.0 devel + runtime bases, `CUDA_ARCHS=61`) +→ 2.42 GB. A compose fragment swaps the pilot server's image, pins +`NVIDIA_VISIBLE_DEVICES` to the server card (UUID `GPU-a733…e630`) and reserves the +device; the full chain validates `config --quiet` with and without the fragment, and +the recreation passed the full file chain + boot env (never defaults). + +**The engine's verdict** (`speed --model --json` on the CUDA image): +`device: "gpu"`, `agreement: 0.99977` (≥ 0.999 required). Model times GPU vs CPU on +this card: picture **0.145** vs 0.668 s, frame 0.079 vs 0.364 s, screen 0.313 vs +1.440 s, text 0.029 vs 0.151 s. The CUDA chain ships inside `/opt/digup` +(libcublasLt.so.11 → 11.11.3.6, libcudart 11.8.89, libnccl 2.15.5, libnvrtc 11.8) — +`libcuda.so.1` comes from the host driver through the toolkit, as always. The job +resumed from checkpoint: searchable 2,279 carried over, no re-embedding of completed +files. Utilization sampling during a real embedding stretch (0.15 s interval, 300 s, +pinned card): **1,567 samples, 720 busy, peak 95 %, duty 45.9 %, VRAM peak 1,697 MiB** +— passes the gate on both thresholds (peak > 20 %; duty ≥ 10 % with the model resident). +Throughput window 3,151 → 3,383 files in 120 s = **1.93 files/s** vs 1.68 files/s on +the CPU flavor: the GPU model time is 4.6× faster, the net is +15 % — the per-file CPU +pipeline (video decode, PDF parsing, HDD streaming) dominates end to end. Finish +forecast: 128,590 ÷ 1.93 = 66,512 s ≈ **18.5 h**. Index on disk: 1 MB at 3,383 files +(quantized vectors) → the full run lands ≈ 40 MB; disk is not the constraint. + +**UI (suite-asserted):** the access key moved from a load-time `window.prompt` into +the gear's panel — Settings → Access: a field + Save/Clear kept in localStorage, sent +as a bearer header only, never in a URL; a 401 with no stored key pulses the gear; the +embed widget (no settings panel) keeps its prompt. Suite with the new assertions: +**117 passed, 1 deselected, 0 skipped** (121.57 s, then 125.84 s). + +**App (digup_nextcloud 0.2.2):** the progress line rides right of the spinner circle +(one flex group; bar-wrap can't split them); during a live run the denominator is the +engine's own `index_files` — the job record's total arrives at sync end (`files: null` +mid-walk), and the app invents no numbers. + +## 2026-11-07 — WebDAV shape: the dav mount container, zero host sudo, gates green + +**Shape (stage-12 §11.7, the user's no-sudo directive):** compose `--profile dav` runs a +`dav` service (`deploy/dav/`, davfs2 + tini, privileged, `/dev/fuse`) that mounts each +user's dav view read-only at `/mnt/dav/`; the bind rides the server's media bind +`:ro,rslave`, so mounts propagate live. `docker compose up -d --build --profile dav` **is** +the setup — no fstab, no host davfs2, no host secrets file, no sudo. The script +`deploy/nextcloud-webdav.sh` stays as the documented host-mount alternative. + +**Pilot (ai-ubuntu-computer, x86_64, kernel 7.0; corpus2 public fixtures, users +alice/bob/carol):** stack = nextcloud :8090 + server :8791 + dav, project +`digup-server-nextcloud`, roots `DIGUP_SERVER_ROOTS=/media/dav/alice:/media/dav/bob:/media/dav/carol`, +index `data/ncdav-index`. + +Measured facts (numbers only): + +- **davfs2 image fixes:** (1) the mounting env needs a passwd entry for the tree owner + (uid 990) — "can't read user data base" without it (image carries `useradd -r -u 990`); + (2) reads die with "reading directory: Invalid argument" on this kernel with davfs2 + 1.7.x defaults — `buf_size 64` in `davfs2.conf` fixes it (Launchpad #2129131); + (3) `printf '%s\n' a b` splits config lines — literal-`\n` printf required. +- **Engine walk does not cross mount points:** `estimate /media/dav` = 0 files in + 590 ms; `estimate /media/dav/alice` = 62 files in 2 043 ms. Roots are therefore the + per-user colon list (§11.5's shape). +- **Mount state after bring-up:** 3 mounts; server sees 65/56/50 files; estimate 153 + files; full index job → `done`; ACL battery over dav roots: alice 6/3/3, bob 4/6, + carol 2/3/3 hits (shared dog deduped to the viewer's own mount path; carol's "dog + running" answers document-only hits — no shared photos), preview 200 + open 200 for + the recipient / 404 unshared, token scopes 401/200/403 + admin estimate 200, z-order + descending, token never in HTML. +- **Mount-flap, no server restart:** dav stop → unmount trap line → server sees 0 files + → estimate 0 files (readable all true); dav start → 3 mounts back → 65/56/50 files, + estimate 153 — same running server process, index untouched. +- **Graceful stop (compose 90 s grace):** server exit **143** + `Application shutdown + complete`; dav exit 0 + `OK all dav views unmounted`; restart → health 200 at 10 s, + search "dog running" 5 hits, top z 5.0 (index on the bind survived; no prune needed). +- **No mutation:** fixture checksums 36/36 identical after the whole session; the only + churn was Nextcloud's own appdata preview cache (28 entries, regenerated by Nextcloud). +- **App passwords (NC 28 measured):** `user:resetpassword --password-from-env` reads + `OC_PASS`; `user:auth-tokens:add --password-from-env` reads `NC_PASS`; failed dav + logins throttle the source IP (429) — `security:bruteforce:reset ` clears it; + trusted_domains must include the compose service name the dav container dials + (400 "Bad Request" refusal otherwise). +- **Suite re-run on this disk after the dav work:** 117 passed, 1 deselected, **0 + skipped**, 110.88 s (image `digup-server-test:local` unchanged — server code untouched + by this shape). +- Compose merge shapes validated: plain, `--profile dav`, lan merge (both env shapes), + nvidia+lan, base — all `docker compose config --quiet` green; CI carries the dav lines. + +## 2026-11-07 — Stage 12 field pilot green: Nextcloud 28 + DigUp server, one box, ACL measured + +**Stack (ai-ubuntu-computer, measured numbers):** `nextcloud:28-apache` (loopback :8090) + +`digup-server:local` (loopback :8791), compose project `digup-server-nextcloud`; corpus2 +fixtures only (36 photos under three users, 25 planted junk copies inside +`appdata_`), public content, no personal data anywhere in the run. + +Gates, in the order `docs/implementation/stage-12-integrations.md` lists them: + +- **Junk exclusion:** `POST /estimate` (admin token) → 37 indexable of the mounted tree + (36 images + 1 document; appdata and the skeleton seeds counted 0, `left_out: 2`) — after + the fix that matters: `estimate_argv` now carries the job's `--exclude` flags (an estimate + without them promised 120 files against a 36-file job; pinned by a new suite test). +- **Index:** job `736362e4a1c0` `done` in 40 s → 45 searchable, 45 vectors, 1 failed file + (the engine's own counts, relayed). +- **Search:** five queries, 3 hits / 3 strong each, 12.9–28.6 ms. +- **Token scopes:** tokenless search 401; read-token index/estimate 403; admin estimate 200; + health 200; principal log lines land (`principal=nextcloud`, `principal=admin`) — after + the audit logger got its own stdout handler (uvicorn's config leaves application loggers + handlerless). +- **App ACL (the §11.1 boundary, over the app's own API, session-authenticated):** alice 3 + dog hits (own), bob 3 office hits (own), carol 1 tabby (own); **shared files map**: bob + 1 shared dog hit, alice 1 shared office hit (recipient-view paths); carol 0 for dogs / + office; bob 0 leakage on the office-desk keyword that names alice's tree. Preview of a + mapped hit serves `image/jpeg` to the recipient; the same file id answered **404** for a + user it was not shared with (core's own check, not ours). +- **App surfaces:** page 200 with the top-bar entry in core's app state; admin settings + section + form render (non-admin 403 on both the form and the save route); Test + connection answers 200 with the engine's live counts (46 files, 45 searchable, 45 vectors, + build_tag b11461); the token appears in **no** page or settings HTML. +- **Read-only contract:** 36/36 user files byte-identical to the pre-pilot baseline + (`sha256 -c`); the only changed tree content is Nextcloud's own preview cache under + `appdata_*`; the DB diff over `oc_appconfig` is appconfig rows only + (`enabled`, `installed_version`, `types`, four app keys) — no new tables, no file writes + (source grep: no Filesystem/DB/write API in `lib/`). +- **Restart/stop:** `docker restart` (Nextcloud) and stop/start (server, `stop_grace_period + 90s`) survive: exit **143** + `Application shutdown complete`, health 200 after, search + + preview green post-restart. + +**Suite with the two fixes: 117 passed, 1 deselected, 0 skipped, 111.09 s** (the new +estimate-argv test included; the deselected test remains the `-m slow` timing run). + +Measured Nextcloud 28 facts (the app repo documents them): the bootable class is +`OCA\\AppInfo\Application` (a class elsewhere never boots); section/setting registration +goes through `IManager::registerSection/registerSetting` at boot (the info.xml `` +map is not read); the public section interface is `IIconSection`; page and form responses are +`TemplateResponse` (`AppTemplate` and `OCP\AppFramework\Template` do not exist at 28); +`getSharedWith`'s recipient list crashes on a non-null node (limit `-1`, node `null`); +`IShare::getTarget()` is the source path, not a recipient-view path (the mapping reads the +share's node id out of the asking user's view root); `Node::getRelativePath()` is not in the +Node API; the SSRF guard's per-request flag is inert on 28 for app requests — +`allow_local_servers => true` in config.php is the deployment shape (the app's probe detail +says so); the core preview route is `/core/preview.php?fileId=…`. + +## 2026-11-05 — 1.8M-tree bench run COMPLETE: 131,972-file library indexed, job record `done` + +**LAN opt-in shape added + field-proven (same day):** `deploy/docker-compose.lan.yml` +appends a LAN host binding (env-forced `DIGUP_LAN_BIND`; the loopback mapping stays — +compose merges `ports` lists). Merged with the GPU override on the live stack: healthy, +UI + searches served on the host's LAN address, 10 hits/10 strong in 0.71–0.81 s. Three +new merge shapes (base+lan, lan+nvidia, ghcr+lan, RFC 5737 addresses) pinned in the +workflow's schema-check step. The field run also **measured the ghost-row trap live**: +a recreation launched with a subset of the compose chain and default env moved the media +bind one folder off the roots the index rows carry — every row fell out of scope, the +API answered 0 hits (keyword queries included, 0 hits/0 strong, `meaning:true`) while +health/files/vectors/job-log all stayed normal. Diagnosis: `ls /media/photos` showed the +tree's folders one level deeper than the row roots in the index (`files/` nesting vs +131,972 rows rooted at the mount top). Fix by semantics, no reindex, no prune: relaunch +with the **same file chain and same env as the original boot** — mount back at the rows' +level → 10/10 strong restored, all three job records intact. Lesson pinned in +`server/README.md`: recreations must always pass the full chain + env, never defaults. + +**Run #13 (`7c2329b`, the LAN shape): `server` SUCCESS 12 min 58 s (15:48:50→16:01:48, +now schema-checking all seven merge shapes) + `publish` SUCCESS 1 h 21 m 44 s +(16:01:51→17:23:35) — both tags republished carrying the LAN override file. The +operator-facing compose family is fully green on CI at the LAN tip: CPU, GPU, pull-CPU, +pull-GPU, LAN, LAN+GPU, LAN+pull. + +Measured on this box (GTX 1070 pinned, compose stack `deploy/`, engine untouched; full +sampler table + arithmetic in `server/data/bench-18m-stats.md`, git-ignored): + +* **Job 3 `ab6686ba178a` ended `done`** (engine's own fields): 20,225.0 s = 5 h 37 m 05 s, + files_done 7,097, files_failed 1, errors 0; final index: **files 131,972 · searchable + 131,758 · pending 0 · unfinished 0 · vectors 403,831 · 1.39 GB on disk**. +* Whole library: three job records — 19,007 + 105,439 + 7,097 = **131,543 files completed**, + **421 failed** (video decode refusals the engine recorded), **0 server-side errors**; + engine working time 3,611 + 36,104 + 20,225 = **59,940 s = 16 h 39 min**, average + **2.195 files/s**; wall clock 16 h 50 m 27 s with ≈8 min idle gaps. Stop/resume proven + **3×** (ceiling → `interrupted` kept → one `POST /index` → walk continues). +* Cost-model check: estimate forecast 31,613.6 + 145,961.7 = 177,575.3 s; measured 59,940 s + = 33.75% of forecast (2.96× the forecast rate, GPU-assisted). +* GPU (sampler, during jobs): peak 99% util (job 2) / 90% (job 3), model resident + 1,780–1,802 MiB on the 1070; container peak 207.75% CPU, 3.436 GiB RAM. +* Search latency at the finished index (limit 10): **0.6875–0.8004 s**, five queries, + all **10 hits / 10 strong / meaning true**. +* Disk watch at end: `/` 7.0 G free (99% used) — the index fit with room; no pruning ran. + +## 2026-11-05 (night) — CI `server` job green end-to-end; `:feat-server` live in ghcr, pull-and-run proven + +**Run #9 (`6d97cd5`): `server` SUCCESS + `publish` SUCCESS — the whole pipeline is green** +(CPU + CUDA images pushed off the warm caches, hygiene step green with the fixed heredoc). +CI/publish objective closed. Registry: `feat-server` (568 MB) + `feat-server-cuda` (3.67 GB). +**Run #10 (`19b21d1`, docs-only tip): `server` SUCCESS + `publish` SUCCESS again** — +CPU + CUDA pushed off the warm caches, hygiene green. The branch tip is green +end-to-end; the ghcr tags now embed exactly the pushed engine. + +Measured on the fork's Server workflow (branch `feat/server`), runs #2–#6 (all numbers from +the run timelines): + +* **Run #6 (commit `ad4b682`): `server` job SUCCESS, 12 min 52 s, all 17 steps green** — + drift guard, Vendor-in-base-image seed 4 m 47 s (cache warm), model pins, engine + + `swift test` in the builder 2 m 05 s, CLI headless 34 s, corpus 15 s, clip gate, test + image 12 s, **contract suite 3 min 03 s with zero skips**, **headless proof 1 min 27 s**, + all four compose merge shapes, boot + `/api/v1/health` + UI + graceful stop (exit 143), + logs as artifacts. +* Four environment failures were diagnosed on the runner and fixed in server-side scope, + each reproduced on this box first: (1) Vendor-seed ABI trap → build Vendor in the + builder's own base image (`6b9d9a4`); (2) Wikimedia blocks Azure ranges → the two pinned + corpus videos joined the pinned photos in git (`808826f`); (3) hosted runner images ship + **no ffmpeg** (Tools list has MediaInfo, apt list jumps `ftp`→`g++`) → corpus step + installs apt's ffmpeg, and `build.sh` now capability-probes libx264 + mp4, probing out + the engine's decode-only Vendor ffmpeg (`b6235a7`, `7b65454`); (4) the engine/server + images run as uid 990 and `$RUNNER_TEMP` dirs deny them writes → CI scratch dirs get + 777 and the proof got its own hermetic index (`ad4b682`). +* **`publish` (run #6): CPU image built and pushed — `ghcr.io/ocean572/digup-server:feat-server` + is live** (public repo, anonymous pulls verified: token flow + tags list + manifest 200). + **Pull-and-run proved on this box:** `docker pull` → compose merge of + `docker-compose.yml + docker-compose.ghcr.yml` with `DIGUP_IMAGE_TAG=feat-server` on port + 8788 → Up **(healthy)**, `/api/v1/health` `ok:true` (engine b11461, encoder ready, index + created), UI served at `/` (7,197 bytes with DigUp/app.js), `docker stop --time 15` → + **exit 143 + `Application shutdown complete`**, compose down clean. +* **CUDA image (`-cuda`) fix (run #6 died at 2 m 44 s):** `build-llama.sh` passes + `CUDA_ARCHS` verbatim to `-DCMAKE_CUDA_ARCHITECTURES`, CMake lists are **semicolon**- + separated, and the publish step's `61,70,75,80,86,89,90` is ONE invalid arch token — + configure died ~35 s × 3 retries + 60 s sleeps = 164 s, matching the step to the second. + Fix lives at the server's boundary (scripts untouched): the Dockerfile's engine-builder + RUN normalizes commas → semicolons (`tr ',' ';'`), plain/`native`/semicolon values pass + through. Verified by configure-only probe on `nvidia/cuda:12.4.1-devel-ubuntu22.04` with + the seeded Vendor: raw commas fail at `ggml-cuda/CMakeLists.txt:72 (enable_language)`, + the semicolon shape prints `CONFIGURE-OK` with compiler-ID cubins for 61/70/75/80/86/89/90. + Shipped in `7400c82`; run #7 step 10 built the CUDA image green in **1 h 28 m 57 s** + (cold), proving the fix live. + +* **CUDA image published and proven in the nvidia shape.** Run #7 step 10 (the CUDA + build-push) went green in **1 h 28 m 57 s** — the arch normalization shipped + `ghcr.io/ocean572/digup-server:feat-server-cuda` (anonymous pull verified; images + on disk here: 568 MB CPU / 3.67 GB CUDA). Boot proof of the merged GPU shape + (`docker-compose.yml + docker-compose.ghcr.yml + docker-compose.ghcr.nvidia.yml + --profile nvidia`, `DIGUP_NVIDIA_GPU` = the box's GTX 1070 UUID, port 8789): + **Up 11 s (healthy)**, `/api/v1/health` 200 first try, encoder `ready`, engine + b11461, model resident on the card (**1780 MiB, 37% util** during prewarm), + `docker stop --time 15` → **exit 143** + `Application shutdown complete`, down clean. +* **Hygiene step fixed (run #7, step 11, exit 1 in 1 s):** the python heredoc body sat + deeper than its delimiters → IndentationError; and version deletion is + `POST …/versions/delete` since GitHub's 2022 verb migration (old `DELETE` → 404). + Body moved to the delimiter column, verb fixed, cleanup made best-effort: a loud + `::warning` + exit 0 — publish has already succeeded by then, so cleanup never reds + a run and never fails silently. Shape verified by extracting the run block from the + YAML and running it (reaches the API: 401 with a dummy token, warning printed, 0). + +## 2026-11-05 — ghcr publish + pull-and-run compose; first fully green suite on x86_64 +``` + +**Suite (this box, suite container, fresh index): 102 passed, 0 failed, 0 skipped, 1 +deselected — 128.55 s.** Two pins moved, both measured here (the engine untouched): + +* **Wakeboard top-3 is pinned as a SET, not an order** (`test_api.py`, + `test_search_contract.py`). Measured x86_64 (engine CLI, corpus index): logan 1.571 > + vienna 1.531 > aarhus-04 0.441; the arm64-Mac measurement ranks vienna first. The top + two are 0.040 z apart — inside the embedding matmul's cross-platform noise. The + portable contract (which three files answer the query, `strong >= 3`, `meaning` true) + stays pinned; the per-platform order is the engine's, and the verbatim pass-through is + pinned by the hit-field/reply-shape tests. +* **Stop-timing window 6.0 s → 1.0 s** (`test_jobs.py`'s resume test). Measured: on a + warm box the scratch embed (8 files) finished **inside** 6 s — the record carried + `files_done` 8/8 at the shutdown instant, so `partial < expected` had become the same + class of timing assumption the test itself already documents for `partial > 0`. 1.0 s + lands inside the model load on every platform (the ~800 MB weight load is the floor + before the first `file` event); `partial = 0` is tolerated, the resume arithmetic + `partial + files_done >= expected` is untouched. + +**Publish → ghcr (Server workflow, new `publish` job):** runs after a green `server` +job on push to `main`/`feat/**` + dispatch. `ghcr.io//digup-server`: +`:latest` + `:latest-cuda` from main, `:` + `:-cuda` from a feature +branch — PR #184's build is `:feat-server`. CUDA shape: build-args +`BASE_IMAGE/CUDA_RUNTIME_IMAGE = nvidia/cuda:12.4.1` + fat `CUDA_ARCHS=61,70,75,80,86,89,90` +(61 compiles deprecated on 12.4.1; CUDA 13 drops it — re-evaluate the base when the line +moves). Pinned actions (login v4.7.0, buildx v4.4.1, build-push v7.4.0, all by commit +SHA), separate gha cache scopes for CPU/CUDA, and a hygiene step that deletes untagged +versions (keep rule: anything carrying a tag). Never publishes on `pull_request`: that +context's token belongs to the BASE repo's registry. x86_64 only, §11's rule. +Server.yml's push `paths:` now includes `Sources/**`, `Package.swift`, `scripts/**` — +an engine change that passes ci.yml republishes the images, ghcr never serves a stale +engine. + +**Pull-and-run (deploy/):** `docker-compose.ghcr.yml` (`build: !reset null` + +`image:`) and `docker-compose.ghcr.nvidia.yml` (profile `nvidia`, `:-cuda`, the +same device reservation + `DIGUP_NVIDIA_GPU` pin as the build override). The +`DIGUP_IMAGE_TAG` knob (default `latest`) selects the build; all DIGUP_* env, knobs and +volumes are the base file's, unchanged. Measured on this box: all four merge shapes pass +`docker compose config --quiet`, and `DIGUP_IMAGE_TAG=feat-server` resolves to +`ghcr.io/ocean572/digup-server:feat-server-cuda` in the GPU merge. CI validates the same +four shapes in the schema step. + +**Registry facts (measured 2026-11-05 via the GitHub API):** `Ocean572/DigUp` is a fork +of `ARahim3/DigUp`; PR #184 is open, head `Ocean572:feat/server` ← base `ARahim3:main`, +mergeable (28 commits), **0 check-runs on head and merge commit, 0 workflow runs in the +fork** — Actions is not enabled on the fork, so no CI (including `publish`) can run +until it is switched on in the fork's Settings → Actions. The package owner will be the +fork's owner (`ghcr.io/ocean572/digup-server`); the repo is public, so images pull +anonymously once the first green publish lands them. + +## 2026-11-04 — Stage 9 measured: **GPU measured on this box** (Gate B all green on x86_64 + a GTX 1070) + +First x86_64 measurement round ever for this PR (merge commit `fabb2ba4` has 0 CI check-runs; +every suite number above is an arm64-Mac measurement). The box: `ai-ubuntu-computer` — x86_64, +kernel 7.0.0-38-generic, Ryzen 9 7950X (nproc 20), 64 GiB RAM, driver 580.178.04, **GTX 1070 +(Pascal sm_61, pinned for all server work by UUID)**; the box's RTX 4070 serves other work and +was never touched. Full run: `server/tests/gpu-handoff.sh` with +`CUDA_BASE=nvidia/cuda:11.8.0-devel-ubuntu22.04 CUDA_RUNTIME_IMAGE=…-runtime-ubuntu22.04 +CUDA_ARCHS=61 GPU_SPEC=device=GPU-a7336580-…` → `server/data/gpu-handoff-20261104-1254.log`, +wall 4 m 15.7 s, header **"GPU: measured on this box"**. Gate B's six gates: + +* **PASS** NVIDIA Container Toolkit (`docker run --gpus all … nvidia-smi`); +* **PASS** CUDA engine-payload build on the devel base — `scripts/build-llama.sh` finds `nvcc` + itself, `libggml-cuda.so` lands in the payload (`/out`); +* **PASS** `ldd libggml-cuda.so` resolves `libcudart.so.11.0`, `libcublas.so.11`, + `libcublasLt.so.11`, `libnccl.so.2`; `libcuda.so.1` unresolved in the builder **by design** — + it comes from the host driver through the toolkit at run time; +* **PASS** the engine measured the device: `speed --model --json` → `backends: CUDA, CPU`, + `device: gpu`, cosine agreement **0.99972** (≥ 0.999 required); picture **0.143 s** on the + card vs **~0.589 s** on CPU, whole run 0.39 s vs ~1.7 s; +* **PASS** the utilization gate: the card **worked during a real index job** — peak **96 %**, + busy **37 %** of 91 samples, model resident in card VRAM (392 MB) for 91/91 samples; +* Gate A's compose gates repeated green on x86_64: healthcheck, HTTP search, `docker stop` + exit **143** + `Application shutdown complete`. + +### What it took to get there (all configuration/docs/suite — README §10: the engine untouched) + +Measured root causes, each with its gate line: + +1. **`CUDA_ARCHS` must be a plain number.** The builder's cmake is Ubuntu 22.04's apt 3.22.1: + inside `nvidia/cuda:11.8.0-devel-ubuntu22.04` it rejects `6.1` **and** `native` alike + (`native` needs cmake ≥ 3.24 *and* a GPU visible at build time, which the builder never + has); `61` configures clean. `gpu-handoff.sh` never passed the arg at all — Gate B could + not have passed on this base. Now: `--build-arg CUDA_ARCHS="${CUDA_ARCHS:-native}"`. +2. **The CUDA-libs copy named files that do not exist.** A CUDA *runtime* base ships SONAME + chains (`libcublasLt.so.11 → .so.11.11.3.6`, `libcudart.so.11.0 → .so.11.8.89`, + `libnvrtc.so.11.2 → …11.8.89`); the dotted names belong to `-dev` packages — a COPY of + them dies "not found" (measured). The `cuda-libs` stage now assembles `/out` with + `cp -a` globs over the five lib families **plus `libnccl.so.2`** (llama b11461's + `libggml-cuda` links nccl; ldd asks for it): 955 M shipped, the Dockerfile's own ldd gate + passes. +3. **Flavor-switch cache hygiene.** A CUDA build leaves `Vendor/llama-build/CMakeCache.txt` + with `GGML_CUDA=ON`; the next CPU build reuses it and configure dies "CUDA Toolkit not + found". The Dockerfile deletes just that file before `build-llama.sh` (the script's stamp + embeds the flavor and wipes `lib/`; objects are kept). CPU↔CUDA alternation costs a + configure, never a full rebuild. +4. **The nvidia Compose override built nothing CUDA.** It now declares `target: + server-runtime-cuda` + the three build-args, so `--profile nvidia up -d --build` produces + the CUDA image itself; Gate B's compose call runs with `--build` (a CPU image with the + device visible is exactly the false positive the utilization gate exists to catch). All + three compose shapes validate (`config --quiet`, base / nvidia+profile / nvidia-without). +5. **The handoff script checked `/opt/digup` on the payload image** — the `engine-payload` + stage lands in `/out` (the runtime stages COPY it to `/opt/digup`). Checks now use `/out`. +6. **The utilization gate must be job-aware.** `/health` answers 503 during the searcher's + ~20–30 s prewarm (the container is healthy, the API is honest) — the gate now waits for a + 200, then POSTs `/index` with retry. And sampling cadence is measured: small-corpus GPU + embedding is ~150 ms kernel bursts between P8 idle — a 2 s sampler sees 0–12 % on a + provably-GPU job (392 MB model in VRAM, engine costs at the GPU's 0.15 s/image rate). The + gate samples at 0.15 s across the job and passes on peak > 20 % **or** busy-duty ≥ 10 % + with the model on the card; a CPU-embedded job is flat — zero busy samples, ~0 VRAM. + +**Estimate's ceiling is now the operator's.** Measured on the bench tree: a full crawl of +187,055 files on the 2 TB HDD crossed the endpoint's hardcoded 120 s and answered +502 "digup estimate timed out". `POST /estimate` now reads `DIGUP_SERVER_ESTIMATE_TIMEOUT_S` +(default 120 — every existing pin keeps; the suite adds `test_the_estimate_timeout_is_an_operator_knob`). +**The same for a job's ceiling: `DIGUP_SERVER_JOB_TIMEOUT_S`.** Measured on the bench: at the +default 3600 s a 131,972-file job recorded **interrupted with its partial counters** +(`files_done 19,007, files_failed 0, errors 0`) when the runner's deadline expired — README §5's +stop semantics proven on a real tree, and the next job resumed (measured: it resumed and kept +embedding on the 1070). Default 3600 kept; pinned in `tests/test_jobs.py`. + +**Gate A's suite on x86_64: 2 failed, 99 passed, 1 deselected, 0 SKIPPED in 154.81 s** (101 +tests with the knob pin; the two failures below). The +two failures are the wakeboard top-3 pins (`tests/test_api.py:58`, `tests/test_search_contract.py:50`): +this box ranks `wakeboard-logan.jpg` (z 1.571) above `wakeboarding-vienna.webm` (z 1.531), a +0.040 margin — the pins are arm64-Mac measurements and the 2.0σ z-gate over 15 files is the +engine's, identical on every platform (README §"fewer strong hits on tiny corpora"). The +jobs stop-timing pin passed under load (it is load-sensitive by construction). The pin +decisions (clip commit, corpus2 re-bake, ranking baseline, timing assumption) sit with the +maintainer; **no test was edited to make the suite green**. + +## 2026-11-04 — The UI's state-dependent layout, and the counters an interrupted job always carries + +Second round on `feat/server` (PR #184) after the API/security one: the web app's layout, plus a +job-record finding that only the suite caught. **Suite: 100 passed, 1 deselected, 0 SKIPPED in +385.33 s** (arm64, the shipped engine payload, the dev-loop mounts; 4 new tests) — `tests/test_jobs.py` +alone: 16 passed in 160.19 s; `tests/test_media_ui.py`: 19 passed in 6.99 s. **GPU NOT MEASURED:** +this machine has no CUDA/Vulkan device ([docs/implementation/gpu-handoff.md](../docs/implementation/gpu-handoff.md)). + +### The layout: `body[data-page]`, one switch, four states + +`render()` in `server/ui/app.js` sets `body[data-page]` to `idle | busy | results | none`, derived +by `pageMode()` from the state the renderers already use (`hits` present → `results`; nothing typed +or the `code:` hint → `idle`; words in, meaning on its way → `busy`; meaning answered zero → `none`). +`server/ui/style.css` lays out from it: the results grid, the hit counter (`#engine-says`) and the +preview pane exist in `results` **only** — `body:not([data-page="results"]) #results, … #engine-says +{ display: none }`, specificity outranking the grid rule, so there is no `hidden` juggling anywhere. +`#idle` (welcome + `#idle-note` + `#suggest` chips) and `#no-results` (`#no-results-why` + a filter +hint) are centered panels, shown by the same attribute. Measured with Playwright (geometry, not +eyes): `idle` at 1280/1024/1440/390 px → the panel's center equals the viewport center (640/512/720/195) +with `#results`, `#preview` and the counter all `display: none`; `results` → grid +**438.391 px / 657.609 px = 40 % / 60 %** at 1280 and 1440 (the 72 rem content column caps it), +**387.188 / 580.812 px** at 1024, gutter 16 px, grid axis = viewport axis, the first pick +auto-selected, the pane carrying `picture` + `facts`; `busy` → centered "Matching words now…" with +the counter and the suggestions hidden; `none` → centered message quoting the engine's own sentence +for the empty answer, plus its `warning` when the engine sent one. +Below 900 px it is one column (measured 350 px at 390 px) with the pane **below** the list and +`position: static`; `documentElement.scrollWidth == innerWidth` at 390/1024/1280/1440 — no +horizontal scroll. The state fade is inside `@media (prefers-reduced-motion: no-preference)`. +Keyboard flow unchanged and verified in the browser: ↑↓ move the selection (measured 1→2), ↩ opens, +Space opens the big look, Esc closes it, Esc-on-text returns to `idle`. Idle suggestions fill `#q` +and call `queryChanged()` — the one search route, no second path (`test_media_ui.py` pins it). + +*Three bugs found by measuring, none of them visible by reading the code:* (1) `render()` did not +run at startup, so `body[data-page]` was unset and **every** panel was gated off — a blank page — +and `state.groups` did not exist for the first `render()` to iterate; (2) `replaceChildren(array)` +stringifies its argument, so the chips reached the page as the text `[object HTMLButtonElement]` +(it is variadic: spread the array); (3) the mobile `#preview { position: static }` lost to the later +`#preview { position: sticky }` at equal specificity — the override is now `body #preview`. +A preview the engine has not rendered is now a **message plus the file's metadata**: every preview +`img`/`video`/`audio` has an `onerror` that replaces the picture with `noPreview(hit)` while +`facts(hit)` (name, folder, kind, size, date, z, cosine, keyword rank, segment, Open/Download) stays +in the pane. Verified with a dead handle: `.message` + `.facts`, no blank panel. + +### Jobs: counters exist from the first moment of the walk + +The suite caught `test_a_stop_mid_job_is_interrupted_and_the_next_job_resumes` failing +(`assert partial is not None`), in two rounds: + +1. **The counts were the worker thread's private knowledge.** `run_sync()` collected events in a + local list, and `shutdown()` marks the job from **its own thread** — so *interrupted* was + published with no counters at all. `IndexService.events()` is now drained in `_run_one`, and + `_progress()` merges the engine's counts into the live record **as they arrive** (in-memory on + every `file`/`error` event; into the job log on the engine's `idle` batch totals — a handful per + job, so a record recovered from the log after a hard kill carries the last totals it saw). + `_mark` merges into that record, never replaces it, so every ending (`EngineExited`, timeout, + `shutdown()`, an escaped runner exception) publishes the counts the engine really emitted. +2. **A job stopped during the model load has completed zero files** — the truth is `files_done: 0`, + never an absent counter, so the counters are published when the walk starts. The test's + `assert 0 < partial < expected` was a timing assumption (this suite's stop lands 6 s into a walk + whose first minute is the model load); it is now `0 <= partial < expected`, with the engine's own + arithmetic kept: `partial + files_done ≥ expected` (measured 8 + 7 = 15 in Stage 8). + +And one contract truth the suite surfaced under load: a writer stopped mid-model-load keeps its +flock while the engine closes its stream, and a request in that window is answered **`busy`** — the +honest 409, with `_active` cleared so the next request runs. The resume test asked for the walk +exactly once and asserted that single answer was `done`; it now asks the resume until it is +accepted, which is README §5's contract ("one writer, answered busy, the next job resumes"). +`tests/test_jobs.py` (+2 unit tests: the counter merge, and log-recovery totals after a hard kill). + +### Notes + +* `server/README.md`: the four-state layout paragraph with the measured px. +* Dev discipline learned the hard way: **never run two container suites at once** — two model loads + saturate the VM, widen the flock window (the `busy` race above) and starve live searches. Run the + heavy suites one at a time, and take UI measurements on a quiet machine. +* Suite count went 96 → 100 (UI pins +2, jobs +2). The run that reported `1 failed, 99 passed, + 1 deselected in 433.99 s` used the pre-fix `jobs.py` (its container imported the module before the + fix landed) and is superseded by the 100-passed run above. + +## 2026-11-04 — The API as the product, the security seams, and the Stage-9 handoff + +The overnight round on `feat/server` (PR #184): API surface, security, and the handoff the GPU +box needs. **Suite: 96 passed, 1 deselected, 0 SKIPPED in 222.11 s** (the handoff script's own +run, twice: 224.38 s / 222.11 s, EXIT=0) — 21 new tests, zero skips, arm64, the shipped engine +payload. **GPU NOT MEASURED: this machine has no CUDA/Vulkan device; see +[docs/implementation/gpu-handoff.md](../docs/implementation/gpu-handoff.md).** + +* **API default `limit` = 40** (`api.py`), the engine's own default (`SearchService.swift:130`). + The UI already sent 40; a direct API client that named no limit was quietly getting a third of + the engine's answer. Pinned twice: the typed default, and the reply against an explicit 40. +* **`server/docs/API.md`** — the API reference as an artifact of its own (schemas, media + identifiers, limits/pagination, the error table, curl examples, versioning, security, + multi-client use, deploy shape), independent of the UI. `tests/test_api_docs.py` fails if an + endpoint, the `limit` default row, or the OpenAPI/docs paths drift out of it. +* **Error shapes pinned** (`tests/test_api.py`): one `detail` string for 404/409/502/503, a + **list of objects** for 422 (the UI renders each item; printing the raw array is the bug). +* **One API process, said out loud** (README + API.md): `media_id` handles live in this process's + LRU and the job runner is a thread inside it, so `--workers >1`/replicas 404 handles and let + two writers race. Searches are concurrent inside the one process; only process count is fixed. +* **Host/Origin validation** (`security.py`, opt-in, off by default): + `DIGUP_SERVER_ALLOWED_HOSTS` names the URL hosts a deployment serves → a foreign `Host` gets + 400, a browser's state-changing request from another origin gets 403, non-browser clients (no + `Origin`) pass, read-only GETs are unchecked, ports are not compared, no wildcard. Installed + inert by default, so a localhost box behaves exactly as before. 11 tests in + `tests/test_security.py` pin the whole table. +* **Bearer tokens with a read/admin split** (`DIGUP_SERVER_TOKEN`, `DIGUP_SERVER_ADMIN_TOKEN`): + read scope covers `/api/v1` except `health`/`docs`/`openapi.json` (the container's + `HEALTHCHECK` calls `/health` with no credential); admin scope covers `POST /index` and + `POST /estimate`. `401` + `WWW-Authenticate: Bearer` with nothing, `401` for the wrong token, + `403` for the read token on an admin endpoint, `hmac.compare_digest` comparison. The UI prompts + once on a 401, keeps the token in `localStorage` (`digupBearer`), retries once, and does not + loop. Off by default; Compose passes the three knobs through unset. +* **Media-handle semantics pinned** (`tests/test_media_handles.py` + one API test): handles are + per-reply (two replies never share one), the cache is bounded and a true LRU, and handles die + with the process — clients key their own records on `path` + `kind` + `segment_id` (+ `loc`). +* **The interrupted-job counts fix** (a real finding from this round's suite run): + `run_sync()` raises `EngineExited` on EOF-without-`exit` and loses the events it collected, so + a stop landing during the model load published an *interrupted* record with **no counts** — the + suite caught it as `assert partial is not None` under load. The runner now drains the event + stream as it arrives and marks *interrupted* with the engine's real partial counts. The + resume test's arithmetic was corrected to the engine's own (measured in Stage 8: 8 + 7 = 15): + the interrupted job embedded `partial`, the resumed job embeds the rest (`unchanged` files are + not `done`), so `partial + files_done ≥ expected` — the old `files_done ≥ partial` was the + lucky version and fails whenever the stop lands past the halfway point. +* **Stage-9 handoff** (`docs/implementation/gpu-handoff.md` + + `server/tests/gpu-handoff.sh`): Gate A (the CPU baseline: pins verified against + `ModelFiles.swift`, the contract suite with a zero-skip gate, Compose boot, the image's own + healthcheck, a search, a non-137 stop with `Application shutdown complete`) and Gate B (the + card: toolkit test, CUDA payload `ldd`, `speed --model --json`'s measured `device`, and + **GPU utilization sampled during a real index job** — "detected the device, embedded on CPU" + is a failing gate). Gate A ran green here on arm64 twice; Gate B runs on the x86_64 NVIDIA box. + The script prints `GPU NOT MEASURED` in those words whenever Gate B did not prove it. + +## 2026-11-03 — Stage 11: the review round — parity proven, suite 75, CI that can't pass by luck + +Adversarial review of the whole PR (backend; UI + media; build/CI/docs), every real finding +fixed **and pinned by a test** — the finding→fix→test table is +[docs/implementation/review-round.md](../docs/implementation/review-round.md), the stage +record with all measurements is +[stage-11-review-round.md](../docs/implementation/stage-11-review-round.md). + +* **Suite: 75 passed, 1 deselected, 0 SKIPPED in 235.15 s (EXIT=0)** on the final committed + tree (`suite24.log`) — reproduced 75/1/0 in 228.12 s (`suite23.log`). Buckets 17 + search-contract + 19 API + 6 index-contract + 13 job + 17 UI/media + 4 CLI = 76 collected, + CI deselects the slow model timing. +* **The harness fact that changed the test image:** plain `docker run` makes pytest **PID 1**, + and ParentWatch ends every engine helper whose parent is PID 1 (`ParentWatch.swift:19`) — + measured `7 failed, 9 passed, 59 errors`, all helpers dead `exit 0, no stderr`. The test + image now carries `tini` as ENTRYPOINT, so the suite is PID-1-safe by construction and CI + needs no `--init`. +* **Parity (the user's "100% like the app" question):** the app's own adapter (`SearchService`) + on a `sqlite3 .backup` snapshot of the live corpus2 index answers the same queries as the + live HTTP API with **identical hits/strong/z** — water 1 (2.09), harbor 2 (3.87/2.35), + "a sunset over water" 3 (3.51/3.37/2.28), wakeboarding 3 (4.15/4.06/3.09), fractal 1 + (3.59) — plus the keyword-first path. Logs: `stage11-parity-direct.log`, + `parity_backup.log`. Probe lesson kept: a read-only mount of a **hot WAL index** (and a + plain `cp` of it) is not a snapshot — it answered 0 hits, a probe artifact, not a server bug. +* **"Fewer responses" quantified:** `zGate = 2.0` (`Searcher.swift:116`) and the engine's + default limit 40 (`SearchService.swift:130`) are identical on Mac/shell/server; corpus2 is + 61 files, so 1–3 ≥2σ hits per query is the corpus's size, not a cap (documented in + `server/README.md` + `server/tests/corpus2/README.md`). +* **Live proof battery (`stage11-proof.sh`): PASSED, 11 gates, 0 fails** — compose config + valid; container healthy via the image's own `healthcheck.py` (503 = alive); + `restart: unless-stopped`; published `{127.0.0.1 8787}`; healthcheck proven both + directions (0 on live, non-zero with nothing listening); 12 health polls in 1 s (the + StatusCache TTL, no subprocess per poll); index job `done · 61/61 · vectors 65 · 0.1 s`; + searches with the engine's z verbatim (3-strong, to z 4.46); `nosniff` on 200 + 206; + `Range: bytes=0-` → `206 bytes 0-124176/124177` (8 MiB cap, streamed); 416; 404; + estimate `≈263.9 s · 61 files · calibrated true`; UI markers pinned present in served bytes. + Log: `stage11-proof.log`. +* **UI verified in a live browser (Playwright, no vision model):** Space on the focused gear + opens Settings (aria `true`, focus into the panel), Esc closes (aria `false`, focus back to + the gear), Space on a focused row opens the big look with media, Space on a focused link is + native activation; 3 hits + preview fields verbatim; video `…/file#t=12.5` ("▶ at 0:12", + z 3.25); job line `done · searchable 61/61 · vectors 65 · 0.1 s`; estimate renders; dark + theme applies + persists; console clean (favicon silenced with a `data:` icon — no + external URL, the pin test stays green). Screenshot: `.playwright-mcp/stage11-dark-results.png`. +* **Backend fixes (pinned by tests):** `shutdown()` self-deadlock; runner thread surviving a + `TimeoutError`; child reaped on a failed handshake; `ok:false` without `error` is a + refusal; render beside the read loop (2 slots, lock-free); `Range` capped + streamed + + `fstat` + `is_relative_to` + `nosniff`; `TimeoutExpired` → quoted 502; job-log rotation is + one rename (timestamp suffix) and replay skips non-records; `HitCache` is a true LRU; + `StatusCache` TTL; `SearchReply` `extra="allow"`. +* **UI fixes (browser-proven):** Space keeps native activation for buttons/links (the + reviewer's blocker); a failed job poll answers honestly and stops (one loop per click); + FastAPI list-`detail` renders readably; one `closeSettings()` syncs aria + focus; estimate + generation token; roving tabindex + aria; chip contrast AA (4.54:1); `.badge.warn` cascade; + audio control selector; mobile footer wrap; `maxlength=512`. +* **CI/images:** 180-min ceiling; vendor built host-side via the repo-pinned `actions/cache` + and seeded through `.ci-vendor/`; drift guard FIRST (revision + names + sha256s); + zero-skip gate + clip-exists gate; boot step polls with the image's own `healthcheck.py`; + `proof.sh` counts failures and gates the eval ALL row (R@1 ≥ 0.60, R@5 ≥ 0.85). Measured + honesty: base Debian 13 trixie, glibc 2.41-12+deb13u4 (newer than the payload's 2.35 — + forward, never back); no `/data/locks`; inert host/port ENVs gone. +* **Commit shape (local, on `feat/server` — origin is upstream, nothing pushed):** + `6ac98a3` backend+tests · `431059c` UI · `5830e8d` images/CI · docs. Additive invariant + held: `git diff --name-status main..HEAD` = 61 A + 1 M (`.gitignore`); `Sources/`, + `shell/`, `website/`, root README, upstream workflows untouched. +* **Open (honest):** CI's first green run is pending (no push to upstream per the git-safety + rule — the workflow is written and YAML-validated, 17 steps); Stage 9 (GPU override) and + Stage 10 (PhotoPrism comparison) remain. + +## 2026-11-02 — UI reorganized around the search (the user's layout brief), in the browser + +* **The page is a search:** centered search box with the hit-limit select at its right + end (arrow evenly spaced off the edge — selects render their own chevron per theme), + theme-colored kind chips in one evenly spaced centered row beneath it (filled in the + kind's color when ticked), results bottom-left, preview right, keys footer. Header: + title + searchable count + activity + a gear. +* **Settings panel (the gear):** Appearance, Index now + prune missing, Estimate, and the + engine's counts (vectors, encoder, kinds, index size, build tag, roots). Estimate stays + honest and short — one note line, the forecast renders above the results, the long + paragraph the user pointed at is gone. The panel closes on Esc, outside click, and on + opening the estimate. `#settings[hidden]{display:none}` — the same `[hidden]` lesson as + `#big`. +* **`el()` fixed:** the helper took an element as its text arg at three call sites and + stringified it — the "[object HTMLElement]" text the user saw (estimate rows, preview + messages). Children are real nodes/text nodes now; verified clean in the browser + (`[object HTMLElement]` = false, zero console errors). +* **"water shows only the harbor photo" is the engine, not the UI:** `Searcher.zGate = 2.0` + (`Sources/DigUpKit/Searcher.swift:116`) filters sub-2σ candidates inside the binary — + the Linux app with the same index answers the same single hit. Measured live: 1 hit, + z 2.09. The UI publishes the engine's answer unchanged (the server's core rule). + Panel parity re-audited against `panel.js`: `limit: 40` (the engine default — the + select is a superset), the WEAKER fold, groups, keyboard flow, footer summary. +* Browser battery re-run: typing → 5 hits (4 strong) → click → preview, settings open, + estimate clean, theme flip, 0 console errors; layout verdict from the vision model: + all checks pass (header tidy, centered search, chips even, panel neat). +* **Geometry pass (the "all off centered" complaint) — measured, not eyeballed:** + bounding-box audit in Playwright at 1280 px and 390 px. After: search input center + offset **0 px**, chips row **0 px**, statusline −5 px, gear's icon **0/0** in its circle + (inline SVG, theme-colored via `currentColor`), settings panel anchored under the gear + (right edges equal at 1196) with **all control left edges at one x (911)**, footer text + box aligned to the content column, `overflowX: 0` at both widths, search row stacks + centered on mobile. One axis, verified by numbers. The single-column system: header + grid (title/gear, centered status row), search row `1fr auto 1fr` (the input's center + IS the page's center; limit + code at its right end), settings as a label/control + grid with section rules. Caught by the same audit: the gear's **svg child** broke the + outside-click close (`target !== gear`) — `closest("#gear")` now; the panel-open + geometry proved it (a collapsed 0×0 rect = the click had opened and re-closed it). + +## 2026-11-02 — Dark theme made readable, in a real browser (Playwright is up) + +* **The complaint ("The UI is way too dark. I cannot see anything for the dark theme") had a + root cause found in the browser, not in the palette:** the big-look overlay `#big` set + `display: flex`, which outranks the UA's `[hidden] { display: none }` — so a `position: fixed` + layer at 78% black covered the whole page permanently, dimming everything under it and + intercepting every click (Playwright: "div#big intercepts pointer events"). Fixed with + `#big[hidden] { display: none }`; click → preview, Space big-look, Esc all verified live. +* **Palettes are now explicit per theme — no system colors, no OS dependence:** light/dark var + blocks per `data-theme` (+ a `prefers-color-scheme` block for System), each setting its own + `color-scheme`, so checkboxes, selects and scrollbars render *in* the theme; + `accent-color` on the checkboxes. The text pairs are measured WCAG ratios, not eyeballed: + on dark, text `#e8eaed` on `#202124` = 13.4:1, muted `#b7bbc1` = 8.4, accent `#8ab4f8` = 7.6, + kind colors ≥ 6.4; on light, text `#1a1a1a` on white = 17.4, muted `#5f6368` = 5.9. Panels + got surfaces (`--panel` + `--edge` borders) so structure stops dissolving — hairlines stay + deliberately quiet (~1.8:1, they are separators, not text). Screenshot verdict from the + vision model after the fix: 9/10 readable, both themes, results + preview + estimate. +* **The UI ships `cache-control: no-cache` (`UIFiles` in `api.py`):** measured in the browser — + without it, Chrome answered reloads from its memory cache for minutes, serving a stylesheet + the server no longer had, so a deploy never reached the page. Etag in, 304 out, revalidation + is cheap. Pinned by a test. +* **`EngineRefused` — the engine's refusal is a 503, in its words:** the Stage-10 battery hit + the searcher mid-transition right after a done job: the engine answers `{"ok":false, + "error":…}` (`SearchService.swift:106,115`) and `api.py` crashed `KeyError: 'hits'` (500). + Now `SearchService._checked` raises `EngineRefused` on any `ok:false` reply, the API answers + 503 with the engine's sentence, the job runner survives a refused post-job refresh (the job + stays *done*, with a "searcher refresh deferred" note), and startup's prewarm is best-effort + the same way. Proven by the API pair of tests — a refusing injected service, a stub refusal + reply, no engine mocks; the proof battery gates on "searchable == files · encoder ready" and + retries 503s, printing each refusal it met. +* Statusline: `index NaN KB` — app.js ran the engine's human string through `bytes()`; now it + shows the engine's own words verbatim (`index 201 KB`). +* **Suite: 56 passed, 1 deselected in 192.51 s (EXIT=0)** — the 53 plus the 503 pair and the + revalidation rule. Live proof: [stage-10-ui-parity.md](../docs/implementation/stage-10-ui-parity.md). + +## 2026-11-02 — Stage 10 done: the web UI reaches desktop parity; corpus2 for live testing + +* **UI parity (`server/ui/`, rewritten against `shell/ui/panel.js`):** keyword-first + typing (words at once with `prefix_last`, `embed_ahead`, the `meaning:true` upgrade + after 120 ms, generation + final-generation guards); strong/weaker fold; kind + checkboxes → `kinds` (none ticked = all = the engine's default); `code:` prefix + + code checkbox as one state; limit select (40 = the engine's default); grouped rows + with lazy engine thumbnails; a preview pane for the selection (photo, video frame + that plays *from* `#t=loc`, PDF page, audio at the moment, matched passage via + `/text`) with facts + a plain-language reason built from the engine's own fields; + arrows/↩/Space/Esc keyboard flow with the big look; Open + Download; Estimate panel + (`POST /estimate`, the `calibrated` truth); activity dot; summary + keys footer. +* **Appearance (the theme ask):** System / Light / Dark select → `data-theme` + CSS + variables, kept in `localStorage`; System tracks `prefers-color-scheme` exactly as + the OS-following desktop does. +* **`GET /media/{id}/file` — the app's "Open" over HTTP:** server-issued handles only, + roots-containment on the resolved path, media type from **magic bytes**, inline for + media kinds, attachment + `text/plain` for code/doc (a source file named `index.html` + never arrives as markup), `Accept-Ranges: bytes` with 206/416 — what makes + `` does that. The corpus is the input the headless proof, +# the adapter's tests and the web app's acceptance run share, so the assertions in corpus.json +# name these files exactly. See README.md for provenance and licenses. +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +MEDIA="${1:-$HERE/media}" +UA="DigUpServer-corpus/1.0 (dev corpus setup; contact: https://github.com/ARahim3/DigUp)" +mkdir -p "$MEDIA/photos" "$MEDIA/videos" "$MEDIA/clips" + +# ffmpeg: only the synthetic clip runs it; the photos and videos need nothing. The clip needs +# an ENCODER (libx264) and the mp4 muxer, so a candidate is accepted only when it lists both. +# The engine's own Vendor ffmpeg (scripts/build-ffmpeg.sh) is a decode-only build — its encoder +# list is pcm_s16le,mjpeg,ppm,rawvideo, no mp4 muxer — and is probed out by this check. +# Measured on CI run #3 (2026-11-05): the unprobed Vendor pick (the step-4 cache restores +# Vendor/ into the workspace) died on the clip with ffmpeg's exit 8, "Unknown encoder 'libx264'" +# — a structural mismatch, not a network or ABI one: the engine never encodes x264. An explicit +# FFMPEG=… is honoured as given. +ffmpeg_makes_the_clip() { # true when "$1" encodes libx264 and muxes mp4 (the clip's demands) + local out + out="$("$1" -hide_banner -encoders 2>/dev/null)" && [[ "$out" == *" libx264 "* ]] || return 1 + out="$("$1" -hide_banner -muxers 2>/dev/null)" && [[ "$out" == *" mp4 "* ]] +} +FFMPEG="${FFMPEG:-}" +if [[ -z "$FFMPEG" ]]; then + for candidate in "$HERE/../../../Vendor/ffmpeg/bin/ffmpeg" "$(command -v ffmpeg 2>/dev/null || true)"; do + [[ -n "$candidate" && -x "$candidate" ]] || continue + if ffmpeg_makes_the_clip "$candidate"; then FFMPEG="$candidate"; break; fi + done +fi + +digest() { # sha256 of a file, 64 hex digits (coreutils' tool, with the BSD one as a fallback) + sha256sum "$1" 2>/dev/null | cut -c1-64 || shasum -a 256 "$1" | cut -c1-64 +} + +get() { # get : fetch once, verify against the pin, keep + local out="$1" url="$2" sha="$3" + if [[ -f "$out" && "$(digest "$out")" == "$sha" ]]; then + echo " ok ${out##*/}"; return 0 + fi + echo " get ${out##*/}" + curl --fail --silent --show-error --location --proto '=https' -A "$UA" --retry 3 --retry-delay 2 \ + -o "$out" "$url" + local got; got="$(digest "$out")" + if [[ "$got" != "$sha" ]]; then echo " sha256 mismatch for ${out##*/}: got $got, pinned $sha" >&2; exit 1; fi + echo " ok ${out##*/}" +} + +while IFS=$'\t' read -r name title role license artist page url bytes sha; do + case "$role" in + *-video) get "$MEDIA/videos/${name%.*}.webm" "$url" "$sha" ;; + *) get "$MEDIA/photos/$name" "$url" "$sha" ;; + esac +done < "$HERE/manifest.tsv" + +# Three 6-second scenes in one 18-second MP4: test bars, a fractal, testsrc2. The boundaries are +# known exactly, which is what the timestamp assertions in corpus.json need. +if [[ -x "$FFMPEG" ]]; then + "$FFMPEG" -y -hide_banner -loglevel error -f lavfi -i "smptebars=size=640x360:duration=6:rate=25" \ + -f lavfi -i "mandelbrot=size=640x360:rate=25" -f lavfi -i "testsrc2=size=640x360:rate=25" \ + -filter_complex "[0][1][2]concat=n=3:v=1:a=0" -t 18 -c:v libx264 -pix_fmt yuv420p \ + -preset veryfast -crf 30 "$MEDIA/clips/scenes.mp4" + echo " made scenes.mp4" +else + echo " no ffmpeg with libx264 + mp4: scenes.mp4 not built (install ffmpeg, or set FFMPEG=…)" >&2 +fi +echo "corpus in $MEDIA" diff --git a/server/tests/corpus/corpus.json b/server/tests/corpus/corpus.json new file mode 100644 index 00000000..5011fdaf --- /dev/null +++ b/server/tests/corpus/corpus.json @@ -0,0 +1,22 @@ +{ + "base": "~/DigUpTestbed", + "queries": [ + { "q": "wakeboarding", "expect": ["photos/wakeboarding-aarhus-04.jpg", "photos/wakeboard-logan.jpg"], "tag": "image" }, + { "q": "a person riding a wakeboard behind a boat", "expect": ["photos/wakeboarding-aarhus-04.jpg", "photos/wakeboard-logan.jpg"], "tag": "image" }, + { "q": "wakeboarding", "expect": ["videos/wakeboarding-vienna.webm"], "tag": "video", "kind": "video" }, + { "q": "hiking", "expect": ["photos/hiking trail at Námafjall.jpg"], "tag": "image" }, + { "q": "hiking in the mountains", "expect": ["photos/hiking trail at Námafjall.jpg"], "tag": "image" }, + { "q": "hiking", "expect": ["videos/pct-hiking-southern-oregon.webm"], "tag": "video", "kind": "video" }, + { "q": "people sitting around a campfire", "expect": ["photos/people-surrounding-campfire.jpg"], "tag": "image" }, + { "q": "a dog running on the beach", "expect": ["photos/a-dog-running-on-the-beach.jpg"], "tag": "image" }, + { "q": "a boat on the water at sunset", "expect": ["photos/boats-at-sunset-pakse-laos.jpg"], "tag": "image" }, + { "q": "someone fishing from a dock", "expect": ["photos/fishing-from-dock-oregon.jpg", "photos/man-on-dock-with-fish-1900.jpg"], "tag": "image" }, + { "q": "Christmas dinner", "expect": ["photos/christmas-eve-dinner-table.jpg"], "tag": "image" }, + { "q": "a cat sitting on a sofa", "expect": ["photos/cat-sitting-on-a-sofa.jpg"], "tag": "image" }, + { "q": "an office desk with a computer monitor", "expect": ["photos/office-desk-with-monitor.jpg"], "tag": "image" }, + { "q": "cars parked in a parking lot", "expect": ["photos/cars-in-a-parking-lot.jpg"], "tag": "image" }, + { "q": "television test bars", "expect": ["clips/scenes.mp4@0-6"], "tag": "video", "kind": "video" }, + { "q": "a fractal", "expect": ["clips/scenes.mp4@6-12"], "tag": "video", "kind": "video" }, + { "q": "a test pattern with a moving ramp and numbers", "expect": ["clips/scenes.mp4@12-18"], "tag": "video", "kind": "video" } + ] +} diff --git a/server/tests/corpus/manifest.tsv b/server/tests/corpus/manifest.tsv new file mode 100644 index 00000000..7463297c --- /dev/null +++ b/server/tests/corpus/manifest.tsv @@ -0,0 +1,14 @@ +a-dog-running-on-the-beach.jpg File:A dog running on the beach.jpg dog-beach CC BY-SA 4.0 Bruce Emmerling https://commons.wikimedia.org/w/index.php?curid=74405587 https://commons.wikimedia.org/wiki/Special:FilePath/A_dog_running_on_the_beach.jpg?width=1024 169153 cbb6c74cf3337ee9c088714670023614fe0b83d285fcd644e5e0d91b78eaa4eb +christmas-eve-dinner-table.jpg File:Christmas Eve dinner table with Christmas food 03.jpg christmas-dinner CC BY-SA 4.0 Chris Olszewski https://commons.wikimedia.org/w/index.php?curid=76375335 https://commons.wikimedia.org/wiki/Special:FilePath/Christmas_Eve_dinner_table_with_Christmas_food_03.jpg?width=1024 577192 8d9ff574374f389a804e4afdc5228cbfb52f45f5f427cd3c240e5c506c152d93 +boats-at-sunset-pakse-laos.jpg File:Clouds over the Xe Don River with boats at sunset in Pakse Laos.jpg boat-sunset CC BY-SA 4.0 Basile Morin https://commons.wikimedia.org/w/index.php?curid=139113170 https://commons.wikimedia.org/wiki/Special:FilePath/Clouds_over_the_Xe_Don_River_with_boats_at_sunset_in_Pakse_Laos.jpg?width=1024 202126 2ba12cda5759af582052a72a135c0d20ad974ae59070d53478728057fd23fd7e +cat-sitting-on-a-sofa.jpg File:Female cat sitting down on sofa.jpg control CC BY-SA 4.0 Smirkybec https://commons.wikimedia.org/w/index.php?curid=95606043 https://commons.wikimedia.org/wiki/Special:FilePath/Female_cat_sitting_down_on_sofa.jpg?width=1024 555767 42aadb196f8bc2bc49ca3ff4f8b4d24f7aff1161cd9f32dc0afd7708682a1c43 +fishing-from-dock-oregon.jpg File:Fishing from Dock (Columbia County, Oregon scenic images) (colDA0033).jpg fishing-dock Attribution Gary Halvorson, Oregon State Archives https://commons.wikimedia.org/w/index.php?curid=19097738 https://commons.wikimedia.org/wiki/Special:FilePath/Fishing_from_Dock_%28Columbia_County%2C_Oregon_scenic_images%29_%28colDA0033%29.jpg?width=1024 39627 d78d488bfb87b00f4052c0f53d0284c84af1fc10064b681d4b05a81898e06476 +hiking trail at Námafjall.jpg File:Hiking trail at Námafjall Mountain, Iceland, 20240716 1137 1368.jpg hiking CC BY 4.0 Jakub Hałun https://commons.wikimedia.org/w/index.php?curid=158279693 https://commons.wikimedia.org/wiki/Special:FilePath/Hiking_trail_at_N%C3%A1mafjall_Mountain%2C_Iceland%2C_20240716_1137_1368.jpg?width=1024 317646 744342750d7e983f8f8fb25a99ee9d48ca188b13eee18d8692f3a5c3094917c6 +man-on-dock-with-fish-1900.jpg File:Man on dock standing next to large sturgeon hanging from a hook, vicinity of the lower Columbia River, ca 1900 (INDOCC 634).jpg fishing-dock Public domain John Fletcher Ford https://commons.wikimedia.org/w/index.php?curid=78930761 https://commons.wikimedia.org/wiki/Special:FilePath/Man_on_dock_standing_next_to_large_sturgeon_hanging_from_a_hook%2C_vicinity_of_the_lower_Columbia_River%2C_ca_1900_%28INDOCC_634%29.jpg?width=1024 78052 9cd9873cbdb28f3aae4238fd0ee2503da661d9f9f4107d8210e3ec116b8be148 +cars-in-a-parking-lot.jpg File:ParkingLot 20250511 145525.jpg control CC BY 4.0 Ka23 13 https://commons.wikimedia.org/w/index.php?curid=178449520 https://commons.wikimedia.org/wiki/Special:FilePath/ParkingLot_20250511_145525.jpg?width=1024 120194 f640239ac48e1748635fdbcecea48184da680f34d10a7fa5466255d1d23028b1 +people-surrounding-campfire.jpg File:People surrounding campfire.jpg campfire CC BY-SA 4.0 V3N0M V1V14N https://commons.wikimedia.org/w/index.php?curid=130671623 https://commons.wikimedia.org/wiki/Special:FilePath/People_surrounding_campfire.jpg?width=1024 84949 ac9d23e39f552cd51181935d9a8d9fa7478c1d29228a8b81a19cb1ab980ce384 +pct-hiking-southern-oregon.webm File:The Pacific Crest Trail (PCT) in southern Oregon (23690497321).webm hiking-video Public domain BLM Oregon & Washington https://commons.wikimedia.org/w/index.php?curid=165941251 https://commons.wikimedia.org/wiki/Special:FilePath/The_Pacific_Crest_Trail_%28PCT%29_in_southern_Oregon_%2823690497321%29.webm 33743271 9422c8d172bfa7b4d6b8ce338ac3bd1eb41365bbfdc21f6b5ac6c0c9a1366654 +wakeboard-logan.jpg File:Wakeboard logan.jpg wakeboarding Public domain Original uploaded by Logan wake (Transfered by T137) https://commons.wikimedia.org/w/index.php?curid=12638846 https://commons.wikimedia.org/wiki/Special:FilePath/Wakeboard_logan.jpg?width=1024 73640 cd569ede524854592a3b008bda0521089381611d7fd58974227c2d5c0ded5f2e +wakeboarding-aarhus-04.jpg File:Wakeboarding Watersports Aarhus Ø docklands Denmark 04.jpg wakeboarding CC BY-SA 4.0 Ciara Ní Riain https://commons.wikimedia.org/w/index.php?curid=192773108 https://commons.wikimedia.org/wiki/Special:FilePath/Wakeboarding_Watersports_Aarhus_%C3%98_docklands_Denmark_04.jpg?width=1024 277135 961ddbcc33d7c8e81fb84436f36ecff86ff8359b30cdd879aae77477b211ad59 +wakeboarding-vienna.webm File:Wakeboarding in Vienna.webm wakeboarding-video CC BY-SA 3.0 Rpoth https://commons.wikimedia.org/w/index.php?curid=32821212 https://commons.wikimedia.org/wiki/Special:FilePath/Wakeboarding_in_Vienna.webm 59380542 79db23e6724af006e681b09a65b0fe1f220a5d18b6ec8cb78252bb4e1d27942a +office-desk-with-monitor.jpg File:Workspace setup with a computer monitor and desk lamp in an office environment.jpg control CC BY 2.0 Shixart1985 https://commons.wikimedia.org/w/index.php?curid=198703486 https://commons.wikimedia.org/wiki/Special:FilePath/Workspace_setup_with_a_computer_monitor_and_desk_lamp_in_an_office_environment.jpg?width=1024 215746 a27bbfaa4061b777678f4aba139999df6af2765d1965214d8442d674d8984ebe diff --git a/server/tests/corpus/media/photos/a-dog-running-on-the-beach.jpg b/server/tests/corpus/media/photos/a-dog-running-on-the-beach.jpg new file mode 100644 index 00000000..e9d01d3d Binary files /dev/null and b/server/tests/corpus/media/photos/a-dog-running-on-the-beach.jpg differ diff --git a/server/tests/corpus/media/photos/boats-at-sunset-pakse-laos.jpg b/server/tests/corpus/media/photos/boats-at-sunset-pakse-laos.jpg new file mode 100644 index 00000000..54fa2eaa Binary files /dev/null and b/server/tests/corpus/media/photos/boats-at-sunset-pakse-laos.jpg differ diff --git a/server/tests/corpus/media/photos/cars-in-a-parking-lot.jpg b/server/tests/corpus/media/photos/cars-in-a-parking-lot.jpg new file mode 100644 index 00000000..72fe4d36 Binary files /dev/null and b/server/tests/corpus/media/photos/cars-in-a-parking-lot.jpg differ diff --git a/server/tests/corpus/media/photos/cat-sitting-on-a-sofa.jpg b/server/tests/corpus/media/photos/cat-sitting-on-a-sofa.jpg new file mode 100644 index 00000000..e61a6d6d Binary files /dev/null and b/server/tests/corpus/media/photos/cat-sitting-on-a-sofa.jpg differ diff --git a/server/tests/corpus/media/photos/christmas-eve-dinner-table.jpg b/server/tests/corpus/media/photos/christmas-eve-dinner-table.jpg new file mode 100644 index 00000000..d121d0a4 Binary files /dev/null and b/server/tests/corpus/media/photos/christmas-eve-dinner-table.jpg differ diff --git a/server/tests/corpus/media/photos/fishing-from-dock-oregon.jpg b/server/tests/corpus/media/photos/fishing-from-dock-oregon.jpg new file mode 100644 index 00000000..e08d3467 Binary files /dev/null and b/server/tests/corpus/media/photos/fishing-from-dock-oregon.jpg differ diff --git "a/server/tests/corpus/media/photos/hiking trail at N\303\241mafjall.jpg" "b/server/tests/corpus/media/photos/hiking trail at N\303\241mafjall.jpg" new file mode 100644 index 00000000..84ade675 Binary files /dev/null and "b/server/tests/corpus/media/photos/hiking trail at N\303\241mafjall.jpg" differ diff --git a/server/tests/corpus/media/photos/man-on-dock-with-fish-1900.jpg b/server/tests/corpus/media/photos/man-on-dock-with-fish-1900.jpg new file mode 100644 index 00000000..82b7b413 Binary files /dev/null and b/server/tests/corpus/media/photos/man-on-dock-with-fish-1900.jpg differ diff --git a/server/tests/corpus/media/photos/office-desk-with-monitor.jpg b/server/tests/corpus/media/photos/office-desk-with-monitor.jpg new file mode 100644 index 00000000..17d2ac12 Binary files /dev/null and b/server/tests/corpus/media/photos/office-desk-with-monitor.jpg differ diff --git a/server/tests/corpus/media/photos/people-surrounding-campfire.jpg b/server/tests/corpus/media/photos/people-surrounding-campfire.jpg new file mode 100644 index 00000000..d4952227 Binary files /dev/null and b/server/tests/corpus/media/photos/people-surrounding-campfire.jpg differ diff --git a/server/tests/corpus/media/photos/wakeboard-logan.jpg b/server/tests/corpus/media/photos/wakeboard-logan.jpg new file mode 100644 index 00000000..b9cdcfb0 Binary files /dev/null and b/server/tests/corpus/media/photos/wakeboard-logan.jpg differ diff --git a/server/tests/corpus/media/photos/wakeboarding-aarhus-04.jpg b/server/tests/corpus/media/photos/wakeboarding-aarhus-04.jpg new file mode 100644 index 00000000..4f558422 Binary files /dev/null and b/server/tests/corpus/media/photos/wakeboarding-aarhus-04.jpg differ diff --git a/server/tests/corpus/media/videos/pct-hiking-southern-oregon.webm b/server/tests/corpus/media/videos/pct-hiking-southern-oregon.webm new file mode 100644 index 00000000..983c39d6 Binary files /dev/null and b/server/tests/corpus/media/videos/pct-hiking-southern-oregon.webm differ diff --git a/server/tests/corpus/media/videos/wakeboarding-vienna.webm b/server/tests/corpus/media/videos/wakeboarding-vienna.webm new file mode 100644 index 00000000..f6f0ddee Binary files /dev/null and b/server/tests/corpus/media/videos/wakeboarding-vienna.webm differ diff --git a/server/tests/corpus2/README.md b/server/tests/corpus2/README.md new file mode 100644 index 00000000..5def6c60 --- /dev/null +++ b/server/tests/corpus2/README.md @@ -0,0 +1,50 @@ +# corpus2 — 60 Commons photographs for the server's own live testing + +The frozen corpus (`server/tests/corpus`) is exactly 15 files, and the suite's ranking +assertions (`corpus.json`) depend on that exact set: the engine's z-scores normalize over +the live candidate rows, so adding photos there would change every pinned ranking. **Extra +test images therefore live here, in a second corpus** — 60 photographs, 3 for each of 20 +subjects, taken from Wikimedia Commons (the same source `corpus/` uses), fetched as 1400 px +renders. + +* `titles.tsv` — the Commons search hits, curated by hand (name ↔ wiki title). +* `build.sh` — fetches the pins in `manifest.tsv` and verifies each sha256. + `BAKE=1 build.sh` (re)writes the manifest from `titles.tsv` after a fresh fetch. +* The photos themselves are **not in git** (≈15 MB of bytes the manifest pins; the + `.gitignore` entry keeps them out) — one `build.sh` run materializes them, like the + corpus's videos. + +It is *testing* material, for the running server: point Compose at it and the UI has a real +gallery to browse — 60 photos, meaning search, previews, opening files: + +```bash +server/tests/corpus2/build.sh # fetch + verify the pinned renders +cd deploy +DIGUP_INDEX_DIR=$HOME/Projects/digup-data/index-corpus2 \ +DIGUP_MODELS_DIR=$HOME/Projects/digup-data/models \ +DIGUP_MEDIA_DIR=$PWD/../server/tests/corpus2/media \ +docker compose up -d +# then: Index now in the UI (http://127.0.0.1:8787/) — 60 photos, ≈ 8 min on 7 CPU cores +``` + +The suite does **not** run against corpus2: its assertions are the frozen corpus's. This +corpus is for live proofs, UI runs, and the manual sweep of every endpoint — a bigger, +varied, licensed corpus with the same discipline (pinned bytes, honest source notes). + +**Expect fewer strong hits per query than a real library.** 61 files is a small gallery, +and the engine publishes a hit only above its own z-gate of 2.0 sigma over the indexed +set (`zGate` in `Sources/DigUpKit/Searcher.swift`, identical on Mac, shell and server). +On a personal library of thousands of files the same engine answers many more matches +above 2σ; the server adds no cap of its own, and the UI shows the engine's counts +verbatim. A 1-hit answer here is the size of the corpus, not a filter, a limit, or a bug. + +## Provenance and licenses + +Every file is a Wikimedia Commons file, fetched through +`https://commons.wikimedia.org/wiki/Special:FilePath/?width=1400` (the wiki's own +thumbnail service). Each title is a Commons search hit (`action=query&list=search`, +namespace 6), picked by hand from 20 subject queries. License and author are on each +file's Commons page — e.g. `https://commons.wikimedia.org/wiki/File:<title>` — most are +CC BY / CC BY-SA / public-domain; the `File:` names carry the attribution the uploader +chose. The pinned sha256 in `manifest.tsv` is of the 1400 px render as fetched (2026-11-02). +Fetch user-agent: `DigUpServer-corpus2/1.0`. diff --git a/server/tests/corpus2/build.sh b/server/tests/corpus2/build.sh new file mode 100755 index 00000000..3a6c9df7 --- /dev/null +++ b/server/tests/corpus2/build.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# corpus2: 60 Wikimedia Commons photographs (3 per subject, 20 subjects) for the *running* +# server to search — the frozen corpus stays exactly 15 files because the suite's ranking +# assertions normalize over that set. Photos are pinned by sha256 (manifest.tsv) and kept +# out of git; this script fetches them as 1400 px renders from Commons and verifies. +# +# server/tests/corpus2/build.sh fetch the pinned photos (manifest.tsv), verify +# BAKE=1 server/tests/corpus2/build.sh fetch from titles.tsv and (re)write the pins +# +# It indexes nothing: `digup index <dir>` does that. See README.md for provenance/licenses. +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +MEDIA="${1:-$HERE/media}" +WIDTH="${WIDTH:-1400}" +UA="DigUpServer-corpus2/1.0 (dev test corpus setup; contact: https://github.com/ARahim3/DigUp)" +mkdir -p "$MEDIA/photos" + +digest() { # sha256 of a file, 64 hex digits (coreutils' tool, with the BSD one as a fallback) + sha256sum "$1" 2>/dev/null | cut -c1-64 || shasum -a 256 "$1" | cut -c1-64 +} + +url_of() { # the wiki's own thumbnail service, the title percent-encoded + local t="${1#File:}" + local enc; enc="$(python3 -c 'import sys,urllib.parse; print(urllib.parse.quote(sys.argv[1], safe=""))' "$t")" + echo "https://commons.wikimedia.org/wiki/Special:FilePath/${enc}?width=${WIDTH}" +} + +fetch() { # fetch <out> <url>: Commons renders are redirects; retry politely + curl --fail --silent --show-error --location --proto '=https' -A "$UA" --retry 3 --retry-delay 2 \ + -o "$2" "$1" # note: <out> is the second argument here ($2), the url is $1 +} + +if [[ "${BAKE:-0}" == 1 ]]; then + tmp="$(mktemp)" + printf '# corpus2 pins: name, Commons title, fetch URL (%s px render), sha256 of the render\n' "$WIDTH" > "$tmp" + while IFS=$'\t' read -r name title; do + case "$name" in ''|'#'*) continue ;; esac + url="$(url_of "$title")" + out="$MEDIA/photos/$name" + echo " bake ${name}" + fetch "$url" "$out" + printf '%s\t%s\t%s\t%s\n' "$name" "$title" "$url" "$(digest "$out")" >> "$tmp" + done < "$HERE/titles.tsv" + mv "$tmp" "$HERE/manifest.tsv" + echo "baked $(grep -vc '^#' "$HERE/manifest.tsv") pins into manifest.tsv" + exit 0 +fi + +[[ -s "$HERE/manifest.tsv" ]] || { echo "no manifest.tsv pins: run BAKE=1 $0" >&2; exit 1; } + +get() { # get <out> <url> <sha256>: fetch once, verify against the pin, keep + local out="$1" url="$2" sha="$3" + if [[ -f "$out" && "$(digest "$out")" == "$sha" ]]; then + echo " ok ${out##*/}"; return 0 + fi + echo " get ${out##*/}" + fetch "$url" "$out" + local got; got="$(digest "$out")" + if [[ "$got" != "$sha" ]]; then echo " sha256 mismatch for ${out##*/}: got $got, pinned $sha" >&2; exit 1; fi + echo " ok ${out##*/}" +} + +while IFS=$'\t' read -r name title url sha; do + case "$name" in ''|'#'*) continue ;; esac + get "$MEDIA/photos/$name" "$url" "$sha" +done < "$HERE/manifest.tsv" + +# The synthetic clip travels with corpus2 too (its scene boundaries are known to the +# second): a copy of the corpus's built clip, so the running server has an H.264 video +# whose moments the web UI opens at #t=… (the corpus's webm files stay in corpus/). +if [[ -f "$HERE/../corpus/media/clips/scenes.mp4" ]]; then + mkdir -p "$MEDIA/clips" + cp -f "$HERE/../corpus/media/clips/scenes.mp4" "$MEDIA/clips/scenes.mp4" + echo " clip scenes.mp4 copied from corpus" +fi + +echo "corpus2 in $MEDIA: $(find "$MEDIA/photos" -type f | wc -l | tr -d ' ') photos" diff --git a/server/tests/corpus2/manifest.tsv b/server/tests/corpus2/manifest.tsv new file mode 100644 index 00000000..44db022a --- /dev/null +++ b/server/tests/corpus2/manifest.tsv @@ -0,0 +1,61 @@ +# corpus2 pins: name, Commons title, fetch URL (1400 px render), sha256 of the render +wakeboarding-bolesworth.jpg File:Wakeboarding on Bolesworth Lake - geograph.org.uk - 5437012.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Wakeboarding%20on%20Bolesworth%20Lake%20-%20geograph.org.uk%20-%205437012.jpg?width=1400 a44414b7e54c43ba75734d9c3b99486abbab203d5ad1769aea1ffa4ee0f88bda +wakeboarding-lake-travis.jpg File:Wakeboarding on Lake Travis (295772578).jpg https://commons.wikimedia.org/wiki/Special:FilePath/Wakeboarding%20on%20Lake%20Travis%20%28295772578%29.jpg?width=1400 20287d6649137bf5852806364a5c04c7ac557dd7259f341167e9a9a088fb6062 +wakeboarding-lake-powell.jpg File:Wakeboarding in Lake Powell.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Wakeboarding%20in%20Lake%20Powell.jpg?width=1400 8ee5bc38c33b1ca30281d4751b9b80083a4f39df4bc1d9576c053aa2f70fee9c +hiking-biskupska-kupa.jpg File:Hiking trail to Biskupská kupa, Czechia.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Hiking%20trail%20to%20Biskupsk%C3%A1%20kupa%2C%20Czechia.jpg?width=1400 538fa8e5fbda450452cad18d9439dcadc044ee98bdff844a0ace32151eb69879 +hiking-confluence-forest.jpg File:Hiking trail through forest in Confluence Natural Area.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Hiking%20trail%20through%20forest%20in%20Confluence%20Natural%20Area.jpg?width=1400 2f0d55b84eb2b7067af52344ec4713dc7a3f4762d7ff7c650a685c73dcdc85f6 +forest-hiking-trail.jpg File:Forest Hiking Trail.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Forest%20Hiking%20Trail.jpg?width=1400 35123fe744afff04b83d82fce769e5837afae8dc2611c41192b8ef834e5f2d7e +cat-on-sofa.jpg File:Female cat sitting down on sofa.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Female%20cat%20sitting%20down%20on%20sofa.jpg?width=1400 9328d5fbb4db9d94b34d481178dcb02753737688091c6d8182059dc81fdf5ec3 +calico-cat-sofa.jpg File:Calico cat sitting on the sofa.Calico color, calico breed.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Calico%20cat%20sitting%20on%20the%20sofa.Calico%20color%2C%20calico%20breed.jpg?width=1400 b7ea9c47a72af7e0784317a8211e52a9626b549036a975a80547a4f6c30784be +tabby-cat-couch.jpg File:Orange Tabby Cat sitting on a couch.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Orange%20Tabby%20Cat%20sitting%20on%20a%20couch.jpg?width=1400 140275a8cca615fea5d81b161995794141435e6d04da8fbc96605eaa39a25da9 +dog-running-beach.jpg File:Running dog at the beach.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Running%20dog%20at%20the%20beach.jpg?width=1400 3faa5482b1a246d361a674a997f3160ce5992f93413453909fba5f62babbb227 +dog-running-beach-2.jpg File:A dog running on the beach.jpg https://commons.wikimedia.org/wiki/Special:FilePath/A%20dog%20running%20on%20the%20beach.jpg?width=1400 de1a8fcfe2e356e923e6b8971e1f242082ea63db4f7e9c4458f535ba142218e0 +jack-russell-beach.jpg File:Jack Russell Terrier Eddi running at the beach.JPG https://commons.wikimedia.org/wiki/Special:FilePath/Jack%20Russell%20Terrier%20Eddi%20running%20at%20the%20beach.JPG?width=1400 b8068b2f3b1f50043f77293de52fb2294f15880916de34e293f84556df9dbfe5 +office-desk-monitor-lamp.jpg File:Workspace setup with a computer monitor and desk lamp in an office environment.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Workspace%20setup%20with%20a%20computer%20monitor%20and%20desk%20lamp%20in%20an%20office%20environment.jpg?width=1400 6109d41ea18695d8d78481c75c0cc8d674c06df0c01dc11f63e86b28b03a00c6 +office-desk-blue-ceiling.jpg File:EFTA00001748 - Wooden desk with a white chair a computer monitor and various office supplies under a blue-painted ceiling.jpg https://commons.wikimedia.org/wiki/Special:FilePath/EFTA00001748%20-%20Wooden%20desk%20with%20a%20white%20chair%20a%20computer%20monitor%20and%20various%20office%20supplies%20under%20a%20blue-painted%20ceiling.jpg?width=1400 20d99569a35a2c224157de1e13e907dc983df2beeaec379a29db7472da02b16e +office-desk-cluttered.jpg File:EFTA00002168 - Cluttered desk with a computer monitor printer keyboard and various office supplies including a funnel and a label maker.jpg https://commons.wikimedia.org/wiki/Special:FilePath/EFTA00002168%20-%20Cluttered%20desk%20with%20a%20computer%20monitor%20printer%20keyboard%20and%20various%20office%20supplies%20including%20a%20funnel%20and%20a%20label%20maker.jpg?width=1400 1b655bd1a11c05bb67327f2cd6995046c2b32a92930d1c44b1377a1f8eb39055 +sunset-fisherman.jpg File:Sunset and the Fisherman.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Sunset%20and%20the%20Fisherman.jpg?width=1400 1419d4508df67552aa30c0a7aa077d6f0f4d781e53e02669e4488afcbea02b26 +harbor-sunset-boats.jpg File:Scenic waterfront view at sunset with mountains and boats in a bustling harbor.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Scenic%20waterfront%20view%20at%20sunset%20with%20mountains%20and%20boats%20in%20a%20bustling%20harbor.jpg?width=1400 12acef6c3e1771144efbdd49b0aa13b21e0b69805de7cb86c357d8407f78ed03 +copper-harbor-sunset.jpg File:Copper Country Trail - Copper Harbor Sunset - NARA - 7717779.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Copper%20Country%20Trail%20-%20Copper%20Harbor%20Sunset%20-%20NARA%20-%207717779.jpg?width=1400 c39a4d98ad65531b14bcff5866fef3e071a580aad46ab03da80bcf1bd4ddd265 +moroccan-village-snow.jpg File:Timahdite, a Moroccan mountain village trapped in snow (Copernicus).jpg https://commons.wikimedia.org/wiki/Special:FilePath/Timahdite%2C%20a%20Moroccan%20mountain%20village%20trapped%20in%20snow%20%28Copernicus%29.jpg?width=1400 93c78d5b24354dcc990108a8161ef0efbe4f742b4d0059e00c2ca27599eee303 +snow-valley-village.jpg File:Village , valleys and mountains coved with snow in winter season.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Village%20%2C%20valleys%20and%20mountains%20coved%20with%20snow%20in%20winter%20season.jpg?width=1400 c10ea8edd09947369b895cd458ef9f2e0ea8b15cf21ce15b7e5967cde0bbacf4 +korisos-snow-mountains.jpg File:Korisos with the nearby snow covered mountains.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Korisos%20with%20the%20nearby%20snow%20covered%20mountains.jpg?width=1400 51c9abaa0f833a18ec8a3d7512355e2b7586898e24271f378649b3d1214f6b84 +brigg-veg-stall.jpg File:Vegetable Stall, Brigg Market - geograph.org.uk - 1725905.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Vegetable%20Stall%2C%20Brigg%20Market%20-%20geograph.org.uk%20-%201725905.jpg?width=1400 472e08617c9ee176a978b50c8b429267935be42192cc583545de189be931b744 +romford-market-stall.jpg File:Outdoor market fruit and vegetable stall Market Place Romford London 01.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Outdoor%20market%20fruit%20and%20vegetable%20stall%20Market%20Place%20Romford%20London%2001.jpg?width=1400 3c383c06d7f0885e1d8577ab87ac4dd73c109c75b6b8f19b9946e055b5a29f71 +godollo-market-stall.jpg File:Vegetable stall. Municipal Market. - Gödöllő.JPG https://commons.wikimedia.org/wiki/Special:FilePath/Vegetable%20stall.%20Municipal%20Market.%20-%20G%C3%B6d%C3%B6ll%C5%91.JPG?width=1400 dd8ccf566695f23ceab4e9ea7eb53fc414f5bc5d4d71b650a55f650bb6ef8a4d +fims-library-shelves.jpg File:Book shelves in the FIMS Graduate Library.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Book%20shelves%20in%20the%20FIMS%20Graduate%20Library.jpg?width=1400 69bde6195b85b277372f6183ac87050e7cc8d9c46008ac9971fa49c66dab8850 +chifley-library-shelves.jpg File:Book shelves on the top floor of the Chifley Library August 2026.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Book%20shelves%20on%20the%20top%20floor%20of%20the%20Chifley%20Library%20August%202026.jpg?width=1400 1dacd5a4599536ece8c2045acb4beb808ef9ae9749e1dd1bf1e197caed3e41e0 +mukono-library-shelves.jpg File:Lutex Community Library, Mukono, Book Shelves 02.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Lutex%20Community%20Library%2C%20Mukono%2C%20Book%20Shelves%2002.jpg?width=1400 0cc406c70c5d754e32b269bf317e0c8d359a93ba6dea9e0f62cd566e6f48bc08 +parking-gross-gerau.jpg File:Parking lot, Gross-Gerau (P1090665).jpg https://commons.wikimedia.org/wiki/Special:FilePath/Parking%20lot%2C%20Gross-Gerau%20%28P1090665%29.jpg?width=1400 f39aeb7523be624b1c9036790227853bdd2b3f8dcf7ea7cd7a09c0005fa5859c +parking-monthieu.jpg File:Parking Monthieu.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Parking%20Monthieu.jpg?width=1400 99693ccf2e95386490bf46fcce98db2a3ef4396051e24a1adcaa8e5f5e4a1da6 +parking-cusat.jpg File:Parking lot, CUSAT.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Parking%20lot%2C%20CUSAT.jpg?width=1400 146417b7dbf006d89a972fe40c7c712d7ebfa710b42f2d6f582ef3d96a332a1d +villagers-around-fire.jpg File:Villagers around a fire.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Villagers%20around%20a%20fire.jpg?width=1400 0d287be4b2406ef376030f537e370a92b635256919658ea9e546905580b471d8 +campfire-bramble-bield.jpg File:Campfire at Bramble Bield.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Campfire%20at%20Bramble%20Bield.jpg?width=1400 7c840d88e895e882e005cff5c26ccb12f23fdba26b88b5d1daa0723f19e8ee3e +campfire-night.jpg File:Campfire (night).jpg https://commons.wikimedia.org/wiki/Special:FilePath/Campfire%20%28night%29.jpg?width=1400 590f20d15ce727b1b6dc78d5c613fa5b22c94a9f77126dc4560567dee87f3cb1 +grandmother-kitchen.jpg File:Grandmother preparing food inside traditional kitchen (edited).jpg https://commons.wikimedia.org/wiki/Special:FilePath/Grandmother%20preparing%20food%20inside%20traditional%20kitchen%20%28edited%29.jpg?width=1400 859b927f5489626c235ea8cb5c4f9c4f721756995d1ed83603dc460848e2fcc6 +indian-kitchen-cooking.jpg File:Preparation of cooking Indian food at kitchen in a household.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Preparation%20of%20cooking%20Indian%20food%20at%20kitchen%20in%20a%20household.jpg?width=1400 74726fb2ea6a0b23aae3ca55e26f5f17b78214deb8cc9b3126db1a32b699e029 +chef-red-onion.jpg File:Chef prepares red onion at kitchen counter for cooking in the morning.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Chef%20prepares%20red%20onion%20at%20kitchen%20counter%20for%20cooking%20in%20the%20morning.jpg?width=1400 c0578323042675939a1dc1b71f8d6ae7df74895476a017485438a284006ecfbe +bicycles-groningen.jpg File:Bicycles parked behind a no bicycle parking sign, Groningen (2020) 02.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Bicycles%20parked%20behind%20a%20no%20bicycle%20parking%20sign%2C%20Groningen%20%282020%29%2002.jpg?width=1400 9e0ce95c256727065fb3e3508589bfd5d9a5767064adff69754f25973c8ed13f +bicycle-new-orleans.jpg File:Forgotten (temorarily) Bicycle, City Park, New Orleans March 2022.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Forgotten%20%28temorarily%29%20Bicycle%2C%20City%20Park%2C%20New%20Orleans%20March%202022.jpg?width=1400 5b5b1afaa5a2ff676faa4cc12096733adea3fe933f3cb55e9fe3662cacd4d377 +bicycle-amsterdam.jpg File:Parked bicycle in the streets of Amsterdam in spring of 2013.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Parked%20bicycle%20in%20the%20streets%20of%20Amsterdam%20in%20spring%20of%202013.jpg?width=1400 f530381f318a8f008137bf34fe6cfcb52f5f6c009b8442b8ccc66382baaddc1a +church-santa-margalida.jpg File:Església Santa Margalida - Santa Margalida - Mallorca - stained-glass windows.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Esgl%C3%A9sia%20Santa%20Margalida%20-%20Santa%20Margalida%20-%20Mallorca%20-%20stained-glass%20windows.jpg?width=1400 7372af9e5b1abc4e99c56fcdcd11678f4d7c5553f167d72eb8176d507bb25dd3 +church-st-etheldreda.jpg File:St Etheldreda's Church Interior, London, UK - Diliff.jpg https://commons.wikimedia.org/wiki/Special:FilePath/St%20Etheldreda%27s%20Church%20Interior%2C%20London%2C%20UK%20-%20Diliff.jpg?width=1400 9c1de19efe2c15bdf2e953a7630dd775587eff37afa23c971bcfe5af853bd6be +church-el-salvador-windows.jpg File:Windows Iglesia de El Salvador - Santa Cruz de La Palma.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Windows%20Iglesia%20de%20El%20Salvador%20-%20Santa%20Cruz%20de%20La%20Palma.jpg?width=1400 0d3fa4d18c40dfc9662311bc1de2023ca197131f56b67d3ae61a7e3045cb159b +desert-road-sinai-peaks.jpg File:Sinai, Egypt, Road through the desert landscape, rocky peaks and valleys.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Sinai%2C%20Egypt%2C%20Road%20through%20the%20desert%20landscape%2C%20rocky%20peaks%20and%20valleys.jpg?width=1400 b75d3e95b851579bb3cc17df456867861259dd4c01f1640fd31811361ddd400e +wadi-rum-desert-highway.jpg File:Desert landscape in Jordan off Desert Highway near Wadi Rum.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Desert%20landscape%20in%20Jordan%20off%20Desert%20Highway%20near%20Wadi%20Rum.jpg?width=1400 1a33a669ca91a820b0e3330300eb83fdaea591986d7caf10a2d9d9c9e79aecb9 +nuweiba-desert-road.jpg File:Nuweiba, Desert road, Sinai, Egypt.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Nuweiba%2C%20Desert%20road%2C%20Sinai%2C%20Egypt.jpg?width=1400 b73aa10535dfb011b280582cd9b7f24cda38ec0ef124db11fcf23024dbef942d +airplane-calgary-gate.jpg File:Parked Viva Aerobus airplane at a gate at Calgary International Airport (YYC).jpg https://commons.wikimedia.org/wiki/Special:FilePath/Parked%20Viva%20Aerobus%20airplane%20at%20a%20gate%20at%20Calgary%20International%20Airport%20%28YYC%29.jpg?width=1400 e0fc94566815ada91e4e6819eb6710a01af93cfe8f87870238e2ee2a6dd2887f +jet-bridge-cdg.jpg File:Jet bridge, Paris-Charles de Gaulle Airport, Roissy (SIAE2156).jpg https://commons.wikimedia.org/wiki/Special:FilePath/Jet%20bridge%2C%20Paris-Charles%20de%20Gaulle%20Airport%2C%20Roissy%20%28SIAE2156%29.jpg?width=1400 d36a6c1aa3ae06e2588009f171c0cef1bc0caa9d3bb464f43e7feee2aa3689fe +airport-gate-recife.jpg File:Recife international airport gate to the airplane, pernambuco state of Brazil.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Recife%20international%20airport%20gate%20to%20the%20airplane%2C%20pernambuco%20state%20of%20Brazil.jpg?width=1400 d5b940dc929c90c3ef61e176a4122536483ac987d0a6b8a0879abada9776ec05 +jesenik-train-station.jpg File:Jeseník train station 2025.01.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Jesen%C3%ADk%20train%20station%202025.01.jpg?width=1400 4a8891d2ca841c7af3f4209c01a2a4fe320813a2696ddfa69cf69beab815d1a2 +brussels-luxembourg-platform.jpg File:Bruxelles-Luxembourg train station platform 3 (Belgium).jpg https://commons.wikimedia.org/wiki/Special:FilePath/Bruxelles-Luxembourg%20train%20station%20platform%203%20%28Belgium%29.jpg?width=1400 b39eecf4412958f9c59ed4510fcb918dac06d26be6b65ba1927f47fb55264df1 +pasila-platform-10.jpg File:Platform 10 in Pasila train station in Helsinki.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Platform%2010%20in%20Pasila%20train%20station%20in%20Helsinki.jpg?width=1400 2833cdbd4e27e5178e79096537787d50a12ee0f1eef54bd2a801669478e066f7 +coffee-laptop-desk.jpg File:Coffee-desk-laptop-notebook (24244320481).jpg https://commons.wikimedia.org/wiki/Special:FilePath/Coffee-desk-laptop-notebook%20%2824244320481%29.jpg?width=1400 c04d2e062758e833734e60f46a5fe1f73ef70cdbf139e5478a6ce32ff79ebfc2 +laptop-coffee-plant-white.jpg File:Working at a desk with a laptop coffee cup and a plant on a white surface.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Working%20at%20a%20desk%20with%20a%20laptop%20coffee%20cup%20and%20a%20plant%20on%20a%20white%20surface.jpg?width=1400 e6d321c1f572b2a9288d499aefb38a2535e7f7798126dbac2c9a0d39f3086407 +desk-coffee-laptop-artwork.jpg File:Desk workspace featuring coffee cup, laptop, and plant with artwork in background during a sunny day.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Desk%20workspace%20featuring%20coffee%20cup%2C%20laptop%2C%20and%20plant%20with%20artwork%20in%20background%20during%20a%20sunny%20day.jpg?width=1400 0e70d8316646e87bfc03ae58ad56eb7f2ab5197d9aa19c24d8c96bec4079a96b +coral-reef-fish-slope.jpg File:Coral reef fish swim above the coral slope.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Coral%20reef%20fish%20swim%20above%20the%20coral%20slope.jpg?width=1400 77be1503640acb76188d99074631154ac79ca7d22b8d6202b8049e40341c8366 +coral-reef-underwater.jpg File:Underwater photo of coral reef.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Underwater%20photo%20of%20coral%20reef.jpg?width=1400 9885f1cd64113c0fc638994fb1f283e8428f057c3bb40b8a9a65ec885fe84c76 +coral-reef-colorful.jpg File:Colorful underwater landscape of a coral reef.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Colorful%20underwater%20landscape%20of%20a%20coral%20reef.jpg?width=1400 614eb7c04ecf51a29a6f3b088a3becc6742dc560ea24537513933399a3d6c1f9 +autumn-tuntorp.jpg File:Forest path through yellow autumn leaves in Tuntorp 2.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Forest%20path%20through%20yellow%20autumn%20leaves%20in%20Tuntorp%202.jpg?width=1400 24dfdf90ab182b36e8d49d1b869c3b8b50887928562bbd5526a91d62a907c3ed +autumn-ehrenbach.jpg File:Forest path, Ehrenbach.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Forest%20path%2C%20Ehrenbach.jpg?width=1400 cdbdf4cad7eccc76b587643f2d404b198615a669ecf64cb487e45ba0aa3c6f32 +autumn-glenarm.jpg File:Autumn leaves, Glenarm forest - geograph.org.uk - 1010256.jpg https://commons.wikimedia.org/wiki/Special:FilePath/Autumn%20leaves%2C%20Glenarm%20forest%20-%20geograph.org.uk%20-%201010256.jpg?width=1400 5022b96a9fb7c6942e3c7c3bfeaceb98bc285962494831f4b825038fbe2b0777 diff --git a/server/tests/corpus2/titles.tsv b/server/tests/corpus2/titles.tsv new file mode 100644 index 00000000..fe610246 --- /dev/null +++ b/server/tests/corpus2/titles.tsv @@ -0,0 +1,62 @@ +# corpus2 titles — Wikimedia Commons search hits, 3 per subject, 20 subjects (60 photos). +# name wiki title. BAKE=1 build.sh fetches each as a 1400 px render and pins its sha256. +wakeboarding-bolesworth.jpg File:Wakeboarding on Bolesworth Lake - geograph.org.uk - 5437012.jpg +wakeboarding-lake-travis.jpg File:Wakeboarding on Lake Travis (295772578).jpg +wakeboarding-lake-powell.jpg File:Wakeboarding in Lake Powell.jpg +hiking-biskupska-kupa.jpg File:Hiking trail to Biskupská kupa, Czechia.jpg +hiking-confluence-forest.jpg File:Hiking trail through forest in Confluence Natural Area.jpg +forest-hiking-trail.jpg File:Forest Hiking Trail.jpg +cat-on-sofa.jpg File:Female cat sitting down on sofa.jpg +calico-cat-sofa.jpg File:Calico cat sitting on the sofa.Calico color, calico breed.jpg +tabby-cat-couch.jpg File:Orange Tabby Cat sitting on a couch.jpg +dog-running-beach.jpg File:Running dog at the beach.jpg +dog-running-beach-2.jpg File:A dog running on the beach.jpg +jack-russell-beach.jpg File:Jack Russell Terrier Eddi running at the beach.JPG +office-desk-monitor-lamp.jpg File:Workspace setup with a computer monitor and desk lamp in an office environment.jpg +office-desk-blue-ceiling.jpg File:EFTA00001748 - Wooden desk with a white chair a computer monitor and various office supplies under a blue-painted ceiling.jpg +office-desk-cluttered.jpg File:EFTA00002168 - Cluttered desk with a computer monitor printer keyboard and various office supplies including a funnel and a label maker.jpg +sunset-fisherman.jpg File:Sunset and the Fisherman.jpg +harbor-sunset-boats.jpg File:Scenic waterfront view at sunset with mountains and boats in a bustling harbor.jpg +copper-harbor-sunset.jpg File:Copper Country Trail - Copper Harbor Sunset - NARA - 7717779.jpg +moroccan-village-snow.jpg File:Timahdite, a Moroccan mountain village trapped in snow (Copernicus).jpg +snow-valley-village.jpg File:Village , valleys and mountains coved with snow in winter season.jpg +korisos-snow-mountains.jpg File:Korisos with the nearby snow covered mountains.jpg +brigg-veg-stall.jpg File:Vegetable Stall, Brigg Market - geograph.org.uk - 1725905.jpg +romford-market-stall.jpg File:Outdoor market fruit and vegetable stall Market Place Romford London 01.jpg +godollo-market-stall.jpg File:Vegetable stall. Municipal Market. - Gödöllő.JPG +fims-library-shelves.jpg File:Book shelves in the FIMS Graduate Library.jpg +chifley-library-shelves.jpg File:Book shelves on the top floor of the Chifley Library August 2026.jpg +mukono-library-shelves.jpg File:Lutex Community Library, Mukono, Book Shelves 02.jpg +parking-gross-gerau.jpg File:Parking lot, Gross-Gerau (P1090665).jpg +parking-monthieu.jpg File:Parking Monthieu.jpg +parking-cusat.jpg File:Parking lot, CUSAT.jpg +villagers-around-fire.jpg File:Villagers around a fire.jpg +campfire-bramble-bield.jpg File:Campfire at Bramble Bield.jpg +campfire-night.jpg File:Campfire (night).jpg +grandmother-kitchen.jpg File:Grandmother preparing food inside traditional kitchen (edited).jpg +indian-kitchen-cooking.jpg File:Preparation of cooking Indian food at kitchen in a household.jpg +chef-red-onion.jpg File:Chef prepares red onion at kitchen counter for cooking in the morning.jpg +bicycles-groningen.jpg File:Bicycles parked behind a no bicycle parking sign, Groningen (2020) 02.jpg +bicycle-new-orleans.jpg File:Forgotten (temorarily) Bicycle, City Park, New Orleans March 2022.jpg +bicycle-amsterdam.jpg File:Parked bicycle in the streets of Amsterdam in spring of 2013.jpg +church-santa-margalida.jpg File:Església Santa Margalida - Santa Margalida - Mallorca - stained-glass windows.jpg +church-st-etheldreda.jpg File:St Etheldreda's Church Interior, London, UK - Diliff.jpg +church-el-salvador-windows.jpg File:Windows Iglesia de El Salvador - Santa Cruz de La Palma.jpg +desert-road-sinai-peaks.jpg File:Sinai, Egypt, Road through the desert landscape, rocky peaks and valleys.jpg +wadi-rum-desert-highway.jpg File:Desert landscape in Jordan off Desert Highway near Wadi Rum.jpg +nuweiba-desert-road.jpg File:Nuweiba, Desert road, Sinai, Egypt.jpg +airplane-calgary-gate.jpg File:Parked Viva Aerobus airplane at a gate at Calgary International Airport (YYC).jpg +jet-bridge-cdg.jpg File:Jet bridge, Paris-Charles de Gaulle Airport, Roissy (SIAE2156).jpg +airport-gate-recife.jpg File:Recife international airport gate to the airplane, pernambuco state of Brazil.jpg +jesenik-train-station.jpg File:Jeseník train station 2025.01.jpg +brussels-luxembourg-platform.jpg File:Bruxelles-Luxembourg train station platform 3 (Belgium).jpg +pasila-platform-10.jpg File:Platform 10 in Pasila train station in Helsinki.jpg +coffee-laptop-desk.jpg File:Coffee-desk-laptop-notebook (24244320481).jpg +laptop-coffee-plant-white.jpg File:Working at a desk with a laptop coffee cup and a plant on a white surface.jpg +desk-coffee-laptop-artwork.jpg File:Desk workspace featuring coffee cup, laptop, and plant with artwork in background during a sunny day.jpg +coral-reef-fish-slope.jpg File:Coral reef fish swim above the coral slope.jpg +coral-reef-underwater.jpg File:Underwater photo of coral reef.jpg +coral-reef-colorful.jpg File:Colorful underwater landscape of a coral reef.jpg +autumn-tuntorp.jpg File:Forest path through yellow autumn leaves in Tuntorp 2.jpg +autumn-ehrenbach.jpg File:Forest path, Ehrenbach.jpg +autumn-glenarm.jpg File:Autumn leaves, Glenarm forest - geograph.org.uk - 1010256.jpg diff --git a/server/tests/gpu-handoff.sh b/server/tests/gpu-handoff.sh new file mode 100755 index 00000000..bbcbc6d5 --- /dev/null +++ b/server/tests/gpu-handoff.sh @@ -0,0 +1,275 @@ +#!/usr/bin/env bash +# Stage 9 handoff proof — run this on an x86_64 host with an NVIDIA GPU (docs/implementation/ +# gpu-handoff.md is the prose version; this is the executable version). +# +# It counts failures and exits non-zero. A run that prints encouraging sentences and exits 0 +# while the GPU was never used is exactly the artifact this project refuses, so: +# +# --cpu-only Gate A only (the CPU baseline on this machine). Gate B is NOT run and the +# script says GPU NOT MEASURED, in those words. +# (no args) Gate A, then Gate B with the card: CUDA build, payload check, speed check, +# utilization sampled DURING a real index job. +# +# Every gate prints `PASS gate: …` / `FAIL gate: …`. Nothing here is a mock: the contract suite +# runs the real engine, and every skipped line is a failure. +set -uo pipefail +cd "$(dirname "$0")/../.." +ROOT="$PWD" + +MODE=all +[ "${1:-}" = "--cpu-only" ] && MODE=cpu-only +OUT="server/data/gpu-handoff-$(date +%Y%m%d-%H%M).log" +mkdir -p server/data +exec > >(tee -a "$OUT") 2>&1 + +fails=0 +gpus_used=0 +pass() { echo "PASS gate: $*"; } +fail() { echo "FAIL gate: $*"; fails=$((fails + 1)); } +note() { echo "note: $*"; } + +DATA="${DIGUP_DATA:-$HOME/digup-data}" +REV=bfcd298762cc34d0357ece5ebdd31791a3a374d8 +CUDA_BASE="${CUDA_BASE:-nvidia/cuda:12.4.1-devel-ubuntu22.04}" +ARCH=$(uname -m) + +echo "== DigUp Stage-9 handoff · mode=$MODE · arch=$ARCH · log=$OUT" +echo + +# ── Gate A — the CPU baseline, measured on this machine ───────────────────────────────── +gate_a_toolchain() { + command -v docker > /dev/null || { fail "docker is not installed"; return; } + docker compose version > /dev/null || { fail "docker compose is not installed"; return; } + [ "$ARCH" = x86_64 ] && pass "docker + compose present, arch=$ARCH" \ + || note "arch is $ARCH — Gate B's CUDA build only makes sense on x86_64 (README §11: never build under emulation)" +} + +gate_a_corpus() { + [ -f server/tests/corpus/media/clips/scenes.mp4 ] || { fail "corpus clip missing: run server/tests/corpus/build.sh"; return; } + [ -f server/tests/corpus2/media/clips/scenes.mp4 ] || note "corpus2 gallery not built (run server/tests/corpus2/build.sh) — live gallery checks skipped" + pass "corpus fixtures present" +} + +gate_a_models() { + local want got + want=$(grep -oE 'sha256: "[0-9a-f]{64}"' Sources/DigUpKit/ModelFiles.swift | grep -oE '[0-9a-f]{64}' | sort) + got=$(sha256sum "$DATA"/models/*.gguf 2>/dev/null | awk '{print $1}' | sort) + if [ -z "$got" ]; then fail "no model files in $DATA/models (docs/implementation/gpu-handoff.md Gate A has the download lines)"; return; fi + if [ "$want" != "$got" ]; then fail "model sha256s do not match Sources/DigUpKit/ModelFiles.swift: engine=$want files=$got"; return; fi + pass "model files verified against the engine's own pins" +} + +gate_a_bind_dirs() { + mkdir -p "$DATA"/{index,models,config} + local owner; owner=$(stat -c '%u' "$DATA/index" 2>/dev/null || echo 0) + if [ "$owner" != 990 ] && [ "$owner" != 0 ]; then + note "$DATA/index is owned by uid $owner; the image's user is 990 — sudo chown -R 990:990 $DATA" + fi + pass "bind folders exist ($DATA)" +} + +gate_a_suite() { + DOCKER_BUILDKIT=1 docker build --target server-test -f server/docker/Dockerfile -t digup-server-test:handoff . \ + || { fail "server-test image build"; return; } + mkdir -p "$DATA/index-suite" + docker run --rm \ + -v "$ROOT/server/tests/corpus/media:/corpus:ro" \ + -v "$ROOT/server/tests/corpus:/corpus-root:ro" \ + -v "$DATA/models:/models:ro" \ + -v "$DATA/index-suite:/index" \ + digup-server-test:handoff /srv/server/.venv/bin/pytest -rs > "$OUT.suite" 2>&1 + local code=$? + tail -25 "$OUT.suite" + if grep -qE '^(SKIPPED|tests/.*SKIPPED)' "$OUT.suite"; then + fail "a contract test SKIPPED (a self-skipping suite is green-by-construction: corpus/models/clip mounts are not optional)" + elif [ $code -ne 0 ] || ! grep -qE '[0-9]+ passed' "$OUT.suite"; then + fail "contract suite (exit $code)" + else + pass "contract suite on $ARCH: $(grep -E '[0-9]+ passed' "$OUT.suite" | tail -1)" + fi +} + +gate_a_compose() { + ( cd deploy && docker compose config --quiet ) || { fail "compose config"; return; } + ( cd deploy && DIGUP_INDEX_DIR="$DATA/index" DIGUP_MODELS_DIR="$DATA/models" \ + DIGUP_CONFIG_DIR="$DATA/config" DIGUP_MEDIA_DIR="$DATA/media/photos" \ + docker compose up -d --build ) > /dev/null 2>&1 \ + || { fail "compose up --build (CPU-only shape)"; return; } + local i alive=0 + for i in $(seq 1 40); do + if ( cd deploy && docker compose exec -T server python /opt/digup/healthcheck.py ) > /dev/null 2>&1; then alive=1; break; fi + sleep 3 + done + [ $alive -eq 1 ] && pass "the CPU-only stack boots and its own healthcheck answers" \ + || fail "healthcheck never answered (the API is the healthcheck: /health 503 counts as alive, so this means the API is not there)" + local body; body=$(curl -sf -X POST localhost:8787/api/v1/search -H 'content-type: application/json' -d '{"q":"wakeboarding"}' || true) + [ -n "$body" ] && pass "a search over HTTP answers" || fail "POST /api/v1/search answered nothing (index it first: POST /index {})" + # The id must be captured BEFORE the stop: `compose ps -q` does not list a stopped service, + # so inspecting after the stop is how this gate used to report "unknown". + local cid code + cid=$( cd deploy && docker compose ps -q server ) + ( cd deploy && docker compose stop -t 90 server ) > /dev/null 2>&1 + code=$(docker inspect "$cid" --format '{{.State.ExitCode}}' 2>/dev/null || echo unknown) + [ "$code" = "137" ] && fail "the container stop landed as SIGKILL (137): the lifespan hung" \ + || note "container stop exit code: $code (143 = a completed SIGTERM stop)" + ( cd deploy && docker compose logs server 2>/dev/null | grep -q "Application shutdown complete" ) \ + && pass "graceful lifespan (Application shutdown complete)" \ + || fail "no 'Application shutdown complete' in the server log" + ( cd deploy && docker compose down -v ) > /dev/null 2>&1 +} + +gate_a_toolchain; gate_a_corpus; gate_a_models; gate_a_bind_dirs; gate_a_suite; gate_a_compose + +if [ "$MODE" = cpu-only ]; then + echo + echo "GPU NOT MEASURED (Gate B skipped: --cpu-only)." + echo "CPU-only baseline: $fails failed gate(s). Log: $OUT" + exit $((fails > 0 ? 1 : 0)) +fi + +# ── Gate B — the card, measured here and now ──────────────────────────────────────────── +gate_b_toolkit() { + command -v nvidia-smi > /dev/null || { fail "no nvidia-smi on the host: the driver is missing"; return; } + docker run --rm --gpus all nvidia/cuda:12.4.1-base-ubuntu22.04 nvidia-smi > /dev/null 2>&1 \ + && pass "NVIDIA Container Toolkit hands the container a device" \ + || fail "NVIDIA Container Toolkit: a --gpus all container cannot run nvidia-smi (install/configure it first)" +} + +gate_b_cuda_build() { + # CUDA_ARCHS reaches scripts/build-llama.sh's cmake as CMAKE_CUDA_ARCHITECTURES, and the + # builder's cmake is the Ubuntu 22.04 apt one (3.22.1, measured): it accepts only plain + # numbers — "6.1" and "native" both fail configure there (native needs cmake >= 3.24 and a + # GPU the builder container does not have). Pass the card's plain number: 61 for Pascal + # sm_61, 89 for Ada sm_89. The default stays native for toolchains that speak it. + DOCKER_BUILDKIT=1 docker build --build-arg BASE_IMAGE="$CUDA_BASE" \ + --build-arg CUDA_ARCHS="${CUDA_ARCHS:-native}" --target engine-payload \ + -f server/docker/Dockerfile -t digup-payload:cuda . > /dev/null 2>&1 \ + || { fail "CUDA engine-payload build on $CUDA_BASE (match the base's CUDA to the host driver: nvidia-smi's 'CUDA Version')"; return; } + local listing; listing=$(docker run --rm digup-payload:cuda sh -c 'ls /out | grep -i cuda' || true) + [ -n "$listing" ] && pass "payload carries the CUDA backend ($listing)" \ + || fail "payload has no CUDA backend: scripts/build-llama.sh did not find nvcc in $CUDA_BASE" + # /out, not /opt/digup: the engine-payload stage lands in /out; the runtime stages COPY it + # to /opt/digup. A check written against /opt/digup fails on the payload image (measured). + docker run --rm digup-payload:cuda sh -c 'ldd /out/libggml-cuda.so' | tee "$OUT.ldd" + if grep -qE 'libcuda\.so\.1.*not found' "$OUT.ldd"; then note "libcuda.so.1 unresolved in the build stage — expected: it comes from the HOST driver through the toolkit at run time"; fi + if grep -qE 'libcudart|libcublas' "$OUT.ldd"; then pass "CUDA runtime libs resolve in the payload (ship these, from a CUDA runtime base, never the toolkit)"; else fail "ldd shows CUDA runtime libs that do not resolve: add exactly them to the runtime stage"; fi +} + +gate_b_device_choice() { + # The engine's own measurement, on the real index: `device` is what the helpers then use. + # On a real Linux host (not Docker Desktop) the bind folder must be writable by the + # container's user: index the corpus AS uid 990 (the runtime user) in a folder any of + # them can write, so the files the Compose gate below uses belong to the same uid. + mkdir -p "$DATA/index-gpu"; chmod 777 "$DATA/index-gpu" 2>/dev/null || true + docker run --rm --gpus "${GPU_SPEC:-all}" -u 990:990 -e HOME=/tmp \ + -v "$ROOT/server/tests/corpus/media:/corpus:ro" -v "$DATA/models:/models:ro" -v "$DATA/index-gpu:/index" \ + -e DIGUP_MODELS=/models digup-payload:cuda sh -c '/out/digup index /corpus --index /index > /dev/null 2>&1; /out/digup speed --model --json --index /index | tail -1' \ + > "$OUT.speed" 2>&1 + if grep -qE '"device"|"seconds"' "$OUT.speed"; then + tail -2 "$OUT.speed" + if grep -q '"device": *"gpu"' "$OUT.speed"; then pass "the engine measured the GPU as the faster device (agreement check inside speed --model)"; gpus_used=1 + else note "the engine measured the processor as faster here (device: cpu) — a real measurement, not a failure; report the numbers"; fi + else + fail "digup speed --model --json answered nothing usable (see $OUT.speed)" + fi +} + +gate_b_utilization() { + # Detecting a device is not using one. Sample the card DURING a real index job. + # Work the fresh walk: a fully-embedded corpus makes the job a no-op sync — nothing to + # sample — so the gate measures a real embedding pass over the corpus. + # + # Sampling cadence is measured (2026-11-04, GTX 1070): GPU embedding of a small corpus is + # a series of ~150 ms kernel bursts separated by P8 idle. A 2 s-interval sampler is blind + # to that shape: over a full 15-file job it saw peak 12%, and 0% across most windows — + # a false FAIL for a server demonstrably embedding on the GPU (model resident in 392 MB + # of card VRAM, engine costs at the GPU's 0.15 s/image rate). The 0.15 s sampler over the + # job sees peak 94% and 39% busy time. A CPU-embedding job is flat: zero busy samples, + # no model VRAM on the card. So: sample at the job's pace, judge busy-time (duty) and + # VRAM residency, and let the old peak>20 stand as the easy pass. + mkdir -p "$DATA/index-gpu"; chmod 777 "$DATA/index-gpu" 2>/dev/null || true + rm -rf "$DATA/index-gpu"/* "$DATA/index-gpu"/.digup-server.writer.lock 2>/dev/null || true + ( cd deploy && DIGUP_INDEX_DIR="$DATA/index-gpu" DIGUP_MODELS_DIR="$DATA/models" \ + DIGUP_CONFIG_DIR="$DATA/config" DIGUP_MEDIA_DIR="$ROOT/server/tests/corpus/media" \ + docker compose -f docker-compose.yml -f docker-compose.nvidia.yml --profile nvidia up -d --build ) > /dev/null 2>&1 + if [ $? -ne 0 ]; then fail "the nvidia-profile Compose override did not come up (author deploy/docker-compose.nvidia.yml on this box, per docs/implementation/gpu-handoff.md §4)"; return; fi + # nvidia-smi -i wants an index/UUID/name; the docker --gpus value carries a device= prefix. + local gpu="${GPU_SPEC:-all}"; gpu="${gpu#device=}" + # /health answers 503 while the searcher warms/respawns (~20–30 s prewarm): the container + # is healthy, the API is honest. The index job needs the API up — count 200 as ready, not 503. + local i code=000 + for i in $(seq 1 40); do + code=$(curl -s -o /dev/null -w '%{http_code}' localhost:8787/api/v1/health 2>/dev/null || true) + [ "$code" = 200 ] && break + sleep 3 + done + if [ "$code" != 200 ]; then + fail "the nvidia-profile API never answered /health with 200 (last: $code) — job was never asked" + ( cd deploy && docker compose -f docker-compose.yml -f docker-compose.nvidia.yml --profile nvidia down -v ) > /dev/null 2>&1 + return + fi + local jobid + jobid=$(python3 - <<'PYEOF' +import json, sys, time, urllib.request +for attempt in range(40): + try: + req = urllib.request.Request('http://127.0.0.1:8787/api/v1/index', data=b'{}', + headers={'content-type': 'application/json'}) + with urllib.request.urlopen(req, timeout=10) as r: + print(json.load(r)["job"]["id"]); sys.exit(0) + except Exception: + time.sleep(3) # warming (503) / transient: ask again; 409 = a job runs — handled below +PYEOF +) + if [ -z "$jobid" ]; then + fail "POST /index answered no job id (the nvidia container is up but the API did not accept the job)" + ( cd deploy && docker compose -f docker-compose.yml -f docker-compose.nvidia.yml --profile nvidia down -v ) > /dev/null 2>&1 + return + fi + local stats + stats=$(python3 - "$jobid" "$gpu" <<'PYEOF' +import json, subprocess, sys, time, urllib.request +jid, gpu = sys.argv[1], sys.argv[2] +busy = tot = peak = vram_hi = 0 +t0 = time.time() +while time.time() - t0 < 300: + out = subprocess.run(['nvidia-smi', '--query-gpu=utilization.gpu,memory.used', + '--format=csv,noheader,nounits', '-i', gpu], + capture_output=True, text=True).stdout.splitlines()[0].split(',') + u, m = int(out[0]), int(out[1]) + tot += 1 + busy += u > 0 + peak = max(peak, u) + vram_hi += m > 200 + try: + js = json.load(urllib.request.urlopen('http://127.0.0.1:8787/api/v1/jobs', timeout=5)) + j = next((x for x in js['jobs'] if x['id'] == jid), None) + if j and j.get('state') == 'done': + break + except Exception: + pass + time.sleep(0.15) +print(f"{peak} {busy} {tot} {vram_hi}") +PYEOF +) + local peak busy tot vram_hi + read -r peak busy tot vram_hi <<< "$stats" + : "${tot:=0}" + ( cd deploy && docker compose -f docker-compose.yml -f docker-compose.nvidia.yml --profile nvidia down -v ) > /dev/null 2>&1 + if [ "${tot:-0}" -eq 0 ]; then fail "nvidia-smi answered nothing during the job (card '$gpu' not visible?)"; return; fi + local duty=$(( 100 * busy / tot )) + if [ "$peak" -gt 20 ] || { [ "$duty" -ge 10 ] && [ "$vram_hi" -ge $(( tot / 2 )) ]; }; then + pass "the card worked during the index job (peak ${peak}%, busy ${duty}% of ${tot} samples, model on card for ${vram_hi}/${tot} samples)" + gpus_used=1 + else + fail "GPU utilization peaked at ${peak}% with ${duty}% busy samples over ${tot} — the container detected the device and embedded on the CPU" + fi +} + +gate_b_toolkit; gate_b_cuda_build; gate_b_device_choice; gate_b_utilization + +echo +if [ "$gpus_used" -eq 1 ]; then echo "GPU: measured on this box (see $OUT.speed and the utilization gate)."; else echo "GPU NOT MEASURED: no gate below proved the card ran the model."; fi +echo "Gates failed: $fails. Full log: $OUT" +[ "$fails" -eq 0 ] && [ "$gpus_used" -eq 1 ] && exit 0 +exit 1 diff --git a/server/tests/proof.sh b/server/tests/proof.sh new file mode 100755 index 00000000..4c12101e --- /dev/null +++ b/server/tests/proof.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# The headless proof (Stage 2): the original DigUp engine, in Linux, with no UI — indexing the +# labeled corpus and answering natural-language searches from it, exactly as the server will. +# +# server/tests/proof.sh # run inside the engine container +# +# Typical invocation on this machine (arm64, CPU only, no GPU, no desktop, no Tauri): +# +# docker run --rm --platform linux/arm64 \ +# -v "$PWD:/src:ro" \ +# -v "$PWD/server/tests/corpus/media:/corpus:ro" \ +# -v "$HOME/Projects/digup-data/models:/models:ro" \ +# -v "$HOME/Projects/digup-data/index:/index" \ +# -e DIGUP_MODELS=/models \ +# digup-engine:local bash /src/server/tests/proof.sh +# +# Nothing here invents anything: every number and every ranking is the engine's own, and the +# logs land under server/data/proof/ (git-ignored runtime evidence) for docs/implementation/. +set -uo pipefail +DIGUP="${DIGUP:-digup}" +CORPUS="${CORPUS:-/corpus}" +INDEX="${INDEX:-/index}" +export DIGUP_MODELS="${DIGUP_MODELS:-/models}" +SRC="${SRC:-/src}" +OUT="${OUT:-$SRC/server/data/proof}" # runtime evidence, git-ignored; docs quote it +mkdir -p "$OUT" "$INDEX" +fails=0 +gate() { gate_msg="$1"; } # the last gate that ran (shown on failure) + +run() { # run <file> <command…>: keep the engine's own words, on screen and in a log. + local file="$1"; shift + echo; echo "══ $*" + # pipefail is set, so this pipeline's status IS the command's: a proof that cannot + # report failure is not evidence (the old version always exited 0 — a run where the + # index died and every search answered nothing was "green", and CI cited its EXIT=0). + if ! "$@" 2>&1 | tee "$OUT/$file.log"; then + fails=$((fails + 1)) + echo "FAILED (gate: ${gate_msg:-command}, exit above) — $OUT/$file.log" | tee -a "$OUT/$file.log" + fi +} + +gate "speed --model" +run speed "$DIGUP" speed --model --when-quiet 5 --index "$INDEX" +gate "estimate (plain — ParentWatch rule, reconnaissance §15)" +# `estimate` plain: proof.sh runs as PID 1's script, and `estimate --json` is a ParentWatch +# helper that exits at once when its parent is PID 1 ("the app is gone") — the server, as a +# long-lived parent, uses `--json`; this script, as a script, must not (reconnaissance §15). +run estimate "$DIGUP" estimate "$CORPUS" --index "$INDEX" +gate "index" +run index "$DIGUP" index "$CORPUS" --index "$INDEX" +gate "status" +run status "$DIGUP" status --index "$INDEX" + +queries=( + "wakeboarding" + "hiking" + "a person riding a wakeboard behind a boat" + "people sitting around a campfire" + "a dog running on the beach" + "a boat on the water at sunset" + "someone fishing from a dock" + "Christmas dinner" + "cars in a parking lot" +) +n=0 +for query in "${queries[@]}"; do + n=$((n + 1)) + run "search-$n" "$DIGUP" search "$query" -n 10 --index "$INDEX" + run "search-$n-json" "$DIGUP" search "$query" -n 10 --json --index "$INDEX" +done + +run search-video-json "$DIGUP" search wakeboarding --kind video -n 10 --json --index "$INDEX" +run search-fractal-json "$DIGUP" search "a fractal" --kind video -n 10 --json --index "$INDEX" +gate "eval (exit)" +run eval env DIGUP_TESTBED="$CORPUS" "$DIGUP" eval "$SRC/server/tests/corpus/corpus.json" -n 10 --index "$INDEX" +# The gate whose numbers the docs quote (Stage 2, corpus.json pins): the ALL row must +# clear R@1 ≥ 0.60 and R@5 ≥ 0.85 — the recorded values (0.71 / 0.88) sit above it with +# margin; a corpus whose ranking collapsed fails here, not in some doc's prose later. +gate "eval ALL row (R@1 ≥ 0.60, R@5 ≥ 0.85)" +if ! awk '/^ALL/{ok = ($3 >= 0.60 && $4 >= 0.85)} END {exit ok ? 0 : 1}' "$OUT/eval.log"; then + fails=$((fails + 1)) + echo "FAILED (gate: eval ALL row below R@1 0.60 / R@5 0.85) — see $OUT/eval.log" +fi +gate "status-after" +run status-after "$DIGUP" status --index "$INDEX" + +echo; echo "proof logs: $OUT" +if [ "$fails" -ne 0 ]; then + echo "PROOF FAILED: $fails gate(s) failed (last: ${gate_msg:-none})" + exit 1 +fi +echo "PROOF PASSED: all gates green" diff --git a/server/tests/test_api.py b/server/tests/test_api.py new file mode 100644 index 00000000..15e3faba --- /dev/null +++ b/server/tests/test_api.py @@ -0,0 +1,355 @@ +"""The API contract (Stage 4): `/api/v1` over the real adapter, the real engine, no mocks. + +Runs in the `server-test` container: TestClient drives the FastAPI app in-process; the +app's lifespan spawns `search-service` as a **direct child of pytest** — a live parent, +which is the ParentWatch rule (reconnaissance §15 item 8) the API must keep in production. + +Every assertion here is about *pass-through*: the API invents nothing and hides nothing — +the engine's ranking, fields, timings, empties and errors reach the client as the engine +made them, minus the protocol's own `id` bookkeeping. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + +from digup_server.api import SearchRequest, create_app + +# The three wakeboard media — the corpus's planted answer to "wakeboarding". Their ORDER +# is not portable: the embedding matmul moves with CPU features. Measured 2026-11-04 on +# x86_64 (the suite container, this box): logan 1.571 > vienna 1.531 > aarhus-04 0.441; +# the arm64-Mac measurements the strict pin carried before ranked vienna first — the top +# two are 0.040 z apart, inside cross-platform noise. What IS portable (and is exactly +# the server's contract) is WHICH three files answer the query, and that they clear the +# engine's gate: the server relays the engine's per-platform order verbatim — that +# pass-through is pinned by the hit-field and reply-shape tests, not by hard-coding one +# machine's ranking here. +WAKEBOARD_TOP3 = {"wakeboarding-vienna.webm", "wakeboard-logan.jpg", "wakeboarding-aarhus-04.jpg"} + + +@pytest.fixture(scope="module") +def client(corpus, index): + from conftest import _cfg + app = create_app(_cfg(index, (corpus,))) + with TestClient(app) as c: # lifespan: spawn search-service, quit() on exit + yield c + + +def paths(hits): + return [Path(h["path"]).name for h in hits] + + +def test_health_publishes_the_engines_identity(client): + body = client.get("/api/v1/health").json() + assert body["ok"] is True + eng = body["engine"] + assert "embeddinggemma-2@" in eng["fingerprints"] + assert eng["build_tag"].startswith("b") # llama.cpp b11461 + assert eng["vectors_version"] == 1 + live = body["live"] + assert live["files"] >= 15 and live["searchable"] == live["files"] + assert live["pending"] == 0 + assert live["encoder"] in {"stopped", "starting", "ready", "unavailable"} + + +def test_status_adds_the_cli_kind_table(client): + body = client.get("/api/v1/status").json() + kinds = {k["kind"]: k["files"] for k in body["kinds"]} + assert kinds.get("image") == 12 and kinds.get("video") == 3 + assert body["build_tag"].startswith("b") + + +def test_search_answers_with_the_engines_ranking(client): + reply = client.post("/api/v1/search", json={"q": "wakeboarding", "limit": 10}).json() + assert set(paths(reply["hits"])[:3]) == WAKEBOARD_TOP3 # the set is portable; the order is the engine's (see WAKEBOARD_TOP3) + assert reply["meaning"] is True and reply["strong"] >= 3 + assert {"embed", "search"} <= set(reply["ms"]) # the engine's own milliseconds + assert "id" not in reply # protocol bookkeeping stays inside + top = reply["hits"][0] + assert {"path", "kind", "segment", "segment_id", "score", "modified", "size", "info"} \ + <= set(top) # hitFields(), verbatim + + +def test_the_honest_empty_survives_the_http_leg(client): + reply = client.post("/api/v1/search", + json={"q": "a person riding a wakeboard behind a boat"}).json() + assert reply["hits"] == [] # the refusal is published, not filled + assert reply["meaning"] is True + + +def test_keyword_only_search_when_meaning_is_off(client): + reply = client.post("/api/v1/search", + json={"q": "wakeboarding", "meaning": False}).json() + assert reply["meaning"] is False + assert reply["hits"] # keyword-only still answers + + +def test_kind_filter_is_the_engines(client): + reply = client.post("/api/v1/search", + json={"q": "wakeboarding", "kinds": ["video"]}).json() + assert reply["hits"] and all(h["kind"] == "video" for h in reply["hits"]) + + +def test_the_default_limit_is_the_engines_default(client): + """The API's default `limit` equals the engine's own (40, `SearchService.swift:130`): a + client that names no limit gets the engine's answer, not a shorter one — the server + invents no cap of its own. Pinned twice: the typed default, and the reply on the same + query against an explicit 40 (the UI sends 40 explicitly, so nothing client-side moves).""" + assert SearchRequest.model_fields["limit"].default == 40 + quiet = client.post("/api/v1/search", json={"q": "wakeboarding"}).json() + explicit = client.post("/api/v1/search", json={"q": "wakeboarding", "limit": 40}).json() + assert paths(quiet["hits"]) == paths(explicit["hits"]) + assert quiet["strong"] == explicit["strong"] + + +def test_error_bodies_are_one_detail_string(client): + """API.md's error table is a promise: every documented error is one JSON object whose + `detail` is safe to show a user verbatim. Pinned on the cheap ones — an unknown job id, + a dead media handle on all three media endpoints, and the validation case whose `detail` + is a **list of objects** (the UI renders each item; a page that prints the raw array is + the bug this pins against).""" + unknown = client.get("/api/v1/jobs/deadbeefdeadbe") + assert unknown.status_code == 404 and isinstance(unknown.json()["detail"], str) + for suffix in ("preview", "text", "file"): + dead = client.get(f"/api/v1/media/deadbeefdeadbe/{suffix}") + assert dead.status_code == 404 and isinstance(dead.json()["detail"], str) + invalid = client.post("/api/v1/search", json={"q": "x", "kinds": ["not-a-kind"]}) + assert invalid.status_code == 422 + assert invalid.headers["content-type"].startswith("application/json") + detail = invalid.json()["detail"] + assert isinstance(detail, list) and detail + assert all({"type", "loc", "msg"} <= set(item) for item in detail) + + +def test_media_ids_are_per_reply_across_the_http_surface(client): + """`docs/API.md`'s rule, pinned over HTTP: the same file in two replies carries two + handles, so a client may not cache a `media_id` across searches (and the engine's own + `path` + `kind` + `segment_id` + `loc` are what it should key on instead).""" + def top(c): + hit = c.post("/api/v1/search", json={"q": "wakeboarding", "limit": 1}).json()["hits"][0] + return Path(hit["path"]).name, hit["media_id"] + (name_a, id_a) = top(client) + (name_b, id_b) = top(client) + assert name_a == name_b and id_a != id_b + + +def test_the_api_validates_what_the_engine_would_silently_drop(client): + bad = client.post("/api/v1/search", json={"q": "x", "kinds": ["not-a-kind"]}) + assert bad.status_code == 422 # engine drops; API says so + assert client.post("/api/v1/search", json={"q": "x" * 513}).status_code == 422 + assert client.post("/api/v1/search", json={"q": "x", "limit": 0}).status_code == 422 + assert client.post("/api/v1/search", json={}).status_code == 422 + + +def test_estimate_is_the_engines_forecast_as_json(client, corpus): + reply = client.post("/api/v1/estimate", json={}).json() + folder = reply["folders"][0] + assert folder["readable"] is True + assert folder["folder"] == str(corpus) + assert folder["files"]["image"] == 12 # the engine's own counts + + +def test_openapi_is_the_versioned_public_surface(client): + spec = client.get("/api/v1/openapi.json").json() + assert all(p.startswith("/api/v1/") for p in spec["paths"]) + assert {"health", "status", "search", "estimate"} <= \ + {p.rsplit("/", 1)[1] for p in spec["paths"]} + + +def test_engine_refusal_becomes_a_503_not_a_keyerror(corpus, index): + """A searcher mid-transition answers `{"ok":false,"error":…}` — the Stage-10 battery + hit exactly that as a 500 (`KeyError: 'hits'` in `api.py`). The refusal is the + engine's own words; they surface verbatim as a 503, and nothing reads `hits`/`files` + out of an answer that has none. No lifespan, no engine: the injected service refuses.""" + from conftest import _cfg + from digup_server.engine.protocol import EngineRefused + + app = create_app(_cfg(index, (corpus,))) + + class RefusingSearch: + def search(self, *args, **kwargs): + raise EngineRefused("search", "index is reloading, try again") + def status(self): + raise EngineRefused("status", "index is reloading, try again") + + app.state.search = RefusingSearch() + c = TestClient(app) # no `with`: no lifespan — the state stays + r = c.post("/api/v1/search", json={"q": "anything", "limit": 1}) + assert r.status_code == 503 + assert "index is reloading" in r.json()["detail"] + assert c.get("/api/v1/status").status_code == 503 + + +def test_a_refusal_reply_is_raised_in_the_adapter(index, corpus): + """The same fact one layer down: `SearchService` raises `EngineRefused` on an + `ok:false` reply (the engine's refusal, `SearchService.swift:106,115`) — stub proc, + no engine, the reply is the engine's refusal shape verbatim.""" + from conftest import _cfg + from digup_server.engine.protocol import EngineRefused + from digup_server.engine.search import SearchService + + svc = SearchService(_cfg(index, (corpus,))) + + class StubProc: + alive = True + def request(self, fields, *, timeout=None): + return {"id": 1, "ok": False, "error": "index is reloading"} + + svc._proc = StubProc() + with pytest.raises(EngineRefused) as raised: + svc.search("anything") + assert "index is reloading" in str(raised.value) + +def test_status_shows_the_servers_own_roots(client, corpus): + """Stage 10: the UI shows *where this server looks*. These are the deployment's own + configured paths (env) — publishing them is presentation, not path acceptance (§7).""" + assert client.get("/api/v1/status").json()["roots"] == [str(corpus)] + + +def test_startup_prewarms_the_searcher(client): + """Stage 7: the lifespan prewarms at startup (the engine's `prewarm` op), so a + freshly booted server answers the first search with meaning + real z-scores — + no cold-encoder keyword fallback in front of a first-time user.""" + status = client.get("/api/v1/status").json() + assert status["vectors"] > 0 # vectors opened at startup, not lazily + reply = client.post("/api/v1/search", json={"q": "wakeboarding", "limit": 3}).json() + assert reply["meaning"] is True and reply["strong"] >= 3 + + +# ── review-round units (this file's fixtures stay real: app + engine, no mocks) ──────── + + +def test_open_and_huge_ranges_are_capped_at_the_served_span(): + """`bytes=0-` is what every browser media element sends first. Uncapped, it read + the whole file into RAM (a 20 GB clip asked for a 20 GB buffer). Open ranges, + huge explicit ranges and huge suffix ranges all cap at MAX_RANGE_SPAN; the + 200 (no-range) path is untouched — `save as` still gets the whole file.""" + from digup_server.api import parse_range, MAX_RANGE_SPAN + big, huge = (1 << 30), (2 << 30) + assert parse_range("bytes=0-", huge) == (0, MAX_RANGE_SPAN - 1) + assert parse_range("bytes=-5000000000", big) == (big - MAX_RANGE_SPAN, big - 1) + assert parse_range("bytes=1000-9000000000", huge) == (1000, 1000 + MAX_RANGE_SPAN - 1) + assert parse_range("bytes=0-99", 500) == (0, 99) # normal spans untouched + assert parse_range("bytes=-40", 500) == (460, 499) + + +def test_hit_cache_is_a_true_lru_get_refreshes(): + """The docstring promised LRU; eviction was insertion-order. The UI re-searches + per keystroke-pause (up to 40 handles per search): after ~26 searches the handles + of the rows on screen were evicted, and their thumbnails/preview/file 404'd with + no recovery path. `get()` must move_to_end.""" + from digup_server.api import HitCache + cache = HitCache(capacity=3) + a = cache.put("/a.jpg", "image", 1, None, False) + b = cache.put("/b.jpg", "image", 2, None, False) + c = cache.put("/c.jpg", "image", 3, None, False) + assert cache.get(a) is not None # refresh a + d = cache.put("/d.jpg", "image", 4, None, False) # evicts b, not a + assert cache.get(a) is not None and cache.get(c) is not None + assert cache.get(d) is not None + assert cache.get(b) is None # the LRU victim: b + + +def test_search_reply_publishes_fields_this_model_has_never_seen(): + """`extra="allow"`: an engine field this model has never seen rides through to + the client, not dropped at the Pydantic boundary (hits already ride raw; the + top level must answer the engine's growth the same way).""" + from digup_server.api import SearchReply + reply = SearchReply(hits=[], strong=0, meaning=True, ms={}, fresh_engine_field={"a": 1}) + assert reply.model_dump()["fresh_engine_field"] == {"a": 1} + + +def test_a_slow_estimate_answers_busy_never_parallel(client, monkeypatch): + """`estimate` walks the whole tree (120 s timeout) as a sync endpoint: a second + UI tab, a user retry, a retry during a job all stacked full-tree crawls with no + mutual exclusion. Now the second click is a 409, the engine's one-crawl rule.""" + import threading + import time + import digup_server.api as api_mod + + def crawling(config, **kwargs): + time.sleep(3.0) # a slow crawl, in flight + return [{"path": str(config.roots[0]), "files": 15}] + + monkeypatch.setattr(api_mod, "estimate_lines", crawling) + box = [] + first = threading.Thread(target=lambda: box.append( + client.post("/api/v1/estimate", json={}))) + first.start() + time.sleep(0.3) # crawl #1 running + second = client.post("/api/v1/estimate", json={}) + assert second.status_code == 409 + assert "already running" in second.json()["detail"] + first.join(30) + assert box[0].status_code == 200 # crawl #1 finished honest + third = client.post("/api/v1/estimate", json={}) + assert third.status_code == 200 # the lock released + + +def test_the_estimate_timeout_is_an_operator_knob(client, monkeypatch): + """The 120 s default fits corpus-sized trees; a real media tree on a spinning disk + needs minutes (measured 2026-11-04: 187k files on a 2 TB HDD crossed the default → + 502). The route reads DIGUP_SERVER_ESTIMATE_TIMEOUT_S per request — default 120 + keeps every existing pin, an operator's value reaches the CLI call.""" + import digup_server.api as api_mod + + seen = {} + + def counted(config, **kwargs): + seen["timeout"] = kwargs.get("timeout") + return [{"path": str(config.roots[0]), "files": 15}] + + monkeypatch.setattr(api_mod, "estimate_lines", counted) + assert client.post("/api/v1/estimate", json={}).status_code == 200 + assert seen["timeout"] == 120.0 # the default, unchanged + monkeypatch.setenv("DIGUP_SERVER_ESTIMATE_TIMEOUT_S", "900") + assert client.post("/api/v1/estimate", json={}).status_code == 200 + assert seen["timeout"] == 900.0 # the operator's value + + +def test_a_cli_timeout_surfaces_as_a_quoted_502_not_a_500(client, monkeypatch): + """`subprocess.TimeoutExpired` is a SubprocessError, not a TimeoutError — no + handler covered it, so a `digup status` stalling behind the writer's WAL, or an + estimate crossing its 120 s, died as a bare 500 traceback. This file's own rule + (api.py:6): quote the engine. The mapping lives in cli.run, where it covers + every one-shot call at once.""" + import subprocess + import digup_server.engine.cli as cli_mod + + def timed_out(argv, **kwargs): + raise subprocess.TimeoutExpired(argv, 120) + + monkeypatch.setattr(cli_mod.subprocess, "run", timed_out) + reply = client.post("/api/v1/estimate", json={}) + assert reply.status_code == 502 + assert "timed out" in reply.json()["detail"] + + +def test_the_cli_identity_is_served_from_a_short_cache(): + """The UI polls /status every 15 s and the container HEALTHCHECK hits /health + every 30 s — each one used to spawn a `digup status` doing real SQLite counting. + The identity fields move slowly, so they ride a TTL cache: a poll costs an + argument, not a subprocess. A TTL of 0 is the dev escape hatch (every poll + re-reads, e.g. while watching a job's counts land).""" + from digup_server.api import StatusCache + + calls = [] + + def fetch(config): + calls.append(config) + return {"engine": {"fingerprints": "embeddinggemma-2@2188ac1d"}} + + cache = StatusCache(fetch, ttl_s=60.0) + first = cache.get("cfg") + second = cache.get("cfg") + assert len(calls) == 1 # the second poll: the cache, not a subprocess + assert first is second # the same identity dict, unchanged + assert first["engine"]["fingerprints"] == "embeddinggemma-2@2188ac1d" + + live = StatusCache(fetch, ttl_s=0.0) # TTL 0: every poll re-reads + live.get("cfg") + live.get("cfg") + assert len(calls) == 3 # 1 + 2: nothing served stale diff --git a/server/tests/test_api_docs.py b/server/tests/test_api_docs.py new file mode 100644 index 00000000..20980a62 --- /dev/null +++ b/server/tests/test_api_docs.py @@ -0,0 +1,52 @@ +"""The API reference stays true to the app (docs/API.md is the compatibility promise). + +`create_app()` builds the routes without spawning anything — the engine only starts inside the +lifespan — so this is a cheap tripwire that needs no engine, no corpus and no container: add an +endpoint and forget the docs, and this fails. The parameter *names* in the doc may differ from +the code's (`{id}` vs `{job_id}`), so each `{parameter}` segment matches any non-slash token. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +from digup_server.api import SearchRequest, create_app +from digup_server.engine import EngineConfig + +DOC = Path(__file__).resolve().parent.parent / "docs" / "API.md" + + +def app_paths() -> list[str]: + app = create_app(EngineConfig(index="/nonexistent-index", roots=("/nonexistent-root",))) + return sorted({r.path for r in app.routes if r.path.startswith("/api/v1")}) + + +def test_every_api_endpoint_is_documented(): + text = DOC.read_text(encoding="utf-8") + missing = [] + for path in app_paths(): + pattern = "/".join("[^/`]+" if seg.startswith("{") else re.escape(seg) + for seg in path.split("/")) + if not re.search(pattern, text): + missing.append(path) + assert not missing, f"endpoints missing from server/docs/API.md: {missing}" + + +def test_the_documented_default_limit_is_the_models_default(): + """The doc's `limit` row and the model's default cannot drift apart (step 1's decision: + the API default equals the engine's own default, 40).""" + default = SearchRequest.model_fields["limit"].default + rows = [line for line in DOC.read_text(encoding="utf-8").splitlines() + if line.startswith("| `limit` |")] + assert rows, "API.md documents no `limit` parameter row" + assert all(f"**{default}**" in line for line in rows), \ + f"API.md's limit row(s) do not carry the model's default ({default}): {rows}" + + +def test_the_api_version_and_the_openapi_surface_are_named_in_the_docs(): + text = DOC.read_text(encoding="utf-8") + app = create_app(EngineConfig(index="/nonexistent-index", roots=("/nonexistent-root",))) + assert app.openapi_url and app.docs_url + for url in (app.openapi_url, app.docs_url): + assert url in text, f"API.md does not name {url}" diff --git a/server/tests/test_cli.py b/server/tests/test_cli.py new file mode 100644 index 00000000..f1af6b6c --- /dev/null +++ b/server/tests/test_cli.py @@ -0,0 +1,46 @@ +"""The one-shot CLI the server reads: `status`, `estimate --json` + plain, `speed --model --json`.""" + +from __future__ import annotations + +import pytest + +from digup_server.engine import cli +from conftest import _cfg + + +def test_status_parses_the_engines_lines(corpus, index): + status = cli.status(_cfg(index, (corpus,))) + assert status["index"] == str(index / "index.sqlite") + assert status["vectors"] >= 15 + assert status["build_tag"] and status["build_tag"].startswith("b") + assert "embeddinggemma-2@" in status["fingerprints"] + assert status["index_size"] + kinds = {k["kind"]: k["files"] for k in status["kinds"]} + assert kinds.get("image") == 12 and kinds.get("video") == 3 + + +def test_estimate_is_the_engines_own_paragraph(corpus, index): + text = cli.estimate_text(_cfg(index, (corpus,))) + assert "first index" in text and "images" in text + assert "can't read" not in text + + +def test_estimate_json_answers_for_the_server_as_parent(corpus, corpus_root, index): + # `estimate --json` is a ParentWatch helper: live parent (pytest here; uvicorn in prod) + # → one JSON line per folder; parent PID 1 → it exits at once, "the app is gone", by + # design. The Stage-2 "silent estimate --json" report was that PID-1 invocation, not an + # engine fault (reconnaissance §15 item 8). The server is the parent, so it uses --json. + answer = cli.estimate_json(_cfg(index, (corpus_root,))) + assert answer["readable"] is True + assert answer["skipped"] >= 1 and answer["left_out"] # 29 skipped files, counted + media = cli.estimate_json(_cfg(index, (corpus,))) + assert media["files"]["image"] == 12 + + +@pytest.mark.slow +def test_speed_model_json_reports_the_device(corpus, index): + reply = cli.speed_model(_cfg(index, (corpus,)), when_quiet_s=2.0, tries=2, timeout=600.0) + assert reply["event"] == "speed" + assert reply["measured"] is True + assert {"kinds", "seconds", "load", "total", "tries", "checks"} <= set(reply) + assert reply["kinds"] # per-kind multipliers, the engine's diff --git a/server/tests/test_index_contract.py b/server/tests/test_index_contract.py new file mode 100644 index 00000000..439fed8d --- /dev/null +++ b/server/tests/test_index_contract.py @@ -0,0 +1,123 @@ +"""The index-service contract: plain-line commands, JSON events, the single writer, reload. + +Events are the engine's counts as it emits them (`IndexService.swift`): a sync that names +its numbers, file events that name files, a summary, and the exit event when the helper +lingers out. A missing folder is *reported* and indexing goes on — never "everything +deleted". The server's flock makes a second writer ask and lose, by design. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from conftest import _cfg +from digup_server.engine import IndexBusy, IndexService, SearchService, cli + + +def events_of(events, name): + return [e for e in events if e.get("event") == name] + + +def test_sync_events_count_real_files(scratch, scratch_index): + _folder, events = scratch_index + syncs = events_of(events, "sync") + assert syncs, "the engine emitted no sync event" + first = syncs[0] + # First sync counts the crawl; the engine re-syncs between batches (`new` = queued for + # embedding), and the counts ride every count event. The post-index truth rides `idle`. + assert first["new"] == 6 # 4 photos + the clip + notes.txt — the engine's own count + assert {"added", "changed", "moved", "removed", "unchanged", "ms", "started", "left_out", + "files", "searchable", "pending", "vectors", "unfinished"} <= set(first) + assert first["added"] == 6 + files = events_of(events, "file") + assert len(files) == 6 + assert all(f["state"] == "done" for f in files) + assert all({"path", "kind", "seconds", "done", "total", "new", "note", "code"} <= set(f) + for f in files) + kinds = {Path(f["path"]).name: f["kind"] for f in files} + assert kinds["notes.txt"] == "doc" and kinds["scenes.mp4"] == "video" + summaries = events_of(events, "summary") + assert len(summaries) >= 2 and all("states" in s for s in summaries) # batches as they land + idle = events_of(events, "idle")[-1] + assert idle["files"] == 6 and idle["searchable"] == 6 and idle["pending"] == 0 + assert idle["vectors"] >= 6 # the measured totals, not an invented percent + assert events[-1]["event"] == "exit" # lingered out: the normal end + assert not events_of(events, "error") + + +def test_counts_land_in_the_index(scratch, scratch_index): + folder, _ = scratch_index + status = cli.status(_cfg(folder, (scratch,))) + assert sum(k["files"] for k in status["kinds"]) == 6 + by_kind = {k["kind"]: k["files"] for k in status["kinds"]} + assert by_kind == {"image": 4, "video": 1, "doc": 1} + assert status["vectors"] >= 6 # every file embedded at least one segment + + +def test_a_second_writer_is_told_busy(scratch, scratch_index): + folder, _ = scratch_index + first = IndexService(_cfg(folder, (scratch,)), linger_s=30.0) + first.start() + try: + second = IndexService(_cfg(folder, (scratch,))) + with pytest.raises(IndexBusy): + second.start() + finally: + first.stop() + first.finish() + + +def test_missing_root_is_reported_not_fatal(tmp_path_factory, scratch): + folder = tmp_path_factory.mktemp("missing-index") + gone = str(tmp_path_factory.mktemp("gone") / "not-here") # never existed + service = IndexService(_cfg(folder, (scratch, gone)), linger_s=2.0) + service.start() + service.sync() + events = list(service.events(timeout=900.0)) + service.finish() + missing = events_of(events, "missing") + assert missing and gone in missing[-1]["folders"] + assert not events_of(events, "error") + assert events_of(events, "file") # the real folder was indexed regardless + + +def test_the_text_a_hit_points_at(scratch, scratch_index): + folder, _ = scratch_index + service = SearchService(_cfg(folder, (scratch,))) + service.start() + try: + service.prewarm() # start the query encoder, load the vectors + hits = service.search("the stories continued after midnight, long after the marshmallows", + kinds=["doc"], limit=5)["hits"] + top = hits[0] + assert Path(top["path"]).name == "notes.txt" + text = service.text(top["segment_id"])["text"] + assert "campfire stories" in text # the passage the hit points at, verbatim + finally: + service.quit() + + +def test_reload_after_indexing(scratch, scratch_index): + folder, _ = scratch_index + service = SearchService(_cfg(folder, (scratch,))) + service.start() + try: + before = service.status()["vectors"] # 0: a fresh searcher loads vectors lazily + # A new file arrives, indexing runs through the single writer… + (scratch / "more").mkdir(exist_ok=True) + (scratch / "more" / "bicycle.txt").write_text( + "A blue bicycle leans against the garage wall in the evening light.\n", encoding="utf-8") + writer = IndexService(_cfg(folder, (scratch,)), backfill=True, linger_s=2.0) + writer.run_sync(timeout=900.0) + # …`reload` reopens the index, `prewarm` loads the vectors (the shell's own sequence) + # and the engine reports a count that moved. + reply = service.reload() + assert reply["ok"] is True + warm = service.prewarm() + assert warm["vectors"] > before + hits = service.search("blue bicycle garage", kinds=["doc"], limit=5)["hits"] + assert Path(hits[0]["path"]).name == "bicycle.txt" + finally: + service.quit() diff --git a/server/tests/test_integrations.py b/server/tests/test_integrations.py new file mode 100644 index 00000000..4427ddbe --- /dev/null +++ b/server/tests/test_integrations.py @@ -0,0 +1,325 @@ +"""Stage 12 integration groundwork (docs/implementation/stage-12-integrations.md): the +named-token map, the CORS allowlist, the principal log, the embed widget, and the +Nextcloud compose shape. The middleware tests are engine-free (pure ASGI, the +test_security pattern); the last test drives the token map through the real API with +the real engine. The rules they pin: + +* `DIGUP_SERVER_READ_TOKENS=name:token,…` — every entry guards the read scope, each is + attributable by name, one entry's revocation touches no other; +* the CORS allowlist is exact origins only, inert when unset, and it is the boundary + that lets an allowlisted browser widget POST (the guard's cross-origin exception is + set ONLY by a CORS-passed preflight, stage-12 §11.4); +* every guarded request lands in the log with its accepted principal; +* the embed widget follows the same UI rules as the full UI (no external URLs, the + built Lit+Vite shape served same-origin, server-issued media handles, the token + never in a `<img>`-visible URL). +""" + +from __future__ import annotations + +import json +import logging +import re + +import pytest +from fastapi.testclient import TestClient + +from digup_server.security import ( + Auth, + BearerAuth, + Cors, + HostOriginGuard, + PrincipalLog, + allowed_origins_from_env, + auth_from_env, + read_tokens_from_env, +) + + +async def echo(scope, receive, send) -> None: + """Minimal ASGI app: answers 200 with what the middlewares wrote on the scope, so a + request that passed is distinguishable from one refused — and the accepted principal + (BearerAuth) and the CORS pass-mark (Cors) are readable from the answer.""" + body = json.dumps({"ok": True, "method": scope["method"], "path": scope["path"], + "principal": scope.get("digup.principal"), + "cors_allowed": scope.get("digup.cors_allowed", False)}).encode() + await send({"type": "http.response.start", "status": 200, + "headers": [(b"content-type", b"application/json")]}) + await send({"type": "http.response.body", "body": body}) + + +# ── the named-token map: parsing, scopes, revocation-isolation ─────────────────────────── + +def test_the_map_parses_names_and_tokens(): + assert read_tokens_from_env({}) == {} + assert read_tokens_from_env({"DIGUP_SERVER_READ_TOKENS": " "}) == {} + m = read_tokens_from_env({"DIGUP_SERVER_READ_TOKENS": "nc-a:tokA, nc-b:tokB,NC-A:dup"}) + assert m == {"nc-a": "tokA", "nc-b": "tokB"} # names lowercase; FIRST wins on dupes + assert read_tokens_from_env({"DIGUP_SERVER_READ_TOKENS": "nc-a:tokA,broken-no-colon"}) == \ + {"nc-a": "tokA"} # a typo is skipped, loudly: + + +def test_a_dropped_map_entry_names_itself_in_the_log(caplog): + with caplog.at_level(logging.WARNING, logger="digup.auth"): + read_tokens_from_env({"DIGUP_SERVER_READ_TOKENS": "nc-a:tokA,typo-entry"}) + assert any("typo-entry" in r.message for r in caplog.records), \ + "a silently-dropped token is an outage nobody asked for" + + +def test_auth_from_env_assembles_all_three_knobs(): + a = auth_from_env({"DIGUP_SERVER_TOKEN": "plain", "DIGUP_SERVER_ADMIN_TOKEN": "adm", + "DIGUP_SERVER_READ_TOKENS": "nc-a:tokA,nc-b:tokB"}) + assert a.read == "plain" and a.admin == "adm" + assert a.readers == {"nc-a": "tokA", "nc-b": "tokB"} + assert a.enabled is True + assert a.token_names("tokB") == ("nc-b",) + assert a.token_names("adm") == ("admin",) # admin covers the read scope too + assert a.token_names("nope") == () + + +def bearer_client(auth: Auth) -> TestClient: + return TestClient(PrincipalLog(BearerAuth(echo, auth=auth))) + + +def test_map_tokens_guard_reads_and_admin_stays_admins_alone(): + both = bearer_client(Auth(readers={"nc-a": "tokA", "nc-b": "tokB"}, admin="adm")) + for token in ("tokA", "tokB"): + assert both.post("/api/v1/search", + headers={"Authorization": f"Bearer {token}"}).status_code == 200 + refused = both.post("/api/v1/index", headers={"Authorization": f"Bearer {token}"}) + assert refused.status_code == 403 and "admin" in refused.json()["detail"] + assert both.post("/api/v1/index", + headers={"Authorization": "Bearer adm"}).status_code == 200 + assert both.post("/api/v1/search", headers={"Authorization": "Bearer nope"}).status_code == 401 + assert both.post("/api/v1/search").status_code == 401 + + +def test_each_request_is_attributed_to_the_principal_that_asked(caplog): + both = bearer_client(Auth(read="plain", readers={"nc-a": "tokA", "nc-b": "tokB"}, admin="adm")) + with caplog.at_level(logging.INFO, logger="digup.auth"): + assert both.post("/api/v1/search", headers={"Authorization": "Bearer tokA"}).status_code == 200 + assert both.post("/api/v1/search", headers={"Authorization": "Bearer tokB"}).status_code == 200 + assert both.post("/api/v1/search", headers={"Authorization": "Bearer plain"}).status_code == 200 + assert both.post("/api/v1/search", headers={"Authorization": "Bearer adm"}).status_code == 200 + assert both.post("/api/v1/index", headers={"Authorization": "Bearer adm"}).status_code == 200 + lines = [r.message for r in caplog.records if "principal=" in r.message] + assert "principal=nc-a" in lines[0] and "principal=nc-b" in lines[1] + assert "principal=read" in lines[2] + assert "principal=admin (read scope)" in lines[3] # an admin token on the read side says so + assert "principal=admin" in lines[4] + # health/docs stay out of the audit line: they are the healthcheck and the docs page + both.get("/api/v1/health") + assert not any("/api/v1/health" in r.message for r in caplog.records) + + +def test_an_unguarded_deployment_still_logs_its_principals_as_none(caplog): + open_book = bearer_client(Auth()) + with caplog.at_level(logging.INFO, logger="digup.auth"): + assert open_book.post("/api/v1/search").status_code == 200 + assert any(r.message.endswith("principal=none") and "/api/v1/search" in r.message + for r in caplog.records) + + +# ── the CORS allowlist: exact origins, inert by default, the widget's boundary ─────────── + +def test_the_cors_allowlist_is_exact_and_off_by_default(): + assert allowed_origins_from_env({}) is None + assert allowed_origins_from_env({"DIGUP_SERVER_ALLOWED_ORIGINS": " "}) is None + origins = allowed_origins_from_env( + {"DIGUP_SERVER_ALLOWED_ORIGINS": "http://Nextcloud.Lan:8081/, http://a.example"}) + assert origins == frozenset({"http://nextcloud.lan:8081", "http://a.example"}) + inert = TestClient(Cors(echo)) + preflight = inert.options("/api/v1/search", headers={"Origin": "http://a.example"}) + assert "access-control-allow-origin" not in preflight.headers + plain = inert.get("/api/v1/status", headers={"Origin": "http://a.example"}) + assert "access-control-allow-origin" not in plain.headers # off = no CORS headers, period + + +def test_an_allowlisted_origin_gets_preflight_and_decorated_answers(): + cors = TestClient(Cors(echo, allowed_origins=frozenset({"http://nc.example"}))) + pre = cors.options("/api/v1/search", headers={"Origin": "http://nc.example", + "access-control-request-method": "POST"}) + assert pre.status_code == 204 + assert pre.headers["access-control-allow-origin"] == "http://nc.example" + assert "POST" in pre.headers["access-control-allow-methods"] + assert "authorization" in pre.headers["access-control-allow-headers"] + assert pre.headers["vary"] == "Origin" + posted = cors.post("/api/v1/search", headers={"Origin": "http://nc.example"}) + assert posted.status_code == 200 + assert posted.headers["access-control-allow-origin"] == "http://nc.example" + assert posted.json()["cors_allowed"] is True # the guard's exception is marked + # an origin NOT on the list: the middleware passes through, decorates nothing + off = cors.post("/api/v1/search", headers={"Origin": "http://evil.example"}) + assert off.status_code == 200 + assert "access-control-allow-origin" not in off.headers + assert off.json()["cors_allowed"] is False + # a request with no Origin (server-to-server: PHP, curl, agents) is untouched + assert "access-control-allow-origin" not in cors.get("/api/v1/status").headers + + +# ── the composed stack: Cors → Guard → Bearer, the widget's real path ─────────────────── + +def widget_stack() -> TestClient: + """The stack create_app installs, over the echo app: the operator names the hosts it + serves and the one widget origin it embeds, with the read token for the widget.""" + app = Cors(HostOriginGuard(BearerAuth(echo, auth=Auth(read="tokA")), + allowed_hosts=frozenset({"nc.example", "127.0.0.1"})), + allowed_origins=frozenset({"http://nc.example"})) + return TestClient(app) + + +def test_the_allowlisted_widget_posts_cross_origin_with_the_bearer_token(): + stack = widget_stack() + widget = {"Host": "nc.example:8787", "Origin": "http://nc.example", + "Authorization": "Bearer tokA"} + ok = stack.post("/api/v1/search", headers=widget) + assert ok.status_code == 200 and ok.json()["principal"] == "read" + assert ok.headers["access-control-allow-origin"] == "http://nc.example" + # the preflight lands before the POST and passes the guard (OPTIONS is not state-changing) + pre = stack.options("/api/v1/search", headers={"Host": "nc.example:8787", + "Origin": "http://nc.example"}) + assert pre.status_code == 204 + + +def test_an_unallowlisted_browser_still_hits_the_same_origin_rule(): + """The guard's cross-origin POST refusal survives CORS: a page the operator never + allowlisted gets no exception, token or no token.""" + stack = widget_stack() + refused = stack.post("/api/v1/search", headers={"Host": "nc.example:8787", + "Origin": "http://evil.example", + "Authorization": "Bearer tokA"}) + assert refused.status_code == 403 + # server-to-server (no Origin at all) remains the normal, unblessed path + assert stack.post("/api/v1/search", headers={"Host": "nc.example:8787", + "Authorization": "Bearer tokA"}).status_code == 200 + # the host allowlist outranks everything: a stale Host is refused before tokens + assert stack.post("/api/v1/search", headers={"Host": "evil.example", + "Origin": "http://nc.example", + "Authorization": "Bearer tokA"}).status_code == 400 + + +def test_the_app_installs_all_four_middlewares_knobs_from_env(monkeypatch): + from digup_server.api import create_app + from digup_server.engine import EngineConfig + app = create_app(EngineConfig(index="/nonexistent-index", roots=("/nonexistent-root",))) + classes = [m.cls for m in app.user_middleware] + for cls in (BearerAuth, HostOriginGuard, Cors, PrincipalLog): + assert cls in classes, f"{cls.__name__} must be installed" + assert app.user_middleware[0].cls is PrincipalLog # added last = outermost: logs every guarded request + assert next(m for m in app.user_middleware if m.cls is Cors).kwargs["allowed_origins"] is None + assert next(m for m in app.user_middleware if m.cls is BearerAuth).kwargs["auth"].enabled is False + monkeypatch.setenv("DIGUP_SERVER_READ_TOKENS", "nc-a:tokA") + monkeypatch.setenv("DIGUP_SERVER_ALLOWED_ORIGINS", "http://nc.example") + app = create_app(EngineConfig(index="/nonexistent-index", roots=("/nonexistent-root",))) + assert next(m for m in app.user_middleware + if m.cls is Cors).kwargs["allowed_origins"] == frozenset({"http://nc.example"}) + assert next(m for m in app.user_middleware + if m.cls is BearerAuth).kwargs["auth"].readers == {"nc-a": "tokA"} + + +# ── the embed widget: the same UI rules as the full UI, over the same static mount ────── + +def embed_client(): + from digup_server.api import create_app + from digup_server.engine import EngineConfig + return TestClient(create_app(EngineConfig(index="/nonexistent-index", + roots=("/nonexistent-root",)))) + + +def test_the_embed_page_is_served_and_dependency_free(): + client = embed_client() + page = client.get("/embed.html") + assert page.status_code == 200 + html = page.text + assert "DigUp" in html and "embed.js" in html and "embed.css" in html + assert not re.search(r'(src|href)="(https?:)?//', html), "the widget must load no external URLs" + # The built shape (2026-11-08 migration): ES-module entry, and the shared measured + # palette rides with the widget's own CSS graph (Vite extracts the tokens into a + # hashed stylesheet the html links — the widget page stays self-contained). + assert re.search(r'type="module"', html) + assert re.search(r'href="\./(assets/[^"]+\.css)"', html) + assert 'name="referrer" content="none"' in html # no referrer leaves the widget + for asset in ("/embed.css", "/embed.js"): + assert client.get(asset).status_code == 200 + # the shared static mount decorates every embed byte the same way (no stale UI) + assets = ["/embed.html", "/embed.css", "/embed.js"] + assets += ["/" + p for p in re.findall(r'href="\./((?:chunks|assets)/[^"]+)"', html)] + for asset in assets: + reply = client.get(asset) + assert reply.status_code == 200, asset + assert reply.headers["cache-control"] == "no-cache", asset + assert "etag" in reply.headers, asset + + +def test_the_embed_widget_uses_the_published_api_and_hides_the_token(): + # The widget's bundle: the embed entry plus the chunks its html links (the api + # service is a shared module — the pins below name the rules, and the rules are + # tested as behavior in src/embed.test.js / src/services/api.test.js). + client = embed_client() + html = client.get("/embed.html").text + js = client.get("/embed.js").text + "\n" + "\n".join( + client.get("/" + p).text for p in re.findall(r'href="\./((?:chunks|assets)/[^"]+\.js)"', html)) + assert "fetch(" in js and "/api/v1/search" in js and "/api/v1/embed_ahead" in js + # embed.html DECLARES the widget; the entry wires that element (the double-mount + # bug — an unwired twin above the working widget — is pinned out, 2026-11-08). + assert html.count("<digup-embed></digup-embed>") == 1 + assert 'document.querySelector("digup-embed")' in js + assert "media_id" in js and "/preview" in js and "/file" in js # server-issued handles + assert "sessionStorage" in js and "history.replaceState" in js # the token leaves the URL + assert "URL.createObjectURL" in js and "IntersectionObserver" in js + assert "postMessage" in js and "digup-host" in js # the host drives the theme + assert "embed_ahead" in js and "meaning" in js # keyword-first typing + assert 'This DigUp server asks for its read token' in js # 401 asks once + + +# ── the token map over the REAL API + engine: what the Nextcloud app's calls do ────────── + +def test_estimate_argv_carries_the_excludes_the_job_will_honour(): + """Stage-12 §11.3, measured on the Nextcloud pilot: the forecast must describe the + tree the indexer WOULD walk. An estimate that counted the folders the job excludes + promised work nobody will do (the pilot counted 120 with 36 real files).""" + from digup_server.engine import EngineConfig + cfg = EngineConfig(index="/idx", roots=("/media/photos",), excluded=("/media/photos/appdata_x",), + excluded_extensions=("heic",)) + argv = cfg.estimate_argv() + assert argv[:3] == ["digup", "estimate", "/media/photos"] + assert argv.count("--exclude") == 1 and "--exclude" in argv + assert argv[argv.index("--exclude") + 1] == "/media/photos/appdata_x" + assert argv[argv.index("--exclude-ext") + 1] == "heic" + assert argv[-1] == "--json" + plain = EngineConfig(index="/idx", roots=("/r",)).estimate_argv() + assert "--exclude" not in plain and "--exclude-ext" not in plain # knobs inert when unset + # Measured 2026-11-08 on the pilot: the job path was the gap — index-service ran with + # no excludes and walked the tree's appdata (1.47M preview files) while estimate + # (carrying them) reported 131,972. The job must carry the SAME list. + job = cfg.index_argv() + assert job[:3] == ["digup", "index-service", "/media/photos"] + assert job.count("--exclude") == 1 and job.count("--exclude-ext") == 1 + assert job[job.index("--exclude") + 1] == "/media/photos/appdata_x" + assert job[job.index("--exclude-ext") + 1] == "heic" + plain_job = EngineConfig(index="/idx", roots=("/r",)).index_argv() + assert "--exclude" not in plain_job and "--exclude-ext" not in plain_job + + +def test_the_map_protects_the_real_api_end_to_end(corpus, index, monkeypatch): + from conftest import _cfg + from digup_server.api import create_app + monkeypatch.setenv("DIGUP_SERVER_READ_TOKENS", "nc-a:tokA,nc-b:tokB") + monkeypatch.setenv("DIGUP_SERVER_ADMIN_TOKEN", "adm") + with TestClient(create_app(_cfg(index, (corpus,)))) as c: + assert c.get("/api/v1/health").status_code == 200 # health: open, always + assert c.post("/api/v1/search", json={"q": "wakeboarding"}).status_code == 401 + for token in ("tokA", "tokB"): + reply = c.post("/api/v1/search", json={"q": "wakeboarding", "limit": 3}, + headers={"Authorization": f"Bearer {token}"}) + assert reply.status_code == 200 and reply.json()["hits"] + assert c.post("/api/v1/index", + headers={"Authorization": f"Bearer {token}"}).status_code == 403 + assert c.get("/api/v1/media/deadbeefcafe/preview", + headers={"Authorization": f"Bearer {token}"}).status_code == 404 + # POST /index with the admin token starts a real writer job on the session index — + # that path is pinned end-to-end in test_jobs.py over a scratch index, not here. + assert c.post("/api/v1/estimate", json={}, + headers={"Authorization": "Bearer adm"}).status_code == 200 + assert c.post("/api/v1/estimate", json={}).status_code == 401 + assert c.get("/api/v1/jobs", headers={"Authorization": "Bearer tokA"}).status_code == 200 diff --git a/server/tests/test_jobs.py b/server/tests/test_jobs.py new file mode 100644 index 00000000..6ebad9c9 --- /dev/null +++ b/server/tests/test_jobs.py @@ -0,0 +1,478 @@ +"""The index-job contract (Stage 5): one writer, answered busy, honest states, recovery. + +Runs in the `server-test` container over the **scratch corpus** (conftest: 4 photos + the +clip + notes.txt, already indexed): a new file arrives, `POST /api/v1/index` runs the real +`index-service` under the adapter's flock, the job's counts are the engine's own events, +and after `done` the searcher answers for the new file — the `reload → prewarm` sequence +the Stage-3 suite proved, wired by the app itself. No mocks. + +The interrupted-recovery test is the rule from README §5 verbatim: *an interrupted job is +recovered as interrupted, never as success* — a log line `running` with no ending becomes +`interrupted` at the next startup. +""" + +from __future__ import annotations + +import json +import os +import threading +import time +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + +from digup_server.api import create_app +from digup_server.engine import EngineConfig +from digup_server.jobs import JobRunner + +from conftest import _cfg + +NEW_SENTENCE = "A lighthouse stands above a stormy sea at dusk, its beam sweeping the waves.\n" + + +def test_an_interrupted_record_always_carries_the_counts_it_saw(): + """The rule the stop-mid-job test and README §5 both promise: an interrupted job reports + the counts the engine really emitted. The live job record is seeded at zero the first + moment of the walk (`_run_one`) and `_live_counts` publishes missing keys as None — a + job stopped before any file finished says **0**, not nothing (measured: a stop landing + during the model load took the `EngineExited` path and published an interrupted record + with no counts at all, which is why the counters ride the live record from the first + moment and every ending reads them from there, never from a thread-local list).""" + runner = JobRunner.__new__(JobRunner) # the counts bookkeeping, no worker + runner._jobs = {"j1": {"id": "j1"}} + runner._lock = threading.Lock() + runner._progress("j1", files_done=0, files_failed=0, errors=0) # _run_one's seeding + counts = runner._live_counts("j1") + assert counts["files_done"] == 0 and counts["files_failed"] == 0 and counts["errors"] == 0 + assert counts["files"] is None and counts["vectors"] is None # no `idle` totals yet + + +def test_the_job_timeout_is_an_operator_knob(scratch, scratch_index, tmp_path_factory, monkeypatch): + """3600 s is the runner's patience for corpora (the scratch walk finishes in seconds). + A real tree's first pass outlives an hour — measured 2026-11-04: a 131,972-file job + recorded *interrupted with its partial counters* at 19,007 when the default expired + (and the next job resumed — README §5). The operator owns the ceiling: + DIGUP_SERVER_JOB_TIMEOUT_S, read at the lifespan; the default stays 3600.""" + folder, _ = scratch_index + monkeypatch.setenv("DIGUP_SERVER_CONFIG", str(tmp_path_factory.mktemp("job-timeout-config"))) + monkeypatch.setenv("DIGUP_SERVER_JOB_TIMEOUT_S", "900") + with TestClient(create_app(_cfg(folder, (scratch,)))) as c: + assert c.app.state.jobs._timeout == 900.0 # the operator's value + monkeypatch.delenv("DIGUP_SERVER_JOB_TIMEOUT_S") + with TestClient(create_app(_cfg(folder, (scratch,)))) as c: + assert c.app.state.jobs._timeout == 3600.0 # the default, unchanged + + +@pytest.fixture(scope="module") +def jobs_client(scratch, scratch_index, tmp_path_factory): + """The app over the scratch corpus, with its own job log folder.""" + folder, _ = scratch_index + config_dir = tmp_path_factory.mktemp("job-config") + saved = os.environ.get("DIGUP_SERVER_CONFIG") + os.environ["DIGUP_SERVER_CONFIG"] = str(config_dir) + try: + with TestClient(create_app(_cfg(folder, (scratch,)))) as c: + yield c + finally: + if saved is None: + os.environ.pop("DIGUP_SERVER_CONFIG", None) + else: + os.environ["DIGUP_SERVER_CONFIG"] = saved + + +def wait_job(client, job_id, timeout=300.0): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + job = client.get(f"/api/v1/jobs/{job_id}").json()["job"] + if job["state"] in ("done", "interrupted", "busy"): + return job + time.sleep(0.5) + pytest.fail(f"job {job_id} never reached a final state within {timeout:g} s") + + +def test_job_runs_and_the_new_file_searches_back(jobs_client, scratch): + # A busy window first: the job POSTed here is live, and a second request is answered. + first = jobs_client.post("/api/v1/index", json={}) + assert first.status_code == 202 + live = first.json()["job"] + assert live["state"] == "queued" + busy = jobs_client.post("/api/v1/index", json={}) + assert busy.status_code == 409 # answered, never queued blind + assert live["id"] in busy.json()["detail"] + assert wait_job(jobs_client, live["id"])["state"] == "done" # quiet corpus: counts, 0 new + + # Now the real stretch: a new file, indexed by a job, searchable after it. + (scratch / "lighthouse.txt").write_text(NEW_SENTENCE, encoding="utf-8") + started = jobs_client.post("/api/v1/index", json={}) + assert started.status_code == 202 + job = wait_job(jobs_client, started.json()["job"]["id"]) + assert job["state"] == "done" + assert job["files_done"] >= 1 # the engine's file events, counted + assert job["errors"] == 0 + assert job["searchable"] >= 7 and job["pending"] == 0 # the `idle` totals, the engine's + reply = jobs_client.post("/api/v1/search", + json={"q": "lighthouse stormy sea at dusk", "kinds": ["doc"], + "limit": 5}).json() + assert Path(reply["hits"][0]["path"]).name == "lighthouse.txt" # reload→prewarm happened + + +def test_job_log_holds_the_engines_counts(jobs_client): + jobs = jobs_client.get("/api/v1/jobs").json()["jobs"] + done = [j for j in jobs if j["state"] == "done"] + assert done, "no completed job in the log" + last = done[-1] + assert {"id", "state", "started", "ended", "seconds", "files", "searchable", + "pending", "vectors", "files_done", "files_failed", "errors"} <= set(last) + assert last["vectors"] >= 7 # every file embedded, the engine's + + +def test_unknown_job_is_404(jobs_client): + assert jobs_client.get("/api/v1/jobs/not-a-job").status_code == 404 + + +def test_a_run_never_logged_finished_recovers_as_interrupted(scratch, scratch_index, tmp_path): + folder, _ = scratch_index + config_dir = tmp_path / "config" + config_dir.mkdir() + (config_dir / "jobs.jsonl").write_text( + json.dumps({"id": "abc123", "state": "running", "started": 0}) + "\n", encoding="utf-8") + runner = JobRunner(_cfg(folder, (scratch,)), config_dir=str(config_dir)) + try: + assert runner.job("abc123")["state"] == "interrupted" # never success — README §5 + lines = [json.loads(l) for l in + (config_dir / "jobs.jsonl").read_text().splitlines()] + assert lines[-1]["id"] == "abc123" and lines[-1]["state"] == "interrupted" + finally: + runner.shutdown() + +CANARY = "A red canoe rests on a glass-still lake under the northern lights.\n" + + +def test_plain_sync_retires_walked_root_ghosts(jobs_client, scratch): + """Found by suite 11: the engine's plain sync retires rows for files missing under + the roots it walks — the walk *is* the scope, and a vanished file inside it is + removed at sync. (Rows whose paths are outside the walked roots are a different + animal: the next test pins them, with prune.)""" + before = jobs_client.get("/api/v1/status").json()["searchable"] + + (scratch / "canary.txt").write_text(CANARY, encoding="utf-8") + job = jobs_client.post("/api/v1/index", json={}).json()["job"] + assert wait_job(jobs_client, job["id"])["state"] == "done" + assert jobs_client.get("/api/v1/status").json()["searchable"] == before + 1 + hits = jobs_client.post("/api/v1/search", json={ + "q": "red canoe glass-still lake northern lights", "kinds": ["doc"], + "limit": 3}).json()["hits"] + assert any(h["path"].endswith("canary.txt") for h in hits) # indexed, searchable + + (scratch / "canary.txt").unlink() # the file is gone + job = jobs_client.post("/api/v1/index", json={}).json()["job"] # plain sync… + assert wait_job(jobs_client, job["id"])["state"] == "done" + assert jobs_client.get("/api/v1/status").json()["searchable"] == before # …row gone + hits = jobs_client.post("/api/v1/search", json={ + "q": "red canoe glass-still lake northern lights", "kinds": ["doc"], + "limit": 3}).json()["hits"] + assert not any(h["path"].endswith("canary.txt") for h in hits) # row gone, hits gone + + +def test_out_of_scope_rows_survive_plain_sync_and_retire_under_prune(tmp_path): + """The compose-proof ghost, at scratch scale: rows whose paths live OUTSIDE the + server's roots (the old root scope after a remount) keep counting in + `files`/`searchable`/`vectors` — plain syncs walk the current roots and never + touch them; diluting z for live hits is what that dilution looked like in the + proof. `prune: true` — the engine's `--prune` — retires them: "files no longer + found under these roots" includes paths outside the roots. The persisted job + record echoes the flag, and survives a runner restart (replay).""" + s2 = tmp_path / "old-root" + s2.mkdir() + (s2 / "canary2.txt").write_text(CANARY, encoding="utf-8") + idx = tmp_path / "ghost-index" + idx.mkdir() + nowater = {"DIGUP_SERVER_PREWARM": "0"} # status ops don't need the model + saved = os.environ.get("DIGUP_SERVER_PREWARM") + + os.environ["DIGUP_SERVER_PREWARM"] = "0" + os.environ["DIGUP_SERVER_CONFIG"] = str(tmp_path / "job-config") + try: + # A server whose roots INCLUDE the folder: the canary is indexed, searchable. + with TestClient(create_app(_cfg(idx, (s2,)))) as scope_a: + job = scope_a.post("/api/v1/index", json={}).json()["job"] + assert wait_job(scope_a, job["id"])["state"] == "done" + assert scope_a.get("/api/v1/status").json()["searchable"] == 1 + + # The server AFTER the remount: its roots no longer include that folder. + empty = tmp_path / "new-root" + empty.mkdir() + with TestClient(create_app(_cfg(idx, (empty,)))) as scope_b: + status = scope_b.get("/api/v1/status").json() + assert status["searchable"] == 1 and status["files"] == 1 # the ghost counts + + job = scope_b.post("/api/v1/index", json={}).json()["job"] # plain sync… + assert wait_job(scope_b, job["id"])["state"] == "done" + assert scope_b.get("/api/v1/status").json()["searchable"] == 1 # …row SURVIVES + + started = scope_b.post("/api/v1/index", json={"prune": True}) + job_id = started.json()["job"]["id"] + assert started.json()["job"].get("prune") is True # echoed live + assert wait_job(scope_b, job_id)["state"] == "done" + assert scope_b.get("/api/v1/status").json()["searchable"] == 0 # retired + assert scope_b.get(f"/api/v1/jobs/{job_id}").json()["job"].get("prune") is True + finally: + if saved is None: + os.environ.pop("DIGUP_SERVER_PREWARM", None) + else: + os.environ["DIGUP_SERVER_PREWARM"] = saved + + +def test_progress_lands_on_the_live_record_and_survives_the_final_mark(tmp_path): + """The failure this pins, measured in the full suite (97 passed, 1 failed, 306 s): + a job stopped by `shutdown()` is marked from the **shutdown** thread, which knows nothing + about the worker thread's local event list — so `interrupted` went out with **no counts** + (`assert partial is not None`). Counts now ride the live record as they arrive, and + `_mark` merges into it, never replaces.""" + runner = JobRunner(_cfg(str(tmp_path / "idx"), (str(tmp_path / "root"),)), + config_dir=str(tmp_path / "cfg")) + try: + with runner._lock: + runner._active = "p1" + runner._jobs["p1"] = {"id": "p1", "state": "running", "started": 1.0} + runner._progress("p1", files_done=3, files_failed=1, errors=0) + runner._progress("p1", files_done=4) # a later event: merge, not reset + runner._mark("p1", "interrupted", "server stopped") + record = runner.job("p1") + assert record["state"] == "interrupted" and record["why"] == "server stopped" + assert record["files_done"] == 4 and record["files_failed"] == 1 and record["errors"] == 0 + finally: + runner.shutdown() + + +def test_a_hard_killed_job_recovers_with_the_totals_the_log_last_saw(tmp_path): + """The durable half of the same rule: the engine's `idle` totals ride in the log, so a + record recovered after a hard kill is *interrupted* **and** carries the last totals it + published (here files 9, files_done 2) — not an empty shell.""" + cfg, idx, root = str(tmp_path / "c"), str(tmp_path / "i"), str(tmp_path / "r") + first = JobRunner(_cfg(idx, (root,)), config_dir=cfg) + with first._lock: + first._active = "k1" + first._jobs["k1"] = {"id": "k1", "state": "running", "started": 1.0} + first._progress("k1", log=True, files_done=2, files=9, searchable=9, pending=7, vectors=2) + with first._lock: + first._active = None # the log's last line stays `running`: a hard kill + first.shutdown() + + second = JobRunner(_cfg(idx, (root,)), config_dir=cfg) + try: + recovered = second.job("k1") + assert recovered["state"] == "interrupted" # never success (README §5) + assert recovered["files"] == 9 and recovered["files_done"] == 2 + assert recovered["vectors"] == 2 and recovered["pending"] == 7 + finally: + second.shutdown() + + +def test_a_stop_mid_job_is_interrupted_and_the_next_job_resumes(tmp_path, scratch): + """Stage 8: `shutdown()` sends the engine's `stop` — the writer closes its stream at + the safe point **between files** — and the job records *interrupted* with its + partial counts, never success. SQLite stays whole: the next writer takes the lock, + resumes the walk (files already embedded are unchanged) and finishes it.""" + idx = tmp_path / "resume-index" + cfg_dir = tmp_path / "resume-cfg" + expected = sum(1 for f in scratch.rglob("*") if f.is_file()) # session scratch grows: + runner = JobRunner(_cfg(str(idx), (scratch,)), config_dir=str(cfg_dir)) # assert vs live + job_id = runner.start()["id"] + # Land the shutdown in the model-load phase — the one window every platform must + # cross (~800 MB of weights before the engine's first `file` event). The "embeds + # ~1 min" budget was Stage 8's cold measurement; measured 2026-11-05 on a warm box + # the whole scratch embed (8 files) finished *inside* 6 s — files_done 8/8 at the + # shutdown instant, and `partial < expected` became a timing assumption, not a fact. + # 1.0 s is under every measured embed phase on every platform, and above no load + # floor: the record is then partial = 0 (tolerated below) on a cold runner, a + # small partial on a warm one — the interrupted-then-resume contract holds both. + time.sleep(1.0) + runner.shutdown() # this shutdown lands mid-walk + job = runner.job(job_id) + assert job["state"] == "interrupted" + assert "stopped" in job.get("why", "") # truthful: never success + partial = job.get("files_done") + assert partial is not None, "an interrupted record always carries its counters" + + runner2 = JobRunner(_cfg(str(idx), (scratch,)), config_dir=str(tmp_path / "resume-cfg2")) + try: + # One writer, answered busy: a writer stopped mid-model-load keeps its flock for a + # moment while the engine closes its stream, and the server answers `busy` (the honest + # 409) for a request in that window. Measured: under machine load that window is + # seconds. The contract is "answered busy, then the next job runs the walk", so the + # resume is asked until it is accepted — not assumed to land on the first try. + deadline = time.monotonic() + 420.0 + done = None + while time.monotonic() < deadline: + job2 = runner2.start()["id"] + while time.monotonic() < deadline and runner2.job(job2)["state"] in ("queued", "running"): + time.sleep(2.0) + done = runner2.job(job2) + if done["state"] != "busy": + break + time.sleep(3.0) # the writer is closing: the flock frees + assert done["state"] == "done" + assert done["searchable"] == expected and done["pending"] == 0 # index == the walk + assert done["vectors"] >= 10 + # The engine's own arithmetic, measured in Stage 8 (8 + 7 = 15): the interrupted job + # embedded `partial` files, the resumed job embeds the REST (files already in the index + # come back `unchanged`, not `done`), so the two records sum to the walk. `partial` may + # legitimately be **0**: this shutdown lands inside the model load that opens every + # walk (a job stopped before the engine's first `file` event has completed zero + # files — that is the truth, and the record says 0: jobs.py publishes the counters + # when the walk starts). Demanding `partial > 0` was a timing assumption, not a fact. + assert 0 <= partial < expected + assert partial + done["files_done"] >= expected + finally: + runner2.shutdown() + + +def test_the_job_log_rotates_and_history_survives_rotation(scratch, scratch_index, tmp_path): + """Stage 8: the job log rotates at size (`DIGUP_SERVER_JOB_LOG_SIZE`); replay reads + ALL rotations oldest-first, so the job history outlives any rotation.""" + folder, _ = scratch_index + cfg = str(tmp_path / "rot-cfg") + runner = JobRunner(_cfg(folder, (scratch,)), config_dir=cfg, log_rotate_bytes=900) + ids = [] + for _ in range(6): + job_id = runner.start()["id"] + ids.append(job_id) + deadline = time.monotonic() + 120.0 + while time.monotonic() < deadline and runner.job(job_id)["state"] in ("queued", "running"): + time.sleep(0.5) + assert runner.job(job_id)["state"] == "done" + runner.shutdown() + + files = [f.name for f in (tmp_path / "rot-cfg").iterdir() if f.name.startswith("jobs.jsonl")] + assert len(files) >= 2 # rotation happened + replay = JobRunner(_cfg(folder, (scratch,)), config_dir=cfg, log_rotate_bytes=900) + try: + history = {j["id"]: j["state"] for j in replay.jobs()} + assert set(ids) <= set(history) # every job visible + assert all(state == "done" for state in history.values()) # none ghosted, none lost + finally: + replay.shutdown() + + +def test_the_knobs_are_read_from_the_environment(scratch, scratch_index): + """`DIGUP_SERVER_BACKFILL=0` reaches the engine argv (IndexService's `--backfill` + off) and `DIGUP_SERVER_JOB_LOG_SIZE` reaches the runner — the settings are env- + driven, per `.env.example`; defaults stay the shipping ones (backfill on, 1 MiB).""" + folder, _ = scratch_index + saved = {k: os.environ.get(k) for k in + ("DIGUP_SERVER_BACKFILL", "DIGUP_SERVER_JOB_LOG_SIZE", "DIGUP_SERVER_PREWARM")} + os.environ["DIGUP_SERVER_BACKFILL"] = "0" + os.environ["DIGUP_SERVER_JOB_LOG_SIZE"] = "5000" + os.environ["DIGUP_SERVER_PREWARM"] = "0" + try: + with TestClient(create_app(_cfg(folder, (scratch,)))) as c: + runner = c.app.state.jobs + assert runner._backfill is False + assert runner._rotate == 5000 + cfg = EngineConfig(index=str(folder), roots=(str(scratch),)) + assert "--backfill" not in cfg.index_argv() + assert "--backfill" in cfg.index_argv(backfill=True) + finally: + for k, v in saved.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + + +def test_shutdown_mid_job_finishes_and_marks_the_job_interrupted(scratch, scratch_index, tmp_path): + """The reviewer's repro of a self-deadlock: `shutdown()` called `_mark()` from inside + `with self._lock:` and `_mark` takes that same plain (non-reentrant) Lock — a docker + stop while a job was running hung the lifespan forever (job never logged *interrupted*, + `service.quit()` never ran, and every job poll consumed a threadpool thread until the + API stopped answering). The shape that hits it: a job whose remaining work outlives + the worker's join grace — one long file's embed on a 15 s stop.""" + folder, _ = scratch_index + config_dir = tmp_path / "shutdown-cfg" + runner = JobRunner(_cfg(folder, (scratch,)), config_dir=str(config_dir)) + runner._run_one = lambda job_id: time.sleep(25.0) # outlives the 20 s join grace + job = runner.start()["id"] + time.sleep(0.5) # the worker has picked it up + began = time.monotonic() + runner.shutdown() # deadlocked: this never returns + assert time.monotonic() - began < 30.0, "shutdown self-deadlocked (jobs.py:136/248)" + assert runner.job(job)["state"] == "interrupted" # README §5, on the stop path + assert runner.job(job).get("why") == "server stopped" + + +def test_a_crashing_job_runner_leaves_indexing_alive(scratch, scratch_index, tmp_path): + """`prewarm` answers with a 60 s TimeoutError — a `TimeoutError` is NOT a + `RuntimeError`, and the old `except RuntimeError` let it (and any other escape from + `_run_one`, e.g. an OSError from spawn) kill the worker thread silently: `_active` + stayed set forever, every later `POST /index` answered 409 naming a job that could + never run, and the queued job never left `queued`. Recovery was a container restart.""" + folder, _ = scratch_index + config_dir = tmp_path / "crash-cfg" + runner = JobRunner(_cfg(folder, (scratch,)), config_dir=str(config_dir)) + state = {"first": True} + + def crash_once(job_id): + if state["first"]: + state["first"] = False + raise TimeoutError("search-service: no reply to 'prewarm' within 60 s") + JobRunner._run_one(runner, job_id) # the real path from here on + + runner._run_one = crash_once + first = runner.start()["id"] + deadline = time.monotonic() + 30.0 + while time.monotonic() < deadline and runner.job(first)["state"] == "queued": + time.sleep(0.2) + assert runner.job(first)["state"] == "interrupted" # honest, not wedged + assert "prewarm" in runner.job(first).get("why", "") + assert runner._worker.is_alive() # the thread SURVIVED the crash + second = runner.start()["id"] # old code: JobBusy forever here + deadline = time.monotonic() + 300.0 + while time.monotonic() < deadline and runner.job(second)["state"] in ("queued", "running"): + time.sleep(0.5) + assert runner.job(second)["state"] == "done" # indexing works after the crash + runner.shutdown() + + +def test_a_prewarm_timeout_after_done_keeps_the_job_done(scratch, scratch_index, tmp_path): + """A job is *done* when the writer finished: the refresh behind it (reload → + prewarm) timing out is a note on a done job — never a dead runner thread, never a + state that claims more than happened.""" + folder, _ = scratch_index + config_dir = tmp_path / "prewarm-cfg" + + def after_done(): + raise TimeoutError("search-service: no reply to 'prewarm' within 60 s") + + runner = JobRunner(_cfg(folder, (scratch,)), config_dir=str(config_dir), + after_done=after_done) + job = runner.start()["id"] + deadline = time.monotonic() + 300.0 + while time.monotonic() < deadline and runner.job(job)["state"] in ("queued", "running"): + time.sleep(0.5) + done = runner.job(job) + assert done["state"] == "done" + assert "searcher refresh deferred" in done.get("why", "") + assert runner._worker.is_alive() + runner.shutdown() + + +def test_replay_ignores_lines_that_are_not_job_records(scratch, scratch_index, tmp_path): + """A hand-edited/merged log must not brick the boot: the old replay did + `record["id"]` on any parsed JSON — a bare `5` raised, `__init__` propagated, and + the app refused to start. Torn tails and non-record lines are skipped; the one + real `running` record still recovers as *interrupted*.""" + folder, _ = scratch_index + config_dir = tmp_path / "junk-cfg" + config_dir.mkdir() + (config_dir / "jobs.jsonl").write_text( + "[1, 2, 3]\n5\nnull\n\"a string\"\n{\"state\":\"running\"}\n" # valid JSON, not records + '{"id":"abc123","state":"running","started":0}\n' + '{"id":"real", "state":"done", "ended":1}\n' + "truncat", # torn tail + encoding="utf-8") + runner = JobRunner(_cfg(folder, (scratch,)), config_dir=str(config_dir)) + assert runner.job("abc123")["state"] == "interrupted" + assert runner.job("real")["state"] == "done" + runner.shutdown() diff --git a/server/tests/test_media_handles.py b/server/tests/test_media_handles.py new file mode 100644 index 00000000..3792e76a --- /dev/null +++ b/server/tests/test_media_handles.py @@ -0,0 +1,48 @@ +"""The media-handle semantics `docs/API.md` promises clients (step: media identifiers). + +These are the properties a multi-application client designs against, pinned at the unit level +(no engine, no lifespan): handles are per-reply (two replies never share one), the cache is +bounded and a real LRU (a browsing session refreshes its own on-screen handles), and handles +die with the process. `tests/test_media_ui.py` pins the HTTP side (`404` for an unissued id, +previews, text, `/file`); this file pins the rules the doc states. +""" + +from __future__ import annotations + +from digup_server.api import HitCache + + +def put(cache: HitCache, path: str = "/media/photos/a.jpg") -> str: + return cache.put(path, "image", 1, None, False) + + +def test_handles_are_per_reply_never_global(): + """The same file in two replies gets two handles: a client may not cache a `media_id` + across searches, and may not share one between clients.""" + cache = HitCache(capacity=8) + first, second = put(cache), put(cache) + assert first != second + assert cache.get(first)["path"] == cache.get(second)["path"] + + +def test_the_cache_is_bounded_and_a_true_lru(): + """A bounded cache: oldest-unused goes. `get()` refreshes, which is what keeps the handles + a session is looking at alive while it keeps searching (the Stage-11 fix).""" + cache = HitCache(capacity=2) + a, b = put(cache), put(cache) + assert cache.get(a) is not None # refresh: a is now the most recently used + c = put(cache) + assert cache.get(c) is not None + assert cache.get(b) is None # b is the least recently used: gone + assert cache.get(a) is not None + + +def test_handles_are_the_process_lifespan_not_a_durable_identifier(): + """A new cache is what a restart is: every handle from before it answers as unknown. A + client keys its own records on the engine's identity fields (path + kind + segment_id, + plus loc for a moment), never on a media_id.""" + cache = HitCache(capacity=4) + issued = put(cache) + assert cache.get(issued) is not None + after_restart = HitCache(capacity=4) + assert after_restart.get(issued) is None diff --git a/server/tests/test_media_ui.py b/server/tests/test_media_ui.py new file mode 100644 index 00000000..db881fd0 --- /dev/null +++ b/server/tests/test_media_ui.py @@ -0,0 +1,441 @@ +"""The UI and media surface (Stages 6 and 10; the Lit migration of 2026-11-08): +same-origin frontend, server-issued handles, and the file itself with ranges honored. + +The rules this pins (README §6, §7, §10.4's 2026-11-08 amendment): the UI is the +**built Lit + Vite frontend** served same-origin (no external URLs; Node exists at +build time only, the runtime ships static files); the stable asset names (`app.js`, +`style.css`, `embed.js`, `embed.css`) are the pinned contract FastAPI serves from +`server/ui/dist`; the media endpoints accept **only ids the server issued from +published hits**, never client paths; previews are the engine's own JPEGs from +`render`, served only from `<index>/previews`; the text endpoint quotes the passage +the engine indexed, verbatim. + +The string pins moved with the migration: the vanilla UI kept ids and strings in +served HTML; the Lit UI renders them inside shadow templates, so the pins read the +served BUNDLE (Lit `html``/`css`` template literals survive minification verbatim — +the pinned strings must stay byte-identical in the templates, and the pins below +say which ones are load-bearing). Pins scoped "in html" check the thin shell; +"in bundle" checks the entry plus the hashed chunks the shell links. +""" + +from __future__ import annotations + +import re + +import pytest +from fastapi.testclient import TestClient + +from conftest import _cfg +from digup_server.api import create_app + + +@pytest.fixture(scope="module") +def media_client(corpus, index): + with TestClient(create_app(_cfg(index, (corpus,)))) as c: + yield c + + +@pytest.fixture(scope="module") +def doc_client(scratch, scratch_index): + """Same app shape over the scratch corpus — the one with a text document.""" + folder, _ = scratch_index + with TestClient(create_app(_cfg(folder, (scratch,)))) as c: + yield c + + +def _page(client) -> str: + return client.get("/").text + + +def _bundle(client) -> str: + """Every JS byte the built page loads: the entry plus the hashed chunks the html + links. Module boundaries are a build detail — the string pins read the bundle a + browser actually runs, not one chosen file.""" + html = _page(client) + paths = ["/app.js"] + paths += re.findall(r'href="\./((?:chunks|assets)/[^"]+\.js)"', html) + return "\n".join(client.get(p).text for p in paths) + + +def _styles(client) -> str: + """Every CSS byte + the component CSS: Lit ships `css`` chunks as RUNTIME styles + (template literals in the bundle, verbatim — the reason the layout pins below read + them there, not from a static file); the static files carry tokens + the frame.""" + html = _page(client) + css_paths = re.findall(r'href="\./(assets/[^"]+\.css)"', html) + parts = [client.get("/style.css").text] + [client.get("/" + p).text for p in css_paths] + return "\n".join(parts) + "\n" + _bundle(client) + + +def test_root_serves_the_lit_ui(media_client): + page = media_client.get("/") + assert page.status_code == 200 + html = page.text + bundle = _bundle(media_client) + assert "DigUp" in html and "app.js" in html and "style.css" in html + # Same-origin discipline (README §10.4): no external scripts/styles, ever. + assert not re.search(r'(src|href)="(https?:)?//', html), "UI must not load external URLs" + # ES-module entry: the built bundle (the migration's served shape; vanilla was a + # plain classic script — the flip is pinned here so it can never silently regress). + assert re.search(r'type="module"', html) + appjs = media_client.get("/app.js") + assert appjs.status_code == 200 and "fetch(" in bundle + assert "media_id" in bundle # it uses the server's handles + assert media_client.get("/style.css").status_code == 200 + # Stage-10 parity with the desktop panel: appearance choice, kind filters, code mode, + # keyword-first typing (prefix_last then the meaning upgrade), the keys footer. + assert 'data-theme="system"' in html # the thin shell carries it + assert 'id="theme"' in bundle and "system" in bundle + assert '"screenshot","Screenshots"' in bundle and 'id="code"' in bundle + assert "prefix_last" in bundle and "embed_ahead" in bundle + assert "localStorage" in bundle # the theme is kept in the browser + assert "↑↓" in bundle and "Space" in bundle + # API key lives in Settings (2026-11-08 UX brief, round 2): the section is ONLY the + # minted-key rows with Copy and the Regenerate button — no paste field, no Save, + # no Clear (the key is server-generated at runtime; the applications paste it into + # THEIR settings, this browser takes the admin key itself on Regenerate). The embed + # widget (no settings panel of its own) keeps its prompt (test_integrations covers). + assert "API key" in bundle and "Copy" in bundle + assert 'id="api-key"' not in html and 'id="api-key"' not in bundle + assert 'id="api-key-save"' not in bundle and 'id="api-key-clear"' not in bundle + assert "digupBearer" in bundle + assert "window.prompt" not in appjs.text # main UI: no load-time prompt + # Stage 13 (reworked) + the 2026-11-08 round-3 brief: the minted keys render as + # ONE label + the key in a field + icon actions (⧉ copy, ↻ regenerate). The round-4 + # brief put the index counts into the status card; the round-5 brief took the + # "Looking in: <roots>" line out of the panel entirely (the card's two tiles carry + # files/searchable; the roots stay in GET /status). The gear's section order is the + # operator's reading order; the round-4 layout brief renamed the sections to + # Appearance · Indexing · Index status · API access, each heading carrying a small + # icon INSIDE the title span — so the order pin rides the heading ids plus the + # heading text. Rows keep one row height, equal padding and borders (the user's + # "clean and orderly, equal size rows"). + assert 'id="appkeys"' in bundle and "Total files" in bundle # keys rows; the card's tiles + assert 'id="api-key-sec"' in bundle + assert "Set in the server's environment" in bundle # env-keyed shape says so + assert "Regenerate" in bundle and "/api/v1/tokens" in bundle + assert "/api/v1/targets" in bundle + css = _styles(media_client) + row = css[css.index(".appkey-line"):] + assert "grid-template-columns" in row and "min-height" in row + # One row height for every gear control (round-3 styling ask): the panel's controls + # and key rows share the token-defined 2rem row (`--row-height: 2rem`). + assert "--row-height: 2rem" in css + assert "min-height: var(--row-height)" in css + pos = [bundle.index(f'id="{s}"') for s in + ("set-appearance", "set-indexing", "set-status", "set-apikey")] + assert pos == sorted(pos) + for h in (" Appearance</span>", " Indexing</span>", " Index status</span>", + " API access</span>"): + assert h in bundle + # Round-4 layout brief: ONE distinct card (the index status — directory, the two + # counts side by side, wrapping badges); the estimates row is text + right-aligned + # switch; the sections ride 1 px dividers, not boxes. + assert 'class="status-card"' in bundle and 'class="stats"' in bundle + assert 'class="set-state"' in bundle # heading state (round-6) + assert "border-top: 1px solid var(--line)" in css + # The main page keeps no status nag (2026-11-08 UX ask): counts live in the panel. + assert 'id="statusline"' not in html and 'id="statusline"' not in appjs.text + assert "status unavailable" not in bundle + # Estimate (round-6 brief): the toggle and the forecast rows are GONE — the Estimate + # button rides the Index now row, and the INDEXING heading carries the live state: + # a TRUE CIRCLE spinner (ring + rotating bright quadrant, no glyph) while the engine + # walks, ✓ when it finished, the brief update beside it (the job's own words). + # No main-page estimate popup, as always. + assert 'id="estimate-spin"' in bundle and ".estimate-spin" in css + assert "estimating — the engine is walking the folders…" not in bundle + assert appjs.text.count('id="estimate"') == 0 # the forecast box is gone + assert 'class="set-head-row"' in bundle and "state-done" in bundle + assert "digup-settings" in bundle and "digup-big" in bundle # panel-side, never the main page + assert css.index(".estimate-spin") < css.index("@keyframes spin") + assert "border-radius: 50%" in css[css.index(".estimate-spin"):css.index("@keyframes spin")] + assert "flex-wrap: wrap" in css[css.index(".set-facts"):] # badges wrap naturally (round-4 brief) + # Round 7 (the layout brief): the panel NEVER scrolls — `overflow-y: auto` made a + # clipping box (controls cut at the edges, a scrollbar stealing width, the modal + # able to leave the viewport). The API-key row's grid is minmax()-bounded: the old + # `9rem ... auto auto` minimum (~24.6 rem) sat ABOVE the ~21 rem control column — + # a ~120 px overflow past the panel's right edge, growing on narrow windows. + assert "overflow-x: clip" in css and "overflow-y: auto" not in css + row_grid = css[css.index(".appkey-line { display: grid"):] + assert "grid-template-columns: minmax(4rem, 6rem) minmax(0, 1fr)" in row_grid + assert "min-content min-content" in row_grid[:220] and ".appkey-line > * { min-width: 0; }" in row_grid + # Round 6 (the user's brief): the API-key section is the key ITSELF with Copy — + # the keyless gear is shown the minted ticket (`GET /tokens` key-free in a minted + # deployment) — and Regenerate rides the key row's right edge (not a loose button + # under the section). The row grid: label | key | Copy | Regen. + assert 'id="appkeys"' in bundle + assert '"keys-regenerate"' in bundle and "digup:regenerate" in bundle + assert "#keys-regenerate { margin-left: auto; }" in css + # Round 6 (the user's brief): the estimates toggle + the forecast section were + # removed — the Estimate button rides the Index row, the heading carries the state. + # The CONTROLLER keeps its gate keys (object-literal keys survive minification): the + # switch is gone, the browser key and the click-again cancel stay. + assert 'id="estimate-toggle"' not in bundle and "estimateOn" in bundle + assert "estimate-ctl" not in css and "select#theme { width: 100%" in css + # A finished crawl re-reads the index list: the panel shows the targets that exist + # NOW (no stale rows survive a completed job). Key names in an object literal + # survive minification; the controller's export keys are pinned. + assert "loadAccess" in bundle and "retried" in bundle + # Round 7: the gear ADOPTS the minted admin key it is shown — its own controls + # (Index now, Estimate, Regenerate) are admin-scope by the minted shape's design, + # so the operator's gear uses the key the server just displayed to it. A stale + # stored key is dropped only where a keyless main read proves the minted shape. + assert '"digupBearer"' in bundle and "setItem" in bundle and "removeItem" in bundle + + +def test_the_space_key_keeps_buttons_and_links_activated(media_client): + """The reviewer found the hijack: the document-level Space handler (Quick Look) + `preventDefault()`ed for every focused element except a few tags — and native + button activation fires on keyup, so the keydown kill disabled the gear, the ✕, + Index now, Estimate, "all", the Esc close, and the Open/Download links for every + keyboard user, while opening the big-look over the page. The guard is a closest() + over the activation targets, kept pinned here — in shadow DOM it runs over the + event's composed path, the selector string verbatim.""" + bundle = _bundle(media_client) + assert 'closest("button, a, [role=\'button\'], input, select, textarea")' in bundle + + +def test_the_layout_is_gated_on_the_pages_own_state(media_client): + """The preview pane takes space only when it has something to show. + + `body[data-page]` — written by the shell at every render — is the ONE switch: the + results grid, the hit counter and the preview pane exist in `results` only; idle, + busy and none center a panel and reserve nothing. The shell MIRRORS the switch onto + its host attribute (the panels live in its shadow, where a body selector cannot + reach), so the pins below ride on `:host([data-page=…])` in style.css plus the + `document.body.dataset.page` write in the bundle. Pinned on the source, because a + stylesheet edit is exactly what quietly puts the empty panel back (measured before + this change: a 574×192 px empty preview box with nothing typed, at 1280 px).""" + html = _page(media_client) + css = _styles(media_client) + bundle = _bundle(media_client) + appjs = media_client.get("/app.js").text + for panel in ('id="idle"', 'id="idle-note"', 'id="no-results"', 'id="no-results-why"', + 'id="suggest"', 'id="engine-says"'): + assert panel in bundle, f"the state panel {panel} is gone from the templates" + assert 'id="no-results" hidden' not in html # CSS gates it: one mechanism, not two + assert ':host([data-page="results"]) #results' in css + assert ':host([data-page="idle"]) #idle' in css and ':host([data-page="none"]) #no-results' in css + assert ':host([data-page="busy"]) #suggest' in css # busy is not a shopping list + assert "document.body.dataset.page" in appjs + assert "grid-template-columns: minmax(0, 2fr) minmax(0, 3fr)" in css # 40 % list, 60 % preview + assert "grid-template-columns: minmax(0, 1fr)" in css # narrow screens: one column + assert "#preview { position: static; }" in css # a sticky pane on one column covers the list + assert "@media (prefers-reduced-motion: no-preference)" in css # the state fade is guarded + + +def test_the_first_paint_is_a_state_and_a_missing_preview_says_so(media_client): + """Two failures this pins, both found by measuring the page rather than reading it: + the layout must PAINT a state at startup (without it `body[data-page]` is unset and + EVERY panel is gated off — the blank page), and a preview the engine has not + rendered is a message plus the file's metadata, never an empty pane. + + The Lit shape: the shell paints and syncs the state in `firstUpdated` (Lit runs it + with the first render — the vanilla startup `render();` line, carried over as a + lifecycle), and `main.js` carries the initial results slice (its `groups: []`).""" + appjs = media_client.get("/app.js").text + bundle = _bundle(media_client) + html = _page(media_client) + assert "firstUpdated" in appjs, "the shell must paint at startup: the first paint is a state" + # The served shape: the html DECLARES the shell element, and the entry wires THAT + # element. A second `createElement` mount is the double page a browser shows (an + # unwired twin renders its bare template above the working one — measured 2026-11-08): + # exactly one declaration in the page, one wiring line in the entry, append only as + # the fallback branch. + assert html.count("<digup-app></digup-app>") == 1, "the shell is declared once in the page" + assert 'document.querySelector("digup-app")' in appjs, "the entry wires the DECLARED element" + assert "document.body.append(" in appjs, "the createElement path survives as the fallback mount" + assert re.search(r"groups:\s*\[\]", appjs), "the initial state needs groups: the first render iterates it" + assert "No preview of this file" in bundle + assert bundle.count("@error=") >= 5, "photo, PDF, video and thumbnail previews must announce a missing render" + assert 'class="facts"' in bundle, "the metadata sits beside the preview, not behind it" + # the centered "no results" panel quotes the engine's own sentence, the same string + # the counter line uses — both publishers carry it verbatim + assert bundle.count("the engine found no good matches for this and refused to guess") >= 2 + # suggestions take the ONE search route: they only ever emit `digup:set-query`, the + # shell hands the phrase to digup-search, which emits the query-change the + # controller runs — no second route to the engine exists (AGENTS.md). + assert "wakeboarding on a lake" in appjs + assert appjs.count("digup:set-query") >= 1 + # Exactly two appearances of the route name in the shell graph: the emitter template + # (digup-search) and the shell's listener — the controller is the only consumer. + assert appjs.count("digup:query-change") == 2, "the ONE search route: one emitter, one listener" + + +def test_the_ui_survives_a_failed_job_poll(media_client): + """S1: the job poll loop died on its first failed fetch (server restart, engine + mid-respawn 503) — unhandled rejection, the job line froze at its last state, + and "indexing…" stayed in the footer forever. It now answers honestly and clears + the indexing state; repeated busy-clicks poll one loop, not N (the dedupe token), + and every settings-close path keeps aria-expanded true to the panel's state.""" + bundle = _bundle(media_client) + assert "job poll failed" in bundle + assert "jobPoll" in bundle # one poll loop per live job + assert "aria-expanded" in bundle + html = _page(media_client) + assert 'id="settings-close"' in bundle + assert _styles(media_client).count("color-mix(in srgb, var(--kc) 10%") == 1 + + +def test_ui_files_revalidate_never_go_stale(media_client): + """Measured in a live browser during the Stage-10 proof: without a `cache-control`, + Chrome served a stylesheet the server no longer had for minutes — reload answered + from its memory cache, and a UI fix never reached the page. UIFiles answers every + UI byte `cache-control: no-cache` (the etag makes the revalidation a cheap 304). + + The migration made the asset list dynamic (hashed chunks), so the walk discovers + what the built page links and decorates EVERY byte: html, entry, chunks, styles.""" + html = _page(media_client) + assets = ["/", "/style.css", "/app.js"] + assets += ["/" + p for p in re.findall(r'href="\./((?:chunks|assets)/[^"]+)"', html)] + assert len(assets) >= 5, "the built page must link its chunks/styles (the stable-name contract)" + for path in assets: + reply = media_client.get(path) + assert reply.status_code == 200, path + assert reply.headers["cache-control"] == "no-cache", path + assert "etag" in reply.headers, path + + +def test_search_issues_media_handles(media_client): + reply = media_client.post("/api/v1/search", json={"q": "wakeboarding", "limit": 3}).json() + ids = [h["media_id"] for h in reply["hits"]] + assert len(ids) == len(reply["hits"]) == len(set(ids)) + assert all(len(i) == 12 for i in ids) + + +def test_preview_serves_the_engines_jpeg(media_client): + hit = media_client.post("/api/v1/search", + json={"q": "wakeboard photo", "kinds": ["image"], "limit": 1} + ).json()["hits"][0] + reply = media_client.get(f"/api/v1/media/{hit['media_id']}/preview?pixels=256") + assert reply.status_code == 200 + assert reply.headers["content-type"] == "image/jpeg" + assert reply.content[:2] == b"\xff\xd8" # a real JPEG, from render + + +def test_video_preview_is_a_frame_at_the_moment(media_client): + hit = media_client.post("/api/v1/search", + json={"q": "hiking", "kinds": ["video"], "limit": 1} + ).json()["hits"][0] + assert hit["loc"] is not None # the engine's seconds + reply = media_client.get(f"/api/v1/media/{hit['media_id']}/preview?pixels=256") + assert reply.status_code == 200 and reply.content[:2] == b"\xff\xd8" + + +def test_the_passage_a_doc_hit_points_at(doc_client): + hit = doc_client.post("/api/v1/search", + json={"q": "the stories continued after midnight marshmallows", + "kinds": ["doc"], "limit": 1}).json()["hits"][0] + body = doc_client.get(f"/api/v1/media/{hit['media_id']}/text").json() + assert "campfire stories" in body["text"] # the engine's text, verbatim + + +def test_media_ids_the_server_never_issued_are_404(media_client): + assert media_client.get("/api/v1/media/deadbeefcafe/preview").status_code == 404 + assert media_client.get("/api/v1/media/deadbeefcafe/text").status_code == 404 + + +def test_embed_ahead_answers_ok(media_client): + assert media_client.post("/api/v1/embed_ahead", json={"q": "wake"}).json()["ok"] is True + + +# ── Stage 10: GET /media/{id}/file — the desktop's "Open", over HTTP ─────────────────── +# Same discipline as /preview: only server-issued handles; type from the file's magic +# bytes; ranges honored (that is what makes <video …#t=loc> seek); code/doc content is +# text/plain and an attachment — markup the user's own files carry never runs here. + +def test_file_streams_the_photo_itself(media_client): + hit = media_client.post("/api/v1/search", + json={"q": "wakeboard photo", "kinds": ["image"], "limit": 1} + ).json()["hits"][0] + reply = media_client.get(f"/api/v1/media/{hit['media_id']}/file") + assert reply.status_code == 200 + assert reply.headers["content-type"] == "image/jpeg" # sniffed from the bytes + assert reply.content[:3] == b"\xff\xd8\xff" # the real file, not a render + assert reply.headers["accept-ranges"] == "bytes" + assert reply.headers["content-disposition"].startswith("inline;") + assert "filename*=UTF-8''" in reply.headers["content-disposition"] + + +def test_ranges_are_honored_for_video_seeking(media_client): + hit = media_client.post("/api/v1/search", + json={"q": "hiking", "kinds": ["video"], "limit": 1} + ).json()["hits"][0] + whole = media_client.get(f"/api/v1/media/{hit['media_id']}/file") + size = int(whole.headers["content-length"]) + head = media_client.get(f"/api/v1/media/{hit['media_id']}/file", headers={"range": "bytes=0-99"}) + assert head.status_code == 206 + assert head.content == whole.content[:100] + assert head.headers["content-range"] == f"bytes 0-99/{size}" + tail = media_client.get(f"/api/v1/media/{hit['media_id']}/file", headers={"range": "bytes=-40"}) + assert tail.status_code == 206 and tail.content == whole.content[-40:] + assert tail.headers["content-range"] == f"bytes {size - 40}-{size - 1}/{size}" + open_end = media_client.get(f"/api/v1/media/{hit['media_id']}/file", + headers={"range": f"bytes={size - 10}-"}) + assert open_end.status_code == 206 and len(open_end.content) == 10 + unsatisfiable = media_client.get(f"/api/v1/media/{hit['media_id']}/file", + headers={"range": f"bytes={size}-{size + 99}"}) + assert unsatisfiable.status_code == 416 + assert unsatisfiable.headers["content-range"] == f"bytes */{size}" + + +def test_open_serves_a_documents_text_as_plain_text(doc_client): + hit = doc_client.post("/api/v1/search", + json={"q": "the stories continued after midnight marshmallows", + "kinds": ["doc"], "limit": 1}).json()["hits"][0] + reply = doc_client.get(f"/api/v1/media/{hit['media_id']}/file") + assert reply.status_code == 200 + assert reply.headers["content-type"].startswith("text/plain") # never markup, never script + assert reply.headers["content-disposition"].startswith("attachment;") + assert "campfire stories" in reply.text + + +def test_file_for_an_unissued_handle_is_404(media_client): + assert media_client.get("/api/v1/media/deadbeefcafe/file").status_code == 404 + + +def test_video_file_arrives_with_a_playable_media_type(media_client): + """scenes.mp4 (H.264, browser-streamable): the magic sniff reads `ftyp` at offset 4 + and the brand at 8-12. Measured lesson: reading the brand at 8 against `ftyp` sent + every real MP4 to octet-stream — Chrome sniffs past that, Safari is not obliged to.""" + hit = media_client.post("/api/v1/search", + json={"q": "color bars test pattern", "kinds": ["video"], + "limit": 3}).json()["hits"][0] + reply = media_client.get(f"/api/v1/media/{hit['media_id']}/file") + assert reply.status_code == 200 + assert reply.headers["content-type"] in {"video/mp4", "video/x-matroska"} + assert reply.headers["accept-ranges"] == "bytes" + + +def test_open_range_streams_the_video_rather_than_buffering_it(media_client): + """`Range: bytes=0-` is what every browser media element sends first. The old + handler computed the full span and `read()` it — a whole-file RAM buffer (a 20 GB + clip asked for a 20 GB read) — and answered 206 with the ENTIRE file, so seeking + never streamed. Now: open ranges are capped at MAX_RANGE_SPAN and the span is + streamed in chunks from its own file handle.""" + from digup_server.api import MAX_RANGE_SPAN + hit = media_client.post("/api/v1/search", + json={"q": "wakeboarding", "kinds": ["video"], "limit": 1} + ).json()["hits"][0] + full = media_client.get(f"/api/v1/media/{hit['media_id']}/file") + size = len(full.content) + open_range = media_client.get(f"/api/v1/media/{hit['media_id']}/file", + headers={"range": "bytes=0-"}) + assert open_range.status_code == 206 + expected_end = min(size - 1, MAX_RANGE_SPAN - 1) + assert open_range.headers["content-range"] == f"bytes 0-{expected_end}/{size}" + assert len(open_range.content) == expected_end + 1 # the span, streamed, complete + + +def test_file_carries_nosniff(media_client): + """The magic-bytes typing above refuses to call script script; `nosniff` is the + cheap belt that says the browser must not guess a type on its own — including + for the text/plain path used by code/doc hits.""" + hit = media_client.post("/api/v1/search", + json={"q": "wakeboard photo", "kinds": ["image"], "limit": 1} + ).json()["hits"][0] + reply = media_client.get(f"/api/v1/media/{hit['media_id']}/file") + assert reply.headers["x-content-type-options"] == "nosniff" + ranged = media_client.get(f"/api/v1/media/{hit['media_id']}/file", + headers={"range": "bytes=0-2"}) + assert ranged.headers["x-content-type-options"] == "nosniff" diff --git a/server/tests/test_search_contract.py b/server/tests/test_search_contract.py new file mode 100644 index 00000000..b3ea1423 --- /dev/null +++ b/server/tests/test_search_contract.py @@ -0,0 +1,243 @@ +"""The search-service contract: ops, reply fields, honest empties, the idle-exit restart. + +Every expectation here is what the Stage-2 proof *observed the engine saying* +(docs/implementation/stage-2-headless-proof.md) — the corpus pins, the field names, the +rankings, the refusal. If upstream changes this protocol, these tests fail loudly; the +server never papers over it. +""" + +from __future__ import annotations + +import os +import time +from pathlib import Path + +import pytest + +from digup_server.engine import EngineConfig, SearchService + +BASE_FIELDS = {"path", "kind", "segment", "segment_id", "score", "modified", "size", "info"} +MAYBE_FIELDS = {"z", "cosine", "keyword_rank", "loc", "loc_end", "focus", "excerpt", + "strong", "meaning", "ms", "warning", "id", "ok", "error"} + + +def paths(hits): + return [Path(h["path"]).name for h in hits] + + +def test_ready_handshake(search, index): + ready = search["ready"] + assert ready["event"] == "ready" + assert Path(ready["index"]).name == index.name # the engine names the index it opened + assert search["service"].running + + +def test_status_op_counts_are_the_index(search, index): + status = search["service"].status() + assert {"encoder", "vectors", "code_vectors", "index", "files", "searchable", + "pending", "unfinished"} <= set(status) + assert status["index"] == str(index / "index.sqlite") + assert status["files"] >= 15 # the corpus is in there + assert status["searchable"] == status["files"] # every file indexed `done` + assert status["pending"] == 0 + assert status["encoder"] in {"stopped", "starting", "ready", "unavailable"} + + +def test_wakeboarding_ranks_the_wakeboards(search): + reply = search["service"].search("wakeboarding", limit=10) + assert {"hits", "strong", "meaning", "ms"} <= set(reply) + assert reply["meaning"] is True # vectors were used + assert set(paths(reply["hits"])[:3]) == {"wakeboarding-vienna.webm", "wakeboard-logan.jpg", + "wakeboarding-aarhus-04.jpg"} + # The SET is the contract (the three planted wakeboard media answer the query). The + # order is platform-dependent (embedding matmul vs CPU features; measured 2026-11-04: + # x86_64 logan 1.571 > vienna 1.531, arm64 vienna first — 0.040 z apart, inside noise). + assert reply["strong"] >= 3 + + +def test_hit_fields_are_the_engines_ones(search): + hits = search["service"].search("wakeboarding", limit=5)["hits"] + for hit in hits: + assert BASE_FIELDS <= set(hit), f"missing fields in {hit}" + assert set(hit) - BASE_FIELDS <= MAYBE_FIELDS, f"invented fields in {hit}" + assert {"z", "cosine", "keyword_rank"} <= set(hits[0]) # the Stage-2 proof's scored hits + + +def test_hiking_finds_a_moment_in_the_video(search): + reply = search["service"].search("hiking", limit=10) + top = reply["hits"][0] + assert Path(top["path"]).name == "pct-hiking-southern-oregon.webm" + assert top["segment"] == "frame" + assert 0.0 <= top["loc"] <= float(top["info"]["duration"]) + assert top["loc_end"] >= top["loc"] + assert {"embed", "search"} <= set(reply["ms"]) # real milliseconds, per phase + + +def test_kind_filter_is_the_engines(search): + hits = search["service"].search("wakeboarding", kinds=["video"], limit=10)["hits"] + assert hits and all(h["kind"] == "video" for h in hits) + + +def test_no_good_matches_stays_an_empty_answer(search): + # Stage 2's honest miss: the engine refuses to rank below its bar. The server publishes + # the emptiness, it never fills it with the best vector it can find. + reply = search["service"].search("a person riding a wakeboard behind a boat", limit=10) + assert reply["hits"] == [] + assert reply["meaning"] is True + assert "warning" not in reply # the encoder worked; the bar held + + +def test_controls_stay_below_the_matches(search): + reply = search["service"].search("a cat sitting on a sofa", limit=10) + hits = reply["hits"] + assert paths(hits)[0] == "cat-sitting-on-a-sofa.jpg" + assert not any("wakeboard" in p.lower() for p in paths(hits)) + + +def test_summaries_count_the_corpus(search): + roots = search["service"].summaries()["roots"] + assert len(roots) == 1 + root = roots[0] + assert {"path", "missing", "unreadable", "files", "indexed", "pending", "failed", + "skipped", "unfinished", "kinds", "skipped_reasons"} <= set(root) + assert root["missing"] is False + assert root["files"] >= 15 + assert root["kinds"].get("image") == 12 and root["kinds"].get("video") == 3 + + +def test_embed_ahead_answers_ok(search): + reply = search["service"].embed_ahead("wake") + assert reply["ok"] is True # the engine's own {ok:true} + + +def test_unknown_op_is_the_engines_error(search): + reply = search["service"].raw({"op": "definitely-not-an-op"}) + assert reply["ok"] is False + assert "unknown op" in reply["error"] + + +def test_render_makes_a_jpeg_beside_the_index(search, index): + hit = search["service"].search("wakeboard photo", kinds=["image"], limit=3)["hits"][0] + reply = search["service"].render(hit["path"], kind="image", pixels=256) + image = Path(reply["image"]) + assert str(image).startswith(str(index / "previews")) # the engine's own previews folder + assert image.is_file() + assert image.read_bytes()[:2] == b"\xff\xd8" # a real JPEG, not a promise + + +def test_idle_exit_is_a_restart_not_a_failure(corpus, index): + # `--encoder-idle 2`: the engine releases the model at 2 s idle and exits itself at 4 s + # (`{"event":"exit","reason":"idle"}`) — and the next request finds it answering again. + service = SearchService(EngineConfig(index=str(index), roots=(str(corpus),), + binary="digup", + models_dir=os.environ.get("DIGUP_SERVER_MODELS"), + encoder_idle=2.0)) + service.start() + try: + first = service.search("wakeboarding", limit=3)["hits"] + assert first + assert service.running + time.sleep(6) # 2 × idle: the engine ends itself + assert not service.running + again = service.search("wakeboarding", limit=3)["hits"] # transparent respawn + assert paths(again) == paths(first) + finally: + service.quit() + + +# ── adapter supervision units: what a failed spawn must never leave behind ───────────── +# (unit-level by design: these paths are about leaks and exception mapping, and a fake +# proc makes them deterministic; the live protocol above stays mock-free.) + + +def test_a_failed_handshake_reaps_the_child_it_spawned(monkeypatch, tmp_path): + """`_spawn_locked` assigned `self._proc` only AFTER a successful handshake — a + `start()` that raised (ready timeout on a huge index, a `ready` mismatch) left the + fresh child running with its 3 pipes and 2 reader threads unreferenced until its + own idle-exit (2 × `--encoder-idle` = 600 s), and every retried request spawned + another one.""" + from digup_server.engine import search as search_mod + from digup_server.engine.protocol import EngineExited + + closed = [] + + class LeakProc: + def __init__(self, argv, *, name, env=None): + self._name = name + + def start(self): + raise EngineExited(self._name, 1, ["handshake boom"]) + + def close(self): + closed.append(True) + + monkeypatch.setattr(search_mod, "JsonLinesProcess", LeakProc) + service = SearchService(EngineConfig(index=str(tmp_path), roots=("/corpus",))) + with pytest.raises(EngineExited): + service.search("wakeboarding") + assert closed == [True] # the child was reaped, not orphaned + assert service._proc is None + + +def test_a_missing_engine_binary_answers_clierror(tmp_path): + """FileNotFoundError from Popen must reach the API as a 502 quoting the spawn, + not a bare 500 traceback (api.py:6: quote the engine, invent nothing).""" + from digup_server.engine.cli import CLIError + service = SearchService(EngineConfig(index=str(tmp_path), roots=("/corpus",), + binary="/no/such/digup-binary")) + with pytest.raises(CLIError, match="cannot start the engine helper"): + service.search("wakeboarding") + + +def test_a_refusal_without_an_error_field_is_still_a_refusal(): + """`ok:false` ALONE: the old guard demanded `"error" in reply`, so a refusal + shaped `{"ok":false,"id":4}` sailed through `_checked` and then api.py's + `stripped["hits"]` raised KeyError — the exact 500 the Stage-10 battery hit. + The guard must fire on the shape, not on one field of it.""" + from digup_server.engine.protocol import EngineRefused + with pytest.raises(EngineRefused): + SearchService._checked({"ok": False, "id": 4}, {"op": "search"}) + ok = {"hits": [], "strong": 0, "meaning": True, "ms": {}, "id": 5} + assert SearchService._checked(ok, {"op": "search"})["hits"] == [] + + +def test_renders_run_beside_searching_not_ahead_of_it(tmp_path): + """The engine runs `render` beside its read loop on two slots, answers routed by + `id` — "made beside the requests (typing never waits on one)" (SearchService + .swift:52-56, 99-113). The adapter held the serialize-lock across the engine's + 120 s of ffmpeg work, undoing that design: a 1024-px preview froze every + keystroke-search behind it, and a 40-hit gallery serialized 40 renders.""" + import threading + + class SlowRenderProc: + """The protocol allocates the request `id` itself (`protocol.py: request()` — + `fields` carries no id), routes the reply by it, and strips it before returning. + This fake answers with a fresh id, out of order, like the engine does.""" + + def __init__(self): + self._ids = iter(range(1, 10_000)) + + def request(self, fields, *, timeout): + rid = next(self._ids) + if fields["op"] == "render": + time.sleep(1.2) # a slow ffmpeg frame, engine-side + return {"id": rid, "image": "/index/previews/x.jpg"} + return {"id": rid, "hits": [], "strong": 0, "meaning": True, "ms": {}} + + service = SearchService(EngineConfig(index=str(tmp_path), roots=("/corpus",))) + service._proc = SlowRenderProc() + box = {} + + def do_render(): + box["reply"] = service.render("/media/photos/x.jpg", pixels=1024) + + thread = threading.Thread(target=do_render) + thread.start() + time.sleep(0.15) # the render is in flight + began = time.monotonic() + reply = service.search("wakeboarding", limit=10) + typing_ms = time.monotonic() - began + thread.join(10) + assert reply["hits"] == [] # answered — and answered FAST + assert typing_ms < 0.9, f"typing waited {typing_ms:.2f} s behind the render" + assert box["reply"]["image"].endswith("x.jpg") # out-of-order id-routing intact diff --git a/server/tests/test_security.py b/server/tests/test_security.py new file mode 100644 index 00000000..fd30e3b2 --- /dev/null +++ b/server/tests/test_security.py @@ -0,0 +1,153 @@ +"""The Host/Origin guard (server/digup_server/security.py): pure ASGI, no engine, no lifespan. + +The behavior this pins is the *shape* of the opt-in: loopback default = guard off (nothing +changes for today's deployment), guard on = the named hosts only, browsers' state-changing +requests same-origin only, non-browser clients (no `Origin`) still welcome. Those are the rules +a LAN/proxy deployment must not quietly lose. +""" + +from __future__ import annotations + +import json + +import pytest +from fastapi.testclient import TestClient + +from digup_server.security import ( + Auth, + BearerAuth, + HostOriginGuard, + allowed_hosts_from_env, + auth_from_env, +) + + +async def echo(scope, receive, send) -> None: + """A minimal ASGI app: answers 200 with the method and path, so a request that passes the + guard is distinguishable from one the guard refused.""" + body = json.dumps({"ok": True, "method": scope["method"], "path": scope["path"]}).encode() + await send({"type": "http.response.start", "status": 200, + "headers": [(b"content-type", b"application/json")]}) + await send({"type": "http.response.body", "body": body}) + + +def client_for(allowed: frozenset[str] | None) -> TestClient: + return TestClient(HostOriginGuard(echo, allowed_hosts=allowed)) + + +def test_the_guard_is_off_when_no_hosts_are_named(): + assert allowed_hosts_from_env({}) is None + assert allowed_hosts_from_env({"DIGUP_SERVER_ALLOWED_HOSTS": " "}) is None + free = client_for(None) + assert free.get("/x", headers={"Host": "anything.example"}).status_code == 200 + assert free.post("/x", headers={"Host": "anything.example", + "Origin": "http://elsewhere.example"}).status_code == 200 + + +def test_named_hosts_are_enforced_case_insensitively_without_ports(): + allowed = allowed_hosts_from_env({"DIGUP_SERVER_ALLOWED_HOSTS": "A.example, 127.0.0.1:8787"}) + assert allowed == frozenset({"a.example", "127.0.0.1"}) + guarded = client_for(allowed) + assert guarded.get("/x", headers={"Host": "a.example"}).status_code == 200 + assert guarded.get("/x", headers={"Host": "127.0.0.1:8787"}).status_code == 200 + refused = guarded.get("/x", headers={"Host": "evil.example"}) + assert refused.status_code == 400 + assert isinstance(refused.json()["detail"], str) + assert refused.headers["content-type"].startswith("application/json") + + +def test_no_wildcard_is_understood(): + """`*` is a host name here, not a wildcard: a list is the point of the setting, and a + deployment that means "every host" leaves the setting unset.""" + assert allowed_hosts_from_env({"DIGUP_SERVER_ALLOWED_HOSTS": "*"}) == frozenset({"*"}) + assert client_for(frozenset({"*"})).get("/x", headers={"Host": "a.example"}).status_code == 400 + + +def test_state_changing_requests_from_another_origin_are_refused(): + guarded = client_for(frozenset({"127.0.0.1"})) + same = {"Host": "127.0.0.1:8787", "Origin": "http://127.0.0.1:8787"} + assert guarded.post("/x", headers=same).status_code == 200 + cross = {"Host": "127.0.0.1:8787", "Origin": "http://evil.example"} + refused = guarded.post("/x", headers=cross) + assert refused.status_code == 403 and isinstance(refused.json()["detail"], str) + # non-browsers send no Origin at all: curl, Immich, Home Assistant, agents + assert guarded.post("/x", headers={"Host": "127.0.0.1:8787"}).status_code == 200 + # a read-only GET carries no Origin in browsers, and is not checked + assert guarded.get("/x", headers={"Host": "127.0.0.1:8787", + "Origin": "http://evil.example"}).status_code == 200 + + +def test_the_api_installs_the_guard_inert_by_default(): + """create_app() always installs the middleware: the switch is configuration + (`DIGUP_SERVER_ALLOWED_HOSTS`), never a code change. No engine is spawned by building the + app — that happens in the lifespan — so this stays a fast unit test.""" + from digup_server.api import create_app + from digup_server.engine import EngineConfig + app = create_app(EngineConfig(index="/nonexistent-index", roots=("/nonexistent-root",))) + guards = [m for m in app.user_middleware if m.cls is HostOriginGuard] + assert len(guards) == 1 + assert guards[0].kwargs.get("allowed_hosts") is None # unset env → guard off + + +# ── the bearer gate: off by default, read scope everywhere, admin scope on index/estimate ── + +def auth_client(read: str | None = None, admin: str | None = None) -> TestClient: + return TestClient(BearerAuth(echo, auth=Auth(read=read, admin=admin))) + + +def test_no_tokens_means_no_auth_at_all(): + assert auth_from_env({}) == Auth(read=None, admin=None) + open_book = auth_client() + assert open_book.post("/api/v1/index").status_code == 200 + assert open_book.get("/api/v1/search").status_code == 200 + + +def test_read_token_covers_the_search_side_and_asks_401_first(): + guarded = auth_client(read="s3cret") + naked = guarded.post("/api/v1/search") + assert naked.status_code == 401 + assert naked.headers["www-authenticate"] == "Bearer" + assert isinstance(naked.json()["detail"], str) + ok = guarded.post("/api/v1/search", headers={"Authorization": "Bearer s3cret"}) + assert ok.status_code == 200 + assert guarded.post("/api/v1/search", headers={"Authorization": "Bearer wrong"}).status_code == 401 + assert guarded.post("/api/v1/search", headers={"Authorization": "s3cret"}).status_code == 401 + + +def test_health_and_the_docs_page_stay_open_so_the_healthcheck_and_the_openapi_work(): + guarded = auth_client(read="s3cret", admin="adm1n") + for path in ("/api/v1/health", "/api/v1/docs", "/api/v1/openapi.json", + "/", "/index.html", "/app.js"): + assert guarded.get(path).status_code == 200, path + + +def test_index_and_estimate_are_admin_scope(): + both = auth_client(read="s3cret", admin="adm1n") + assert both.post("/api/v1/index").status_code == 401 + assert both.post("/api/v1/estimate").status_code == 401 + read_only = both.post("/api/v1/index", headers={"Authorization": "Bearer s3cret"}) + assert read_only.status_code == 403 # the right token, the wrong scope + assert "admin" in read_only.json()["detail"] + assert both.post("/api/v1/index", headers={"Authorization": "Bearer adm1n"}).status_code == 200 + assert both.post("/api/v1/estimate", headers={"Authorization": "Bearer adm1n"}).status_code == 200 + assert both.get("/api/v1/jobs", headers={"Authorization": "Bearer s3cret"}).status_code == 200 + + +def test_an_admin_only_deployment_leaves_search_open_to_the_shared_clients(): + """A deployment that names only the admin token is a localhost box that wants a *guard on + its mutating endpoints*: the read side stays open for every client on the machine.""" + admin_only = auth_client(admin="adm1n") + assert admin_only.get("/api/v1/status").status_code == 200 + assert admin_only.post("/api/v1/search").status_code == 200 + assert admin_only.post("/api/v1/index").status_code == 401 + assert admin_only.post("/api/v1/index", + headers={"Authorization": "Bearer adm1n"}).status_code == 200 + + +def test_the_api_installs_the_bearer_gate_inert_by_default(): + from digup_server.api import create_app + from digup_server.engine import EngineConfig + app = create_app(EngineConfig(index="/nonexistent-index", roots=("/nonexistent-root",))) + gates = [m for m in app.user_middleware if m.cls is BearerAuth] + assert len(gates) == 1 + assert gates[0].kwargs["auth"].enabled is False diff --git a/server/tests/test_targets.py b/server/tests/test_targets.py new file mode 100644 index 00000000..d15ba802 --- /dev/null +++ b/server/tests/test_targets.py @@ -0,0 +1,512 @@ +"""Stage 13 — named indexes and minted application keys (targets.py, mint_tokens, +target-scoped BearerAuth, the API's per-target supervision). + +The promise under test is the user's flow, end to end: the compose server runs +standalone over its media index; an application (Nextcloud's bridge) gets its own minted +key, pastes it into its settings, and every call that carries it — status, search, index, +jobs, media — answers the application's OWN index. The media index is untouched by that +application's crawl, one writer per index keeps both crawls independent, and no token +ever answers a target it was not minted for. +""" +from __future__ import annotations + +import json +import os +import stat +import time + +import pytest +from fastapi.testclient import TestClient + +from conftest import _cfg +from digup_server.security import Auth, BearerAuth, PrincipalLog, mint_tokens, read_tokens_from_env +from digup_server.targets import MAIN_TARGET, build_registry, load_targets_file, parse_app_targets + +CANARY = "A red canoe rests on a glass-still lake under northern lights.\n" + + +# ── registry: parsing, validation, merging ───────────────────────────────────────────── + +def test_the_registry_parses_the_env_shape_exactly_as_documented(): + targets = parse_app_targets("nc=/media/nc:/media/nc2:/idx/nc;dc=/media/dc:/idx/dc") + assert targets["nc"].roots == ("/media/nc", "/media/nc2") + assert targets["nc"].index == "/idx/nc" + assert targets["dc"].roots == ("/media/dc",) and targets["dc"].index == "/idx/dc" + + +def test_the_registry_refuses_every_shape_that_could_silently_break_a_target(): + for bad in ("x=/media/a", # no index folder + "media=/media/a:/idx", # reserved name + "nc=relative/path:/idx/nc", # root not absolute + "nc=/media/a:relative-index", # index not absolute + "nc=/media/a:", # empty index + "=/media/a:/idx", # empty name + "nc=/media/a:/idx/nc;;dc=/media/b"): # a later entry is broken + with pytest.raises(ValueError): + parse_app_targets(bad) + + +def test_registry_merges_the_file_and_the_env_with_env_winning(tmp_path): + (tmp_path / "targets.json").write_text(json.dumps({ + "dc": {"roots": ["/media/from-file"], "index": "/idx/file-dc"}, + "fl": {"roots": ["/media/fl"], "index": "/idx/fl"}, + }), encoding="utf-8") + reg = build_registry({"DIGUP_SERVER_APP_TARGETS": "dc=/media/env:/idx/env-dc"}, + main_roots=("/media/photos",), main_index="/idx/main", + config_dir=str(tmp_path)) + assert reg[MAIN_TARGET].main and reg[MAIN_TARGET].roots == ("/media/photos",) + assert reg["dc"].roots == ("/media/env",) and reg["dc"].index == "/idx/env-dc" + assert reg["fl"].roots == ("/media/fl",) # file-only target joins the registry + + +def test_a_target_may_not_be_the_main_index_itself(tmp_path): + (tmp_path / "targets.json").write_text( + json.dumps({"x": {"roots": ["/media/a"], "index": "/idx/main"}}), encoding="utf-8") + with pytest.raises(ValueError, match="separate indexes"): + build_registry({}, main_roots=("/media/photos",), main_index="/idx/main", + config_dir=str(tmp_path)) + + +# ── minted keys: inert alone, stable once, env-authoritative, 0600 ───────────────────── + +def test_minting_is_inert_without_targets_a_standalone_server_stays_exactly_it_was(tmp_path): + auth = mint_tokens({}, str(tmp_path), ()) + assert not auth.enabled + assert not (tmp_path / "tokens.json").exists() # nothing written, nothing minted + + +def test_minted_keys_fire_for_targets_are_restart_stable_and_live_0600(tmp_path): + first = mint_tokens({}, str(tmp_path), ("nc",)) + assert first.enabled and first.admin and first.readers["app:nc"] + assert set(first.minted) == {"admin", "app:nc"} + second = mint_tokens({}, str(tmp_path), ("nc",)) # a restart re-reads the file + assert second.admin == first.admin and second.readers["app:nc"] == first.readers["app:nc"] + # File-backed keys are server-generated keys: displayable on every boot, not only + # the boot that generated them (the UI shows what the operator never typed). + assert set(second.minted) == {"admin", "app:nc"} + mode = stat.S_IMODE(os.stat(tmp_path / "tokens.json").st_mode) + assert mode == 0o600 + stored = json.loads((tmp_path / "tokens.json").read_text(encoding="utf-8")) + assert stored["admin"] == first.admin and stored["readers"]["app:nc"] == first.readers["app:nc"] + + +def test_environment_tokens_are_authority_mint_only_fills_the_gaps(tmp_path): + env = {"DIGUP_SERVER_ADMIN_TOKEN": "adm", "DIGUP_SERVER_READ_TOKENS": "app:nc:toknc"} + auth = mint_tokens(env, str(tmp_path), ("nc", "dc")) + assert auth.admin == "adm" and "admin" not in auth.minted # env admin kept, not echoed + assert auth.readers["app:nc"] == "toknc" and "app:nc" not in auth.minted + assert "app:dc" in auth.minted # gap filled for the rest + stored = json.loads((tmp_path / "tokens.json").read_text(encoding="utf-8")) + assert "admin" not in stored # env secrets never stored + assert stored.get("readers", {}).get("app:nc") is None # ditto + assert stored["readers"]["app:dc"] == auth.readers["app:dc"] + + +def test_rotating_minted_keys_mints_new_values_and_leaves_env_keys_intact(tmp_path): + from digup_server.security import rotate_minted + d = str(tmp_path) + first = mint_tokens({"DIGUP_SERVER_ADMIN_TOKEN": "adm-env", + "DIGUP_SERVER_READ_TOKENS": "app:dc:tokdc"}, d, ("nc", "dc")) + rotated = rotate_minted(first, d) + assert rotated is not None + assert rotated.admin == "adm-env" # env admin: kept, never rotated + assert rotated.readers["app:dc"] == "tokdc" # env app key: never rotated + assert rotated.readers["app:nc"] != first.readers["app:nc"] # minted: a fresh value + assert rotated.minted == ("app:nc",) + stored = json.load(open(os.path.join(d, "tokens.json"), encoding="utf-8")) + assert stored["readers"]["app:nc"] == rotated.readers["app:nc"] + assert "admin" not in stored and stored["readers"].get("app:dc") is None + assert rotated.token_names(first.readers["app:nc"]) == () # the old key dies at the live auth + assert rotated.token_names(rotated.readers["app:nc"]) == ("app:nc",) + assert stat.S_IMODE(os.stat(os.path.join(d, "tokens.json")).st_mode) == 0o600 + + +def test_rotation_offers_nothing_where_nothing_is_minted(tmp_path): + from digup_server.security import rotate_minted + assert rotate_minted(mint_tokens({"DIGUP_SERVER_TOKEN": "plain"}, str(tmp_path), ()), + str(tmp_path)) is None + + +def test_read_token_names_may_carry_colons_the_last_one_is_the_separator(): + readers = read_tokens_from_env({"DIGUP_SERVER_READ_TOKENS": "nc-a:tokA,app:nc:tokB"}) + assert readers == {"nc-a": "tokA", "app:nc": "tokB"} + + +# ── the bearer gate binds tokens to targets (unit, no engine) ────────────────────────── + +async def _echo(scope, receive, send): + body = json.dumps({"target": scope.get("digup.target"), + "principal": scope.get("digup.principal")}).encode() + await send({"type": "http.response.start", "status": 200, + "headers": [(b"content-type", b"application/json")]}) + await send({"type": "http.response.body", "body": body}) + + +def gated_client(auth: Auth) -> TestClient: + return TestClient(PrincipalLog(BearerAuth(_echo, auth=auth))) + + +def test_app_tokens_answer_their_own_target_and_nothing_else(): + auth = Auth(admin="adm", readers={"app:nc": "toknc"}) + c = gated_client(auth) + nc = {"Authorization": "Bearer toknc"} + adm = {"Authorization": "Bearer adm"} + # The binding IS the token's default: plain paths answer its own index (the bridge + # needs no target parameter), and any explicit target it does not match is refused. + r = c.get("/api/v1/status", headers=nc) + assert r.status_code == 200 and r.json()["target"] == "nc" + assert c.get("/api/v1/status", params={"target": "nc"}, headers=nc).status_code == 200 + assert c.get("/api/v1/status", params={"target": "media"}, headers=nc).status_code == 403 + assert c.get("/api/v1/status", params={"target": "other"}, headers=nc).status_code == 403 + assert c.post("/api/v1/index", params={"target": "nc"}, headers=nc).status_code == 200 + assert c.post("/api/v1/index", headers=nc).status_code == 200 # own target by default + assert c.post("/api/v1/index", params={"target": "media"}, headers=nc).status_code == 403 + assert c.post("/api/v1/index", headers=adm).status_code == 200 # media: admin's own + assert c.post("/api/v1/index", params={"target": "nc"}, headers=adm).status_code == 200 + assert c.get("/api/v1/status", params={"target": "nc"}, headers=adm).status_code == 200 + plain = gated_client(Auth(read="plain", admin="adm")) + assert plain.get("/api/v1/status", params={"target": "nc"}, + headers={"Authorization": "Bearer plain"}).status_code == 403 + assert plain.get("/api/v1/status", headers={"Authorization": "Bearer plain"}).status_code == 200 + + +def test_the_principal_log_names_the_application_that_asked(caplog): + import logging + c = gated_client(Auth(admin="adm", readers={"app:nc": "toknc"})) + with caplog.at_level(logging.INFO, logger="digup.auth"): + c.get("/api/v1/status", params={"target": "nc"}, headers={"Authorization": "Bearer toknc"}) + lines = [r.message for r in caplog.records if "principal=" in r.message] + assert any("principal=app:nc" in line for line in lines) + # The audit line names WHICH index was addressed (named indexes, stage 13). + assert any("target=nc" in line for line in lines) + + +# ── end to end: minted key → target machinery, over the real engine ──────────────────── + +def test_an_application_key_answers_only_its_own_index_end_to_end(scratch, tmp_path, monkeypatch): + from digup_server.api import create_app + + main_index = tmp_path / "main-index" + app_index = tmp_path / "app-index" + monkeypatch.setenv("DIGUP_SERVER_APP_TARGETS", f"nc={scratch}:{app_index}") + monkeypatch.setenv("DIGUP_SERVER_CONFIG", str(tmp_path / "server-config")) + with TestClient(create_app(_cfg(main_index, (tmp_path / "media",)))) as c: + keys = json.loads((tmp_path / "server-config" / "tokens.json").read_text(encoding="utf-8")) + adm, nctok = keys["admin"], keys["readers"]["app:nc"] + nc = {"Authorization": f"Bearer {nctok}"} + admin = {"Authorization": f"Bearer {adm}"} + + # The target's status answers the target's roots, the media index stays the app's. + st = c.get("/api/v1/status", headers=nc).json() + assert st["roots"] == [str(scratch)] and st["target"] == "nc" + assert c.get("/api/v1/status", headers=admin).json()["roots"] == [str(tmp_path / "media")] + + # The application queues its OWN crawl — and never the media crawl. + assert c.post("/api/v1/index", headers=nc).status_code == 202 + assert c.post("/api/v1/index", params={"target": "media"}, headers=nc).status_code == 403 + job_id = c.get("/api/v1/jobs", headers=nc).json()["jobs"][-1]["id"] + assert wait_final(c, job_id, nc) == "done" + st = c.get("/api/v1/status", headers=nc).json() + assert st["files"] > 0 and st["searchable"] > 0 + assert c.get("/api/v1/status", headers=admin).json()["files"] == 0 # media untouched + + # Its own search answers from its own index; the media index has none of its files. + hits = c.post("/api/v1/search", json={"q": "campfire stories", "limit": 5}, + headers=nc).json()["hits"] + assert hits and all(h["path"].startswith(str(scratch)) for h in hits) + assert c.post("/api/v1/search", json={"q": "campfire stories"}, + headers=admin).json()["hits"] == [] + + # Media handles resolve their target: the app's handle previews through its + # engine. The preview contract is the engine's: it renders image/pdf/video + # segments, and a text doc answers the honest "unreadable: can't decode + # image" refusal forever — so preview the IMAGE hit (and poll: right after a + # job the searcher is mid reload→prewarm, and refusals are states, pinned + # 503s in test_api, that clients retry). + media_id = next(h["media_id"] for h in hits if h["kind"] == "image") + assert wait_status(c, f"/api/v1/media/{media_id}/preview", nc, 200) == 200 + assert wait_status(c, f"/api/v1/media/{media_id}/preview", admin, 200) == 200 + + # Jobs are per target: the app sees its own history; its log is its own file. + assert all(j["state"] in ("done", "interrupted") + for j in c.get("/api/v1/jobs", headers=nc).json()["jobs"]) + assert c.get("/api/v1/jobs?target=media", headers=nc).status_code == 403 + assert (tmp_path / "server-config" / "jobs-nc.jsonl").exists() + + # Targets listing: admin sees both with counts; the app sees only its own. + # The counts come from the CLI's own table (kinds total + vectors) — no engine + # spawn for a listing. + rows = {t["name"]: t for t in c.get("/api/v1/targets", headers=admin).json()["targets"]} + assert set(rows) == {MAIN_TARGET, "nc"} + assert rows["nc"]["files"] > 0 and rows["nc"]["searchable"] > 0, \ + (rows["nc"]["files"], rows["nc"]["searchable"]) + assert rows[MAIN_TARGET]["files"] == 0, rows[MAIN_TARGET]["files"] + assert {t["name"] for t in c.get("/api/v1/targets", headers=nc).json()["targets"]} == {"nc"} + + # Tokens: minted values shown to admin only, never to an app token. + t = c.get("/api/v1/tokens", headers=admin).json() + assert t["admin"] == adm and t["apps"][0]["token"] == nctok + assert c.get("/api/v1/tokens", headers=nc).status_code == 403 + + +def test_two_targets_crawl_independently_one_writer_per_index(scratch, tmp_path, monkeypatch): + """The busy guarantee is per index: a running job on target A never blocks target B + (the flock lives beside each index folder) and never blocks the media crawl.""" + from digup_server.api import create_app + from digup_server.jobs import JOB_LOG + + app_a, app_b = tmp_path / "idx-a", tmp_path / "idx-b" + roots_a = tmp_path / "media-a" + roots_a.mkdir() + monkeypatch.setenv("DIGUP_SERVER_APP_TARGETS", f"a={scratch}:{app_a};b={roots_a}:{app_b}") + monkeypatch.setenv("DIGUP_SERVER_CONFIG", str(tmp_path / "config")) + with TestClient(create_app(_cfg(tmp_path / "main-index", (roots_a,)))) as c: + keys = json.loads((tmp_path / "config" / "tokens.json").read_text(encoding="utf-8")) + adm = {"Authorization": f"Bearer {keys['admin']}"} + tok_a = {"Authorization": f"Bearer {keys['readers']['app:a']}"} + tok_b = {"Authorization": f"Bearer {keys['readers']['app:b']}"} + + job = c.post("/api/v1/index", headers=tok_a).json()["job"] + assert job["state"] in ("queued", "running") + # Same target, second request: the honest 409 (one writer per index). + busy = c.post("/api/v1/index", headers=tok_a) + assert busy.status_code == 409 and job["id"] in busy.json()["detail"] + # A different target answers 202 right away — independent writers, independent logs. + assert c.post("/api/v1/index", headers=tok_b).status_code == 202 + # The media crawl is independent of both. + assert c.post("/api/v1/index", headers=adm).status_code == 202 + for tok in (tok_a, tok_b): + deadline = time.monotonic() + 900 + while time.monotonic() < deadline: + jobs = c.get("/api/v1/jobs", headers=tok).json()["jobs"] + if jobs and jobs[-1]["state"] in ("done", "interrupted"): + break + time.sleep(1) + assert jobs[-1]["state"] == "done" + for name in ("a", "b"): + assert (tmp_path / "config" / f"jobs-{name}.jsonl").exists() + assert (tmp_path / "config" / JOB_LOG).exists() + + +def wait_final(client, job_id, headers, timeout=900.0) -> str: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + state = client.get(f"/api/v1/jobs/{job_id}", headers=headers).json()["job"]["state"] + if state in ("done", "interrupted", "busy"): + return state + time.sleep(0.5) + pytest.fail(f"job {job_id} never reached a final state within {timeout:g} s") + + +def wait_status(client, url, headers, want, timeout=120.0) -> int: + """Poll a GET until it answers `want` — a searcher mid reload→prewarm answers the + engine's 503 refusal, which the API surfaces verbatim (test_api pins it); clients + retry, and so does this contract test. The failure names the LAST answer's body: + the engine's own words are the diagnosis.""" + deadline = time.monotonic() + timeout + code, body = None, "" + while time.monotonic() < deadline: + r = client.get(url, headers=headers) + code, body = r.status_code, r.text[:200] + if code == want: + return code + time.sleep(1.0) + pytest.fail(f"{url} never answered {want} within {timeout:g} s " + f"(last {code}: {body})") + + +# ── POST /api/v1/tokens — the UI's Regenerate (admin scope, minted keys only) ───────── + +def test_the_admin_can_regenerate_the_minted_keys_and_the_old_values_die(tmp_path, + monkeypatch): + from digup_server.api import create_app + monkeypatch.setenv("DIGUP_SERVER_APP_TARGETS", f"nc={tmp_path}/media:{tmp_path}/app-index") + monkeypatch.setenv("DIGUP_SERVER_CONFIG", str(tmp_path / "server-config")) + with TestClient(create_app(_cfg(tmp_path / "main-index", (tmp_path / "media",)))) as c: + keys = json.load(open(tmp_path / "server-config" / "tokens.json", encoding="utf-8")) + adm, old_nc = keys["admin"], keys["readers"]["app:nc"] + A = {"Authorization": f"Bearer {adm}"} + assert c.get("/api/v1/tokens", headers=A).json()["admin"] == adm + r = c.post("/api/v1/tokens", headers=A) + assert r.status_code == 200 and r.json()["rotated"] is True + new_admin, new_nc = r.json()["admin"], r.json()["apps"][0]["token"] + assert new_admin != adm and new_nc != old_nc + # The rotation takes effect immediately — the old values die, the new ones answer: + assert c.get("/api/v1/status", headers={"Authorization": f"Bearer {old_nc}"} + ).status_code == 401 + assert c.get("/api/v1/status", headers=A).status_code == 401 + st = c.get("/api/v1/status", headers={"Authorization": f"Bearer {new_nc}"}) + assert st.status_code == 200 and st.json()["target"] == "nc" + assert c.post("/api/v1/tokens", + headers={"Authorization": f"Bearer {new_nc}"}).status_code == 403 + fresh = json.load(open(tmp_path / "server-config" / "tokens.json", encoding="utf-8")) + assert fresh["admin"] == new_admin and fresh["readers"]["app:nc"] == new_nc + assert stat.S_IMODE(os.stat(tmp_path / "server-config" / "tokens.json").st_mode) == 0o600 + + +def test_an_env_keyed_deployment_rotates_nothing(tmp_path, monkeypatch): + from digup_server.api import create_app + monkeypatch.setenv("DIGUP_SERVER_ADMIN_TOKEN", "adm") + monkeypatch.delenv("DIGUP_SERVER_APP_TARGETS", raising=False) + monkeypatch.delenv("DIGUP_SERVER_TOKEN", raising=False) + monkeypatch.setenv("DIGUP_SERVER_CONFIG", str(tmp_path / "server-config")) + with TestClient(create_app(_cfg(tmp_path / "main-index", (tmp_path / "media",)))) as c: + A = {"Authorization": "Bearer adm"} + t = c.get("/api/v1/tokens", headers=A).json() + assert t["admin"] is None and t["admin_configured"] is True # never echoed + # The typed-secrets shape stays guarded: a keyless call reaches the endpoint + # (the gate's open door is the main index's READS; the keys surface is not a + # read) and the endpoint refuses it — 403, keys shown to the admin token only. + # Round 6 draws the open-DISPLAY line at SERVER-MINTED keys only. + assert c.get("/api/v1/tokens").status_code == 403 + r = c.post("/api/v1/tokens", headers=A) + assert r.status_code == 400 and "environment" in r.json()["detail"] + assert not (tmp_path / "server-config" / "tokens.json").exists() + + +# ── the minted shape answers the MAIN index's reads openly (2026-11-08 user brief) ──── + +def test_a_minted_key_deployment_answers_main_reads_openly(tmp_path, monkeypatch): + """The user's model: the standalone box is searchable with no key; the minted keys + are what the APPLICATIONS get. Keys typed in the environment (the guarded shape) + keep guarding every call — test_security pins that shape unchanged.""" + from digup_server.api import create_app + monkeypatch.setenv("DIGUP_SERVER_APP_TARGETS", f"nc={tmp_path}/media:{tmp_path}/app-index") + monkeypatch.setenv("DIGUP_SERVER_CONFIG", str(tmp_path / "server-config")) + with TestClient(create_app(_cfg(tmp_path / "main-index", (tmp_path / "media",)))) as c: + keys = json.load(open(tmp_path / "server-config" / "tokens.json", encoding="utf-8")) + nc = {"Authorization": f"Bearer {keys['readers']['app:nc']}"} + # No key: the main index answers reads (status, search, targets). + st = c.get("/api/v1/status").json() + assert st["target"] == MAIN_TARGET and st["files"] == 0 + assert c.post("/api/v1/search", json={"q": "wakeboarding", "limit": 5} + ).status_code == 200 + names = {t["name"] for t in c.get("/api/v1/targets").json()["targets"]} + assert MAIN_TARGET in names # the gear's rows + # What the keys are FOR stays guarded: the index operations, the keys surface, + # and the application's OWN index (the open door is the main index's reads). + assert c.post("/api/v1/index", json={}).status_code == 401 + assert c.post("/api/v1/estimate", json={}).status_code == 401 + assert c.get("/api/v1/status", params={"target": "nc"}).status_code == 401 + # Round 6 (the ticket model): a minted deployment shows its keys to the gear + # WITHOUT a key in the browser — the same open shape as the main page's reads; + # an application's own key never lists the server's keys. + tk = c.get("/api/v1/tokens").json() + assert tk["admin"] == keys["admin"] and any(a["name"] == "nc" for a in tk["apps"]) + assert c.get("/api/v1/tokens", headers=nc).status_code == 403 + # The app key keeps answering its own target and nothing else. + assert c.get("/api/v1/status", headers=nc).json()["target"] == "nc" + assert c.get("/api/v1/status", params={"target": "media"}, + headers=nc).status_code == 403 + + +def test_an_open_principal_never_previews_an_applications_files(tmp_path, monkeypatch, scratch): + """The minted shape opens the MAIN index's reads — media handles included, so the + media-id path must stay scoped: a key-less request for a handle whose file lives in + an application's roots is refused; the application's own key previews it.""" + from digup_server.api import create_app + monkeypatch.setenv("DIGUP_SERVER_APP_TARGETS", f"nc={scratch}:{tmp_path}/app-index") + monkeypatch.setenv("DIGUP_SERVER_CONFIG", str(tmp_path / "server-config")) + with TestClient(create_app(_cfg(tmp_path / "main-index", (tmp_path / "media",)))) as c: + keys = json.load(open(tmp_path / "server-config" / "tokens.json", encoding="utf-8")) + nc = {"Authorization": f"Bearer {keys['readers']['app:nc']}"} + assert c.post("/api/v1/index", headers=nc).status_code == 202 + job_id = c.get("/api/v1/jobs", headers=nc).json()["jobs"][-1]["id"] + assert wait_final(c, job_id, nc) == "done" + hits = c.post("/api/v1/search", json={"q": "campfire stories", "limit": 5}, + headers=nc).json()["hits"] + media_id = next(h["media_id"] for h in hits if h["kind"] == "image") + assert wait_status(c, f"/api/v1/media/{media_id}/preview", nc, 200) == 200 + assert c.get(f"/api/v1/media/{media_id}/preview").status_code == 403 + assert c.get(f"/api/v1/media/{media_id}/file").status_code == 403 + assert c.get(f"/api/v1/media/{media_id}/text").status_code == 403 + + +# ── stage 14: the standalone mints its key; applications register at runtime ───────── + +def test_the_standalone_mints_its_api_key_when_asked(tmp_path, monkeypatch): + """DIGUP_SERVER_MINT_KEYS=1: the key exists at boot with no application registered — + the gear's Copy row. Main reads stay open; the key guards index/estimate/keys.""" + from digup_server.api import create_app + monkeypatch.setenv("DIGUP_SERVER_MINT_KEYS", "1") + monkeypatch.delenv("DIGUP_SERVER_APP_TARGETS", raising=False) + monkeypatch.delenv("DIGUP_SERVER_TOKEN", raising=False) + monkeypatch.delenv("DIGUP_SERVER_ADMIN_TOKEN", raising=False) + monkeypatch.setenv("DIGUP_SERVER_CONFIG", str(tmp_path / "server-config")) + with TestClient(create_app(_cfg(tmp_path / "main-index", (tmp_path / "media",)))) as c: + path = tmp_path / "server-config" / "tokens.json" + keys = json.load(open(path, encoding="utf-8")) + adm = keys["admin"] + A = {"Authorization": f"Bearer {adm}"} + assert adm and stat.S_IMODE(os.stat(path).st_mode) == 0o600 + assert c.get("/api/v1/status").status_code == 200 # main reads: open + assert c.post("/api/v1/index", json={}).status_code == 401 # mutations: keyed + assert c.post("/api/v1/estimate", json={}).status_code == 401 + # Round 6: the minted key is DISPLAYED with no key in the browser — the gear's + # Copy row is the ticket; the admin call shows the same value. + tk = c.get("/api/v1/tokens").json() + assert tk["admin"] == adm and tk["apps"] == [] + t = c.get("/api/v1/tokens", headers=A).json() + assert t["admin"] == adm and t["apps"] == [] + assert c.post("/api/v1/estimate", headers=A, json={}).status_code == 200 + # An application's key never lists the server's keys. + reg = c.post("/api/v1/targets", headers=A, + json={"name": "demo", "roots": [str(tmp_path / "media")]}) + assert reg.status_code == 200 + assert c.get("/api/v1/tokens", + headers={"Authorization": f"Bearer {reg.json()['token']}"}).status_code == 403 + + +def test_an_application_registers_at_a_running_server_and_gets_its_own_key(tmp_path, + scratch, + monkeypatch): + """The user's flow as one call: standalone key → POST /targets(name, roots) → + private minted key + own index folder (default <config>/index-<name>) → its crawl + runs in the background; the main page never sees it; the registration survives + restart (targets.json + tokens.json).""" + from digup_server.api import create_app + monkeypatch.setenv("DIGUP_SERVER_MINT_KEYS", "1") + monkeypatch.setenv("DIGUP_SERVER_CONFIG", str(tmp_path / "server-config")) + with TestClient(create_app(_cfg(tmp_path / "main-index", (tmp_path / "media",)))) as c: + keys = json.load(open(tmp_path / "server-config" / "tokens.json", encoding="utf-8")) + A = {"Authorization": f"Bearer {keys['admin']}"} + r = c.post("/api/v1/targets", headers=A, json={"name": "nc", "roots": [str(scratch)]}) + assert r.status_code == 200, r.text + reg = r.json() + assert reg["generated"] is True + assert reg["index"] == str(tmp_path / "server-config" / "index-nc") + N = {"Authorization": f"Bearer {reg['token']}"} + assert c.get("/api/v1/status", headers=N).json()["target"] == "nc" + assert c.get("/api/v1/status", params={"target": "media"}, + headers=N).status_code == 403 + # its OWN crawl, in the background — and never anyone else's + assert c.post("/api/v1/index", headers=N).status_code == 202 + job_id = c.get("/api/v1/jobs", headers=N).json()["jobs"][-1]["id"] + assert wait_final(c, job_id, N) == "done" + assert c.get("/api/v1/status", headers=N).json()["files"] > 0 + assert c.get("/api/v1/status").json()["files"] == 0 # main untouched + # persisted: a restart re-reads both files (the keys and the registration) + tj = json.load(open(tmp_path / "server-config" / "targets.json", encoding="utf-8")) + assert tj["nc"]["roots"] == [str(scratch)] + tk = json.load(open(tmp_path / "server-config" / "tokens.json", encoding="utf-8")) + assert tk["readers"]["app:nc"] == reg["token"] + # the startup validator runs at runtime: every refusal quotes its rule + bad = [({"name": "media", "roots": [str(scratch)]}, 400), + ({"name": "nc", "roots": [str(scratch)]}, 409), + ({"name": "Bad Name", "roots": [str(scratch)]}, 400), + ({"name": "x", "roots": ["relative/root"]}, 400), + ({"name": "y", "roots": [str(scratch)], + "index": str(tmp_path / "main-index")}, 400)] + for body, want in bad: + assert c.post("/api/v1/targets", headers=A, json=body).status_code == want + # key-less and application-keyed requests cannot register anything + assert c.post("/api/v1/targets", + json={"name": "z", "roots": [str(scratch)]}).status_code == 403 + assert c.post("/api/v1/targets", headers=N, + json={"name": "z", "roots": [str(scratch)]}).status_code == 403 + # the open principal never previews the application's files + hits = c.post("/api/v1/search", headers=N, json={"q": "campfire stories", + "limit": 5}).json()["hits"] + media_id = next(h["media_id"] for h in hits if h["kind"] == "image") + assert wait_status(c, f"/api/v1/media/{media_id}/preview", N, 200) == 200 + assert c.get(f"/api/v1/media/{media_id}/preview").status_code == 403 diff --git a/server/ui/embed.html b/server/ui/embed.html new file mode 100644 index 00000000..eabf1711 --- /dev/null +++ b/server/ui/embed.html @@ -0,0 +1,32 @@ +<!doctype html> +<!-- DigUp embed widget — a compact search page for hosts that embed it (a Nextcloud app + page, an Immich workspace panel, a dashboard iframe). Lit component <digup-embed>, + built by Vite to the pinned names `embed.js` / `embed.css`, same origin as the API, + no external URLs — the same UI rules as index.html (test_integrations pins this + file too). Thin shell: the widget renders in its own shadow. + + Query parameters (all optional): + token the read-scope bearer token (docs/API.md "Authentication"): it moves from + the URL into sessionStorage and the URL line is rewritten, so the secret + never lands in bookmarks, history, or the host app's logs; + limit results per search (1–100, the engine's own cap); + q runs this search as soon as the page is ready. + The host may post {source:"digup-host", theme:"light"|"dark"} to set the theme; + the widget posts {source:"digup-embed", event:"ready"} once, and answers + {source:"digup-host", event:"theme"} on request. + A 401 with a stored token clears it and asks once — the same rule as the full UI. --> +<html lang="en" data-theme="system"> +<head> + <meta charset="utf-8"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <meta name="referrer" content="none"> + <link rel="icon" href="data:,"> + <title>DigUp — embed + + + + + + + diff --git a/server/ui/index.html b/server/ui/index.html new file mode 100644 index 00000000..2f2cd929 --- /dev/null +++ b/server/ui/index.html @@ -0,0 +1,33 @@ + + + + + + + + + DigUp — semantic media search + + + + + + + diff --git a/server/ui/package-lock.json b/server/ui/package-lock.json new file mode 100644 index 00000000..f3c81084 --- /dev/null +++ b/server/ui/package-lock.json @@ -0,0 +1,2018 @@ +{ + "name": "digup-ui", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "digup-ui", + "version": "0.1.0", + "dependencies": { + "lit": "^3.2.0" + }, + "devDependencies": { + "jsdom": "^26.0.0", + "vite": "^7.0.0", + "vitest": "^3.0.0" + } + }, + "node_modules/@asamuzakjp/css-color": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-3.2.0.tgz", + "integrity": "sha512-K1A6z8tS3XsmCMM86xoWdn7Fkdn9m6RSVtocUrJYIwZnFVkng/PvkEoWtOWmP+Scc6saYWHWZYbndEEXxl24jw==", + "dev": true, + "dependencies": { + "@csstools/css-calc": "^2.1.3", + "@csstools/css-color-parser": "^3.0.9", + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3", + "lru-cache": "^10.4.3" + } + }, + "node_modules/@csstools/color-helpers": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-5.1.0.tgz", + "integrity": "sha512-S11EXWJyy0Mz5SYvRmY8nJYTFFd1LCNV+7cXyAgQtOOuzb4EsgfqDufL+9esx72/eLhsRdGZwaldu/h+E4t4BA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@csstools/css-calc": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@csstools/css-calc/-/css-calc-2.1.4.tgz", + "integrity": "sha512-3N8oaj+0juUw/1H3YwmDDJXCgTB1gKU6Hc/bB502u9zR0q2vd786XJH9QfrKIEgFlZmhZiq6epXl4rHqhzsIgQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@csstools/css-parser-algorithms": "^3.0.5", + "@csstools/css-tokenizer": "^3.0.4" + } + }, + "node_modules/@csstools/css-color-parser": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@csstools/css-color-parser/-/css-color-parser-3.1.0.tgz", + "integrity": "sha512-nbtKwh3a6xNVIp/VRuXV64yTKnb1IjTAEEh3irzS+HkKjAOYLTGNb9pmVNntZ8iVBHcWDA2Dof0QtPgFI1BaTA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "dependencies": { + "@csstools/color-helpers": "^5.1.0", + "@csstools/css-calc": "^2.1.4" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@csstools/css-parser-algorithms": "^3.0.5", + "@csstools/css-tokenizer": "^3.0.4" + } + }, + "node_modules/@csstools/css-parser-algorithms": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@csstools/css-parser-algorithms/-/css-parser-algorithms-3.0.5.tgz", + "integrity": "sha512-DaDeUkXZKjdGhgYaHNJTV9pV7Y9B3b644jCLs9Upc3VeNGg6LWARAT6O+Q+/COo+2gg/bM5rhpMAtf70WqfBdQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@csstools/css-tokenizer": "^3.0.4" + } + }, + "node_modules/@csstools/css-tokenizer": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@csstools/css-tokenizer/-/css-tokenizer-3.0.4.tgz", + "integrity": "sha512-Vd/9EVDiu6PPJt9yAh6roZP6El1xHrdvIVGjyBsHR0RYwNHgL7FJPyIIW4fANJNG6FtyZfvlRPpFI4ZM/lubvw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.2.tgz", + "integrity": "sha512-XExcO+dvLKvVtNTibSTBej1NCAbaGhWn9Ww1ZPx80qsahhPFe/8jgWP0IchNe0F3HwkU7n8ejhH8bjonqht8mQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.2.tgz", + "integrity": "sha512-kXXoiPVVGQcnIYGOeaovwOURpniDBpSq4A03qkQ+BMQqtGG6HYap3xne9C1O1yo4TR3qxlCX5IqqmX6fFo2Lqg==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.2.tgz", + "integrity": "sha512-5YfKeeI8qWfBZIX+u2xZC3Zlb3Os/gLS2sbEKM+I4ZOcsWmHS2WLysCcQZDAFRslDUU5Oiq44gf6PYN1vGwG5A==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.2.tgz", + "integrity": "sha512-O387ite7SzUyCcy3JQX4P4bLtEA7bLLkx+esve5JHnyYfNTxcVpXZo9jhdB0lTKN44gztELTdU7nS8Nr16Fs1Q==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.2.tgz", + "integrity": "sha512-n4KqkOQrraxHJcgjM1RvwbigfQKIKJVpM7xp+KsxiyUSrRdIXnt73VhrPAx0fV44hgfmIVKjxMN9J1t5jySVkw==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.2.tgz", + "integrity": "sha512-uq6suIWYP37qzGddBKPw5QEQPi6HiLGsO7UmkpfyaYNQ3D+rN6w6WfwH+nuqcGXWvawGwxOEroO4YGnFh95azw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.2.tgz", + "integrity": "sha512-n+I0BTSRIoy+d6RPKnEVwql5UwBJolytvY4mAOIEJorKlqgPII8ix6slVVrfZ5Tnj7glIZvloylbB/EJPMWEXw==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.2.tgz", + "integrity": "sha512-78XJTJkvPs0kz2w61301PJjXl4g7q3JqiYMZ/M/yVI73EHBrCRTgkhu9oqG7vPqq+a/yadEW8aD+agKlk5xrmg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.2.tgz", + "integrity": "sha512-XlDnu2q5yoqems+xay6wSAcg9DDD7K9RLKZEBOMZm3ckNpJBvOX20tSfby8KfrrhINDyv9V2YVZKY/SpoGJI8w==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.2.tgz", + "integrity": "sha512-pW4AC0P3it8c7do9MVM4p51FzHzdM/TZrerurgRcHJ2WTa1VQ1CIq18xncfpBJw4ojkiZZrKW2yIBWBP92j6Ug==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.2.tgz", + "integrity": "sha512-CYbnj78HsIeA+DhgUKgFCfvNsTHFhMMrinUrMZpDXJXKN8T3XViTZ/+wtHeVxEWY8ewSzTFN+nRmSwO2tZaLUQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.2.tgz", + "integrity": "sha512-buwkd8nsph4R+ajRvw0qM5Hja/TXQow3ptzWO2EbG/cqcIkHloRrdlBtQlshyYGTNFvfkfJ5tpPLVkY4DtsPfQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.2.tgz", + "integrity": "sha512-ZVykbDyk7519VwiNb9Lcj9m8XM6v5V9uKPvrEMkkEedVewf+0itkhahp4HDpgERXhwLRpWFypsGbG/J8s0QjJA==", + "cpu": [ + "mips64el" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.2.tgz", + "integrity": "sha512-CAXl+Dtd9UUuJd8pKKdwh6MLm3MUMiqMPmhZ3tTSXPqfyQ3vDl6R5hZdZ/kYojK4ofXtdfSv1tFq8XzWx3heNQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.2.tgz", + "integrity": "sha512-GeXCej4IQtU1B+QlDV8W/RRvbzI3O/Stss+/bCXv4lZls5WGRtu2a+3JkA3i4qIUlMXpcHebWpF8AkJhATowuA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.2.tgz", + "integrity": "sha512-3H1weTYZPxt/WOhByszQZybS9w5lKzUn1FDMsgEChbHWQwHYQQRfBxgCcZvPhjHfKyJjIievvMmEUawJrdY9Dg==", + "cpu": [ + "s390x" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.2.tgz", + "integrity": "sha512-4xTZr1FUmSoQW4XIWmit3tzQrUTZM+N3P0XV8xROKYF50XfI7xeO90+1bZvNwxIufQ9hDQVRJH5YhgPVF8A/HQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.2.tgz", + "integrity": "sha512-sSATRjPeDBg3pdgHoQfoYBob11Kk1FGa9lui5RIHZCoCkJa9QKlvl3/vKz2usCmYYjs7ymJR/2Nnsqe+Hjt5nw==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.2.tgz", + "integrity": "sha512-lqnzCV+mM0gIADaKihiCg6ifgfU2L3h5E33rNQBN1Y4MaVGnzryzmvvf7UHxprpQdE8hpqLolJ9Rl+SkIRDpyw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.2.tgz", + "integrity": "sha512-AL2qJILH7lNjrDmCQDvdxMfAUIv8KMNZOvrwAQ8i8//ntL9FflhOyMJ8OZSMBb8/AWXe3/5v5S20y3zCoZWKoQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.2.tgz", + "integrity": "sha512-QtiuPytchRyC4rwUKhexJdQKvDuZ6hWloi3igqPQNUJCS1/v9EiO3UTOXR6A3FoMo4fnAKbWJdqaIwhOzh8qEw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.2.tgz", + "integrity": "sha512-WkhYDmpTjLvGlScA1rwjRUmhl4k8oXR3cIbtqWmELgU/dFeHHlEllxDvdWcNJV9rbzCexB5vz8gtNewWLgCT7Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.2.tgz", + "integrity": "sha512-GPMSkTOtMnv2U2F8gxe4Io6qmVs+YKyp832Etqqxr0hFngmXQ3rzwytelm3GIn7T4VviRUlf3sOgBOiTdvaf7g==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.2.tgz", + "integrity": "sha512-PIhhEkE9uPBleRBrQEJpUn7MBnibZzbGzYWPmY3x+YoVg/95zbjB4CxPPOQ8l5tYYM4mMaCthF8/1DIfBQQyWQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.2.tgz", + "integrity": "sha512-YmJbfTlvU7Sdn9BB+4PRES4oB6pxgS37MAONj+hBr/cpXS1aBPKXxNnDbu+QCWPj0o9dgyxeq79g6c5P8KeuYA==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.2.tgz", + "integrity": "sha512-5ebpxr3nWMzrL/rnUI755Jkuee0bHL/Gq0WTF9lvcpv73wAp5eu8MfBUgWK9bhWvZjj7yX8etf/8tI8Ney695g==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", + "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==", + "dev": true + }, + "node_modules/@lit-labs/ssr-dom-shim": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@lit-labs/ssr-dom-shim/-/ssr-dom-shim-1.6.0.tgz", + "integrity": "sha512-VHb0ALPMTlgKjM6yIxxoQNnpKyUKLD04VzeQdsiXkMqkvYlAHxq9glGLmgbb889/1GsohSOAjvQYoiBppXFqrQ==" + }, + "node_modules/@lit/reactive-element": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@lit/reactive-element/-/reactive-element-2.1.2.tgz", + "integrity": "sha512-pbCDiVMnne1lYUIaYNN5wrwQXDtHaYtg7YEFPeW+hws6U47WeFvISGUWekPGKWOP1ygrs0ef0o1VJMk1exos5A==", + "dependencies": { + "@lit-labs/ssr-dom-shim": "^1.5.0" + } + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.68.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.68.0.tgz", + "integrity": "sha512-Ta2n0ntLfzP0LqwwcXOIh2z0mnHjlxMS+h4Y9lo+w33Quyf9KFKEi9QZ2IKqeEBKr0qxNpXc/DQjz5KVIs8AEA==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.68.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.68.0.tgz", + "integrity": "sha512-iFDtS0oMaBvKbtrRKXmRcBSsI0UaWL2bWpd62iHzb0A8RSPrVz47mUGE0CfrrD0c/7/7dUhNtoN3cc6aEUn1Ow==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-x64": { + "version": "4.68.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-x64/-/rollup-android-x64-4.68.0.tgz", + "integrity": "sha512-OnYETr+cZYSbesBD1tCvUjrBUTkjSaOcewRUoIHqD0XI3yn5i9WpdFaE3LZwrMlq0yeJNU0N5sxfHK/fNVMqlQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.68.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.68.0.tgz", + "integrity": "sha512-2qavWZtFeon/6oMkjsBF3OH7ENKgRTW4xRTwmz9VJeNUYhmPitpR9Jd4sAAc2OZJryHs2neoovMTCYnNvhvANQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.68.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.68.0.tgz", + "integrity": "sha512-QpgJO+GwNQDPt6uYcFSjZTFVyHNIeumN2am7RISDR9Nf3e/h8sur9cIfplPGihZPUn+sTw9j3JErZIUkHg8+Pw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.68.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.68.0.tgz", + "integrity": "sha512-TERqEn6P9bp4lsTyoFjT3nLKV6G++76HhGb4k6dRGcs+jvhVcTGaS2zkeRdLLRjwKxpBtdCZe11kUQ2hNGgFSA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.68.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.68.0.tgz", + "integrity": "sha512-RBjDrqb7h5l6Xgdgz5eKYvbpnUkii/z5ejWuZmHJsXORi0X0tTtFzEEgJj2DX995kW18c6T9LGFn/deEAMXDcQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.68.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.68.0.tgz", + "integrity": "sha512-HtLgXBqjF1Ampt9eJjzSmoGDvNb4LP8Y2aPXQZeQ5qhyExXLmoXJB9Jfm/nPi/8fJ3UFn4sZ7pQKyr8V+oTN8g==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.68.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.68.0.tgz", + "integrity": "sha512-q51xJyqjaqCNtDjYD7xNVwsPa2TcVfAIBeP6E20wicKZ6XYp8Taz9SUPYhIENojAQIAoV7cO3mUdMUgWYEAuhQ==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.68.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.68.0.tgz", + "integrity": "sha512-wejnVabl92oFC/o4sr/EEJuJZMHtMq4C+LBIPphN4EUiVHe9qvZNw+biqhVHpX2ggs1+Z3ugXGwJxT+qhyaaYQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.68.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.68.0.tgz", + "integrity": "sha512-tyu+hrsCWHHqV6+2tligV/recJH+HVvNV2Sp4nS1C+MKQPkNXRovKIzyHy0V0xXjm6igCLgsT2vTky5CvWP37g==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.68.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.68.0.tgz", + "integrity": "sha512-5gjJNglKDfgs3OmnGi/RVZQPhEfRoCIHxwSAP9ngmdnxreu0Vl9lI4gxmFBXOhKdftpS82xRdc/7Dp5/A1F0qg==", + "cpu": [ + "loong64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-musl": { + "version": "4.68.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.68.0.tgz", + "integrity": "sha512-2IBxTUds33rFYwOIEyf77wfW8WQThwiQQ1ofXSraZQG7PiIq7BnQBf/ge0qEI/Ny/2cSc7SKkGO/wUDqmYXqTg==", + "cpu": [ + "loong64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.68.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.68.0.tgz", + "integrity": "sha512-mE/Pj2PPTZIVHkNNuHq+eYoU3td8LCm73zjrXoBWkNlnuULJS2l/fxwgGPpN4WUCNZgKZ1IXwrnnETPyOcWPmg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-musl": { + "version": "4.68.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.68.0.tgz", + "integrity": "sha512-dYuykUkAiTv+RQdP3Ib47XcFiImkBWiRpTGSQWAMfZPQyoQYGe8Je4IUeZYNUQWIojmRCUnt5xsH4BbUb2uqSw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.68.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.68.0.tgz", + "integrity": "sha512-OnPDDFXLqJewY56T6ow8sA4wuzbqlliD7QJgkrx2KX+pWOpjI3usYXgB0pwK3nsgTF2odCw9IuJs0Uis/jy2qA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.68.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.68.0.tgz", + "integrity": "sha512-CqoQf06cK0brrRfDqH7Mt8G4MdteFGbjBdC61zEMaSVna6xo6QMUixa9bAkAD4a92g3ElPbQAkLMUzdf+Lk8dg==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.68.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.68.0.tgz", + "integrity": "sha512-LPHNmE8GwiyELGLUTkqj4zsP1YEHzGdtwnKxAcdP/swsNxXsfxPtpe5nrD/KGxr136xPB+eWv2r/SGj9md0pnQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.68.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.68.0.tgz", + "integrity": "sha512-GDOmvgephpRCNyNGRQOZh0KD2gTyRVM1Bb1iOo759xeCUpCAoRCmJioOOSu2MwyWfNOocGd0V5I9vYLfteC6/A==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.68.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.68.0.tgz", + "integrity": "sha512-HADRxkvl5eg3hbemCmv81UwTfdQy/CiSJDOabLE+p5tpm3OTjuRtzpLXy5zPfGY9FQD4Qv7qHptXmG/YjmNmyw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openbsd-x64": { + "version": "4.68.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.68.0.tgz", + "integrity": "sha512-iERrpqu1VEfyX736HLKiXRbwGDkCV/iybARY/mn14KcI6oKhJGBx1e+bU+g+umTKghtiGSPaHkwwpII4/N3RjA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "openbsd" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.68.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.68.0.tgz", + "integrity": "sha512-hBn57IqAbpXpyktckonwePTaFOQS6THph8ZnpRO3eZHY4LncuZGjSsseigBnKuSMKKpLnJJ7PemDvXEzniXC+A==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.68.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.68.0.tgz", + "integrity": "sha512-pJWb7mIw9Z25Hl3Q6nHFjl7AnczfOpm2gbOFYguSRv6TiH7DzsoAk/thymb/URSQu9OUEK7uaDMC5Xfm+7TAww==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.68.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.68.0.tgz", + "integrity": "sha512-pnrhmoyF2otH2Mezol14/ELUFGuRddOCz6+I+J0pq2Itnp4UxkjABQcXawCNMaKNsCRiuL90BtNzbEd0fHjpIg==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.68.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.68.0.tgz", + "integrity": "sha512-1dJ3U+ktdtaIdPaOZKwIX57+bKQk4iAjs5M1wQWLKRB6Wglp98qf8LUeuN4u6bGlIYouZKaySZJ5Qd4OalUMnA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.68.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.68.0.tgz", + "integrity": "sha512-e/NRJFl48wL5DFNSzQrvm4kaB5dzNNmqWjIJ6And2Sjuw4klO3w3TXMvCAEF1tHTrQQgpOD4QVkBE/92/yBGrQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true + }, + "node_modules/@types/estree": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.10.tgz", + "integrity": "sha512-lTyOMnOcmRBzdCv6/4zUZCrEhV0xrqQiUZBZtVcbmLHBXKLnN8q89v/wKRF/Q0HbwVsqbIG3o5HDoFl9sduM+g==", + "dev": true + }, + "node_modules/@types/trusted-types": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", + "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==" + }, + "node_modules/@vitest/expect": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-3.2.7.tgz", + "integrity": "sha512-E8eBXaKibuvH2pSZErOjdVb5vF4PbKYcrnluBTYxEk1l/VhhwZg1kZQsdtjq+CsF5CFydf2Rdkz7jDHKSisi3w==", + "dev": true, + "dependencies": { + "@types/chai": "^5.2.2", + "@vitest/spy": "3.2.7", + "@vitest/utils": "3.2.7", + "chai": "^5.2.0", + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/mocker": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-3.2.7.tgz", + "integrity": "sha512-Trr0hYO9CM3Wj6ksWHRhK9IZpIY6wTMO5u/MqXurMxT57sWBaOPEtP3Oq60ihZuh5JsiagKfz95OcxdEP6dBrA==", + "dev": true, + "dependencies": { + "@vitest/spy": "3.2.7", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.17" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/pretty-format": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-3.2.7.tgz", + "integrity": "sha512-KUHlwqVu0sRlhCdyPdQ/wBoTfRahjUky1MubOmYw9fWfIZy1gNoHpuaaQBPAaMaVYdQYHJLurzj8ECCj5OwTqA==", + "dev": true, + "dependencies": { + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/runner": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-3.2.7.tgz", + "integrity": "sha512-sB9y4ovltoQP+WaUPwmSxO9WIg9Ig694Di5PalVPsYHklAdE027mehpWF2SQSVq+k6sFgaivbTjTJwZLSHbedA==", + "dev": true, + "dependencies": { + "@vitest/utils": "3.2.7", + "pathe": "^2.0.3", + "strip-literal": "^3.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/snapshot": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-3.2.7.tgz", + "integrity": "sha512-7C+MwShwtBSI5Buwoyg3s/iY1eHL9PKAf+O1wVh/TdnjXUtkoL/9YQtre90i4MtNXM6edP1wJ2zOBpfCyhIS7g==", + "dev": true, + "dependencies": { + "@vitest/pretty-format": "3.2.7", + "magic-string": "^0.30.17", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/spy": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-3.2.7.tgz", + "integrity": "sha512-Q2eQGI6d2L/hBtZ0qNuKcAGid68XK6cv1xsoaIma6PaJhHPoqcEJhYpXZ/5myCMqkNgtP6UKuBhbc0nHKnrkuQ==", + "dev": true, + "dependencies": { + "tinyspy": "^4.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/utils": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-3.2.7.tgz", + "integrity": "sha512-x6BDOd7dyo3PFLY3I9/HJ25X/6OurhGXk2/B9gOZNPF7XDVjeBK4k01lQE5uvDpbuheErh91qYuE1E2OEjK3Rw==", + "dev": true, + "dependencies": { + "@vitest/pretty-format": "3.2.7", + "loupe": "^3.1.4", + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "dev": true, + "engines": { + "node": ">= 14" + } + }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/cac": { + "version": "6.7.14", + "resolved": "https://registry.npmjs.org/cac/-/cac-6.7.14.tgz", + "integrity": "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/chai": { + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/chai/-/chai-5.3.3.tgz", + "integrity": "sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==", + "dev": true, + "dependencies": { + "assertion-error": "^2.0.1", + "check-error": "^2.1.1", + "deep-eql": "^5.0.1", + "loupe": "^3.1.0", + "pathval": "^2.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/check-error": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/check-error/-/check-error-2.1.3.tgz", + "integrity": "sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==", + "dev": true, + "engines": { + "node": ">= 16" + } + }, + "node_modules/cssstyle": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-4.6.0.tgz", + "integrity": "sha512-2z+rWdzbbSZv6/rhtvzvqeZQHrBaqgogqt85sqFNbabZOuFbCVFb8kPeEtZjiKkbrm395irpNKiYeFeLiQnFPg==", + "dev": true, + "dependencies": { + "@asamuzakjp/css-color": "^3.2.0", + "rrweb-cssom": "^0.8.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/data-urls": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-5.0.0.tgz", + "integrity": "sha512-ZYP5VBHshaDAiVZxjbRVcFJpc+4xGgT0bK3vzy1HLN8jTO975HEbuYzZJcHoQEY5K1a0z8YayJkyVETa08eNTg==", + "dev": true, + "dependencies": { + "whatwg-mimetype": "^4.0.0", + "whatwg-url": "^14.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decimal.js": { + "version": "10.6.0", + "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.6.0.tgz", + "integrity": "sha512-YpgQiITW3JXGntzdUmyUR1V812Hn8T1YVXhCu+wO3OpS4eU9l4YdD3qjyiKdV6mvV29zapkMeD390UVEf2lkUg==", + "dev": true + }, + "node_modules/deep-eql": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-5.0.2.tgz", + "integrity": "sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/entities": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", + "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", + "dev": true, + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/es-module-lexer": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.7.0.tgz", + "integrity": "sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==", + "dev": true + }, + "node_modules/esbuild": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz", + "integrity": "sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==", + "dev": true, + "hasInstallScript": true, + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.2", + "@esbuild/android-arm": "0.28.2", + "@esbuild/android-arm64": "0.28.2", + "@esbuild/android-x64": "0.28.2", + "@esbuild/darwin-arm64": "0.28.2", + "@esbuild/darwin-x64": "0.28.2", + "@esbuild/freebsd-arm64": "0.28.2", + "@esbuild/freebsd-x64": "0.28.2", + "@esbuild/linux-arm": "0.28.2", + "@esbuild/linux-arm64": "0.28.2", + "@esbuild/linux-ia32": "0.28.2", + "@esbuild/linux-loong64": "0.28.2", + "@esbuild/linux-mips64el": "0.28.2", + "@esbuild/linux-ppc64": "0.28.2", + "@esbuild/linux-riscv64": "0.28.2", + "@esbuild/linux-s390x": "0.28.2", + "@esbuild/linux-x64": "0.28.2", + "@esbuild/netbsd-arm64": "0.28.2", + "@esbuild/netbsd-x64": "0.28.2", + "@esbuild/openbsd-arm64": "0.28.2", + "@esbuild/openbsd-x64": "0.28.2", + "@esbuild/openharmony-arm64": "0.28.2", + "@esbuild/sunos-x64": "0.28.2", + "@esbuild/win32-arm64": "0.28.2", + "@esbuild/win32-ia32": "0.28.2", + "@esbuild/win32-x64": "0.28.2" + } + }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "dependencies": { + "@types/estree": "^1.0.0" + } + }, + "node_modules/expect-type": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.4.0.tgz", + "integrity": "sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==", + "dev": true, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/html-encoding-sniffer": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-4.0.0.tgz", + "integrity": "sha512-Y22oTqIU4uuPgEemfz7NDJz6OeKf12Lsu+QC+s3BVpda64lTiMYCyGwg5ki4vFxkMwQdeZDl2adZoqUgdFuTgQ==", + "dev": true, + "dependencies": { + "whatwg-encoding": "^3.1.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "dev": true, + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "dev": true, + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "dev": true, + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-potential-custom-element-name": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz", + "integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==", + "dev": true + }, + "node_modules/js-tokens": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-9.0.1.tgz", + "integrity": "sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ==", + "dev": true + }, + "node_modules/jsdom": { + "version": "26.1.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-26.1.0.tgz", + "integrity": "sha512-Cvc9WUhxSMEo4McES3P7oK3QaXldCfNWp7pl2NNeiIFlCoLr3kfq9kb1fxftiwk1FLV7CvpvDfonxtzUDeSOPg==", + "dev": true, + "dependencies": { + "cssstyle": "^4.2.1", + "data-urls": "^5.0.0", + "decimal.js": "^10.5.0", + "html-encoding-sniffer": "^4.0.0", + "http-proxy-agent": "^7.0.2", + "https-proxy-agent": "^7.0.6", + "is-potential-custom-element-name": "^1.0.1", + "nwsapi": "^2.2.16", + "parse5": "^7.2.1", + "rrweb-cssom": "^0.8.0", + "saxes": "^6.0.0", + "symbol-tree": "^3.2.4", + "tough-cookie": "^5.1.1", + "w3c-xmlserializer": "^5.0.0", + "webidl-conversions": "^7.0.0", + "whatwg-encoding": "^3.1.1", + "whatwg-mimetype": "^4.0.0", + "whatwg-url": "^14.1.1", + "ws": "^8.18.0", + "xml-name-validator": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "canvas": "^3.0.0" + }, + "peerDependenciesMeta": { + "canvas": { + "optional": true + } + } + }, + "node_modules/lit": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/lit/-/lit-3.3.3.tgz", + "integrity": "sha512-fycuvZg/hkpozL00lm1pEJH5nN/lr9ZXd6mJI2HSN4+Bzc+LDNdEApJ6HFbPkdFNHLvOplIIuJvxkS4XUxqirw==", + "dependencies": { + "@lit/reactive-element": "^2.1.0", + "lit-element": "^4.2.0", + "lit-html": "^3.3.0" + } + }, + "node_modules/lit-element": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/lit-element/-/lit-element-4.2.2.tgz", + "integrity": "sha512-aFKhNToWxoyhkNDmWZwEva2SlQia+jfG0fjIWV//YeTaWrVnOxD89dPKfigCUspXFmjzOEUQpOkejH5Ly6sG0w==", + "dependencies": { + "@lit-labs/ssr-dom-shim": "^1.5.0", + "@lit/reactive-element": "^2.1.0", + "lit-html": "^3.3.0" + } + }, + "node_modules/lit-html": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/lit-html/-/lit-html-3.3.3.tgz", + "integrity": "sha512-el8M6jK2o3RXBnrSHX3ZKrsN8zEV63pSExTO1wYJz7QndGYZ8353e2a5PPX+qHe2aGayfnchQmkAojaWAREOIA==", + "dependencies": { + "@types/trusted-types": "^2.0.2" + } + }, + "node_modules/loupe": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.2.1.tgz", + "integrity": "sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==", + "dev": true + }, + "node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "dev": true + }, + "node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "dev": true, + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + }, + "node_modules/nanoid": { + "version": "3.3.20", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.20.tgz", + "integrity": "sha512-uKdg2G3GNCKQn9byYOpxbGqrT2fGO5KRt5J/8b3pok8rT6qxGWF6hxMyJiEYtAf+FVyYuD9hRaDqX5uPFYJ4ZQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/nwsapi": { + "version": "2.2.28", + "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.28.tgz", + "integrity": "sha512-IlVB7OS7qrOsVYlpnFIkETjMwT9jwvmocJmmM+GZU/PAB3uGi9Ezd7vcWhWBUnSc0ya4ppmQITOyP1ez9gg8cg==", + "dev": true + }, + "node_modules/parse5": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz", + "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==", + "dev": true, + "dependencies": { + "entities": "^6.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/pathe": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", + "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", + "dev": true + }, + "node_modules/pathval": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pathval/-/pathval-2.0.1.tgz", + "integrity": "sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==", + "dev": true, + "engines": { + "node": ">= 14.16" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true + }, + "node_modules/picomatch": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.5.32", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.32.tgz", + "integrity": "sha512-LJlKCCAUFKOuZxipUdYw1NzIITEsPcgyOG9LkvRJvuKj/bRzbuRY0ARXvL0pWGpsZvn8jku+LMep2eU0seHp8g==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "dependencies": { + "nanoid": "^3.3.20", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.2" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/rollup": { + "version": "4.68.0", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.68.0.tgz", + "integrity": "sha512-BIHWnc1Qjdy+WLfOdcSd67sxO4iW6pTnhdZosg+7ywSaiMWxxzKldkkNpUJqeXYAiJzyB14P2cIjqC6tlb6xzw==", + "dev": true, + "dependencies": { + "@types/estree": "^1.0.10" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@rollup/rollup-android-arm-eabi": "4.68.0", + "@rollup/rollup-android-arm64": "4.68.0", + "@rollup/rollup-android-x64": "4.68.0", + "@rollup/rollup-darwin-arm64": "4.68.0", + "@rollup/rollup-darwin-x64": "4.68.0", + "@rollup/rollup-freebsd-arm64": "4.68.0", + "@rollup/rollup-freebsd-x64": "4.68.0", + "@rollup/rollup-linux-arm-gnueabihf": "4.68.0", + "@rollup/rollup-linux-arm-musleabihf": "4.68.0", + "@rollup/rollup-linux-arm64-gnu": "4.68.0", + "@rollup/rollup-linux-arm64-musl": "4.68.0", + "@rollup/rollup-linux-loong64-gnu": "4.68.0", + "@rollup/rollup-linux-loong64-musl": "4.68.0", + "@rollup/rollup-linux-ppc64-gnu": "4.68.0", + "@rollup/rollup-linux-ppc64-musl": "4.68.0", + "@rollup/rollup-linux-riscv64-gnu": "4.68.0", + "@rollup/rollup-linux-riscv64-musl": "4.68.0", + "@rollup/rollup-linux-s390x-gnu": "4.68.0", + "@rollup/rollup-linux-x64-gnu": "4.68.0", + "@rollup/rollup-linux-x64-musl": "4.68.0", + "@rollup/rollup-openbsd-x64": "4.68.0", + "@rollup/rollup-openharmony-arm64": "4.68.0", + "@rollup/rollup-win32-arm64-msvc": "4.68.0", + "@rollup/rollup-win32-ia32-msvc": "4.68.0", + "@rollup/rollup-win32-x64-gnu": "4.68.0", + "@rollup/rollup-win32-x64-msvc": "4.68.0", + "fsevents": "~2.3.2" + } + }, + "node_modules/rrweb-cssom": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.8.0.tgz", + "integrity": "sha512-guoltQEx+9aMf2gDZ0s62EcV8lsXR+0w8915TC3ITdn2YueuNjdAYh/levpU9nFaoChh9RUS5ZdQMrKfVEN9tw==", + "dev": true + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dev": true + }, + "node_modules/saxes": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/saxes/-/saxes-6.0.0.tgz", + "integrity": "sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==", + "dev": true, + "dependencies": { + "xmlchars": "^2.2.0" + }, + "engines": { + "node": ">=v12.22.7" + } + }, + "node_modules/siginfo": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", + "dev": true + }, + "node_modules/source-map-js": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.2.tgz", + "integrity": "sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/stackback": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", + "dev": true + }, + "node_modules/std-env": { + "version": "3.10.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-3.10.0.tgz", + "integrity": "sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==", + "dev": true + }, + "node_modules/strip-literal": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/strip-literal/-/strip-literal-3.1.0.tgz", + "integrity": "sha512-8r3mkIM/2+PpjHoOtiAW8Rg3jJLHaV7xPwG+YRGrv6FP0wwk/toTpATxWYOW0BKdWwl82VT2tFYi5DlROa0Mxg==", + "dev": true, + "dependencies": { + "js-tokens": "^9.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/symbol-tree": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", + "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==", + "dev": true + }, + "node_modules/tinybench": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", + "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "dev": true + }, + "node_modules/tinyexec": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-0.3.2.tgz", + "integrity": "sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==", + "dev": true + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tinypool": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/tinypool/-/tinypool-1.1.1.tgz", + "integrity": "sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg==", + "dev": true, + "engines": { + "node": "^18.0.0 || >=20.0.0" + } + }, + "node_modules/tinyrainbow": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-2.0.0.tgz", + "integrity": "sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==", + "dev": true, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/tinyspy": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/tinyspy/-/tinyspy-4.0.6.tgz", + "integrity": "sha512-u8KszXvGfU68hVcZpRHKG28T0krMuv2G5nDhiHaMLen/gIuFEgIJhaJuO69qjnXg5paSrbPMFfx3brNuN8eVSg==", + "dev": true, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/tldts": { + "version": "6.1.86", + "resolved": "https://registry.npmjs.org/tldts/-/tldts-6.1.86.tgz", + "integrity": "sha512-WMi/OQ2axVTf/ykqCQgXiIct+mSQDFdH2fkwhPwgEwvJ1kSzZRiinb0zF2Xb8u4+OqPChmyI6MEu4EezNJz+FQ==", + "dev": true, + "dependencies": { + "tldts-core": "^6.1.86" + }, + "bin": { + "tldts": "bin/cli.js" + } + }, + "node_modules/tldts-core": { + "version": "6.1.86", + "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-6.1.86.tgz", + "integrity": "sha512-Je6p7pkk+KMzMv2XXKmAE3McmolOQFdxkKw0R8EYNr7sELW46JqnNeTX8ybPiQgvg1ymCoF8LXs5fzFaZvJPTA==", + "dev": true + }, + "node_modules/tough-cookie": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-5.1.2.tgz", + "integrity": "sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A==", + "dev": true, + "dependencies": { + "tldts": "^6.1.32" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/tr46": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-5.1.1.tgz", + "integrity": "sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw==", + "dev": true, + "dependencies": { + "punycode": "^2.3.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/vite": { + "version": "7.3.7", + "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.7.tgz", + "integrity": "sha512-AHEzTORMbi7/7aXp7w7KZ5+DCUdSeLsJz0grY8xHpc9DHdYb0vQu6DqJfm2tOw3gDExNgPgnJTXxM6syYFHaCA==", + "dev": true, + "dependencies": { + "esbuild": "^0.27.0 || ^0.28.0", + "fdir": "^6.5.0", + "picomatch": "^4.0.3", + "postcss": "^8.5.6", + "rollup": "^4.43.0", + "tinyglobby": "^0.2.15" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "lightningcss": "^1.21.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vite-node": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-3.2.4.tgz", + "integrity": "sha512-EbKSKh+bh1E1IFxeO0pg1n4dvoOTt0UDiXMd/qn++r98+jPO1xtJilvXldeuQ8giIB5IkpjCgMleHMNEsGH6pg==", + "dev": true, + "dependencies": { + "cac": "^6.7.14", + "debug": "^4.4.1", + "es-module-lexer": "^1.7.0", + "pathe": "^2.0.3", + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0" + }, + "bin": { + "vite-node": "vite-node.mjs" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/vitest": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-3.2.7.tgz", + "integrity": "sha512-KrxIJ62Fd89gfysR4WotlgZABiz2dqFPgqGzX7s+CwsqLFomRH7777ZcrOD6+WVAh7khPQP41A+BKbpcJFrdEg==", + "dev": true, + "dependencies": { + "@types/chai": "^5.2.2", + "@vitest/expect": "3.2.7", + "@vitest/mocker": "3.2.7", + "@vitest/pretty-format": "^3.2.7", + "@vitest/runner": "3.2.7", + "@vitest/snapshot": "3.2.7", + "@vitest/spy": "3.2.7", + "@vitest/utils": "3.2.7", + "chai": "^5.2.0", + "debug": "^4.4.1", + "expect-type": "^1.2.1", + "magic-string": "^0.30.17", + "pathe": "^2.0.3", + "picomatch": "^4.0.2", + "std-env": "^3.9.0", + "tinybench": "^2.9.0", + "tinyexec": "^0.3.2", + "tinyglobby": "^0.2.14", + "tinypool": "^1.1.1", + "tinyrainbow": "^2.0.0", + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0", + "vite-node": "3.2.4", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@types/debug": "^4.1.12", + "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", + "@vitest/browser": "3.2.7", + "@vitest/ui": "3.2.7", + "happy-dom": "*", + "jsdom": "*" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@types/debug": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + } + } + }, + "node_modules/w3c-xmlserializer": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-5.0.0.tgz", + "integrity": "sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA==", + "dev": true, + "dependencies": { + "xml-name-validator": "^5.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/webidl-conversions": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", + "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", + "dev": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/whatwg-encoding": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz", + "integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==", + "deprecated": "Use @exodus/bytes instead for a more spec-conformant and faster implementation", + "dev": true, + "dependencies": { + "iconv-lite": "0.6.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/whatwg-mimetype": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz", + "integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==", + "dev": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/whatwg-url": { + "version": "14.2.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.2.0.tgz", + "integrity": "sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw==", + "dev": true, + "dependencies": { + "tr46": "^5.1.0", + "webidl-conversions": "^7.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/why-is-node-running": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "dev": true, + "dependencies": { + "siginfo": "^2.0.0", + "stackback": "0.0.2" + }, + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/ws": { + "version": "8.22.2", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.22.2.tgz", + "integrity": "sha512-/rEZ2ejown2N5gxVTHUu9bVVC7ttu0/Tgzf5hbTOixR3Fs7mbbXgn2DbdbcKgJOmpQMtrIUWivtYuVMnp/W/yA==", + "dev": true, + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/xml-name-validator": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-5.0.0.tgz", + "integrity": "sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==", + "dev": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/xmlchars": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", + "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", + "dev": true + } + } +} diff --git a/server/ui/package.json b/server/ui/package.json new file mode 100644 index 00000000..0970aad0 --- /dev/null +++ b/server/ui/package.json @@ -0,0 +1,21 @@ +{ + "name": "digup-ui", + "private": true, + "version": "0.1.0", + "type": "module", + "description": "The DigUp server UI: Lit web components, same-origin with /api/v1, built by Vite, served by the FastAPI static mount. No Node at runtime.", + "scripts": { + "dev": "vite", + "build": "vite build", + "preview": "vite preview", + "test": "vitest run" + }, + "dependencies": { + "lit": "^3.2.0" + }, + "devDependencies": { + "jsdom": "^26.0.0", + "vite": "^7.0.0", + "vitest": "^3.0.0" + } +} diff --git a/server/ui/src/app/estimate-controller.js b/server/ui/src/app/estimate-controller.js new file mode 100644 index 00000000..80073fef --- /dev/null +++ b/server/ui/src/app/estimate-controller.js @@ -0,0 +1,63 @@ +// estimate-controller.js — Estimate: the engine's own forecast for the server's +// folders (POST /estimate). The controller keeps its on/off gate (the browser key +// `digupEstimateOn`, a generation token, a click-again cancel); the panel's shape is +// the round-6 brief: NO switch, NO forecast rows — the gear's Estimate button rides +// the Index now row (the shell forces `on` on every click, so a browser that stored +// "off" is never left with a dead button), and the INDEXING heading carries the +// spinning circle + the brief summary while it runs and when it lands. The main page +// shows no estimate popup at all. +// Generation token (like searches have): a second click during a slow crawl must not +// let the FIRST crawl's late reply write rows into a panel closed and reopened in between. +export function createEstimateController({ api, store, storage = localStorage, + key = "digupEstimateOn" }) { + const seq = { token: 0 }; + + function estimateOn() { + const stored = storage.getItem(key); + return stored === null ? true : stored === "1"; + } + + function setToggle(on) { + storage.setItem(key, on ? "1" : "0"); + if (!on) { // switching off stops a run in flight + seq.token++; + store.set({ estimate: { on: false, spinning: false, rows: null, error: null } }); + } else { + store.set({ estimate: { ...store.get().estimate, on: true } }); + } + } + + async function runEstimate() { + const e = store.get().estimate || {}; + if (!estimateOn()) { // the toggle gates the control (round 7) + store.set({ estimate: { ...e, on: true, spinning: false, rows: null, error: null } }); + return; + } + if (e.spinning || (e.rows && !e.error)) { // click again = cancel + clear + seq.token++; + store.set({ estimate: { on: true, spinning: false, rows: null, error: null } }); + return; + } + const token = ++seq.token; + store.set({ estimate: { ...e, on: true, spinning: true } }); + try { + const {folders} = await api("/api/v1/estimate", {method: "POST", body: JSON.stringify({})}); + if (token !== seq.token) return; // a newer click owns the rows now + store.set({ estimate: { on: true, spinning: false, rows: folders, error: null } }); + } catch (error) { + if (token === seq.token) { + store.set({ estimate: { on: true, spinning: false, rows: null, + error: `estimate failed: ${error.message}` } }); + } + } + } + + function cancel() { // the Esc chain closes the forecast (vanilla parity) + seq.token++; + store.set({ estimate: { ...store.get().estimate, spinning: false, rows: null, error: null } }); + } + + store.set({ estimate: { on: estimateOn(), spinning: false, rows: null, error: null } }); + + return { estimateOn, setToggle, runEstimate, cancel }; +} diff --git a/server/ui/src/app/estimate-controller.test.js b/server/ui/src/app/estimate-controller.test.js new file mode 100644 index 00000000..dfb7e2d7 --- /dev/null +++ b/server/ui/src/app/estimate-controller.test.js @@ -0,0 +1,66 @@ +// estimate-controller.js — the panel's forecast: the toggle gates the control and is +// kept in this browser (digupEstimateOn), the run shows the engine's folders, a slow +// crawl's LATE reply never writes into a panel that was cancelled/restarted (the +// generation token), and the Esc chain's cancel owns its path. +import { describe, expect, it, vi } from "vitest"; +import { createStore } from "./store.js"; +import { createEstimateController } from "./estimate-controller.js"; +import { memoryStorage } from "../lib/test-util.js"; + +const folders = [{ folder: "/photos", readable: true, seconds: 12, files: { image: 3 }, + bytes: 1000, ms: 20, subfolders: [], calibrated: true }]; + +describe("estimate controller", () => { + it("the toggle is kept in this browser, defaulting ON, gating the run", async () => { + const storage = memoryStorage(); + const store = createStore({}); + const api = vi.fn(async () => ({ folders })); + const c = createEstimateController({ api, store, storage }); + expect(c.estimateOn()).toBe(true); + c.setToggle(false); + expect(storage.getItem("digupEstimateOn")).toBe("0"); + expect(store.get().estimate).toMatchObject({ on: false, spinning: false, rows: null }); + await c.runEstimate(); // gated: the engine is not asked + expect(api).not.toHaveBeenCalled(); + c.setToggle(true); + expect(storage.getItem("digupEstimateOn")).toBe("1"); + }); + + it("runs spin → rows; click again cancels and clears (no rows on a cancelled run)", async () => { + const storage = memoryStorage(); + const store = createStore({}); + let resolveRun; + const api = vi.fn(() => new Promise((r) => { resolveRun = r; })); + const c = createEstimateController({ api, store, storage }); + const run = c.runEstimate(); + expect(store.get().estimate.spinning).toBe(true); + await c.runEstimate(); // second click: cancel + clear + expect(store.get().estimate).toMatchObject({ spinning: false, rows: null }); + resolveRun({ folders }); // the first crawl answers LATE + await run; await Promise.resolve(); + expect(store.get().estimate.rows).toBeNull(); // its reply was dropped + }); + + it("a late reply of the NEWEST run lands; cancel() (the Esc chain) drops everything", async () => { + const store = createStore({}); + const storage = memoryStorage(); + let resolveRun; + const api = vi.fn(() => new Promise((r) => { resolveRun = r; })); + const c = createEstimateController({ api, store, storage }); + const run = c.runEstimate(); + resolveRun({ folders }); + await run; await Promise.resolve(); + expect(store.get().estimate.rows).toEqual(folders); + c.cancel(); + expect(store.get().estimate).toMatchObject({ spinning: false, rows: null, error: null }); + }); + + it("a failed estimate shows its honest line, carrying the toggle's state", async () => { + const store = createStore({}); + const api = vi.fn(async () => { throw Object.assign(new Error("walk timed out"), { status: 502 }); }); + const c = createEstimateController({ api, store, storage: memoryStorage() }); + await c.runEstimate(); + expect(store.get().estimate.error).toContain("estimate failed: walk timed out"); + expect(store.get().estimate.spinning).toBe(false); + }); +}); diff --git a/server/ui/src/app/job-controller.js b/server/ui/src/app/job-controller.js new file mode 100644 index 00000000..b0568916 --- /dev/null +++ b/server/ui/src/app/job-controller.js @@ -0,0 +1,70 @@ +// job-controller.js — Index now: one job at a time; states and counts exactly as the +// engine reported (the vanilla app.js pollJob/index-btn logic carried over verbatim). +// The poll loop answers a failed fetch honestly (the S1 lesson: a dead promise froze +// the job line and left "indexing…" in the footer forever), repeated busy-clicks poll +// ONE loop per live job (the dedupe token), and a finished crawl re-reads the status +// and the index list so the panel shows the targets that exist NOW. +export function createJobController({ api, store, refreshStatus, loadAccess, pollMs = 2000 }) { + const seq = { jobPoll: null }; + + async function pollJob(id) { + for (;;) { + let job; + try { + // A failed poll (server restarting, engine mid-respawn 503, network) used to reject + // the loop's promise unhandled: the thread of polling DIED, the job line froze at + // its last state, and "indexing…" stayed in the footer forever — no retry, no + // honest error. Say what happened, clear the indexing state, stop. + job = (await api(`/api/v1/jobs/${id}`)).job; + } catch (error) { + store.set({ job: { state: "interrupted", text: `job poll failed: ${error.message}` }, + indexing: null }); + seq.jobPoll = null; + return; + } + showJob(job); + if (["done", "interrupted", "busy"].includes(job.state)) { + refreshStatus(); + loadAccess(); // a finished crawl moves the engine's numbers: re-read the + return; // index list so the panel shows the targets that exist now + } + await new Promise((resolve) => setTimeout(resolve, pollMs)); + } + } + + function showJob(job) { + const indexing = ["queued", "running"].includes(job.state) ? job : null; + if (!indexing) seq.jobPoll = null; // finished: a later 409 may poll afresh + const counts = [job.state, + job.searchable !== null && job.searchable !== undefined ? `searchable ${job.searchable}/${job.files}` : null, + job.pending ? `pending ${job.pending}` : null, + job.vectors ? `vectors ${job.vectors}` : null, + job.files_done ? `indexed ${job.files_done}` : null, + job.seconds !== undefined ? `${job.seconds} s` : null, + job.why].filter(Boolean); + store.set({ job: { state: job.state, text: counts.join(" · ") }, indexing }); + } + + async function indexNow(prune) { + const pruneFlag = Boolean(prune); // engine --prune: retire rows of files no longer in the folders + try { + const {job} = await api("/api/v1/index", {method: "POST", body: JSON.stringify({prune: pruneFlag})}); + store.set({ job: { state: "queued", text: `job ${job.id} queued` } }); + seq.jobPoll = job.id; + pollJob(job.id); + } catch (error) { + if (error.status === 409) { + const live = error.message.match(/job ([0-9a-f]+)/); // answered busy: poll the live one + if (live && live[1] !== seq.jobPoll) { // repeated clicks used to stack + seq.jobPoll = live[1]; // a second polling loop on the + pollJob(live[1]); // same live id — one loop per job + } + store.set({ job: { state: "running", text: "busy — a job is already running" } }); + } else { + store.set({ job: { state: "interrupted", text: `index request failed: ${error.message}` } }); + } + } + } + + return { indexNow, pollJob, get jobPoll() { return seq.jobPoll; } }; +} diff --git a/server/ui/src/app/job-controller.test.js b/server/ui/src/app/job-controller.test.js new file mode 100644 index 00000000..76a0905d --- /dev/null +++ b/server/ui/src/app/job-controller.test.js @@ -0,0 +1,89 @@ +// job-controller.js — the index job's rules, verbatim from the vanilla app.js: +// one poll loop per live job (the 409 dedupe), an honest answer when a poll fails +// (the S1 lesson: a dead promise froze "indexing…" forever), and a finished crawl +// re-reads the status and the index list. +import { describe, expect, it, beforeEach, afterEach, vi } from "vitest"; +import { createStore } from "./store.js"; +import { createJobController } from "./job-controller.js"; +import { mockReply } from "../lib/test-util.js"; + +const base = () => createStore({ results: { phase: "empty", strong: [], weaker: [], groups: [], + hitsById: new Map(), showWeaker: false, selection: null, lastReply: null }, + filters: { kinds: [], limit: 40 }, isCode: false, text: "", navigated: false, + job: null, indexing: null }); + +describe("indexNow + pollJob", () => { + beforeEach(() => vi.useFakeTimers()); + afterEach(() => vi.useRealTimers()); + + it("queues, polls the states with the engine's counts, and reloads access when done", async () => { + const store = base(); + const refreshStatus = vi.fn(), loadAccess = vi.fn(); + let polls = 0; + const calls = []; + globalThis.fetch = async (path, options = {}) => { + calls.push({ path, options }); + if (path === "/api/v1/index") return mockReply(202, { job: { id: "abc123" } }); + polls++; + return mockReply(200, { job: polls === 1 + ? { state: "running", searchable: 3, files: 10, pending: 2, vectors: 5, files_done: 4, seconds: 12 } + : { state: "done", searchable: 10, files: 10, vectors: 20, files_done: 10, seconds: 30 } }); + }; + const api = async (path, options) => { + const r = await globalThis.fetch(path, options); + return r.json(); + }; + const c = createJobController({ api, store, refreshStatus, loadAccess }); + await c.indexNow(false); + expect(store.get().job.text).toBe("job abc123 queued"); + await vi.advanceTimersByTimeAsync(0); // first poll (no delay before it) + expect(store.get().job.text).toContain("searchable 3/10"); + expect(store.get().job.text).toContain("indexed 4"); + expect(store.get().indexing.state).toBe("running"); + await vi.advanceTimersByTimeAsync(2000); // done at the second poll + expect(store.get().job.state).toBe("done"); + expect(store.get().indexing).toBeNull(); + expect(refreshStatus).toHaveBeenCalledTimes(1); + expect(loadAccess).toHaveBeenCalledTimes(1); + }); + + it("a failed poll says so and clears the indexing state (never a frozen footer)", async () => { + const store = base(); + globalThis.fetch = async (path) => { + if (path === "/api/v1/index") return mockReply(202, { job: { id: "abc123" } }); + return mockReply(503, { detail: "searcher is warming" }); + }; + const api = async (path, options) => { + const r = await globalThis.fetch(path, options); + if (!r.ok) throw Object.assign(new Error("HTTP 503"), { status: 503 }); + return r.json(); + }; + const c = createJobController({ api, store, refreshStatus: vi.fn(), loadAccess: vi.fn() }); + c.indexNow(true); + await vi.advanceTimersByTimeAsync(0); + expect(store.get().job.state).toBe("interrupted"); + expect(store.get().job.text).toContain("job poll failed: HTTP 503"); + expect(store.get().indexing).toBeNull(); + expect(c.jobPoll).toBeNull(); + }); + + it("409 busy: poll the live job ONCE — repeated clicks do not stack loops", async () => { + const store = base(); + let polls = 0; + globalThis.fetch = async (path) => { + if (path === "/api/v1/index") return mockReply(409, { detail: "job live9 is running" }); + polls++; + return mockReply(200, { job: { state: "running", files_done: 1, files: 2, searchable: 1, vectors: 2 } }); + }; + const api = async (path, options) => { + const r = await globalThis.fetch(path, options); + if (!r.ok) throw Object.assign(new Error(r.detail), { status: r.status }); + return r.json(); + }; + const c = createJobController({ api, store, refreshStatus: vi.fn(), loadAccess: vi.fn() }); + await Promise.all([c.indexNow(false), c.indexNow(false), c.indexNow(false)]); // three clicks + expect(store.get().job.text).toBe("busy — a job is already running"); + await vi.advanceTimersByTimeAsync(6000); + expect(polls).toBeLessThanOrEqual(4); // ONE loop: 1 poll per 2 s + }); +}); diff --git a/server/ui/src/app/keys-controller.js b/server/ui/src/app/keys-controller.js new file mode 100644 index 00000000..f82fdbf2 --- /dev/null +++ b/server/ui/src/app/keys-controller.js @@ -0,0 +1,85 @@ +// keys-controller.js — the gear's API key section: the server's minted keys, each row +// a Copy button, and Regenerate (the vanilla loadAccess/regenerateKeys carried over +// verbatim). A minted-key server issues its own keys at boot (nothing to type): the +// main page searches WITHOUT a key; these rows are what the APPLICATIONS take — Copy +// puts a key on the clipboard for pasting into an app's settings (a Nextcloud bridge, +// a dashboard tile), Regenerate mints fresh values (confirm first — the old keys die +// immediately and every application that has one pasted loses access until the new key +// is pasted there; this browser keeps the admin key in localStorage, so it updates +// itself). Deployments whose keys come from the environment are never shown and never +// rotated. The index list rides the same call (GET /targets). +export function createKeysController({ store, api, storage, + TOKEN_KEY = "digupBearer", + confirm = (m) => window.confirm(m) }) { + const seq = { retried: false }; + + async function regenerate() { + if (!confirm("Regenerate the server's keys? The keys shown here stop working " + + "immediately — every application that has one pasted loses access until you " + + "paste its new key into that application. This browser keeps working: its key " + + "updates itself.")) { + return; + } + try { + const t = await api("/api/v1/tokens", {method: "POST"}); + if (t.admin) storage.setItem(TOKEN_KEY, t.admin); + loadAccess(); + } catch (error) { + window.alert(error.message || String(error)); + } + } + + async function loadAccess() { + try { + const t = await api("/api/v1/tokens"); // keyless in a minted deployment: + // The gear's own controls (Index now, Estimate, Regenerate) are admin-scope by + // the minted shape's design — so this browser takes the key it was just SHOWN, + // exactly as Regenerate hands it the new key today. No typing, no paste, no + // popup: the standalone box's own operator uses its own box's key, and a stored + // key that the server no longer recognizes (rotated, re-minted, from another + // deployment) is REPLACED by the value shown now — the gear self-heals. The main + // page's key-free reads are untouched. A server configured by environment shows + // no value (admin null) → nothing is adopted → guarded deployments stay guarded. + if (t.admin && storage.getItem(TOKEN_KEY) !== t.admin) { + storage.setItem(TOKEN_KEY, t.admin); + } + store.set({ keys: { + admin: t.admin || null, + apps: (t.apps || []).filter((a) => a.token), + envFlagged: Boolean(t.admin_configured || t.read_configured), + minted: Boolean(t.admin), + } }); + } catch { + // A stored key that the server refuses. In the MINTED shape the main index's reads + // are open, so a keyless `status` answers 200 — the stored key is stale (rotated, + // or from a server that was re-minted): drop it and re-read, the gear then shows + // the key that exists NOW. In the guarded (environment-keys) shape a keyless + // `status` answers 401, so the stored key is kept: it is the deployment's config, + // not something the UI may discard. One retry only — never a loop. + if (!seq.retried && storage.getItem(TOKEN_KEY)) { + seq.retried = true; + try { + const probe = await fetch("/api/v1/status", {headers: {"content-type": "application/json"}}); + if (probe.status === 200) { + storage.removeItem(TOKEN_KEY); + return loadAccess(); + } + } catch { /* the server is not answering at all: show the rows as empty */ } + } + // A read key / no key: the rows are not for this scope — the panel shows nothing. + store.set({ keys: { admin: null, apps: [], envFlagged: false, minted: false } }); + } + try { + const tg = await api("/api/v1/targets"); + store.set({ targets: (tg.targets || []).map((t) => ({ + name: t.name, main: Boolean(t.main), files: t.files, + searchable: t.searchable, roots: t.roots })) }); + } catch { + store.set({ targets: [] }); + } + } + + function resetRetried() { seq.retried = false; } // a fresh gear open may try the stale-key recovery once + + return { loadAccess, regenerate, resetRetried }; +} diff --git a/server/ui/src/app/search-controller.js b/server/ui/src/app/search-controller.js new file mode 100644 index 00000000..308288ab --- /dev/null +++ b/server/ui/src/app/search-controller.js @@ -0,0 +1,122 @@ +// search-controller.js — keyword-first typing, exactly as the desktop panel types +// (the vanilla app.js logic carried over verbatim): words at once with the last word +// as a prefix, meaning after the pause, the query embedded while you type (embed_ahead), +// strong/weaker grouping, selection bookkeeping. Sequencing tokens (generation, +// finalGeneration, the pause timer) stay HERE — they are sequencing, not app state +// (AGENTS.md §State). The results slice it writes is what digup-results/preview/footer read. +import { KIND_TITLE, WEAKER } from "../lib/format.js"; + +const TYPING_PAUSE = 120; // ms after the last keystroke until meaning results + +export function createSearchController({ api, store, pause = TYPING_PAUSE }) { + const seq = { generation: 0, finalGeneration: -1, timer: null }; + + const results = () => store.get().results; + + function searchBody(text, meaning, prefixLast) { + const s = store.get(); + const f = s.filters; + const body = {q: text, code: s.isCode, meaning, limit: f.limit}; + if (prefixLast) body.prefix_last = true; + if (!s.isCode) { + const kinds = f.kinds; + body.kinds = (kinds.length === 0 || kinds.length === 6) ? null : kinds; // none ticked = all + } + return body; + } + + /** The ONE search route (vanilla queryChanged): the typed text, the code prefix, the + * keyword-first pair of calls, the embed-ahead warm-up, the generation guards. */ + function queryChanged(typed, filters) { + store.set({ filters }); + const trimmed = typed.trim(); + const prefixCode = /^code:/i.test(trimmed); + const code = prefixCode || filters.code; + const text = prefixCode ? trimmed.slice(5).trim() : typed; + store.set({ isCode: code, text }); + const generation = ++seq.generation; + clearTimeout(seq.timer); + store.set({ navigated: false, error: null }); + if (!trimmed || (code && !text)) { + show({hits: [], strong: 0}, code && !text ? "codeHint" : "empty"); + return; + } + // Words now. The last word is probably half-typed, so it matches as a prefix. + api("/api/v1/search", {method: "POST", body: JSON.stringify(searchBody(text, false, true))}) + .then((reply) => { if (generation === seq.generation && seq.finalGeneration !== generation) show(reply, "words"); }) + .catch((error) => { if (generation === seq.generation) failed(error); }); + // The meaning is embedded now, so its vector is ready when typing pauses (embed_ahead). + api("/api/v1/embed_ahead", {method: "POST", body: JSON.stringify({q: text, code})}).catch(() => {}); + seq.timer = setTimeout(() => { + api("/api/v1/search", {method: "POST", body: JSON.stringify(searchBody(text, true, false))}) + .then((reply) => { + if (generation !== seq.generation) return; + seq.finalGeneration = generation; + show(reply, reply.meaning ? "meaning" : "wordsOnly"); + }) + .catch((error) => { if (generation === seq.generation) failed(error); }); + }, pause); + } + + function failed(error) { + store.set({ error: String(error.message || error) }); + } + + // The hits: strong results grouped by kind, the rest folded ("N weaker matches"). + function show(reply, phase) { + const hits = reply.hits || []; + const strong = Math.min(reply.strong ?? hits.length, hits.length); + const s = store.get(); + const prev = s.results; + const strongRows = hits.slice(0, strong); + const weakerRows = hits.slice(strong); + const groups = new Map(); + for (const hit of strongRows) { + const title = KIND_TITLE[hit.kind] || "Files"; + if (!groups.has(title)) groups.set(title, []); + groups.get(title).push(hit); + } + const navigated = s.navigated; + const next = { + phase, strong: strongRows, weaker: weakerRows, + groups: [...groups].map(([title, rows]) => ({title, rows})), + hitsById: new Map(hits.map((hit) => [hit.media_id, hit])), + showWeaker: navigated ? prev.showWeaker : false, + selection: prev.selection, lastReply: reply, + }; + const ids = rowIDs(next); + if (!navigated || !ids.includes(next.selection)) next.selection = ids[0] ?? null; + store.set({ results: next }); + } + + function rowIDs(r = results()) { + return [ + ...r.groups.flatMap((group) => group.rows.map((hit) => hit.media_id)), + ...(r.weaker.length ? [WEAKER] : []), + ...(r.showWeaker ? r.weaker.map((hit) => hit.media_id) : []), + ]; + } + + function move(step) { + const ids = rowIDs(); + if (!ids.length) return; + const s = store.get(); + const at = ids.indexOf(s.results.selection); + const selection = ids[Math.max(0, Math.min(ids.length - 1, (at < 0 ? -1 : at) + step))]; + store.set({ navigated: true, results: { ...s.results, selection } }); + } + + function select(mediaId) { + const s = store.get(); + store.set({ navigated: true, results: { ...s.results, selection: mediaId } }); + } + + function toggleWeaker() { + const s = store.get(); + store.set({ results: { ...s.results, showWeaker: !s.results.showWeaker, selection: WEAKER } }); + } + + const selectedHit = () => { const s = store.get(); return s.results.hitsById.get(s.results.selection) ?? null; }; + + return { queryChanged, move, select, toggleWeaker, selectedHit, rowIDs, failed }; +} diff --git a/server/ui/src/app/search-controller.test.js b/server/ui/src/app/search-controller.test.js new file mode 100644 index 00000000..4a2e6376 --- /dev/null +++ b/server/ui/src/app/search-controller.test.js @@ -0,0 +1,115 @@ +// search-controller.js — the keyword-first loop, verbatim from the desktop panel +// (words at once with the last word as a prefix, meaning after the 120 ms pause, +// embed_ahead warming the vector) and its generation guards. These are the rules +// the vanilla app.js had; the migration keeps them byte-for-byte in behavior. +import { describe, expect, it, beforeEach, afterEach, vi } from "vitest"; +import { createStore } from "./store.js"; +import { createSearchController } from "./search-controller.js"; +import { installFetch, mockReply } from "../lib/test-util.js"; +import { WEAKER } from "../lib/format.js"; + +const hit = (id, kind, z) => ({ media_id: id, kind, path: `/f/${id}.${kind === "image" ? "jpg" : "mp4"}`, z }); +const filters = { kinds: [], limit: 40, code: false }; + +const initial = () => ({ phase: "empty", strong: [], weaker: [], groups: [], + hitsById: new Map(), showWeaker: false, selection: null, lastReply: null }); + +describe("queryChanged — the keyword-first pair + embed_ahead", () => { + beforeEach(() => vi.useFakeTimers()); + afterEach(() => vi.useRealTimers()); + + it("words NOW (prefix_last), the vector warmed, meaning after the pause", async () => { + const store = createStore({ results: initial(), filters, isCode: false, text: "", navigated: false }); + const calls = installFetch({ + "POST /api/v1/search": mockReply(200, { hits: [hit("m1", "image", 3.1)], strong: 1, meaning: true, ms: {} }), + "POST /api/v1/embed_ahead": mockReply(200, {}), + }); + const api = async (path, options) => { + const r = await globalThis.fetch(path, options); + return r.json(); + }; + const c = createSearchController({ api, store }); + c.queryChanged("dog", filters); + // The keyword call + the warm-up fire immediately; meaning waits for the pause. + expect(calls.map((x) => x.path)).toEqual(["/api/v1/search", "/api/v1/embed_ahead"]); + const body = JSON.parse(calls[0].options.body); + expect(body).toMatchObject({ q: "dog", meaning: false, prefix_last: true, limit: 40 }); + expect(body.kinds).toBeNull(); // all kinds = null (the engine's shape) + await vi.advanceTimersByTimeAsync(120); + await vi.waitFor(() => store.get().results.phase === "meaning"); + const meaningCall = calls[2]; + expect(JSON.parse(meaningCall.options.body)).toMatchObject({ q: "dog", meaning: true }); + expect(!("prefix_last" in JSON.parse(meaningCall.options.body))).toBe(true); + expect(store.get().results.phase).toBe("meaning"); + expect(store.get().results.groups).toEqual([{ title: "Images", rows: [expect.objectContaining({ media_id: "m1" })] }]); + expect(store.get().results.selection).toBe("m1"); + }); + + it("empty text asks the engine nothing; code: prefix opens code mode", async () => { + const store = createStore({ results: initial(), filters, isCode: false, text: "", navigated: false }); + const calls = installFetch({}); + const c = createSearchController({ api: async () => ({}), store }); + c.queryChanged(" ", filters); + expect(calls.length).toBe(0); + expect(store.get().results.phase).toBe("empty"); + c.queryChanged("code: rope", { ...filters, code: false }); + await Promise.resolve(); await Promise.resolve(); + expect(store.get().isCode).toBe(true); + expect(store.get().text).toBe("rope"); + expect(store.get().results.phase).not.toBe("empty"); // searching: not the idle page + }); + + it("a late reply from an OLD keystroke never lands (generation guard)", async () => { + const store = createStore({ results: initial(), filters, isCode: false, text: "", navigated: false }); + let resolveFirst; + const first = new Promise((r) => { resolveFirst = r; }); + let n = 0; + const api = async (path, options) => { + n++; + if (n === 1) return first; // the "d" keyword reply, slow + return { hits: [hit("m2", "image", 4)], strong: 1, meaning: true, ms: {} }; + }; + const c = createSearchController({ api, store }); + c.queryChanged("d", filters); + c.queryChanged("do", filters); // typing continues + resolveFirst({ hits: [hit("m1", "image", 2)], strong: 1, meaning: false, ms: {} }); + await first; await Promise.resolve(); await Promise.resolve(); + expect(store.get().results.strong[0].media_id).toBe("m2"); // the late m1 was dropped + }); + + it("navigation walks the rows (strong, the weaker fold sentinel, the unfolded ones)", async () => { + const store = createStore({ results: initial(), filters, isCode: false, text: "", navigated: false }); + const api = async () => ({ hits: [hit("a", "image", 3), hit("b", "image", 2.5), hit("c", "video", 2.2)], + strong: 2, meaning: true, ms: {} }); + const c = createSearchController({ api, store }); + c.queryChanged("x", filters); + await vi.waitFor(() => store.get().results.phase === "meaning"); + expect(c.rowIDs()).toEqual(["a", "b", WEAKER]); + c.move(1); + expect(store.get().results.selection).toBe("b"); + c.move(1); + expect(store.get().results.selection).toBe(WEAKER); + c.toggleWeaker(); + expect(c.rowIDs()).toEqual(["a", "b", WEAKER, "c"]); + c.toggleWeaker(); // folded: the sentinel leaves + expect(c.rowIDs()).toEqual(["a", "b", WEAKER]); + }); +}); + +// (the tests wait on the store directly — vi.waitFor advances fake timers) + +// A promise-shaped api for the plain tests (no fake-timer sequencing). +describe("search-controller — selection bookkeeping", () => { + it("the first row is selected for a fresh reply; a navigated selection survives", async () => { + const store = createStore({ results: initial(), filters, isCode: false, text: "", navigated: false }); + const c = createSearchController({ api: async () => ({ + hits: [hit("a", "image", 3), hit("b", "image", 2)], strong: 1, meaning: true, ms: {} }), store }); + c.queryChanged("x", filters); + await vi.waitFor(() => store.get().results.selection); + expect(store.get().results.selection).toBe("a"); + c.select("b"); + expect(store.get().results.selection).toBe("b"); + expect(store.get().navigated).toBe(true); + expect(c.selectedHit().media_id).toBe("b"); + }); +}); diff --git a/server/ui/src/app/store.js b/server/ui/src/app/store.js new file mode 100644 index 00000000..06e5168e --- /dev/null +++ b/server/ui/src/app/store.js @@ -0,0 +1,52 @@ +// store.js — the shared application state (server/ui/AGENTS.md §State): ONE store, +// one writer per slice, selectors for readers. No state library, no context plumbing: +// the store is created in main.js, handed to digup-app (and to the embed widget), and +// components subscribe through `storeController` — a Lit ReactiveController that +// requestUpdate()s on change. Component-local state (a text passage mid-read, a +// focus flag) stays in the component; sequencing tokens (generation, estimateToken, +// jobPoll) stay inside their controllers — they are sequencing, not app state. + +export function createStore(initial = {}) { + let state = { ...initial }; + const subs = new Set(); + return { + get: () => state, + /** Shallow patch — components re-render through selectors, so a patch is one + * notification. Never mutate `get()`'s object in place from outside a set(). */ + set(patch) { + state = { ...state, ...patch }; + for (const fn of subs) fn(state); + }, + subscribe(fn) { + subs.add(fn); + return () => subs.delete(fn); + }, + }; +} + +/** Lit ReactiveController: re-render `host` when the store moves. `select` maps the + * store to the slice this component reads; the component reads it via `value()`. + * Selectors return ONE value (a reference) or a tuple as an ARRAY — arrays are + * compared element-wise, so `s => [s.keys, s.status]` re-renders only when a member + * reference actually changed (typing does not repaint the settings panel). */ +export function storeController(host, store, select = (s) => s) { + const same = (a, b) => a === b + || (Array.isArray(a) && Array.isArray(b) + && a.length === b.length && a.every((x, i) => x === b[i])); + let value = select(store.get()); + const controller = { + value: () => value, + }; + const unsub = store.subscribe((state) => { + const next = select(state); + if (!same(next, value)) { // selector identity: same slice → no re-render + value = next; + host.requestUpdate(); + } + }); + host.addController({ + hostConnected() {}, + hostDisconnected() { unsub(); }, + }); + return controller; +} diff --git a/server/ui/src/app/store.test.js b/server/ui/src/app/store.test.js new file mode 100644 index 00000000..77d0c735 --- /dev/null +++ b/server/ui/src/app/store.test.js @@ -0,0 +1,43 @@ +// store.js — the ONE store's rules (AGENTS.md §State): shallow patch, one +// notification per patch, unsubscribing, and the storeController's slice equality +// (a tuple selector must NOT repaint on unrelated writes — typing does not touch +// the settings panel). +import { describe, expect, it, vi } from "vitest"; +import { createStore, storeController } from "./store.js"; + +describe("createStore", () => { + it("shallow-patches and notifies subscribers; unsub stops the taps", () => { + const store = createStore({ a: 1, nested: { x: 1 } }); + const seen = []; + const off = store.subscribe((s) => seen.push(s)); + store.set({ a: 2 }); + expect(store.get().a).toBe(2); + expect(store.get().nested).toBe(seen[0].nested); // untouched slice keeps identity + off(); + store.set({ a: 3 }); + expect(seen.length).toBe(1); + }); + + it("a patch is ONE notification (components render through selectors)", () => { + const store = createStore({}); + const fn = vi.fn(); + store.subscribe(fn); + store.set({ a: 1, b: 2 }); + expect(fn).toHaveBeenCalledTimes(1); + }); +}); + +describe("storeController", () => { + it("re-renders on reference change of a tuple selector — and only then", () => { + const store = createStore({ keys: { admin: "a" }, typing: "", other: 1 }); + const host = { requestUpdate: vi.fn(), addController: vi.fn() }; + storeController(host, store, (s) => [s.keys, s.other]); + expect(host.requestUpdate).not.toHaveBeenCalled(); + store.set({ typing: "hello" }); // unrelated write + expect(host.requestUpdate).not.toHaveBeenCalled(); + store.set({ other: 2 }); // a member changed + expect(host.requestUpdate).toHaveBeenCalledTimes(1); + store.set({ keys: store.get().keys }); // same reference + expect(host.requestUpdate).toHaveBeenCalledTimes(1); + }); +}); diff --git a/server/ui/src/app/theme.js b/server/ui/src/app/theme.js new file mode 100644 index 00000000..c91febea --- /dev/null +++ b/server/ui/src/app/theme.js @@ -0,0 +1,12 @@ +// theme.js — appearance: system / light / dark, kept in this browser (the desktop +// follows the OS; a server has no OS to follow, so the choice is the user's, per +// browser). The vanilla app.js rule, verbatim: html[data-theme] + localStorage. +export function createTheme({ storage = localStorage, key = "digup-theme" } = {}) { + function apply(choice) { + document.documentElement.dataset.theme = choice; + storage.setItem(key, choice); + } + const current = () => storage.getItem(key) || "system"; + apply(current()); // apply at boot: the theme must not flash + return { apply, current }; +} diff --git a/server/ui/src/components/digup-app.js b/server/ui/src/components/digup-app.js new file mode 100644 index 00000000..67bf995d --- /dev/null +++ b/server/ui/src/components/digup-app.js @@ -0,0 +1,432 @@ +// digup-app — the application shell: the page tree (header, gear panel, search, +// state panels, results grid, preview, footer, big look), the gear's open/close with +// aria-expanded kept true to the panel's state on every path, the document-level +// keyboard (Esc chain, arrow navigation, the Space guard that never hijacks native +// button/link activation — the reviewer's finding, pinned), the 15 s status poll, +// and the ONE layout switch: `document.body.dataset.page` (idle | busy | results | +// none) written at every render, the first included — the vanilla render() at +// startup rule ("the first paint is a state too"), plus the host mirror the shadow +// panels gate on. All cross-component wiring: the search route, the select/open/ +// preview events, the settings commands → controllers. State lives in the store; +// sequencing lives in the controllers; this file composes them. +import { LitElement, html, css, nothing } from "lit"; +import { storeController } from "../app/store.js"; +import { activityCss, noteCss } from "../styles/controls.js"; +import { emptyAnswer, WEAKER } from "../lib/format.js"; + +import "./digup-search.js"; +import "./digup-suggest.js"; +import "./digup-results.js"; +import "./digup-preview.js"; +import "./digup-big.js"; +import "./digup-footer.js"; +import "./digup-settings.js"; + +const IDLE_NOTE = "Say what you'd see, hear, or read in a file — the meaning, not only the " + + "words. Results appear as you type."; +const BUSY_NOTE = "Matching words now — the meaning results follow when you stop typing."; +const CODE_HINT = "Searching your code — say what the code does, or type a name that's in it."; + +const SUGGESTIONS = ["a dog running on the beach", "people gathered around a campfire", + "a cat sitting on a sofa", "wakeboarding on a lake"]; + +class DigupApp extends LitElement { + static properties = { page: { state: true }, bigOpen: { state: true }, + _gearAttention: { state: true }, _statusTick: { state: true } }; + + static styles = [activityCss, noteCss, css` + :host { display: block; } /* deterministic width for the header's bleed (below) */ + /* header: row 1 — title at the LEFT SCREEN EDGE, gear at the RIGHT SCREEN EDGE (the + round-5 layout brief: the bar bleeds past the 72 rem content column, padding-inline + keeps the 1.25 rem inset); row 2 — the searchable count centered on the same axis + as everything else. The settings panel hangs off the header's right edge, directly + under the gear (the gear IS the anchor). */ + header.bar { position: relative; display: grid; gap: 0.3rem; + margin-inline: calc(50% - 50vw); padding-inline: 1.25rem; } + .bar-top { display: flex; align-items: baseline; justify-content: space-between; gap: 0.8rem; } + h1 { font-size: 1.3rem; margin: 0; } + .bar-center { display: flex; align-items: baseline; justify-content: center; gap: 0.6rem; flex-wrap: wrap; } + #gear { flex: none; width: 2.1rem; height: 2.1rem; display: grid; place-items: center; + border: 1px solid var(--edge); border-radius: var(--radius-pill); padding: 0; + background: var(--field); color: var(--text); cursor: pointer; } + #gear svg { display: block; width: 15px; height: 15px; } + #gear:hover { border-color: var(--accent); } + /* a 401 with no stored key points at the gear: a soft pulse until the panel opens */ + #gear.attention { animation: gear-pulse 1.6s ease-in-out 3; } + @keyframes gear-pulse { 50% { box-shadow: 0 0 0 3px var(--accent); } } + + /* ── the page's states: data-page on body (global layout) mirrored on :host + (this shadow), both written by render(): the ONE switch the vanilla UI had + (body[data-page], set by app.js render(), read by style.css): + idle nothing typed → centered welcome + suggestions; no list, no preview, no counter + busy words matching, meaning on its way → same centered block, its note says so + results the two columns, list ~40 % / preview ~60 % + none meaning answered zero hits → centered message carrying the engine's own sentence + The preview pane reserves NO space for a state with nothing to show. */ + #idle, #no-results { display: none; } + :host([data-page="idle"]) #idle, :host([data-page="busy"]) #idle, + :host([data-page="none"]) #no-results { display: block; } + #results { display: none; } + :host([data-page="results"]) #results { display: grid; } + :host([data-page="busy"]) #suggest { display: none; } /* suggestions are an idle affordance */ + :host([data-code]) #suggest { display: none; } /* …and they are not code-search hints */ + + #idle, #no-results { max-width: 34rem; margin: 3rem auto 3.5rem; padding: 0 1rem; text-align: center; } + #idle h2, #no-results h2 { font-size: 1.18rem; font-weight: 600; margin: 0 0 0.4rem; } + #idle p, #no-results p { color: var(--soft); margin: 0 auto; max-width: 30rem; } + #no-results .hint { margin-top: 0.7rem; font-size: 0.85rem; } + #no-results code { font: 0.85rem var(--font-mono); } + #suggest { display: flex; } + + /* A state change is a layout change, so the panel that arrives fades in: display cannot be + transitioned, and an animation starts when an element gains it. Guarded for motion. */ + @media (prefers-reduced-motion: no-preference) { + #idle, #no-results, #results { animation: rise 0.18s ease-out; } + } + @keyframes rise { from { opacity: 0; transform: translateY(4px); } } + + /* Results list + preview pane, like the desktop panel: the list carries the ranking, the + pane shows the selection the way panel.js shows it. 2fr/3fr is the 40/60 the preview + wants; minmax(0, …) keeps a long file name from widening a track past its share. */ + #results { grid-template-columns: minmax(0, 2fr) minmax(0, 3fr); gap: 1rem; + align-items: start; margin: 0.4rem 0 1rem; } + #preview { border: 1px solid var(--edge); border-radius: 10px; padding: 0.8rem; min-height: 12rem; + background: var(--panel); position: sticky; top: 0.6rem; } + /* Narrow screens: one column, and the sticky pane becomes an ordinary block below the list + (a sticky panel on a single-column page sticks over the list it is meant to follow). */ + @media (max-width: 900px) { + #results { grid-template-columns: minmax(0, 1fr); } + #preview { position: static; } + #idle, #no-results { margin-top: 1.8rem; } + } + + footer#footer { + position: fixed; inset: auto 0 0 0; + background: color-mix(in srgb, var(--bg) 88%, transparent); border-top: 1px solid var(--edge); + backdrop-filter: blur(6px); + } + /* the footer strip bleeds to the screen edges (round-5 layout brief): the summary + rides the left edge, the keyboard instructions ride the RIGHT edge. */ + #footer .foot-in { display: flex; gap: 1rem; justify-content: space-between; flex-wrap: wrap; + max-width: none; margin: 0; padding: 0.35rem 1.25rem; font-size: 0.8rem; color: var(--soft); } + #keys { white-space: nowrap; } + @media (max-width: 720px) { + #keys { white-space: normal; } /* the 48-char hint at 0.8rem overflows a 320 px viewport */ + } + `]; + + constructor() { + super(); + this.page = "idle"; + this.bigOpen = false; + this._gearAttention = false; + this._sel = null; + } + + /* ── composition: main.js wires {store, api, theme, controllers}. The html DECLARES + this element, so its FIRST update runs before the wiring lands: render stays empty + and the boot waits for store + api (2026-11-08 — an unwired shell must paint + nothing, not a bare page). Wiring is a state change, so the setters requestUpdate. ── */ + set store(store) { + if (store === this._store) return; + this._store = store; + this._sel = storeController(this, store, (s) => s); + if (this.renderRoot) { + const settings = this.renderRoot.querySelector("digup-settings"); + if (settings) settings.store = store; + } + this.requestUpdate(); + } + get store() { return this._store; } + + set controllers(c) { this._ctl = c; this.requestUpdate(); } + get controllers() { return this._ctl; } + + set api(api) { this._api = api; this.requestUpdate(); } + get api() { return this._api; } + + set theme(theme) { this._theme = theme; } + get theme() { return this._theme; } + + _s() { return this._sel ? this._sel.value() : {}; } + + /* The layout's state, read off the state the renderers already use: no second store, + no second notion of "what page am I on". Hits → the two columns; nothing typed (or + the code-prefix hint) → the centered welcome; words in, meaning on its way → busy; + meaning answered zero → none. */ + #pageMode() { + const s = this._s(); + const r = s.results || { strong: [], weaker: [] }; + return (r.strong.length + r.weaker.length ? "results" + : (r.phase === "empty" || r.phase === "codeHint") ? "idle" + : r.phase === "words" ? "busy" : "none"); + } + + firstUpdated() { + // The keyboard + click routes belong to the page, not to a wired slice: attach them + // here (they read the store lazily, through _s()). Everything that TALKS — the focus, + // the 15 s status poll, the first refresh — waits for the wiring, in #boot(). + document.addEventListener("keydown", this.#docKey); + document.addEventListener("click", this.#docClick); + this.#boot(); + } + + #boot() { + // The vanilla startup lines `render(); refreshStatus(); setInterval(refreshStatus, + // 15000); $("q").focus()`, run ONCE, on the first update where the entry has wired + // store + api (the html declares this element, so the very first update is unwired). + if (this._booted || !this._store || !this._api) return; + this._booted = true; + // Focus rides on the search child's first render (its updateComplete microtask) — + // a guard alone would silently skip the first paint's focus. + const search = this.renderRoot.querySelector("digup-search"); + if (search) search.updateComplete.then(() => search.focusQuery()); + this._poll = setInterval(() => this.refreshStatus(), 15000); + this.refreshStatus(); + } + + disconnectedCallback() { + super.disconnectedCallback(); + clearInterval(this._poll); + document.removeEventListener("keydown", this.#docKey); + document.removeEventListener("click", this.#docClick); + } + + updated() { + if (this._store) this.#syncPage(); // the layout's ONE switch, at every paint + this.#boot(); // fires on the update where the wiring lands + const s = this._s(); + const settings = this.renderRoot.querySelector("digup-settings"); + if (settings) settings.store = this._store; + if (settings && !settings.hasAttribute("hidden")) settings.theme = this._theme?.current() ?? "system"; + // A store change repaints the shell's children through bindings; the host mirror + // for the shadow panels rides with data-page. + } + + #syncPage() { + const mode = this.#pageMode(); + document.body.dataset.page = mode; + this.dataset.page = mode; + const s = this._s(); + this.toggleAttribute("data-code", Boolean(s.isCode)); + } + + render() { + // Unwired: paint NOTHING (the html declares this element, so the first update can + // precede main.js's wiring — a bare page there is the "double DigUp" symptom). + if (!this._store || !this._api) return null; + const s = this._s(); + const r = s.results || { strong: [], weaker: [], groups: [], showWeaker: false, + selection: null, hitsById: new Map() }; + const mode = this.#pageMode(); + const indexing = s.indexing; + const showActivity = s.encoder === "starting" || Boolean(indexing); + const hit = r.hitsById.get(r.selection) ?? null; + return html` +
    +
    +

    DigUp

    + +
    +
    + ${showActivity ? html` + + + ${indexing ? "indexing…" : "meaning search warming up…"} + ` : null} +
    + this.closeSettings()} + @digup:theme=${(e) => { this._theme?.apply(e.detail.choice); this.requestUpdate(); }} + @digup:index=${(e) => this._ctl?.jobs?.indexNow(e.detail.prune)} + @digup:estimate=${() => { this._ctl?.estimate?.setToggle(true); this._ctl?.estimate?.runEstimate(); }} + @digup:regenerate=${() => this._ctl?.keys?.regenerate()}> + +
    + +
    + this._ctl?.search?.queryChanged(e.detail.text, e.detail)} + @digup:nav=${(e) => this._ctl?.search?.move(e.detail.step)} + @digup:activate=${() => { + if (this._s().results?.selection === WEAKER) this._ctl?.search?.toggleWeaker(); + else this.#openHit(this._ctl?.search?.selectedHit()); + }} + @digup:preview=${() => this.#previewRequest()}> + + +
    +

    Describe it. Dig it up.

    +

    ${mode === "busy" ? BUSY_NOTE + : (r.phase === "codeHint" ? CODE_HINT : IDLE_NOTE)}

    +
    + this.renderRoot.querySelector("digup-search")?.setValue(e.detail.phrase)}> +
    +
    + +
    +

    No results found

    +

    ${this.#noResultsWhy(r)}

    +

    Widening the kind filters, or turning off code, asks the + engine to look in more of your files.

    +
    + +
    + this._ctl?.search?.select(e.detail.mediaId)} + @digup:open=${(e) => this.#openHit(this._s().results?.hitsById?.get(e.detail.mediaId))} + @digup:toggle-weaker=${() => this._ctl?.search?.toggleWeaker()}> + + +
    +
    + +
    +
    + + ↑↓ pick · ↩ open · Space preview · Esc clear · type to search +
    +
    + + { this.bigOpen = false; }}>`; + } + + #noResultsWhy(r) { + if (!r.lastReply) return "Try describing what you'd see or hear in it — a scene, an object, " + + "a moment, words you remember from it."; + const reply = r.lastReply; + return emptyAnswer(reply) + (reply.warning ? ` (${reply.warning})` : ""); + } + + /* ── opening a result (the desktop's Open: the file itself; video plays from its moment) ── */ + #openHit(hit) { + if (!hit) return; + window.open(`/api/v1/media/${hit.media_id}/file`, "_blank", "noopener"); + } + + /* Space previews (the big look) — the vanilla rule: after navigation, on a + selection; on the WEAKER row Space/↩ fold and unfold. */ + #previewRequest() { + const s = this._s(); + if (s.results?.selection === WEAKER) { this._ctl?.search?.toggleWeaker(); return; } + this.bigOpen = !this.bigOpen; + } + + /* ── gear / settings: all five close paths share one function — the gear's + aria-expanded must match the panel's hidden state everywhere (a screen reader + must not report an open dialog whose panel is gone), and focus returns to the + gear (closing must not drop the keyboard user to ). ── */ + #gearClick = () => { + const open = this._settingsOpen !== true; + this._settingsOpen = open; + this._gearAttention = false; // pointing done: the panel is open + this.requestUpdate(); + const settings = this.renderRoot.querySelector("digup-settings"); + if (open) { + settings.focusFirst(); + this._ctl?.keys?.resetRetried(); // a fresh open may try the stale-key recovery once + this._ctl?.keys?.loadAccess(); // keys + indexes, admin scope + } + }; + + closeSettings() { + const settings = this.renderRoot.querySelector("digup-settings"); + const hadFocus = settings && (settings.contains(document.activeElement) + || settings.shadowRoot.contains(document.activeElement)); + this._settingsOpen = false; + this.requestUpdate(); + if (hadFocus) this.renderRoot.getElementById("gear").focus(); + } + + #docClick = (event) => { + if (this._settingsOpen !== true) return; + // closest(), not a bare equality: the gear's svg is the click target, not the button. + const path = event.composedPath(); + if (path.includes(this.renderRoot.getElementById("gear")) + || path.includes(this.renderRoot.querySelector("digup-settings"))) return; + this.closeSettings(); + }; + + /* Keys and events (the footer says them; they work from the search field and the list). */ + #docKey = (event) => { + const first = event.composedPath()[0] || event.target; + const inField = ["INPUT", "SELECT", "TEXTAREA"].includes(first.tagName) + || (first.host && ["INPUT", "SELECT", "TEXTAREA"].includes(first.host.tagName)); + switch (event.key) { + case "Escape": + if (this.bigOpen) { event.preventDefault(); this.bigOpen = false; } + else if (this._settingsOpen === true) { event.preventDefault(); this.closeSettings(); } + else if (this._s().estimate?.rows || this._s().estimate?.spinning) { + event.preventDefault(); this._ctl?.estimate?.cancel(); + } + else if (this.renderRoot.querySelector("digup-search")?.shadowRoot.getElementById("q")?.value) { + this.renderRoot.querySelector("digup-search").clearAndRefocus(); + } + break; + case "ArrowDown": + case "ArrowUp": + if (first.tagName === "DIGUP-SEARCH" || inField) break; // the field has its own handler + event.preventDefault(); + this._ctl?.search?.move(event.key === "ArrowDown" ? 1 : -1); + break; + case " ": + // Native button/link activation happens on keyup: preventing default here (the + // Quick-Look preview) silently killed Space-activation of the gear, ✕, Index now, + // Estimate, "all", the Esc close, and Open/Download — Space is a keyboard user's + // standard activation key, so buttons and links keep it; only bare page areas type-preview. + // In shadow DOM the activation target lives in the composed path: run the guard + // from the deepest node up, keeping the pinned selector verbatim. + for (const node of event.composedPath()) { + if (node instanceof Element && node.closest + && node.closest("button, a, [role='button'], input, select, textarea")) return; + } + if (first.tagName === "DIGUP-SEARCH") break; // typing a space (the field handler above) + const s = this._s(); + if (!s.navigated && inField) return; + if (!s.results?.selection && !inField) return; + event.preventDefault(); + this.#previewRequest(); + break; + } + }; + + /* The 15 s status poll (the engine's numbers live in the settings panel; the main + page keeps only the activity dot — a page that nags "paste your key" on every + poll is noise: the gear points at the key, and the counts are the panel's). */ + async refreshStatus() { + try { + const st = await this._api("/api/v1/status"); + this._store.set({ + encoder: st.encoder, + status: { files: st.files, searchable: st.searchable, + vectors: st.vectors, encoder: st.encoder, kinds: st.kinds || [], + index_size: st.index_size, build_tag: st.build_tag }, + }); + } catch { + // No counts to show: a no-key browser sees the gear point at the key (api() + // raised with the pointer); a mid-reload refusal is a state, the next poll + // re-reads. The main page stays clean. + } + } + + // The api's 401-with-no-key pointer (main.js wires createApiClient.onNoToken here). + noteMissingKey() { this._gearAttention = true; this.requestUpdate(); } +} +if (!customElements.get("digup-app")) customElements.define("digup-app", DigupApp); diff --git a/server/ui/src/components/digup-app.test.js b/server/ui/src/components/digup-app.test.js new file mode 100644 index 00000000..8c19db54 --- /dev/null +++ b/server/ui/src/components/digup-app.test.js @@ -0,0 +1,153 @@ +// digup-app — the shell: composition (store/api/controllers), the ONE layout switch +// (body[data-page] + host mirror, written at the FIRST paint), the gear's open/close +// with aria-expanded true to the panel, the Esc chain, the keyboard routes, and the +// pinned Space guard (buttons/links keep Space activation — the reviewer's finding). +import { describe, expect, it, beforeEach, afterEach, vi } from "vitest"; +import "./digup-app.js"; +import { createStore } from "../app/store.js"; +import { createSearchController } from "../app/search-controller.js"; +import { createKeysController } from "../app/keys-controller.js"; +import { createJobController } from "../app/job-controller.js"; +import { createEstimateController } from "../app/estimate-controller.js"; +import { installFetch, mockReply, memoryStorage } from "../lib/test-util.js"; + +const hit = { media_id: "a", kind: "image", path: "/x/a.jpg", z: 3.2 }; + +async function mount() { + const store = createStore({ + results: { phase: "empty", strong: [], weaker: [], groups: [], hitsById: new Map(), + showWeaker: false, selection: null, lastReply: null }, + filters: { kinds: [], limit: 40 }, isCode: false, text: "", navigated: false, + error: null, job: null, indexing: null, encoder: null, status: null, roots: [], + targets: [], keys: null, + }); + const calls = installFetch({ + "/api/v1/status": mockReply(200, { encoder: "gemma-2", vectors: 20, kinds: [], + index_size: "1 MB", build_tag: "b1", roots: ["/m"] }), + "/api/v1/tokens": mockReply(200, { admin: "adm1n", apps: [] }), + "/api/v1/targets": mockReply(200, { targets: [] }), + }); + const api = async (path, options = {}) => { + const r = await globalThis.fetch(path, options); + if (!r.ok) throw Object.assign(new Error("no"), { status: r.status }); + return r.json(); + }; + const theme = { apply: vi.fn(), current: () => "system" }; + const controllers = { + search: createSearchController({ api, store }), + keys: createKeysController({ store, api, storage: memoryStorage() }), + jobs: createJobController({ api, store, refreshStatus: () => {}, loadAccess: () => {} }), + estimate: createEstimateController({ api, store, storage: memoryStorage() }), + }; + const el = document.createElement("digup-app"); + el.store = store; + el.api = api; + el.theme = theme; + el.controllers = controllers; + document.body.append(el); + await el.updateComplete; + await vi.advanceTimersByTimeAsync(0); // the boot status poll + return { el, store, calls, theme }; +} + +const key = (target, k) => target.dispatchEvent( + new KeyboardEvent("keydown", { key: k, bubbles: true, composed: true, cancelable: true })); + +describe("digup-app", () => { + let ctx; + beforeEach(() => vi.useFakeTimers()); + afterEach(() => { ctx?.el.remove(); vi.useRealTimers(); document.body.dataset.page = ""; }); + + it("the FIRST paint is a state: body[data-page] and the host mirror say idle", async () => { + ctx = await mount(); + expect(document.body.dataset.page).toBe("idle"); + expect(ctx.el.dataset.page).toBe("idle"); + expect(ctx.el.renderRoot.querySelector("digup-search")).not.toBeNull(); + expect(ctx.el.renderRoot.querySelector("h1").textContent).toBe("DigUp"); + expect(ctx.el.renderRoot.getElementById("keys").textContent) + .toBe("↑↓ pick · ↩ open · Space preview · Esc clear · type to search"); + expect(ctx.calls.map((c) => c.path)).toContain("/api/v1/status"); + }); + + it("results arrive → the layout switch moves to results (the ONE switch, both places)", async () => { + ctx = await mount(); + ctx.store.set({ results: { phase: "meaning", strong: [hit], weaker: [], + groups: [{ title: "Images", rows: [hit] }], hitsById: new Map([hit].map((h) => [h.media_id, h])), + showWeaker: false, selection: "a", lastReply: { hits: [hit], strong: 1, meaning: true, ms: {} } } }); + await ctx.el.updateComplete; + expect(document.body.dataset.page).toBe("results"); + expect(ctx.el.dataset.page).toBe("results"); + expect(ctx.el.renderRoot.querySelector("digup-results")).not.toBeNull(); + }); + + it("the gear opens the panel (keys load, aria-expanded true) and Esc closes it (aria-expanded false)", async () => { + ctx = await mount(); + const gear = ctx.el.renderRoot.getElementById("gear"); + gear.click(); + await ctx.el.updateComplete; + const settings = ctx.el.renderRoot.querySelector("digup-settings"); + expect(settings.hasAttribute("hidden")).toBe(false); + expect(gear.getAttribute("aria-expanded")).toBe("true"); + await vi.advanceTimersByTimeAsync(0); + expect(ctx.calls.map((c) => c.path)).toContain("/api/v1/tokens"); + expect(ctx.calls.map((c) => c.path)).toContain("/api/v1/targets"); + key(document, "Escape"); + await ctx.el.updateComplete; + expect(settings.hasAttribute("hidden")).toBe(true); + expect(gear.getAttribute("aria-expanded")).toBe("false"); + }); + + it("suggestion chips take the ONE search route through digup-search", async () => { + ctx = await mount(); + const suggest = ctx.el.renderRoot.querySelector("digup-suggest"); + const search = ctx.el.renderRoot.querySelector("digup-search"); + await search.updateComplete; + const first = suggest.renderRoot.querySelector("button"); + const text = first.textContent; + first.click(); + await search.updateComplete; + expect(search.shadowRoot.getElementById("q").value).toBe(text); + }); + + it("the pinned Space guard: buttons keep Space activation; bare page areas preview", async () => { + ctx = await mount(); + const gear = ctx.el.renderRoot.getElementById("gear"); + // Space ON the gear button: native activation, no Quick-Look. + gear.focus(); + key(gear, " "); + await ctx.el.updateComplete; + expect(ctx.el.bigOpen).toBe(false); + // Space with a selection after navigation, from a bare page area: the big look. + ctx.store.set({ navigated: true, results: { phase: "meaning", strong: [hit], weaker: [], + groups: [{ title: "Images", rows: [hit] }], hitsById: new Map([hit].map((h) => [h.media_id, h])), + showWeaker: false, selection: "a", lastReply: { hits: [hit], strong: 1, meaning: true, ms: {} } } }); + await ctx.el.updateComplete; + key(document, " "); + await ctx.el.updateComplete; + expect(ctx.el.bigOpen).toBe(true); + expect(ctx.el.renderRoot.querySelector("digup-big").hasAttribute("hidden")).toBe(false); + // Esc puts it away (the chain: big first, panel second). + key(document, "Escape"); + await ctx.el.updateComplete; + expect(ctx.el.bigOpen).toBe(false); + expect(ctx.el.renderRoot.querySelector("digup-big").hasAttribute("hidden")).toBe(true); + }); + + it("the 15 s status poll keeps the engine's numbers coming (and stops on disconnect)", async () => { + ctx = await mount(); + const before = ctx.calls.filter((c) => c.path === "/api/v1/status").length; + await vi.advanceTimersByTimeAsync(15000); + const after = ctx.calls.filter((c) => c.path === "/api/v1/status").length; + expect(after).toBeGreaterThan(before); + ctx.el.remove(); + await vi.advanceTimersByTimeAsync(60000); + expect(ctx.calls.filter((c) => c.path === "/api/v1/status").length).toBe(after); + }); + + it("a 401 with no stored key makes the gear point (the api's pointer, not a prompt)", async () => { + ctx = await mount(); + ctx.el.noteMissingKey(); + await ctx.el.updateComplete; + expect(ctx.el.renderRoot.getElementById("gear").className).toContain("attention"); + }); +}); diff --git a/server/ui/src/components/digup-big.js b/server/ui/src/components/digup-big.js new file mode 100644 index 00000000..0bd7eef8 --- /dev/null +++ b/server/ui/src/components/digup-big.js @@ -0,0 +1,58 @@ +// digup-big — the big look (Space, as Quick Look; Esc puts it away): the selection +// over all of it, its video or sound stops with it (the vanilla closeBig cleared the +// box — Lit clears by not rendering, which unmounts the media too). The overlay's +// [hidden]-guard lesson rides in :host([hidden]) (display outranks the UA rule — +// the Stage-10 bug). Clicking the backdrop closes (the vanilla target===big rule); +// clicks inside the panel do not. +import { LitElement, html, css } from "lit"; +import { baseName, folderOf } from "../lib/format.js"; +import { pictureTemplate } from "../lib/media.js"; + +class DigupBig extends LitElement { + static properties = { hit: {} }; + + static styles = css` + :host([hidden]) { display: none; } + :host { position: fixed; inset: 0; background: color-mix(in srgb, #000 78%, transparent); + display: flex; align-items: center; justify-content: center; z-index: var(--z-big); } + .big-panel { background: var(--panel); border: 1px solid var(--edge); border-radius: 12px; + max-width: min(1100px, 92vw); max-height: 90vh; overflow: auto; padding: 0.8rem; } + .head { display: flex; align-items: baseline; gap: 0.6rem; margin-bottom: 0.5rem; } + .head .name { font-weight: 600; } + .head .place { color: var(--soft); font-size: 0.85rem; flex: 1; overflow-wrap: anywhere; } + .view img, .view video { display: block; max-width: 100%; max-height: 74vh; object-fit: contain; + border-radius: 8px; background: #000; } + .view audio { width: 100%; } + .view .passage { font: 0.9rem/1.55 var(--font-mono); + white-space: pre-wrap; overflow-wrap: anywhere; max-height: 70vh; overflow: auto; } + .close { padding: 0.1rem 0.6rem; font-size: 0.8rem; } + `; + + constructor() { + super(); + this._broken = new Set(); + } + + #broken = (hit) => this._broken.has(hit.media_id); + #markBroken = (id) => { this._broken = new Set([...this._broken, id]); }; + + render() { + const hit = this.hit; + if (!hit) return html``; + return html` +
    { if (e.target.classList.contains("overlay")) this.#close(); }}> +
    +
    + ${baseName(hit.path)} + ${folderOf(hit.path)} + +
    +
    ${pictureTemplate(hit, { broken: this.#broken, markBroken: this.#markBroken })}
    +
    +
    `; + } + + #close = () => this.dispatchEvent(new CustomEvent("digup:close", + { bubbles: true, composed: true })); +} +if (!customElements.get("digup-big")) customElements.define("digup-big", DigupBig); diff --git a/server/ui/src/components/digup-embed.js b/server/ui/src/components/digup-embed.js new file mode 100644 index 00000000..445748b6 --- /dev/null +++ b/server/ui/src/components/digup-embed.js @@ -0,0 +1,273 @@ +// digup-embed — the embed widget (embed.html; docs/implementation/stage-12-integrations.md +// §11.4): a compact search page for hosts that embed it (a Nextcloud app page, an Immich +// workspace panel, a dashboard iframe). Same rules as the full UI (README §10.4, +// test_integrations pins): the engine's fields relayed verbatim (hits, z, strong, ms, +// meaning), keyword-first typing (words at once, meaning after the pause, embed_ahead +// warming the vector), previews only for tiles on screen. The read-scope token is the +// host's secret: it enters through ?token=, moves to sessionStorage, and the URL line +// is rewritten so the secret never lands in bookmarks, history, or the host app's logs. +// The host drives the theme by postMessage; the widget answers with `ready`. +import { LitElement, html, css } from "lit"; +import { baseName } from "../lib/format.js"; + +const PAUSE = 160; // ms after the last keystroke until meaning results +const KIND_LABEL = { image: "photo", screenshot: "screenshot", video: "video", audio: "audio", + pdf: "pdf", doc: "document", code: "code" }; +const PREVIEWABLE = new Set(["image", "screenshot", "video"]); + +class DigupEmbed extends LitElement { + static properties = { + _limit: { state: true }, _kinds: { state: true }, _strong: { state: true }, + _reply: { state: true }, _activity: { state: true }, _status: { state: true }, + }; + + static styles = css` + /* The vanilla embed.css values, carried over verbatim into the shadow (the widget + is the whole page, so the page's frame rules ride with it). */ + :host { display: block; } + .embed-bar { display: flex; align-items: center; gap: 0.6rem; padding: 0.6rem 0.9rem 0.2rem; } + #q { flex: 1 1 auto; min-width: 0; font-size: 1rem; padding: 0.45rem 0.7rem; + border: 1px solid var(--edge); border-radius: 6px; background: var(--field); color: var(--text); } + .embed-count { flex: 0 0 auto; display: flex; align-items: center; gap: 0.35rem; + font-size: 0.85rem; color: var(--soft); } + .embed-count select { padding: 0.2rem 0.35rem; border: 1px solid var(--edge); border-radius: 5px; + background: var(--field); color: var(--text); } + .embed-chips { display: flex; flex-wrap: wrap; align-items: center; gap: 0.4rem; + padding: 0.45rem 0.9rem 0.1rem; } + .chip { font: inherit; font-size: 0.82rem; padding: 0.25rem 0.7rem; border: 1px solid var(--line); + border-radius: 999px; background: var(--panel); color: var(--text); cursor: pointer; } + .chip.on { border-color: var(--accent); color: var(--accent); } + .chip-check { display: flex; align-items: center; gap: 0.4rem; cursor: pointer; } + main { padding: 0 0.9rem 1rem; } + #statusline { color: var(--soft); font-size: 0.82rem; min-height: 1.2em; margin: 0.2rem 0; } + .activity { display: inline-flex; align-items: center; gap: 0.35rem; font-size: 0.8rem; color: var(--amber); } + .activity .dot { width: 0.55rem; height: 0.55rem; border-radius: 999px; background: var(--amber); + animation: pulse 1.6s ease-in-out infinite; } + @keyframes pulse { 50% { opacity: 0.35; } } + .embed-grid { list-style: none; margin: 0.6rem 0 0; padding: 0; display: grid; + grid-template-columns: repeat(auto-fill, minmax(9.5rem, 1fr)); gap: 0.7rem; } + .tile { display: flex; flex-direction: column; gap: 0.25rem; min-height: 6.5rem; padding: 0.45rem; + border: 1px solid var(--line); border-radius: 8px; background: var(--panel); cursor: pointer; + position: relative; } + .tile.strong { border-color: var(--good); } + .tile img { width: 100%; aspect-ratio: 4 / 3; object-fit: cover; border-radius: 5px; } + .tile .kind { align-self: flex-start; font-size: 0.7rem; letter-spacing: 0.03em; text-transform: uppercase; + color: var(--soft); padding: 0.05rem 0.45rem; border: 1px solid var(--line); border-radius: 999px; + background: var(--bg); } + .tile .name { font-size: 0.82rem; line-height: 1.25; overflow: hidden; display: -webkit-box; + -webkit-line-clamp: 2; -webkit-box-orient: vertical; overflow-wrap: anywhere; } + .tile .z, .tile .loc { font-size: 0.72rem; color: var(--soft); } + .tile.strong .z { color: var(--good); } + `; + + constructor() { + super(); + this._limit = 40; + this._kinds = ""; + this._strong = false; + this._reply = null; + this._activity = false; + this._status = "connecting…"; + this._previews = new Map(); // media_id → objectURL (previews are a grace, not a promise) + this._tileRefs = new Map(); // media_id → rendered tile element (IntersectionObserver targets) + this._text = ""; + this._timer = null; + this._gen = 0; + this._meaningGen = 0; + } + + /* embed.js wires {api (createEmbedApi client), params (URLSearchParams)}. embed.html + DECLARES this element, so its first update runs unwired: render stays empty and the + boot waits for the api (the same rule as the shell, 2026-11-08). */ + set api(client) { this._api = client; this.requestUpdate(); } + set params(params) { this._params = params; } + + firstUpdated() { this.#boot(); } + + #boot() { + if (this._booted || !this._api) return; + this._booted = true; + // Theme: system by default; the host app (Nextcloud / Immich) drives it by postMessage. + const setTheme = (value) => { + if (value === "light" || value === "dark") document.documentElement.dataset.theme = value; + }; + if (this._params?.get("theme")) setTheme(this._params.get("theme")); + this._hostMsg = (event) => { + const data = event.data; + if (data && data.source === "digup-host" && data.theme) setTheme(data.theme); + }; + window.addEventListener("message", this._hostMsg); + window.parent.postMessage({ source: "digup-embed", event: "ready" }, "*"); + + if (this._params?.get("limit")) this._limit = Number(this._params.get("limit")); + + // Previews only for tiles on screen (the vanilla IntersectionObserver with a 300 px + // margin). jsdom has none — then previews load straight away (the unit-test path). + if (typeof IntersectionObserver !== "undefined") { + this._observer = new IntersectionObserver((seen) => { + for (const entry of seen) { + if (!entry.isIntersecting) continue; + this._observer.unobserve(entry.target); + this.#loadPreview(entry.target.dataset.media); + } + }, { rootMargin: "300px" }); + } + + // start: health (open, no token needed), then the host's ?q= if any + this._api.api("/api/v1/health").then((body) => { + const live = body.live || {}; + this._status = `ready · ${live.searchable ?? "?"} searchable files`; + if (this._params?.get("q")) { this.#run(this._params.get("q")); } + this.requestUpdate(); + }).catch((error) => { this._status = error.message; this.requestUpdate(); }); + } + + disconnectedCallback() { + super.disconnectedCallback(); + if (this._hostMsg) window.removeEventListener("message", this._hostMsg); + this._observer?.disconnect(); + } + + async #runSearch(meaning, myGen) { + try { + const reply = await this._api.api("/api/v1/search", + { method: "POST", body: JSON.stringify({ q: this._text, meaning, limit: this._limit }) }); + if (meaning) this._meaningGen = myGen; + if (myGen < this._meaningGen) return; // a late keyword reply must not replace meaning + this._reply = reply; + this.requestUpdate(); + } catch (error) { + this._status = error.message; + this.requestUpdate(); + } + } + + #run(text) { + this._text = text; + if (!text.trim()) { + this._reply = null; + this._status = "say what you'd see, hear, or read — the meaning, not only the words"; + this.requestUpdate(); + return; + } + this._activity = true; + this._api.api("/api/v1/embed_ahead", + { method: "POST", body: JSON.stringify({ q: text.trim() }) }).catch(() => {}); + const myGen = ++this._gen; + this.#runSearch(false, myGen); // words right away: the keyword path + clearTimeout(this._timer); + this._timer = setTimeout(() => { + this._gen++; + this.#runSearch(true, this._gen); + }, PAUSE + 320); + this.requestUpdate(); + } + + async #loadPreview(id) { + if (!id || this._previews.has(id)) return; + const kind = this._reply?.hits.find((h) => h.media_id === id)?.kind; + if (!PREVIEWABLE.has(kind)) return; + try { + const answer = await this._api.blobOf(`/api/v1/media/${id}/preview?pixels=256`); + this._previews = new Map(this._previews).set(id, URL.createObjectURL(await answer.blob())); + this.requestUpdate(); + } catch { /* the tile keeps the kind label: previews are a grace, not a promise */ } + } + + updated() { + this.#boot(); // fires on the update where embed.js wires the api + // (Re)arm the observer on the tiles that are renderable and unpreviewed. + if (this._observer) { + for (const [id, el] of this._tileRefs) { + if (!this._previews.has(id) && PREVIEWABLE.has(el.dataset.kind)) { + this._observer.observe(el); + } + } + } else if (this._reply) { + // No IntersectionObserver (jsdom): load previews straight away. + for (const hit of this._reply.hits) { + if (PREVIEWABLE.has(hit.kind) && !this._previews.has(hit.media_id)) { + this.#loadPreview(hit.media_id); + } + } + } + } + + async #openHit(hit) { + if (!hit.media_id) return; + try { + const answer = await this._api.blobOf(`/api/v1/media/${hit.media_id}/file`); + const url = URL.createObjectURL(await answer.blob()); + window.open(url, "_blank"); + setTimeout(() => URL.revokeObjectURL(url), 10 * 60 * 1000); + } catch (error) { this._status = error.message; this.requestUpdate(); } + } + + render() { + if (!this._api) return null; // unwired: paint nothing (see #boot) + const reply = this._reply; + let hits = []; + if (reply) { + const strongN = Math.min(reply.strong ?? reply.hits.length, reply.hits.length); + hits = reply.hits.map((hit, at) => ({ hit, strong: at < strongN })); + if (this._kinds) { + const wanted = this._kinds.split(","); + hits = hits.filter((h) => wanted.includes(h.hit.kind)); + } + if (this._strong) hits = hits.filter((h) => h.strong); + } + const ms = (reply && reply.ms) || {}; + const bits = []; + if (reply) { + bits.push(`${reply.hits.length} hits · ${Math.min(reply.strong ?? reply.hits.length, reply.hits.length)} strong`); + if (reply.meaning) bits.push(`embed ${ms.embed ?? 0} ms · search ${ms.search ?? 0} ms`); + if (!reply.meaning) bits.push("words only — pause typing for meaning"); + if (reply.warning) bits.push(reply.warning); + } + return html` +
    + this.#run(e.target.value)}> + + ${this._activity ? html`` : null} +
    + +
    +

    ${reply ? bits.join(" · ") : this._status}

    +
      + ${hits.map(({ hit, strong }) => html` +
    • { if (el) this._tileRefs.set(hit.media_id, el); }} + @click=${() => this.#openHit(hit)}> + ${this._previews.has(hit.media_id) + ? html`` : null} + ${KIND_LABEL[hit.kind] || hit.kind} + ${baseName(hit.path)} + ${hit.z !== undefined ? html`z ${hit.z.toFixed(2)}` : null} + ${hit.loc !== undefined && hit.loc !== null + ? html`at ${Math.round(hit.loc)}s` : null} +
      • `)} +
    +
    `; + } +} +if (!customElements.get("digup-embed")) customElements.define("digup-embed", DigupEmbed); diff --git a/server/ui/src/components/digup-embed.test.js b/server/ui/src/components/digup-embed.test.js new file mode 100644 index 00000000..6b76d1c6 --- /dev/null +++ b/server/ui/src/components/digup-embed.test.js @@ -0,0 +1,104 @@ +// digup-embed — the widget: keyword-first (words at once, meaning after the pause), +// the host's ?q= runs at boot, tiles with the engine's fields (kind, name, z, at), +// previews as a grace (the on-screen rule; jsdom loads them straight), client-side +// kind/strength filters on the chips, the ready postMessage. +import { describe, expect, it, beforeEach, afterEach, vi } from "vitest"; +import "./digup-embed.js"; +import { createEmbedApi } from "../services/api.js"; +import { installFetch, mockReply, memoryStorage } from "../lib/test-util.js"; + +const hits = [ + { media_id: "m1", kind: "image", path: "/x/b.jpg", z: 3.4 }, + { media_id: "m2", kind: "video", path: "/x/c.mp4", z: 1.2, loc: 2 }, +]; + +async function mount(paramsStr = "limit=20&q=dog") { + const store = memoryStorage({ digupEmbedToken: "readTok" }); + const client = createEmbedApi({ sessionStorageRef: store, prompt: () => "" }); + const calls = installFetch({ + "/api/v1/health": mockReply(200, { live: { searchable: 5 } }), + "POST /api/v1/search": mockReply(200, { hits, strong: 1, meaning: true, ms: { embed: 4, search: 2 } }), + "POST /api/v1/embed_ahead": mockReply(200, {}), + "/api/v1/media/m1/preview?pixels=256": { ok: true, status: 200, blob: async () => new Blob(["x"]) }, + "/api/v1/media/m2/preview?pixels=256": { ok: true, status: 200, blob: async () => new Blob(["y"]) }, + }); + const el = document.createElement("digup-embed"); + el.api = client; + el.params = new URLSearchParams(paramsStr); + document.body.append(el); + await el.updateComplete; + await vi.advanceTimersByTimeAsync(0); // health → q run → keyword reply + await el.updateComplete; + return { el, calls, store }; +} + +describe("digup-embed", () => { + let ctx; + beforeEach(() => { + vi.useFakeTimers(); + if (!URL.createObjectURL) { URL.createObjectURL = vi.fn(() => "blob:1"); URL.revokeObjectURL = vi.fn(); } + }); + afterEach(() => { ctx?.el.remove(); vi.useRealTimers(); }); + + it("boots: health answers, the host's ?q= runs keyword-first with the bearer token", async () => { + ctx = await mount(); + expect(ctx.calls.map((c) => c.path)).toContain("/api/v1/health"); + const searchCall = ctx.calls.find((c) => c.path === "/api/v1/search"); + expect(JSON.parse(searchCall.options.body)).toMatchObject({ q: "dog", meaning: false, limit: 20 }); + expect(searchCall.headers.authorization).toBe("Bearer readTok"); + expect(ctx.calls.map((c) => c.path)).toContain("/api/v1/embed_ahead"); + }); + + it("meaning search runs after the pause (PAUSE + settle)", async () => { + ctx = await mount(); + await vi.advanceTimersByTimeAsync(480); + const meaning = ctx.calls.filter((c) => c.path === "/api/v1/search") + .map((c) => JSON.parse(c.options.body)).find((b) => b.meaning); + expect(meaning).toMatchObject({ q: "dog" }); + }); + + it("tiles carry the engine's fields; the strong ones are flagged; previews land", async () => { + ctx = await mount(); + await vi.advanceTimersByTimeAsync(10); // previews (no IntersectionObserver in jsdom) + await ctx.el.updateComplete; + const tiles = ctx.el.shadowRoot.querySelectorAll("li.tile"); + expect(tiles.length).toBe(2); + expect(tiles[0].classList.contains("strong")).toBe(true); + expect(tiles[0].querySelector(".name").textContent).toBe("b.jpg"); + expect(tiles[0].querySelector(".z").textContent).toBe("z 3.40"); + expect(tiles[1].querySelector(".loc").textContent).toBe("at 2s"); + expect(tiles[0].querySelector("img")).not.toBeNull(); + const says = ctx.el.shadowRoot.getElementById("statusline").textContent; + expect(says).toContain("2 hits · 1 strong"); + }); + + it("the chips filter the current hits (client-side, like the vanilla widget)", async () => { + ctx = await mount(); + const videos = ctx.el.shadowRoot.querySelector('button.chip[data-kind="video"]'); + videos.click(); + await ctx.el.updateComplete; + const tiles = ctx.el.shadowRoot.querySelectorAll("li.tile"); + expect(tiles.length).toBe(1); + expect(tiles[0].querySelector(".name").textContent).toBe("c.mp4"); + const strong = ctx.el.shadowRoot.getElementById("strong"); + strong.checked = true; + strong.dispatchEvent(new Event("change")); + await ctx.el.updateComplete; + expect(ctx.el.shadowRoot.querySelectorAll("li.tile").length).toBe(0); // the only hit here is weak + ctx.el.shadowRoot.querySelector('button.chip[data-kind=""]').click(); + await ctx.el.updateComplete; + expect(ctx.el.shadowRoot.querySelectorAll("li.tile.strong").length).toBe(1); + }); + + it("an empty query says the honest line and asks the engine nothing", async () => { + ctx = await mount("limit=20"); // no ?q + const q = ctx.el.shadowRoot.getElementById("q"); + const before = ctx.calls.filter((c) => c.path === "/api/v1/search").length; + q.value = " "; + q.dispatchEvent(new Event("input")); + await ctx.el.updateComplete; + expect(ctx.el.shadowRoot.getElementById("statusline").textContent) + .toContain("say what you'd see, hear, or read"); + expect(ctx.calls.filter((c) => c.path === "/api/v1/search").length).toBe(before); + }); +}); diff --git a/server/ui/src/components/digup-footer.js b/server/ui/src/components/digup-footer.js new file mode 100644 index 00000000..53a23e5d --- /dev/null +++ b/server/ui/src/components/digup-footer.js @@ -0,0 +1,34 @@ +// digup-footer — the summary line: the engine's counts and states, assembled exactly +// as the vanilla renderFooter assembled them (the keys hint rides beside it in the +// shell's static footer markup). "The server's helper: " for a failed search; +// the indexing counts while a job runs. +import { LitElement, html, css } from "lit"; + +class DigupFooter extends LitElement { + static properties = { results: {}, indexing: {}, error: {} }; + + static styles = css` + :host { display: inline; } + .error { color: var(--warn); } + `; + + render() { + const r = this.results || { strong: [], weaker: [] }; + let summary; + if (this.error) summary = `The server's helper: ${this.error}`; + else if (r.phase === "words") summary = "Matching words…"; + else if (r.phase === "wordsOnly") summary = "Words only: meaning search isn't available"; + else if (r.phase === "codeHint") summary = "Searching your code"; + else if (r.phase === "empty") summary = ""; + else summary = `${r.strong.length} ${r.strong.length === 1 ? "result" : "results"}` + + (r.weaker.length ? ` · ${r.weaker.length} weaker` : ""); + if (this.indexing) { + const j = this.indexing; + summary = (summary ? summary + " · " : "") + + `indexing ${j.state}${j.files_done ? ` ${j.files_done}` : ""}${j.files ? `/${j.files}` : ""}` + + (j.searchable !== undefined && j.searchable !== null ? ` · searchable ${j.searchable}` : ""); + } + return html`${summary}`; + } +} +if (!customElements.get("digup-footer")) customElements.define("digup-footer", DigupFooter); diff --git a/server/ui/src/components/digup-preview.js b/server/ui/src/components/digup-preview.js new file mode 100644 index 00000000..5c353e27 --- /dev/null +++ b/server/ui/src/components/digup-preview.js @@ -0,0 +1,69 @@ +// digup-preview — the preview pane: what the desktop shows for the selection (panel.js +// `picture`), via the shared media templates (lib/media.js). The pane is laid out only +// in the `results` state (digup-app gates the section on data-page), so the "no hits +// yet" messages live in the centered idle/busy/none panels — this component draws the +// selection, and nothing else. The WEAKER sentinel row gets the fold message (↩ shows +// or folds them), as the vanilla pane had it. +import { LitElement, html, css } from "lit"; +import { WEAKER } from "../lib/format.js"; +import { pictureTemplate, factsTemplate } from "../lib/media.js"; +import { controlCss } from "../styles/controls.js"; + +class DigupPreview extends LitElement { + static properties = { hit: {}, weakerCount: {}, showWeaker: {} }; + + // controlCss: Open / Download are `a.button-like` (lib/media.js) — the shared button + // primitive, so they render as BUTTONS, not as underlined links (2026-11-08 UI brief). + static styles = [controlCss, css` + :host { display: block; } + .message { display: flex; flex-direction: column; gap: 0.35rem; color: var(--soft); + padding: 2rem 0.5rem; text-align: center; } + .message strong { color: var(--text); } + .picture img, .picture video { display: block; width: 100%; max-height: 60vh; + object-fit: contain; border-radius: 8px; background: #000; } + /* audio hits live in an audio-picture box (the pane gives it that class) — + the old picture selector never matched them, so the control rendered at its ~300 px + default in the pane. */ + .picture audio, .audio-picture audio { width: 100%; margin-top: 0.5rem; } + .moment { color: var(--soft); font-size: 0.85rem; margin-top: 0.25rem; } + .passage, .passage code { font: 0.85rem/1.5 var(--font-mono); + white-space: pre-wrap; overflow-wrap: anywhere; max-height: 46vh; overflow: auto; + border: 1px solid var(--edge); border-radius: 8px; padding: 0.6rem 0.8rem; background: var(--bg); } + pre.excerpt { margin: 0.5rem 0 0; padding: 0.6rem 0.8rem; max-height: 20rem; overflow: auto; + border: 1px solid var(--edge); border-radius: 8px; font: 0.85rem/1.4 var(--font-mono); + white-space: pre-wrap; overflow-wrap: anywhere; background: var(--bg); } + .audio-picture pre.excerpt { margin-top: 0.5rem; } + .facts .title { font-weight: 600; overflow-wrap: anywhere; } + .facts .line { color: var(--soft); font-size: 0.85rem; overflow-wrap: anywhere; } + .facts .reason { margin-top: 0.45rem; font-size: 0.88rem; } + .facts .buttons { display: flex; gap: 0.5rem; margin-top: 0.6rem; } + .note { color: var(--soft); font-size: 0.82rem; margin-top: 0.4rem; } + `]; + + constructor() { + super(); + this._broken = new Set(); + this.hit = null; + } + + #broken = (hit) => this._broken.has(hit.media_id); + #markBroken = (id) => { this._broken = new Set([...this._broken, id]); }; + + updated(changed) { if (changed.has("hit")) this._broken = new Set(); } + + render() { + const hit = this.hit; + if (hit === WEAKER) { + const count = this.weakerCount ?? 0; + return html`
    + ${this.showWeaker ? "Weaker matches" + : `${count} weaker ${count === 1 ? "match" : "matches"}`} + They match much less well than the results above. Press ↩ to + ${this.showWeaker ? "fold them away." : "show them."}
    `; + } + if (!hit) return html``; // defensive: no reply backs this selection + return html`${pictureTemplate(hit, { broken: this.#broken, markBroken: this.#markBroken })} + ${factsTemplate(hit)}`; + } +} +if (!customElements.get("digup-preview")) customElements.define("digup-preview", DigupPreview); diff --git a/server/ui/src/components/digup-results.js b/server/ui/src/components/digup-results.js new file mode 100644 index 00000000..2a7b934d --- /dev/null +++ b/server/ui/src/components/digup-results.js @@ -0,0 +1,166 @@ +// digup-results — the hits: strong results grouped by kind, the rest folded +// ("N weaker matches"), the engine's counter line (engine-says), the roving-tabindex +// listbox. Selection state arrives from the store (the controller owns it); rows talk +// back with digup:select / digup:open / digup:toggle-weaker. Thumbnails are the +// engine's small renders (72 px, lazy — rows must not turn every search into a tab +// storm), and a preview render that 404s becomes the kind icon (the vanilla onerror +// swap). File names and excerpts are text nodes via Lit templates — escaped by the +// DOM, never HTML. +import { LitElement, html, css } from "lit"; +import { baseName, folderOf, clock, KIND_TITLE, WEAKER } from "../lib/format.js"; +import { badgeCss } from "../styles/controls.js"; + +const KIND_ICON = {screenshot: "▣", image: "▣", pdf: "❐", doc: "≡", video: "▶", + audio: "♪", code: ""}; + +class DigupResults extends LitElement { + static properties = { results: {}, _broken: { state: true } }; + + static styles = [badgeCss, css` + /* The hits list (the vanilla style.css rules, carried over verbatim — they live here + now because the list renders in this shadow). Rows are separated by a top border, + the group titles are the small uppercase labels, the thumbnail is a fixed square, + and the selected row carries the accent wash. */ + ul#hits { list-style: none; padding: 0; margin: 0; } + ul#hits > li.group-title { color: var(--soft); font-size: 0.75rem; text-transform: uppercase; + letter-spacing: 0.04em; padding: 0.5rem 0.1rem 0.15rem; } + ul#hits > li.row { + display: flex; gap: 0.6rem; border-top: 1px solid var(--line); padding: 0.5rem 0.4rem; + border-radius: 8px; cursor: pointer; + } + ul#hits > li.row.selected { background: color-mix(in srgb, var(--accent) 14%, var(--bg)); } + ul#hits > li.row:focus-visible { outline: 2px solid var(--accent); outline-offset: 1px; } + img.thumb { width: 2.4rem; height: 2.4rem; object-fit: cover; border: 1px solid var(--line); + border-radius: 6px; flex: none; margin-top: 0.15rem; } + .row-text { min-width: 0; } + .row-title { display: flex; align-items: baseline; gap: 0.35rem; flex-wrap: wrap; } + .row-title .name { font-weight: 550; overflow-wrap: anywhere; } + .row-sub { color: var(--soft); font-size: 0.82rem; overflow-wrap: anywhere; } + .row-sub mark { background: color-mix(in srgb, var(--accent) 25%, var(--bg)); color: inherit; + border-radius: 3px; padding: 0 0.1rem; } + li.weaker { padding: 0.5rem 0.4rem; color: var(--soft); font-size: 0.85rem; cursor: pointer; + border-top: 1px solid var(--line); } + li.weaker .chevron { margin-right: 0.35rem; } + li.weaker.selected { color: var(--text); } + .engine-says { color: var(--soft); font-size: 0.82rem; margin: 0.2rem 0 0.4rem; } + .engine-says .warn { color: var(--warn); } + `]; + + constructor() { + super(); + this.results = { groups: [], weaker: [], showWeaker: false, selection: null, + lastReply: null, hitsById: new Map() }; + this._broken = new Set(); + } + + #toggleWeaker = () => this.dispatchEvent(new CustomEvent("digup:toggle-weaker", + { bubbles: true, composed: true })); + + #rowIDs() { + const r = this.results; + return [ + ...r.groups.flatMap((g) => g.rows.map((hit) => hit.media_id)), + ...(r.weaker.length ? [WEAKER] : []), + ...(r.showWeaker ? r.weaker.map((hit) => hit.media_id) : []), + ]; + } + + updated() { + // Keep the selection in view (vanilla scrollIntoView; jsdom has none — guard it). + this.renderRoot.querySelector(".selected") + ?.scrollIntoView?.({ block: "nearest" }); + } + + render() { + const r = this.results; + const reply = r.lastReply || {hits: [], strong: 0, meaning: true, ms: {}}; + const ms = reply.ms || {}; + return html` +

    + ${reply.hits.length} hits · ${reply.strong} strong + ${ms.embed !== undefined ? html` · embed ${(ms.embed / 1000).toFixed(2)} s` : null} + ${ms.search !== undefined ? html` · search ${(ms.search / 1000).toFixed(2)} s` : null} + ${reply.warning ? html` · ${reply.warning}` : null} + ${!reply.hits.length && r.phase !== "empty" && r.phase !== "codeHint" + ? html` — ${this.#emptyAnswer(reply)}` : null} +

    +
      + ${r.groups.map((group) => html` +
    • ${group.title}
      • + ${group.rows.map((hit) => this.#row(hit))}`)} + ${r.weaker.length ? html` +
    • { if (e.key === "Enter") this.#toggleWeaker(); }}> + ${r.showWeaker ? "▾" : "▸"} + ${r.showWeaker ? "Weaker matches" + : `${r.weaker.length} weaker ${r.weaker.length === 1 ? "match" : "matches"}`} +
      • + ${r.showWeaker ? r.weaker.map((hit) => this.#row(hit)) : null}` : null} +
    `; + } + + #emptyAnswer(reply) { + return reply.meaning + ? "the engine found no good matches for this and refused to guess — that is its answer" + : "no keyword hits (meaning search is warming up)"; + } + + #row(hit) { + const selected = this.results.selection === hit.media_id; + return html` +
  • this.dispatchEvent(new CustomEvent("digup:select", + { bubbles: true, composed: true, detail: { mediaId: hit.media_id } }))} + @dblclick=${() => this.dispatchEvent(new CustomEvent("digup:open", + { bubbles: true, composed: true, detail: { mediaId: hit.media_id } }))} + @keydown=${(e) => { + if (e.key === "Enter") { e.preventDefault(); + this.dispatchEvent(new CustomEvent("digup:open", + { bubbles: true, composed: true, detail: { mediaId: hit.media_id } })); } }}> + ${this.#thumbnail(hit)} +
    +
    + ${baseName(hit.path)} + ${hit.kind} + ${hit.loc !== undefined && hit.loc !== null + ? html`at ${clock(hit.loc)}` : null} + ${hit.z !== undefined ? html`z ${hit.z.toFixed(1)}` : null} +
    +
    + ${hit.excerpt && hit.excerpt.includes("«") + ? this.#excerptSpans(hit.excerpt) // words found + : folderOf(hit.path)} +
    +
    +
    • `; + } + + // The words the keyword search matched, marked with «» by the engine: rendered as + // spans (text nodes), never as HTML. + #excerptSpans(excerpt) { + return html` + ${excerpt.split(/(«[^»]*»)/).filter(Boolean).map((piece) => piece.startsWith("«") + ? html`${piece.slice(1, -1)}` + : html`${piece}`)}`; + } + + // Row thumbnails: the small picture the engine renders (72 px), lazy so only the visible + // rows cost a decode. Videos show their frame in the preview pane, at the moment — a + // frame extraction per row for a whole list is a cost the desktop pays too, and the + // server's engine is the same engine: honest to keep the list light. + #thumbnail(hit) { + const icon = html`${KIND_ICON[hit.kind] || "•"}`; + if (!["image", "screenshot", "pdf"].includes(hit.kind)) return icon; + if (this._broken.has(hit.media_id)) return icon; + return html` { this._broken = new Set([...this._broken, hit.media_id]); }}>`; + } +} +if (!customElements.get("digup-results")) customElements.define("digup-results", DigupResults); diff --git a/server/ui/src/components/digup-results.test.js b/server/ui/src/components/digup-results.test.js new file mode 100644 index 00000000..08f74837 --- /dev/null +++ b/server/ui/src/components/digup-results.test.js @@ -0,0 +1,100 @@ +// digup-results — grouped strong hits, the weaker fold, the listbox semantics +// (aria-selected + roving tabindex), the engine's counter line, thumbnails that fall +// back to the kind icon, and excerpt marks. File names and excerpts stay text nodes. +import { describe, expect, it, beforeEach, afterEach, vi } from "vitest"; +import "./digup-results.js"; +import { WEAKER } from "../lib/format.js"; +import { onceEvent } from "../lib/test-util.js"; + +const a = { media_id: "a", kind: "image", path: "/x/a.jpg", z: 3.2, excerpt: "the «dog» ran" }; +const b = { media_id: "b", kind: "image", path: "/x/b.jpg", z: 2.1 }; +const c = { media_id: "c", kind: "video", path: "/x/c.mp4", z: 2.0, loc: 12 }; + +const fixture = (over = {}) => ({ + phase: "meaning", strong: [a], weaker: [c], groups: [{ title: "Images", rows: [a] }], + hitsById: new Map([a, b, c].map((h) => [h.media_id, h])), + showWeaker: false, selection: "a", + lastReply: { hits: [a, c], strong: 1, meaning: true, ms: { embed: 100, search: 200 } }, + ...over, +}); + +async function mount(results = fixture()) { + const el = document.createElement("digup-results"); + el.results = results; + document.body.append(el); + await el.updateComplete; + return el; +} + +describe("digup-results", () => { + let el; + afterEach(() => el?.remove()); + + it("renders the engine's counter line, the group, the selected row, the fold", async () => { + el = await mount(); + const says = el.shadowRoot.getElementById("engine-says").textContent; + expect(says).toContain("2 hits · 1 strong"); + expect(says).toContain("embed 0.10 s"); + expect(says).toContain("search 0.20 s"); + expect(el.shadowRoot.querySelector(".group-title").textContent).toBe("Images"); + const row = el.shadowRoot.querySelector("li.row"); + expect(row.classList.contains("selected")).toBe(true); + expect(row.getAttribute("aria-selected")).toBe("true"); + expect(row.getAttribute("tabindex")).toBe("0"); + const fold = el.shadowRoot.querySelector("li.weaker"); + expect(fold.textContent).toContain("1 weaker match"); + expect(fold.getAttribute("aria-expanded")).toBe("false"); + expect(fold.getAttribute("tabindex")).toBe("-1"); + }); + + it("the matched words arrive as , the folder as the row's subtitle", async () => { + el = await mount(); + const row = el.shadowRoot.querySelector("li.row"); + expect(row.querySelector("mark").textContent).toBe("dog"); + expect(row.querySelector(".name").textContent).toBe("a.jpg"); + }); + + it("the fold toggles through the event; rows answer click (select) and dblclick/Enter (open)", async () => { + el = await mount(); + const p = onceEvent(document, "digup:toggle-weaker"); + el.shadowRoot.querySelector("li.weaker").click(); + await p; + + const sel = onceEvent(document, "digup:select"); + el.shadowRoot.querySelector("li.row").click(); + expect((await sel).detail.mediaId).toBe("a"); + + const open = onceEvent(document, "digup:open"); + el.shadowRoot.querySelector("li.row").dispatchEvent( + new KeyboardEvent("keydown", { key: "Enter", bubbles: true, composed: true, cancelable: true })); + expect((await open).detail.mediaId).toBe("a"); + }); + + it("an unfolded fold renders the weaker rows with the sentinel selected", async () => { + el = await mount(fixture({ showWeaker: true, selection: WEAKER })); + const rows = el.shadowRoot.querySelectorAll("li.row"); + expect(rows.length).toBe(2); // a + c + const fold = el.shadowRoot.querySelector("li.weaker"); + expect(fold.getAttribute("aria-expanded")).toBe("true"); + expect(fold.getAttribute("aria-selected")).toBe("true"); + expect(fold.classList.contains("selected")).toBe(true); + }); + + it("image thumbnails are the engine's small lazy renders; a broken preview becomes the kind icon", async () => { + el = await mount(); + const img = el.shadowRoot.querySelector("li.row img.thumb"); + expect(img.getAttribute("loading")).toBe("lazy"); + expect(img.getAttribute("src")).toContain("/api/v1/media/a/preview?pixels="); + img.dispatchEvent(new Event("error")); + await el.updateComplete; + expect(el.shadowRoot.querySelector("li.row img.thumb")).toBeNull(); + expect(el.shadowRoot.querySelector("li.row .kind-icon").textContent).toBe("▣"); + }); + + it("an empty meaning answer carries the engine's own sentence verbatim", async () => { + el = await mount(fixture({ phase: "none", strong: [], weaker: [], groups: [], + selection: null, lastReply: { hits: [], strong: 0, meaning: true, ms: {} } })); + expect(el.shadowRoot.getElementById("engine-says").textContent) + .toContain("the engine found no good matches for this and refused to guess — that is its answer"); + }); +}); diff --git a/server/ui/src/components/digup-search.js b/server/ui/src/components/digup-search.js new file mode 100644 index 00000000..3c0f552d --- /dev/null +++ b/server/ui/src/components/digup-search.js @@ -0,0 +1,160 @@ +// digup-search — the search box row (input + limit + code), the kind chips, and the +// ONE search route: every change emits `digup:query-change` with the full filter set +// (text, code, kinds, limit) and the search-controller runs it. The code prefix and +// the box are one state (the controller writes store.isCode; updated() reflects it +// into the box, as app.js did `$("code").checked = code`). In-field keys mirror the +// desktop panel: ↑↓ pick results (digup:nav), ↩ opens the pick (digup:activate), +// Space previews only after navigation (digup:preview). Markup and strings ride the +// vanilla index.html verbatim — test_media_ui greps them from the bundle. +import { LitElement, html, css } from "lit"; +import { controlCss } from "../styles/controls.js"; + +const KINDS = [["screenshot", "Screenshots"], ["image", "Images"], ["pdf", "PDFs"], + ["doc", "Documents"], ["video", "Video"], ["audio", "Audio"]]; + +class DigupSearch extends LitElement { + static properties = { + isCode: {}, + navigated: {}, + hasSelection: {}, + _code: { state: true }, + _limit: { state: true }, + _kinds: { state: true }, + }; + + static styles = [controlCss, css` + :host { display: block; } + /* the search row (round-5 layout brief): the input's center IS the page's center, on + the SAME axis as the chips under it. On wide screens the limit select + code box + leave the flow (absolute, riding the input's right edge) so the 1fr/1fr columns + balance exactly; below 1000 px they drop back into the flow as a centered row. */ + #searchbox { display: grid; grid-template-columns: 1fr auto 1fr; gap: 0.5rem; + align-items: center; margin: 0.9rem auto 0.3rem; max-width: 60rem; position: relative; } + #q { grid-column: 2; width: min(34rem, calc(100vw - 11rem)); padding: 0.55rem 0.7rem; } + #search-side { grid-column: 1 / -1; display: flex; align-items: center; justify-content: center; + gap: 0.5rem; min-width: 0; flex-wrap: wrap; } + @media (min-width: 1000px) { + #search-side { grid-column: 3; position: absolute; right: 0; top: 50%; + transform: translateY(-50%); } + } + #q:focus { outline: 2px solid var(--accent); outline-offset: 1px; } + #code-label { display: inline-flex; align-items: center; gap: 0.35rem; font-size: 0.85rem; + color: var(--soft); white-space: nowrap; } + /* the kind chips: themed per kind, filled when ticked, evenly spaced under the search. */ + #find { display: flex; gap: 0.5rem; margin: 0.5rem 0 0.2rem; justify-content: center; flex-wrap: wrap; } + #find .kind { --kc: var(--soft); display: inline-flex; align-items: center; gap: 0.4rem; + border: 1px solid var(--edge); border-radius: var(--radius-pill); padding: 0.3rem 0.75rem; + font-size: 0.85rem; color: var(--soft); background: var(--panel); cursor: pointer; } + #find .kind:has(input:checked) { color: var(--kc); border-color: var(--kc); + background: color-mix(in srgb, var(--kc) 10%, var(--panel)); } /* 10 %, not 14: the ticked + chip is kind-color text on a kind-tinted panel, and the measured pair (dark audio #c58af9 on + its own tint) is 4.54:1 at 10 % — under 4.5 (AA) at 14 %. Others measure 5.4 and up. */ + #find .kind[data-kind="screenshot"], #find .kind[data-kind="image"] { --kc: var(--kind-image); } + #find .kind[data-kind="video"] { --kc: var(--kind-video); } + #find .kind[data-kind="audio"] { --kc: var(--kind-audio); } + #find .kind[data-kind="pdf"], #find .kind[data-kind="doc"] { --kc: var(--kind-doc); } + #kinds-all { padding: 0.1rem 0.6rem; font-size: 0.78rem; border-radius: var(--radius-pill); } + /* the all button is a TOGGLE (round-7 brief): pressed state while all six are on */ + #kinds-all[aria-pressed="true"] { color: var(--accent); border-color: var(--accent); } + @media (max-width: 720px) { + #searchbox { grid-template-columns: 1fr; } + #q, #search-side { grid-column: 1; } + #q { width: 100%; } + #search-side { justify-content: center; position: static; } + } + `]; + + constructor() { + super(); + this._code = false; + this._limit = 40; + this._kinds = new Set(KINDS.map(([k]) => k)); // all ticked = null = all kinds + this._text = ""; + } + + #emit() { + this.dispatchEvent(new CustomEvent("digup:query-change", { + bubbles: true, composed: true, + detail: { text: this._text, code: this._code, + kinds: KINDS.filter(([k]) => this._kinds.has(k)).map(([k]) => k), + limit: this._limit }, + })); + } + + /** Suggestion chips take the ONE search route: put the phrase in the field and + * run it as a typed query does (the vanilla chip click, verbatim). */ + setValue(phrase) { + this._text = phrase; + const input = this.renderRoot.getElementById("q"); + input.value = phrase; + input.focus(); + this.#emit(); + } + + clearAndRefocus() { // Esc clears the query (vanilla parity) + this._text = ""; + this.renderRoot.getElementById("q").value = ""; + this.renderRoot.getElementById("q").focus(); + this.#emit(); + } + + focusQuery() { this.renderRoot.getElementById("q")?.focus(); } + + updated(changed) { + if (changed.has("isCode")) this._code = this.isCode; // prefix and box are one state + } + + render() { + return html` + + +
    + ${KINDS.map(([kind, label]) => html` + `)} + +
    `; + } + + #fieldKey = (event) => { + // In the field: ↑↓ pick results, ↩ opens the pick, Space types a space (as the desktop + // panel: Space previews only after you've navigated, and only with a selection). + if (event.key === "ArrowDown" || event.key === "ArrowUp") { + event.preventDefault(); + this.dispatchEvent(new CustomEvent("digup:nav", { bubbles: true, composed: true, + detail: { step: event.key === "ArrowDown" ? 1 : -1 } })); + } else if (event.key === "Enter") { + event.preventDefault(); + this.dispatchEvent(new CustomEvent("digup:activate", { bubbles: true, composed: true })); + } else if (event.key === " " && this.navigated && this.hasSelection) { + event.preventDefault(); + this.dispatchEvent(new CustomEvent("digup:preview", { bubbles: true, composed: true })); + } + }; +} +if (!customElements.get("digup-search")) customElements.define("digup-search", DigupSearch); diff --git a/server/ui/src/components/digup-search.test.js b/server/ui/src/components/digup-search.test.js new file mode 100644 index 00000000..e2798079 --- /dev/null +++ b/server/ui/src/components/digup-search.test.js @@ -0,0 +1,107 @@ +// digup-search — the ONE search route: every control change emits `digup:query-change` +// with the full filter set (the search-controller runs it; nothing here fetches). +// The in-field keys mirror the desktop panel: ↑↓ pick, ↩ open, Space preview after +// navigation, code prefix and checkbox one state. +import { describe, expect, it, beforeEach, afterEach, vi } from "vitest"; +import "./digup-search.js"; +import { onceEvent } from "../lib/test-util.js"; + +const key = (target, k) => target.dispatchEvent( + new KeyboardEvent("keydown", { key: k, bubbles: true, composed: true, cancelable: true })); + +async function mount() { + const el = document.createElement("digup-search"); + document.body.append(el); + await el.updateComplete; + return el; +} + +describe("digup-search", () => { + let el; + beforeEach(async () => { el = await mount(); }); + afterEach(() => el.remove()); + + it("typing emits the full filter set with all six kinds by default", async () => { + const p = onceEvent(document, "digup:query-change"); + const q = el.shadowRoot.getElementById("q"); + q.value = "dog"; + q.dispatchEvent(new Event("input")); + const detail = (await p).detail; + expect(detail.text).toBe("dog"); + expect(detail.limit).toBe(40); + expect(detail.code).toBe(false); + expect(detail.kinds).toEqual(["screenshot", "image", "pdf", "doc", "video", "audio"]); + }); + + it("the code checkbox and a kind un-tick ride the same event", async () => { + const p = onceEvent(document, "digup:query-change"); + const code = el.shadowRoot.getElementById("code"); + code.checked = true; + code.dispatchEvent(new Event("change")); + const detail = (await p).detail; + expect(detail.code).toBe(true); + + const p2 = onceEvent(document, "digup:query-change"); + const shot = el.shadowRoot.querySelector("#find .kind[data-kind=screenshot] input"); + shot.checked = false; + shot.dispatchEvent(new Event("change")); + expect((await p2).detail.kinds).toEqual(["image", "pdf", "doc", "video", "audio"]); + }); + + it("the limit select changes the asked-for count", async () => { + const p = onceEvent(document, "digup:query-change"); + const limit = el.shadowRoot.getElementById("limit"); + limit.value = "100"; + limit.dispatchEvent(new Event("change")); + expect((await p).detail.limit).toBe(100); + }); + + it("the all button is a toggle: all six on → click clears all → click restores all", async () => { + const all = el.shadowRoot.getElementById("kinds-all"); + expect(all.getAttribute("aria-pressed")).toBe("true"); // the default: all on + const p1 = onceEvent(document, "digup:query-change"); + all.click(); + expect((await p1).detail.kinds).toEqual([]); // all cleared + expect(all.getAttribute("aria-pressed")).toBe("false"); + expect([...el.shadowRoot.querySelectorAll("#find .kind input")].every((i) => !i.checked)).toBe(true); + const p2 = onceEvent(document, "digup:query-change"); + all.click(); + expect((await p2).detail.kinds).toEqual(["screenshot", "image", "pdf", "doc", "video", "audio"]); + expect(all.getAttribute("aria-pressed")).toBe("true"); + expect([...el.shadowRoot.querySelectorAll("#find .kind input")].every((i) => i.checked)).toBe(true); + }); + + it("suggestion chips take this route: setValue writes the field and emits", async () => { + const p = onceEvent(document, "digup:query-change"); + el.setValue("a campfire"); + const detail = (await p).detail; + expect(detail.text).toBe("a campfire"); + expect(el.shadowRoot.getElementById("q").value).toBe("a campfire"); + }); + + it("in-field keys: ↑↓ nav, ↩ activate, Space previews only after navigation", async () => { + const nav = onceEvent(document, "digup:nav"); + key(el.shadowRoot.getElementById("q"), "ArrowDown"); + expect((await nav).detail.step).toBe(1); + + const act = onceEvent(document, "digup:activate"); + key(el.shadowRoot.getElementById("q"), "Enter"); + await act; + + // Space BEFORE navigation types a space (no preview event). + el.navigated = false; + el.hasSelection = true; + await el.updateComplete; + let previewed = false; + document.addEventListener("digup:preview", () => { previewed = true; }, { once: true, capture: true }); + key(el.shadowRoot.getElementById("q"), " "); + await el.updateComplete; + expect(previewed).toBe(false); + + el.navigated = true; + await el.updateComplete; + const preview = onceEvent(document, "digup:preview"); + key(el.shadowRoot.getElementById("q"), " "); + await preview; + }); +}); diff --git a/server/ui/src/components/digup-settings.js b/server/ui/src/components/digup-settings.js new file mode 100644 index 00000000..2b6d1a99 --- /dev/null +++ b/server/ui/src/components/digup-settings.js @@ -0,0 +1,305 @@ +// digup-settings — the gear panel: the server's own controls, anchored under the gear. +// Four stacked sections (the 2026-11-08 layout brief): APPEARANCE · INDEXING · +// INDEX STATUS · API ACCESS — each with a small icon and a 15–16 px semibold heading, +// 20 px section rhythm on 1 px dividers, 10–12 px radii, muted secondary descriptions, +// restrained blue accents, and ONE distinct card (the index status). The modal's frame — +// its width, its anchor under the gear, the overlay order, the open/close paths — is +// unchanged; only the internal layout is this brief. +// +// All five close paths (gear, ✕, click-outside, Esc, target change) report through +// digup:close so the gear's aria-expanded matches the panel's hidden state everywhere. +// Sections read the store's slices; the controls act through the events below — the +// controllers own what they do. The panel NEVER scrolls (round-7 layout brief): +// overflow-x: clip — a scroller made a clipping box that cut controls at the panel's +// edges, its scrollbar stole ~10 px from the control column, and the panel could +// overflow the viewport. Only the forecast box scrolls, inside its own bounds. +// Every row is the 2 rem row token (round-3); narrow windows fold the grid (§ media). +import { LitElement, html, css } from "lit"; +import { storeController } from "../app/store.js"; +import { duration } from "../lib/format.js"; +import { controlCss, badgeCss, keyRowCss, jobStateCss, spinCss } from "../styles/controls.js"; + +/* Section icons: 15 px, 1.8 stroke, currentColor, aria-hidden (the heading carries the + name). They ride the accent — the panel's one decorative color. */ +const icon = (paths) => html``; +const PALETTE_ICON = icon(html` + + + `); +const DATABASE_ICON = icon(html` + + + `); +const ACTIVITY_ICON = icon(html``); +const KEY_ICON = icon(html` + + `); + +class DigupSettings extends LitElement { + static properties = { theme: {} }; + + static styles = [controlCss, badgeCss, keyRowCss, jobStateCss, spinCss, css` + /* :host is the panel hanging off the header's right edge, directly under the gear + (the gear IS the anchor — header.bar is position:relative). The frame is the + measured one: same width, same anchor, same shadow, overflow-x: clip. */ + :host([hidden]) { display: none; } + :host { position: absolute; right: 1.25rem; top: calc(100% + 0.45rem); z-index: var(--z-panel); + /* right: 1.25rem — the header bar's padding-inline: the panel's edge lines up with + the gear, whose header content edge sits that far in from the bled bar (round-5). */ + width: min(27rem, calc(100vw - 2rem)); + display: flex; flex-direction: column; gap: 0; padding: 0.9rem 0.95rem 1rem; + background: var(--panel); border: 1px solid var(--edge); border-radius: var(--radius-lg); + box-shadow: 0 8px 28px rgb(0 0 0 / 22%); + max-height: calc(100vh - 3rem); overflow-x: clip; } + .set-head { display: flex; justify-content: space-between; align-items: center; + font-size: 0.72rem; text-transform: uppercase; letter-spacing: 0.05em; color: var(--soft); } + #settings-close { padding: 0.1rem 0.55rem; font-size: 0.8rem; line-height: 1.2; } + + /* Section rhythm (the layout brief): 22 px around each 1 px divider (space-md above, + space-md below — the brief's 18–22 px), the heading on its own row. Only INDEX + STATUS is a card — the sections are separated by dividers, not by boxes. */ + .set-sec { display: grid; grid-template-columns: minmax(0, 1fr); gap: var(--space-sm); + padding-top: var(--space-md); margin-top: var(--space-md); + border-top: 1px solid var(--line); } + .set-sec:first-of-type { border-top: 0; padding-top: var(--space-sm); margin-top: var(--space-sm); } + .set-sec > * { min-width: 0; } + .set-title { display: flex; align-items: center; gap: 0.45rem; font-size: 0.95rem; + font-weight: 600; letter-spacing: 0.01em; color: var(--text); } + .set-title svg { width: 15px; height: 15px; flex: none; color: var(--accent); } + .set-line { display: flex; align-items: center; gap: var(--space-md); flex-wrap: wrap; } + select, button, .set-line, .appkey-line { min-height: var(--row-height); box-sizing: border-box; } + .set-line label { display: inline-flex; align-items: center; gap: 0.4rem; + min-height: var(--row-height); color: var(--soft); font-size: var(--text-sm); } + /* The Appearance select IS a control of its section: full width, its edges matching + the rows under it (the native popup opens in the top layer, never clipped). */ + select#theme { width: 100%; max-width: 100%; } + + /* The INDEXING heading carries the live state (round-6 brief): the spinning circle + while the engine works, ✓ when it finished, the brief update at the heading's + right edge. The job's own words ride the jobStateCss pill tones. */ + .set-head-row { display: flex; align-items: center; gap: 0.5rem; + justify-content: space-between; flex-wrap: wrap; } + .set-state { display: inline-flex; align-items: center; gap: 0.45rem; min-width: 0; } + .set-state #job { font-size: var(--text-sm); overflow-wrap: anywhere; } + .state-done { color: var(--good); font-weight: 600; } + .state-err { color: var(--warn); font-weight: 600; } + + /* INDEX STATUS — the one card: the two counts side by side, the capability badges + wrapping naturally below. Semantic tones: healthy green, inactive muted, problem + red. (The roots line left the card, round-5 brief: the counts carry the state.) */ + .status-card { display: grid; gap: var(--space-sm); padding: 0.7rem 0.8rem; + border: 1px solid var(--edge); border-radius: var(--radius-lg); background: var(--bg); } + .stats { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: var(--space-sm); } + .stat { display: grid; gap: 0.15rem; padding: 0.4rem 0.6rem; + border: 1px solid var(--line); border-radius: var(--radius-lg); background: var(--panel); } + .stat-label { color: var(--soft); font-size: var(--text-xs); } + .stat-value { font-size: 1.05rem; font-weight: 600; color: var(--text); + overflow-wrap: anywhere; } + .set-facts { display: flex; gap: 0.35rem; flex-wrap: wrap; align-items: center; + min-height: var(--row-height); box-sizing: border-box; } + .set-facts .badge { margin-right: 0; } + .badge.ok { color: var(--good); border-color: color-mix(in srgb, var(--good) 55%, var(--edge)); } + .badge.amber { color: var(--amber); border-color: color-mix(in srgb, var(--amber) 55%, var(--edge)); } + .badge.err { color: var(--warn); border-color: color-mix(in srgb, var(--warn) 55%, var(--edge)); } + .badge.off { color: var(--soft); } + + /* The panel's boxes ride the 12 px radius (the layout brief's 10–12 px, one value + everywhere): the key rows + key field get it scoped HERE — the shared keyRowCss + keeps the embed widget's measured rows untouched. The field nests at 8 px + (outer radius minus the row's padding). */ + .appkey-line { border-radius: var(--radius-lg); } + .appkey-field { border-radius: var(--radius-md); } + + @media (max-width: 720px) { + /* Narrow windows (round-7 layout brief, kept by the round-6 redesign): the panel + owns the width, the two counts stack, the heading state wraps under the title, + nothing is squeezed and the panel never scrolls. */ + :host { width: calc(100vw - 2rem); } + .stats { grid-template-columns: minmax(0, 1fr); } + } + `]; + + constructor() { + super(); + this.theme = "system"; + this._sel = null; + } + + // The store slice the panel reads (keys, status facts, roots, job line, estimate + // rows) — one subscription through storeController once `store` is set. + set store(store) { + if (store === this._store) return; + this._store = store; + this._sel = storeController(this, store, (s) => [s.keys, s.status, s.job, s.estimate]); + } + get store() { return this._store; } + + render() { + const [keysIn, status, job, estimateRow] = + this._sel ? this._sel.value() : [null, null, null, null]; + const keys = keysIn || { admin: null, apps: [], envFlagged: false, minted: false }; + const e = estimateRow || { spinning: false, rows: null, error: null }; + const jobRun = job && (job.state === "queued" || job.state === "running"); + const estRun = Boolean(e.spinning) && !jobRun; + let state = null; + if (jobRun) state = { spin: true, text: job.text, cls: `job-state job-${job.state}` }; + else if (estRun) state = { spin: true, text: "estimating…", cls: "" }; + else if (job) state = job.state === "done" + ? { done: true, text: job.text, cls: "job-state job-done" } + : { err: true, text: job.text, cls: `job-state job-${job.state}` }; + else if (e.rows) state = { done: true, cls: "", + text: `estimate ≈ ${duration(e.rows.reduce((n, f) => n + (f.seconds || 0), 0))}` + + ` · ${e.rows.length} folder${e.rows.length === 1 ? "" : "s"}` }; + else if (e.error) state = { err: true, cls: "", text: "estimate failed", title: e.error }; + return html` +
    + Settings + +
    + +
    + ${PALETTE_ICON} Appearance + +
    + +
    + +
    + ${DATABASE_ICON} Indexing + ${state ? html` + ${state.spin ? html`` : null} + ${state.done ? html`` : null} + ${state.err ? html`` : null} + ${state.text}` : null} +
    +
    + + + +
    +
    + +
    + ${ACTIVITY_ICON} Index status +
    +
    +
    + Total files + ${status && status.files != null ? status.files : "—"} +
    +
    + Searchable files + ${status && status.searchable != null ? status.searchable : "—"} +
    +
    + + ${status ? html` + ${status.vectors} vectors + encoder ${status.encoder} + ${(status.kinds || []).map((k) => html`${k.kind} ${k.files}`)} + index ${status.index_size} + llama.cpp ${status.build_tag}` : null} + +
    +
    + +
    + ${KEY_ICON} API access + +
    + ${keys.admin ? html` +
    + ev.target.select()}> + + +
    ` : null} + ${keys.apps.map((a) => html` +
    + ${a.name} + ev.target.select()}> + +
    `)} + ${!keys.admin && keys.envFlagged ? html` +
    + Set in the server's environment +
    ` : null} +
    +
    `; + } + + /* Copy (round-5 brief): the panel is reachable over http on the LAN — an insecure + context, where navigator.clipboard does not exist at all. The async clipboard is + used when present, a hidden textarea + execCommand("copy") when not; the button + answers with ✓ (it worked) or — (it did not) for a moment, then Lit restores ⧉. */ + async #copy(text, btn) { + let ok = false; + try { await navigator.clipboard.writeText(text); ok = true; } + catch { + const ta = document.createElement("textarea"); + ta.value = text; ta.setAttribute("readonly", ""); + ta.style.cssText = "position:fixed;top:-3rem;opacity:0;"; + document.body.append(ta); + ta.select(); ta.setSelectionRange(0, text.length); + try { ok = document.execCommand("copy"); } catch { ok = false; } + ta.remove(); + } + btn.textContent = ok ? "✓" : "—"; + setTimeout(() => { if (btn.isConnected) btn.textContent = "⧉"; }, 1500); + } + + #hasKeyRows(keys) { + // A section with nothing in it collapses: the tokenless server mints no keys + // (they appear when an application registers) — no title-only fragment (2026-11-08 + // styling ask, round 3). Regenerate exists only where keys are minted; an + // env-keyed server answers 400 — its keys live in its own config. + return Boolean(keys.admin) || keys.apps.length > 0 || (!keys.admin && keys.envFlagged); + } + + #close = () => this.dispatchEvent(new CustomEvent("digup:close", + { bubbles: true, composed: true })); + + // Keyboard users: the panel focuses its first control when it opens (the vanilla + // gear-open rule), and the Esc chain's "close the forecast" path lives here. + focusFirst() { + const first = this.renderRoot.querySelector("select, button, input"); + first?.focus(); + } +} +if (!customElements.get("digup-settings")) customElements.define("digup-settings", DigupSettings); diff --git a/server/ui/src/components/digup-settings.test.js b/server/ui/src/components/digup-settings.test.js new file mode 100644 index 00000000..404705db --- /dev/null +++ b/server/ui/src/components/digup-settings.test.js @@ -0,0 +1,132 @@ +// digup-settings — the gear panel: the four sections (Appearance · Indexing · Index +// status · API access), the store slice it reads (keys/status/job/estimate), the status +// card (the two count tiles + wrapping badges), the key rows (the key in a field, ⧉ copy +// and ↻ regenerate as icon buttons at its right), the collapse of an empty section, and +// the control events. The roots line left the card (round-5 brief); the target rows left +// in round 3 — the counts ride the tiles from /api/v1/status. +import { describe, expect, it, beforeEach, afterEach, vi } from "vitest"; +import "./digup-settings.js"; +import { createStore } from "../app/store.js"; +import { onceEvent } from "../lib/test-util.js"; + +const state = { + keys: { admin: "adm1n-key", apps: [{ name: "nextcloud", token: "app-key-1" }], + envFlagged: false, minted: true }, + targets: [{ name: "media", main: true, files: 10, searchable: 9, roots: ["/media/photos"] }], + status: { files: 10, searchable: 9, vectors: 20, encoder: "gemma-2", + kinds: [{ kind: "image", files: 5 }], index_size: "12 MB", build_tag: "b11461" }, + roots: ["/media/photos"], + job: { state: "running", text: "searchable 3/10" }, + estimate: { on: true, spinning: true, rows: null, error: null }, +}; + +async function mount(over = {}) { + const store = createStore({ ...state, ...over }); + const el = document.createElement("digup-settings"); + el.store = store; + el.theme = "system"; + document.body.append(el); + await el.updateComplete; + return el; +} + +describe("digup-settings", () => { + let el; + afterEach(() => el?.remove()); + + it("renders the minted key rows: key in a field, ⧉ Copy, ↻ Regenerate at the admin row", async () => { + el = await mount(); + const rows = el.shadowRoot.querySelectorAll("#appkeys .appkey-line"); + expect(rows.length).toBe(2); + expect(rows[0].querySelector(".appkey-field").value).toBe("adm1n-key"); // the key is in the box + const buttons = [...rows[0].querySelectorAll("button")].map((b) => b.textContent.trim()); + expect(buttons).toEqual(["⧉", "↻"]); // icon-only, 2rem squares + expect([...rows[0].querySelectorAll("button")].map((b) => b.getAttribute("aria-label"))) + .toEqual(["Copy this API key to the clipboard", + "Regenerate: new keys now, the old ones stop working"]); + expect(rows[1].querySelector(".appkey-name").textContent).toContain("nextcloud"); + expect(rows[1].querySelector(".appkey-field").value).toBe("app-key-1"); + }); + + it("an env-keyed deployment shows its honest line; a tokenless one collapses the section", async () => { + el = await mount({ keys: { admin: null, apps: [], envFlagged: true, minted: false } }); + expect(el.shadowRoot.getElementById("api-key-sec").hasAttribute("hidden")).toBe(false); + expect(el.shadowRoot.getElementById("api-key-sec").textContent) + .toContain("Set in the server's environment"); + el.remove(); + el = await mount({ keys: { admin: null, apps: [], envFlagged: false, minted: false } }); + expect(el.shadowRoot.getElementById("api-key-sec").hasAttribute("hidden")).toBe(true); + }); + + it("the panel's controls speak through events: theme, estimate toggle, Estimate, Regenerate, index", async () => { + el = await mount(); + const theme = onceEvent(document, "digup:theme"); + const select = el.shadowRoot.getElementById("theme"); + select.value = "dark"; + select.dispatchEvent(new Event("change")); + expect((await theme).detail.choice).toBe("dark"); + + const est = onceEvent(document, "digup:estimate"); + el.shadowRoot.getElementById("estimate-btn").click(); + await est; + + const regen = onceEvent(document, "digup:regenerate"); + el.shadowRoot.getElementById("keys-regenerate").click(); + await regen; + + const idx = onceEvent(document, "digup:index"); + el.shadowRoot.getElementById("index-btn").click(); + expect((await idx).detail.prune).toBe(false); + + const close = onceEvent(document, "digup:close"); + el.shadowRoot.getElementById("settings-close").click(); + await close; + }); + + it("the store slice it reads: the job line, the engine's fact badges, the roots, the spin", async () => { + el = await mount(); + expect(el.shadowRoot.getElementById("job").textContent).toContain("searchable 3/10"); + expect(el.shadowRoot.getElementById("job").classList.contains("job-running")).toBe(true); + const facts = el.shadowRoot.querySelector(".set-facts").textContent; + expect(facts).toContain("20 vectors"); + expect(facts).toContain("image 5"); + expect(facts).toContain("index 12 MB"); + expect(facts).toContain("llama.cpp b11461"); + const rootsLine = el.shadowRoot.getElementById("rootsline"); + expect(rootsLine).toBeNull(); // round-5: the line is gone + const stats = [...el.shadowRoot.querySelectorAll(".stat")].map((s) => s.textContent.replace(/\s+/g, " ").trim()); + expect(stats[0]).toContain("Total files 10"); // the engine's counts, side by side + expect(stats[1]).toContain("Searchable files 9"); + expect(el.shadowRoot.getElementById("targets")).toBeNull(); // the rows are gone + expect(el.shadowRoot.getElementById("estimate-spin")).not.toBeNull(); + }); + + it("the copy button copies: clipboard when present, execCommand fallback on http", async () => { + el = await mount(); + const btn = el.shadowRoot.querySelectorAll("#appkeys .key-icon")[0]; + // jsdom ships no navigator.clipboard — the exact insecure-context (http LAN) shape: + // the click must fall through to the textarea + execCommand path and answer ✓. + const exec = vi.fn(() => true); + el.ownerDocument.execCommand = exec; + btn.click(); + expect(exec).toHaveBeenCalledWith("copy"); + expect(btn.textContent).toBe("✓"); + }); + + it("the heading's brief update: estimate summary + checkmark, the job's running circle, the honest failure", async () => { + el = await mount({ job: null, estimate: { on: true, spinning: false, + rows: [{ folder: "/photos", readable: true, seconds: 12, files: { image: 3 }, + bytes: 2e6, ms: 20, subfolders: [], calibrated: true }], error: null } }); + const job = el.shadowRoot.getElementById("job"); + expect(job.textContent).toContain("estimate ≈ 12 s · 1 folder"); // the engine's seconds, brief + expect(el.shadowRoot.querySelector(".state-done")).not.toBeNull(); + expect(el.shadowRoot.querySelector(".state-err")).toBeNull(); + el.remove(); + el = await mount({ job: null, estimate: { on: true, spinning: false, rows: null, + error: "estimate failed: walk timed out" } }); + expect(el.shadowRoot.getElementById("job").textContent).toBe("estimate failed"); + expect(el.shadowRoot.querySelector(".state-err").getAttribute("title")) + .toBe("estimate failed: walk timed out"); + expect(el.shadowRoot.querySelector(".state-done")).toBeNull(); + }); +}); diff --git a/server/ui/src/components/digup-suggest.js b/server/ui/src/components/digup-suggest.js new file mode 100644 index 00000000..de71d67e --- /dev/null +++ b/server/ui/src/components/digup-suggest.js @@ -0,0 +1,25 @@ +// digup-suggest — the idle suggestions: a click puts the phrase in the field and +// searches it — the exact path a typed query takes (the ONE search route via +// digup:set-query → the shell → digup-search.setValue; nothing here duplicates it). +// These are examples of *how to describe* a file, not promises about a library: they +// describe scenes, which is what the engine's meaning search answers. +import { LitElement, html, css } from "lit"; + +class DigupSuggest extends LitElement { + static properties = { phrases: {} }; + + static styles = css` + :host { display: flex; gap: 0.5rem; flex-wrap: wrap; justify-content: center; margin-top: 1rem; } + button { border: 1px solid var(--edge); border-radius: var(--radius-pill); + padding: 0.3rem 0.8rem; font: inherit; font-size: 0.85rem; + background: var(--field); color: var(--text); cursor: pointer; } + button:hover { border-color: var(--accent); } + `; + + render() { + return (this.phrases || []).map((phrase) => html` + `); + } +} +if (!customElements.get("digup-suggest")) customElements.define("digup-suggest", DigupSuggest); diff --git a/server/ui/src/embed.js b/server/ui/src/embed.js new file mode 100644 index 00000000..b6e68232 --- /dev/null +++ b/server/ui/src/embed.js @@ -0,0 +1,26 @@ +// embed.js — the embed widget's entry (embed.html; stage-12-integrations.md §11.4): +// the host's read token arrives via ?token=, moves to sessionStorage, and the URL line +// is rewritten (history.replaceState) so the secret never lands in bookmarks, history, +// or the host app's logs — the vanilla embed.js boot, carried over verbatim. The +// params (limit, q, theme) hand off to , which owns the widget's loop. +import "./styles/tokens.css"; +import "./styles/embed.css"; +import { createEmbedApi } from "./services/api.js"; +import "./components/digup-embed.js"; + +const params = new URLSearchParams(location.search); +const client = createEmbedApi(); +if (params.get("token")) { + client.adoptUrlToken(params.get("token")); + params.delete("token"); // strip the secret from the visible URL line + const rest = params.toString(); + history.replaceState(null, "", location.pathname + (rest ? "?" + rest : "")); +} + +// embed.html DECLARES the widget; the entry wires that element (a second createElement +// would mount an unwired twin above the working widget — the double page the full UI had +// the same way, measured 2026-11-08). The createElement path is the fallback only. +let widget = document.querySelector("digup-embed"); +if (!widget) { widget = document.createElement("digup-embed"); document.body.append(widget); } +widget.api = client; +widget.params = params; diff --git a/server/ui/src/embed.test.js b/server/ui/src/embed.test.js new file mode 100644 index 00000000..94a191e4 --- /dev/null +++ b/server/ui/src/embed.test.js @@ -0,0 +1,37 @@ +// src/embed.js — the widget's boot as an iframe host lands it: ?token= moves to +// sessionStorage and OUT of the URL line (history.replaceState — the secret never +// lands in bookmarks, history, or the host app's logs), limit/q hand off to the +// widget, and the bearer token rides every call. +import { describe, expect, it, afterEach, vi } from "vitest"; +import { installFetch, mockReply } from "./lib/test-util.js"; + +describe("embed.js entry", () => { + afterEach(() => { + document.querySelector("digup-embed")?.remove(); + vi.useRealTimers(); + }); + + it("adopts ?token=, rewrites the URL line, runs ?q= with the bearer header", async () => { + vi.useFakeTimers(); + sessionStorage.clear(); + history.pushState(null, "", "/embed.html?token=secret123&q=dog&limit=20"); + const calls = installFetch({ + "/api/v1/health": mockReply(200, { live: { searchable: 7 } }), + "POST /api/v1/search": mockReply(200, { hits: [], strong: 0, meaning: true, ms: {} }), + "POST /api/v1/embed_ahead": mockReply(200, {}), + }); + document.body.append(document.createElement("digup-embed")); // embed.html declares it + await import("./embed.js"); + expect(sessionStorage.getItem("digupEmbedToken")).toBe("secret123"); + expect(location.search).toBe("?q=dog&limit=20"); // the secret is GONE from the URL + expect(location.search).not.toContain("secret123"); + const widget = document.querySelector("digup-embed"); + expect(document.querySelectorAll("digup-embed").length).toBe(1); // ONE widget, the wired one + expect(widget).not.toBeNull(); + await widget.updateComplete; + await vi.advanceTimersByTimeAsync(0); + const searchCall = calls.find((c) => c.path === "/api/v1/search"); + expect(JSON.parse(searchCall.options.body)).toMatchObject({ q: "dog", limit: 20, meaning: false }); + expect(searchCall.headers.authorization).toBe("Bearer secret123"); + }); +}); diff --git a/server/ui/src/lib/format.js b/server/ui/src/lib/format.js new file mode 100644 index 00000000..ac88420b --- /dev/null +++ b/server/ui/src/lib/format.js @@ -0,0 +1,40 @@ +// format.js — the vanilla app.js's formatters, carried over verbatim. Every number the +// UI shows is the engine's (counts, seconds, scores); these only format them. +export const seconds = (s) => `${Math.floor(s / 60)}:${String(Math.floor(s % 60)).padStart(2, "0")}`; +export const clock = (s) => (s >= 3600 ? `${Math.floor(s / 3600)}:${String(Math.floor(s / 60 % 60)).padStart(2, "0")}` + + `:${String(Math.floor(s % 60)).padStart(2, "0")}` : seconds(s)); +export const baseName = (path) => path.split("/").pop(); +export const folderOf = (path) => path.slice(0, path.lastIndexOf("/")) || "/"; +export const extension = (path) => (path.split(".").pop() || "").toLowerCase(); +export const bytes = (n) => (n >= 1e6 ? `${(n / 1e6).toFixed(1)} MB` : `${Math.round(n / 1e3)} KB`); +export const duration = (s) => (s >= 3600 ? `${Math.round(s / 3600)} h ${Math.round(s / 60 % 60)} min` + : s >= 60 ? `${Math.round(s / 60)} min ${Math.round(s % 60)} s` : `${Math.round(s)} s`); +export const date = (epoch) => new Date(epoch * 1000).toLocaleDateString(undefined, {year: "numeric", month: "short", day: "numeric"}); + +// The formats browsers stream themselves (H.264/AAC mp4-mov, VP8/VP9 webm, plain audio). +// Everything else keeps the desktop's honest fallback: the picture and the words here, +// the file itself through "Open", which plays it in the user's own player. +export const STREAMABLE_VIDEO = new Set(["mp4", "m4v", "mov", "webm"]); +export const STREAMABLE_AUDIO = new Set(["mp3", "wav", "ogg", "oga", "flac", "m4a", "aac"]); + +export const KIND_TITLE = {screenshot: "Screenshots", image: "Images", pdf: "PDFs", doc: "Documents", + video: "Video", audio: "Audio", code: "Code"}; + +// The selectable row that folds the weaker matches (the sentinel id the arrow keys walk +// onto; the vanilla app.js's string, kept verbatim — tests pin it). +export const WEAKER = "\u0000weaker"; + +export function reasonOf(hit) { + if (hit.keyword_rank !== undefined && hit.keyword_rank !== null && hit.z === undefined) + return `keyword search: the words matched (#${hit.keyword_rank} of the keyword matches)`; + if (hit.keyword_rank !== undefined && hit.keyword_rank !== null) + return `meaning ${hit.z.toFixed(2)}σ · the words matched too (keyword #${hit.keyword_rank})`; + if (hit.z !== undefined) return `meaning match: ${(hit.z).toFixed(2)}σ past the best-mix baseline`; + return "keyword search: the words matched in it"; +} + +// The engine's own sentence for an empty answer, verbatim. One string, both places that +// publish it: the counter line (results) and the centered panel (none). +export const emptyAnswer = (reply) => (reply.meaning + ? "the engine found no good matches for this and refused to guess — that is its answer" + : "no keyword hits (meaning search is warming up)"); diff --git a/server/ui/src/lib/media.js b/server/ui/src/lib/media.js new file mode 100644 index 00000000..bd3e19b1 --- /dev/null +++ b/server/ui/src/lib/media.js @@ -0,0 +1,139 @@ +// media.js — the picture/facts/passage templates the preview pane and the big look +// SHARE (the desktop shows the selection the way panel.js shows it; the server shows +// it the same way, in one place). The vanilla app.js picture()/facts()/textPassage() +// rules carried over verbatim, the honest fallbacks included: a preview the engine has +// not rendered is a message plus the file's metadata, never an empty pane. +import { html, LitElement } from "lit"; +import { baseName, folderOf, extension, bytes, clock, date, STREAMABLE_VIDEO, + STREAMABLE_AUDIO, reasonOf } from "../lib/format.js"; + +const mediaUrls = (hit) => ({ + fileURL: `/api/v1/media/${hit.media_id}/file`, + previewURL: (pixels) => `/api/v1/media/${hit.media_id}/preview?pixels=${pixels}`, +}); + +// A preview that is not there (the engine's render has no JPEG for this segment, the index +// moved, the file left the folders): say so plainly, and keep the file's own metadata and +// its Open/Download — `facts` is a sibling of the picture, so the pane is never blank. +const noPreview = (hit) => html`
    + No preview of this file + The engine has no rendered image for this ${hit.kind} segment. Its details, Open + and Download are below.
    `; + +const cantStream = (suffix) => html`

    This browser can't stream .${suffix} + — ↩ Open plays it in your own player.

    `; + +/** The picture block for a hit. `broken(hit)` reads this render's error state, + * `markBroken(hit)` records an element error (preview render gone, index moved). */ +export function pictureTemplate(hit, { broken, markBroken }) { + const suffix = extension(hit.path); + const { fileURL, previewURL } = mediaUrls(hit); + const onerror = () => markBroken(hit.media_id); + switch (hit.kind) { + case "image": + case "screenshot": { + if (broken(hit)) return noPreview(hit); + return html`
    +
    `; + } + case "pdf": { + const moment = html`
    ${hit.loc + ? `page ${Math.round(hit.loc)}` : "first page"}
    `; + if (broken(hit)) return html`${noPreview(hit)}${moment}`; + return html`
    + ${moment}
    `; // the page the hit was found on + } + case "video": { + const at = hit.loc ?? 0; + const moment = html`
    ▶ at ${clock(at)}
    `; + if (STREAMABLE_VIDEO.has(suffix)) { + return html`
    + ${moment}
    `; + } + if (broken(hit)) return html`${noPreview(hit)}${moment}${cantStream(suffix)}`; + return html`
    + ${moment}${cantStream(suffix)}
    `; + } + case "audio": { + return html`
    + ${STREAMABLE_AUDIO.has(suffix) + ? html`` + : cantStream(suffix)} +
    `; + } + default: { + return html`
    +
    `; + } + } +} + +export function factsTemplate(hit) { + const details = [hit.kind, + hit.size ? bytes(hit.size) : null, + hit.modified ? date(hit.modified) : null, + hit.z !== undefined ? `z ${hit.z.toFixed(2)}` : null, + hit.cosine !== undefined ? `cosine ${hit.cosine.toFixed(3)}` : null, + hit.keyword_rank !== undefined && hit.keyword_rank !== null ? `keyword #${hit.keyword_rank}` : null, + hit.segment !== undefined ? `segment ${hit.segment}` : null].filter(Boolean); + const { fileURL } = mediaUrls(hit); + return html`
    +
    ${baseName(hit.path)}
    +
    ${folderOf(hit.path)}
    +
    ${details.join(" · ")}
    +
    ${reasonOf(hit)}
    +
    + + Open + Download +
    +
    `; +} + +// Transcripts and text (the desktop pulls the passage that matched; same op /text). +// The fetch is token-guarded: a fast selection change must not land an old segment's +// text in the new passage (the vanilla per-render element made this safe; a token makes +// the shared component safe the same way). The call goes through the app's api client +// (the bearer shape stays in services/api.js — main.js calls setPassageApi once). +let _api = null; +export function setPassageApi(apiFn) { _api = apiFn; } + +class DigupPassage extends LitElement { + static properties = { hit: {}, excerptFallback: {}, _text: { state: true } }; + + constructor() { + super(); + this._token = 0; + this._text = null; + } + + updated(changed) { + if (!changed.has("hit")) return; + const token = ++this._token; + this._text = null; + if (!this.hit || !this.hit.media_id) return; + this._text = "reading…"; + const call = _api ?? ((path) => fetch(path, { headers: { "content-type": "application/json" } }) + .then((reply) => reply.ok ? reply.json() : Promise.reject(new Error(`HTTP ${reply.status}`)))); + call(`/api/v1/media/${this.hit.media_id}/text`) + .then((reply) => { + if (token !== this._token || !this.shadowRoot) return; + this._text = (reply.text || "").slice(0, 4000) || "no text for this segment"; + }) + .catch((error) => { + if (token !== this._token || !this.shadowRoot) return; + this._text = `text unavailable: ${error.message}`; + }); + } + + render() { + if (this.hit && this._text === null) { // doc/code passage: excerpt rides first + return html`
    ${this.excerptFallback || ""}
    `; + } + return html`
    ${this._text ?? (this.excerptFallback || "")}
    `; + } +} +if (!customElements.get("digup-passage")) customElements.define("digup-passage", DigupPassage); diff --git a/server/ui/src/lib/test-util.js b/server/ui/src/lib/test-util.js new file mode 100644 index 00000000..8fc279e8 --- /dev/null +++ b/server/ui/src/lib/test-util.js @@ -0,0 +1,31 @@ +// Unit-test helpers (shared, not a test file): a Response-shaped stub for fetch and +// an event capture for the bubbling `digup:*` CustomEvents. +export function mockReply(status, body) { + return { status, ok: status >= 200 && status < 300, json: async () => body }; +} + +export function installFetch(routes) { + const calls = []; + globalThis.fetch = async (path, options = {}) => { + calls.push({ path, options, headers: options.headers || {} }); + const route = routes[path] ?? routes[`${options.method || "GET"} ${path}`]; + if (typeof route === "function") return route(path, options); + if (route === undefined) throw new Error(`unstubbed ${path}`); + return route; + }; + return calls; +} + +/** Resolve when `type` fires on target (capture:true catches across shadow boundaries). */ +export function onceEvent(target, type) { + return new Promise((resolve) => target.addEventListener(type, resolve, { once: true, capture: true })); +} + +export function memoryStorage(initial = {}) { + const map = new Map(Object.entries(initial)); + return { + getItem: (k) => (map.has(k) ? map.get(k) : null), + setItem: (k, v) => map.set(k, String(v)), + removeItem: (k) => map.delete(k), + }; +} diff --git a/server/ui/src/main.js b/server/ui/src/main.js new file mode 100644 index 00000000..5152c12c --- /dev/null +++ b/server/ui/src/main.js @@ -0,0 +1,63 @@ +// main.js — the full UI's entry (server/ui/AGENTS.md §State): composition only. One +// store, the theme applied before the first paint (no flash), the api client wired +// once, the four controllers taking the seams they own, and composes them. +// The vanilla startup lines this mirrors (app.js's tail): `render(); refreshStatus(); +// setInterval(refreshStatus, 15000); $("q").focus()` — the shell's firstUpdated runs +// exactly those (the first paint is a state too, the 15 s status poll, the focus). +import "./styles/tokens.css"; +import "./styles/global.css"; +import { createStore } from "./app/store.js"; +import { createTheme } from "./app/theme.js"; +import { createApiClient } from "./services/api.js"; +import { createSearchController } from "./app/search-controller.js"; +import { createJobController } from "./app/job-controller.js"; +import { createEstimateController } from "./app/estimate-controller.js"; +import { createKeysController } from "./app/keys-controller.js"; +import { setPassageApi } from "./lib/media.js"; +import "./components/digup-app.js"; + +const store = createStore({ + // The initial slice is the idle page: phase "empty" → data-page "idle", the centered + // welcome, no list, no preview. (The vanilla state's shape, carried over.) + results: { phase: "empty", strong: [], weaker: [], groups: [], hitsById: new Map(), + showWeaker: false, selection: null, lastReply: null }, + filters: { kinds: [], limit: 40 }, + isCode: false, + text: "", + navigated: false, + error: null, + job: null, + indexing: null, + encoder: null, + status: null, + roots: [], + targets: [], + keys: null, +}); + +const theme = createTheme(); // apply at boot: the theme must not flash +// The shell element is DECLARED by index.html; the entry wires THAT element. Creating a +// second one mounts a second app — the unwired twin renders its bare template above the +// working page (measured 2026-11-08: a double DigUp header). The createElement path is +// only the fallback (a host that injects the bundle without the shell markup). +let app = document.querySelector("digup-app"); +if (!app) { app = document.createElement("digup-app"); document.body.append(app); } +const client = createApiClient({ onNoToken: () => app.noteMissingKey() }); +app.store = store; +app.api = client.api; +app.theme = theme; + +const search = createSearchController({ api: client.api, store }); +const keys = createKeysController({ store, api: client.api, storage: localStorage }); +const jobs = createJobController({ api: client.api, store, + refreshStatus: () => app.refreshStatus(), loadAccess: () => keys.loadAccess() }); +const estimate = createEstimateController({ api: client.api, store, storage: localStorage }); +app.controllers = { search, jobs, estimate, keys }; + +// Transcripts ride the api client too: the bearer header belongs to services/api.js +// (the vanilla passage fetch had none — with a guarded deployment's key it works now). +setPassageApi(client.api); + +// The declared element is already in the document: wiring the properties above is the +// boot (Lit schedules the first render on the microtask — the shell paints itself idle, +// focused, and starts the 15 s status poll in its firstUpdated). diff --git a/server/ui/src/main.test.js b/server/ui/src/main.test.js new file mode 100644 index 00000000..5d02446b --- /dev/null +++ b/server/ui/src/main.test.js @@ -0,0 +1,37 @@ +// src/main.js — the full UI's boot as served: index.html DECLARES and the +// entry wires that element (store, theme, api, controllers); the first paint is idle +// (body[data-page]), the status poll starts. This is the shape FastAPI's static mount +// hands the browser at `/`. +import { describe, expect, it, afterEach, vi } from "vitest"; +import { installFetch, mockReply } from "./lib/test-util.js"; + +describe("main.js entry", () => { + afterEach(() => { + document.querySelector("digup-app")?.remove(); + vi.useRealTimers(); + }); + + it("boots the app: theme applied, idle first paint, status polling", async () => { + vi.useFakeTimers(); + const calls = installFetch({ + "/api/v1/status": mockReply(200, { encoder: "gemma-2", vectors: 3, kinds: [], + index_size: "1 MB", build_tag: "b1", roots: ["/m"] }), + }); + localStorage.clear(); + // The served shape: index.html DECLARES , the entry wires THAT element. + // The double-mount bug (measured 2026-11-08 in a browser: a DigUp header above a + // working DigUp page) is exactly a second, unwired instance — pinned here. + document.body.append(document.createElement("digup-app")); + await import("./main.js"); + const app = document.querySelector("digup-app"); + expect(document.querySelectorAll("digup-app").length).toBe(1); // ONE app, the wired one + expect(app).not.toBeNull(); + expect(document.documentElement.dataset.theme).toBe("system"); + expect(localStorage.getItem("digup-theme")).toBe("system"); + await app.updateComplete; + await vi.advanceTimersByTimeAsync(0); + expect(document.body.dataset.page).toBe("idle"); + expect(app.dataset.page).toBe("idle"); + expect(calls.map((c) => c.path)).toContain("/api/v1/status"); + }); +}); diff --git a/server/ui/src/services/api.js b/server/ui/src/services/api.js new file mode 100644 index 00000000..62208001 --- /dev/null +++ b/server/ui/src/services/api.js @@ -0,0 +1,109 @@ +// api.js — THE only place the frontend talks HTTP (server/ui/AGENTS.md §API). The +// vanilla app.js/embed.js rules, carried over verbatim; test_media_ui and +// test_integrations pin the header shape, the 401 shapes, the detail-list rendering, +// and the endpoint literals. Components never fetch; controllers call these. +// +// Bearer token, when this deployment asks for one. The shapes (docs/API.md +// "Authentication"): a TOKENLESS server asks for nothing (the main page searches); +// a MINTED-KEY server (apps registered) answers the main index's reads openly and +// guards index/estimate/keys with the minted admin key — the gear's API key rows are +// those keys, and Regenerate hands this browser its new admin key (localStorage, sent +// on every call, never in a URL); a server whose keys the OPERATOR set in the +// environment guards every call — its keys are the deployment's config, shown by +// nothing, so the gear says exactly that. A 401 with a stored key is the server's own +// words ("the bearer token is not this server's token"), shown, not re-prompted — a +// wrong key cannot loop the page. + +const TOKEN_KEY = "digupBearer"; // the full UI keeps its key in localStorage + +/** The full UI's client. getToken/onNoToken are injectable for tests; production uses + * localStorage and the gear-pointing callback (a 401 with no stored key names the + * server's shape in the panel, it never loops a prompt). */ +export function createApiClient({ + getToken = () => localStorage.getItem(TOKEN_KEY), + onNoToken = () => {}, + storage = { + get: (k) => localStorage.getItem(k), + set: (k, v) => localStorage.setItem(k, v), + remove: (k) => localStorage.removeItem(k), + }, +} = {}) { + async function api(path, options = {}, retried = false) { + const token = getToken(); + const headers = {"content-type": "application/json", + ...(token ? {authorization: `Bearer ${token}`} : {}), + ...(options.headers || {})}; + const reply = await fetch(path, {...options, headers}); + if (reply.status === 401 && !token) { + onNoToken(); // the gear names the server's shape + throw Object.assign(new Error( + "This server guards this call with a key the operator set in its environment — " + + "a minted-key server lists its keys under Settings → API key, and a tokenless " + + "server asks for nothing."), + {status: 401}); + } + const body = await reply.json().catch(() => ({})); + if (!reply.ok) { + // FastAPI validation answers a LIST of dicts in `detail` (`new Error(list)` is + // "[object Object]" on the footer): render each {loc, msg}, quote any string, fall + // back to statusText for a non-JSON body (502 from a proxy, an HTML error page). + const d = body.detail; + const message = Array.isArray(d) + ? d.map((x) => (x.loc || []).filter((p) => p !== "body").join(".") + " " + x.msg).join("; ") + : (d || reply.statusText || `HTTP ${reply.status}`); + throw Object.assign(new Error(message), {status: reply.status}); + } + return body; + } + return {api, TOKEN_KEY, storage}; +} + +/** The embed widget's client (embed.html, stage-12): the host's read token arrives via + * ?token=, moves to sessionStorage, and the URL line is rewritten so the secret never + * lands in bookmarks, history, or the host app's logs. A 401 with a stored token + * clears it and asks ONCE — the same rule as the full UI. */ +export function createEmbedApi({ + sessionStorageRef = sessionStorage, + prompt = (msg) => window.prompt(msg), + tokenKey = "digupEmbedToken", +} = {}) { + let prompted = false; + let token = sessionStorageRef.getItem(tokenKey) || ""; + + function adoptUrlToken(value) { // ?token= → storage → strip the URL line + if (!value) return token; + token = value; + sessionStorageRef.setItem(tokenKey, token); + return token; + } + + async function api(path, options = {}) { + const headers = { "content-type": "application/json", ...(options.headers || {}) }; + if (token) headers.authorization = "Bearer " + token; + const answer = await fetch(path, { ...options, headers }); + if (answer.status === 401) { + if (token && !prompted) { // the stored token is stale: clear it and ask once + prompted = true; + sessionStorageRef.removeItem(tokenKey); + const asked = prompt("This DigUp server asks for its read token:"); + token = (asked || "").trim(); + if (token) sessionStorageRef.setItem(tokenKey, token); + return api(path, options); + } + throw new Error("the bearer token was refused"); + } + if (!answer.ok) throw new Error("the server answered " + answer.status); + return answer.json(); + } + + async function blobOf(path) { + const headers = {}; + if (token) headers.authorization = "Bearer " + token; + const answer = await fetch(path, { headers }); + if (!answer.ok) throw new Error("the server answered " + answer.status); + return answer; + } + + return {api, blobOf, adoptUrlToken, get token() { return token; }, + set token(v) { token = v; }, tokenKey}; +} diff --git a/server/ui/src/services/api.test.js b/server/ui/src/services/api.test.js new file mode 100644 index 00000000..42e5f904 --- /dev/null +++ b/server/ui/src/services/api.test.js @@ -0,0 +1,81 @@ +// services/api.js — the pinned header and 401 shapes, the 422 detail-list rendering, +// the widget's token adoption and ask-once recovery (test_media_ui / test_integrations +// pin these rules on the served bundle; these are their behavior tests). +import { describe, expect, it, beforeEach } from "vitest"; +import { createApiClient, createEmbedApi } from "./api.js"; +import { installFetch, mockReply, memoryStorage } from "../lib/test-util.js"; + +describe("createApiClient", () => { + let storage, calls; + beforeEach(() => { + storage = memoryStorage(); + }); + + it("sends the bearer header only when a key is stored, as `Bearer `", async () => { + storage.setItem("digupBearer", "adm1n"); + const { api } = createApiClient({ getToken: () => storage.getItem("digupBearer"), storage }); + calls = installFetch({ "POST /api/v1/index": mockReply(200, { job: { id: "a" } }) }); + await api("/api/v1/index", { method: "POST" }); + expect(calls[0].headers.authorization).toBe("Bearer adm1n"); + expect(calls[0].headers["content-type"]).toBe("application/json"); + }); + + it("a 401 with no stored key: names the server's shape once and points at the gear", async () => { + let pointed = 0; + const { api } = createApiClient({ getToken: () => null, onNoToken: () => { pointed++; } }); + installFetch({ "/api/v1/status": mockReply(401, {}) }); + await expect(api("/api/v1/status")).rejects.toThrow(/operator set in its environment/); + expect(pointed).toBe(1); + }); + + it("FastAPI's 422 detail LIST renders as loc+msg pairs (never [object Object])", async () => { + const { api } = createApiClient({ getToken: () => null }); + installFetch({ "/api/v1/search": mockReply(422, { detail: [ + { loc: ["body", "limit"], msg: "limit > 100" }, + { loc: ["body", "q"], msg: "too short" }, + ] }) }); + await expect(api("/api/v1/search")).rejects.toThrow("limit limit > 100; q too short"); + }); + + it("a string detail and a non-JSON body fall back honestly", async () => { + const { api } = createApiClient({ getToken: () => null }); + installFetch({ "/a": mockReply(502, { detail: "walk timed out" }), + "/b": { status: 500, ok: false, json: async () => { throw new Error("no json"); }, + statusText: "Server Error" } }); + await expect(api("/a")).rejects.toThrow("walk timed out"); + await expect(api("/b")).rejects.toThrow("Server Error"); + }); +}); + +describe("createEmbedApi", () => { + it("?token= adoption stores the secret in sessionStorage (the host's secret shape)", async () => { + const store = memoryStorage(); + const client = createEmbedApi({ sessionStorageRef: store, prompt: () => "" }); + client.adoptUrlToken("readTok"); + expect(store.getItem("digupEmbedToken")).toBe("readTok"); + const calls = installFetch({ "/api/v1/search": mockReply(200, { hits: [] }) }); + await client.api("/api/v1/search"); + expect(calls[0].headers.authorization).toBe("Bearer readTok"); + }); + + it("a 401 with a stored token: clears it, asks ONCE, retries with the answer", async () => { + const store = memoryStorage({ digupEmbedToken: "stale" }); + let asks = 0; + const client = createEmbedApi({ sessionStorageRef: store, + prompt: () => { asks++; return "fresh"; } }); + const calls = installFetch({ "/api/v1/search": async (path, options) => + (options.headers.authorization === "Bearer fresh") + ? mockReply(200, { hits: [] }) : mockReply(401, {}) }); + await client.api("/api/v1/search", { headers: {} }); + expect(asks).toBe(1); + expect(client.token).toBe("fresh"); + expect(store.getItem("digupEmbedToken")).toBe("fresh"); + }); + + it("a second 401 (the asked token refused too) says so — it never loops", async () => { + const store = memoryStorage({ digupEmbedToken: "stale" }); + const client = createEmbedApi({ sessionStorageRef: store, prompt: () => "nope" }); + installFetch({ "/api/v1/health": mockReply(401, {}) }); + await expect(client.api("/api/v1/health")).rejects.toThrow("the bearer token was refused"); + }); +}); diff --git a/server/ui/src/styles/controls.js b/server/ui/src/styles/controls.js new file mode 100644 index 00000000..7d1d67ad --- /dev/null +++ b/server/ui/src/styles/controls.js @@ -0,0 +1,113 @@ +// controls.js — the shared Lit `css` chunks every component imports (server/ui/AGENTS.md +// §Tokens: shared primitives live here; components import, never retype). The rules are +// the vanilla style.css carried over verbatim — same padding, borders, radii, the 2rem +// gear row system — so the built page renders pixel-identical. +import { css } from "lit"; + +export const controlCss = css` + select, input[type="search"] { + font: inherit; color: var(--text); + border: 1px solid var(--edge); border-radius: var(--radius-md); + } + select { /* own arrow, evenly spaced from the edge — not glued to the button */ + appearance: none; -webkit-appearance: none; + background: var(--field) var(--chev) right 0.6rem center / 0.85rem 0.85rem no-repeat; + padding: 0.42rem 1.9rem 0.42rem 0.6rem; + } + input[type="search"] { background: var(--field); } + input[type="checkbox"] { accent-color: var(--accent); } + button, a.button-like { + font: inherit; padding: 0.42rem 0.8rem; border: 1px solid var(--edge); border-radius: var(--radius-md); + background: var(--field); color: var(--text); cursor: pointer; text-decoration: none; + display: inline-block; + } + button:hover, a.button-like:hover { border-color: var(--accent); } +`; + +export const badgeCss = css` + .badge { + display: inline-block; padding: 0.05rem 0.45rem; margin-right: 0.3rem; + border: 1px solid var(--line); border-radius: var(--radius-pill); + font-size: var(--text-badge); color: var(--soft); background: var(--bg); + } + /* The estimate's "can't read it" badge: the .warn class (defined above .badge) and .badge have + the same specificity, so the later .badge won the cascade and the warning rendered in + muted grey — identical to the decorative badges beside it. Two classes win on one. */ + .badge.warn { color: var(--warn); border-color: var(--warn); } + .kind-image, .kind-screenshot { border-color: var(--kind-image); color: var(--kind-image); } + .kind-video { border-color: var(--kind-video); color: var(--kind-video); } + .kind-audio { border-color: var(--kind-audio); color: var(--kind-audio); } + .kind-pdf, .kind-doc, .kind-code { border-color: var(--kind-doc); color: var(--kind-doc); } + .kind-icon { min-width: 1.7rem; text-align: center; } +`; + +export const activityCss = css` + .activity { display: inline-flex; align-items: center; gap: 0.35rem; font-size: 0.8rem; color: var(--amber); } + .activity .dot { width: 0.55rem; height: 0.55rem; border-radius: var(--radius-pill); background: var(--amber); + animation: pulse 1.6s ease-in-out infinite; } + @keyframes pulse { 50% { opacity: 0.35; } } +`; + +export const spinCss = css` + /* Estimate runs in the panel: a true circle — a ring with one bright quadrant, + rotating — beside the button is the progress indicator (2026-11-08 UX brief, + round 5: "perfectly a circle", no glyph). */ + .estimate-spin { display: inline-block; box-sizing: border-box; + width: 1.1rem; height: 1.1rem; border-radius: 50%; + border: 2px solid var(--edge); border-top-color: var(--accent); + animation: spin 1.1s linear infinite; } + @keyframes spin { to { transform: rotate(360deg); } } +`; + +export const keyRowCss = css` + /* Gear rows are a grid: label | value | action ( | action) — every row the same height, + the same padding, the same border; the key text clips with an ellipsis (Copy carries + the whole value), so the panel reads as one clean stack of equal lines (2026-11-08 UX + brief). The API key row carries Regenerate at its right edge (round-6 brief: inline, + not a loose button under the section); label-only rows (the index rows) span full. + Round-7 layout fix, the arithmetic: a 9rem ... auto auto grid has a hard minimum of + ~24.6 rem (label + gaps + two padded buttons), ABOVE the ~21 rem the control column + offers — so the row was ~120 px wider than its column and the buttons sat past the + panel's right edge. Every column is minmax()-bounded instead: label and key shrink + (ellipsis), the buttons keep only their word min (min-content), and min-width: 0 + lets each track collapse. The row can never exceed the column it lives in. */ + .appkey-line { display: grid; grid-template-columns: minmax(4rem, 6rem) minmax(0, 1fr) + min-content min-content; gap: 0.4rem; align-items: center; + min-height: var(--row-height); padding: 0.25rem 0.5rem; + border: 1px solid var(--edge); border-radius: var(--radius-sm); background: var(--field); } + .appkey-line > * { min-width: 0; } + .appkey-line > .appkey-name:only-child { grid-column: 1 / -1; } + #keys-regenerate { margin-left: auto; } + /* Icon-only key actions (2026-11-08 UI brief, round 3): clipboard and refresh, square + in the 2rem row system, names carried in aria-label/title (readers and tooltips). */ + .key-icon { width: var(--row-height); padding: 0; text-align: center; + font-size: 1rem; line-height: 1; } + .appkey-name { color: var(--soft); font-size: var(--text-xs); min-width: 0; overflow: hidden; + text-overflow: ellipsis; white-space: nowrap; } + /* The key lives in a real text box (2026-11-08 UI brief): selectable, copyable, and the + clipboard + refresh buttons ride its right. A row with no label (the minted admin key + — the section title IS its label) starts its field at the column's left edge. */ + .appkey-field { font: var(--text-xs) var(--font-mono); color: var(--text); + background: var(--bg); border: 1px solid var(--edge); border-radius: var(--radius-sm); + padding: 0.3rem 0.45rem; width: 100%; min-width: 0; height: var(--row-height); + box-sizing: border-box; } + .appkey-field:focus-visible { outline: 2px solid var(--accent); outline-offset: 1px; } + .appkey-line > .appkey-field:first-child { grid-column: 1 / 3; } + .appkey-token { font: var(--text-xs) var(--font-mono); + background: var(--bg); border: 1px solid var(--edge); border-radius: var(--radius-sm); + padding: 0.2rem 0.45rem; min-width: 0; overflow: hidden; + text-overflow: ellipsis; white-space: nowrap; } + .appkey-line button { white-space: nowrap; } +`; + +export const jobStateCss = css` + .job-state { padding: 0.1rem 0.5rem; border-radius: var(--radius-md); border: 1px solid var(--edge); } + .job-queued, .job-running { color: var(--amber); } + .job-done { color: var(--good); } + .job-interrupted, .job-busy { color: var(--warn); } +`; + +export const noteCss = css` + .note { color: var(--soft); font-size: var(--text-sm); margin: 0.4rem 0 0; } + .warn-text { color: var(--warn); } +`; diff --git a/server/ui/src/styles/embed.css b/server/ui/src/styles/embed.css new file mode 100644 index 00000000..d954521f --- /dev/null +++ b/server/ui/src/styles/embed.css @@ -0,0 +1,9 @@ +/* embed.css — the embed page's document frame only (the stable pinned name). + The widget's whole interior (search bar, chips, tiles) lives inside 's + shadow, styled by the component with the same measured values the vanilla embed.css + had — carried over verbatim into digup-embed.js. What is left here is the frame the + page itself needs: the embed body has no content column (the full UI's 72 rem + measure is the search page's layout, not the widget's). Tokens ride with + embed.js's import (tokens.css merged into this file at build). */ + +.embed { margin: 0; } diff --git a/server/ui/src/styles/global.css b/server/ui/src/styles/global.css new file mode 100644 index 00000000..cdf0d722 --- /dev/null +++ b/server/ui/src/styles/global.css @@ -0,0 +1,23 @@ +/* global.css — the page frame only: body, focus, and the ONE layout switch. + + The vanilla UI laid out its four page states with `body[data-page]` (app.js render() + set it: idle | busy | results | none) and style.css read it — the preview pane never + reserves space for a state with nothing to show (measured: a 574×192 px empty box at + 1280 px). The migration keeps the mechanism exactly: digup-app sets + `document.body.dataset.page` (pinned string, test_media_ui) AND mirrors it on its own + host attribute, because the state panels render inside the component's shadow where + body-attribute selectors cannot reach. Both are set at every render, including the + first: the first paint is a state too. */ + +* { box-sizing: border-box; } + +body { + margin: 0 auto; max-width: 72rem; padding: 0.9rem 1.25rem 2.5rem; overflow-x: clip; + /* overflow-x: clip — the header bar and the footer strip bleed to the screen edges + (margin-inline: calc(50% - 50vw)); vw is measured past a classic scrollbar, and + clip absorbs that sliver — the page never gets a horizontal scrollbar. */ + font: var(--text-ui)/var(--leading-ui) var(--font-ui); + color: var(--text); background: var(--bg); +} + +:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; } diff --git a/server/ui/src/styles/tokens.css b/server/ui/src/styles/tokens.css new file mode 100644 index 00000000..f575f33d --- /dev/null +++ b/server/ui/src/styles/tokens.css @@ -0,0 +1,56 @@ +/* tokens.css — the DigUp design tokens, global on :root (they inherit across every + shadow boundary; components use var(…) and never hardcode a color). + + The palettes are the vanilla UI's measured values, carried over verbatim (the WCAG + pairs: on dark (#202124) text 13.4:1, muted 8.4:1, accent 7.6:1, kinds 6.4:1+; on + light text 17.4:1, muted 5.9:1). Borders stay quiet on purpose (hairlines ~1.8:1). + Each theme sets its own color-scheme so native controls, selects and scrollbars + render IN the theme. Appearance is system / light / dark, kept per browser + (html[data-theme] set by app/theme.js — the desktop follows the OS, a server has + no OS to follow). New tokens go here; component CSS covers layout only. */ + +:root, :root[data-theme="system"], :root[data-theme="light"] { + --bg: #ffffff; --panel: #f0f1f3; --field: #f5f6f7; --text: #1a1a1a; --soft: #5f6368; + --line: #c8c8c8; --edge: #b4b7bb; + --accent: #1a73e8; --good: #2a6a35; --warn: #a33327; --amber: #7a5a1a; + --kind-image: #3b7a45; --kind-video: #8a4a3b; --kind-audio: #6a4a8a; --kind-doc: #3b5a8a; + --chev: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16'%3E%3Cpath d='M3.5 5.5 8 10.5 12.5 5.5' fill='none' stroke='%231a1a1a' stroke-width='2' stroke-linecap='round' stroke-linejoin='round'/%3E%3C/svg%3E"); + color-scheme: light; +} +:root[data-theme="system"] { color-scheme: light dark; } /* "system" lets the OS pick the controls */ + +:root[data-theme="dark"] { + --bg: #202124; --panel: #2b2f34; --field: #2b2f34; --text: #e8eaed; --soft: #b7bbc1; + --line: #454a50; --edge: #5c636a; + --accent: #8ab4f8; --good: #81c995; --warn: #f28b82; --amber: #f9ab00; + --kind-image: #81c995; --kind-video: #fcad7d; --kind-audio: #c58af9; --kind-doc: #8ab4f8; + --chev: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16'%3E%3Cpath d='M3.5 5.5 8 10.5 12.5 5.5' fill='none' stroke='%23e8eaed' stroke-width='2' stroke-linecap='round' stroke-linejoin='round'/%3E%3C/svg%3E"); + color-scheme: dark; +} +@media (prefers-color-scheme: dark) { + :root[data-theme="system"] { + --bg: #202124; --panel: #2b2f34; --field: #2b2f34; --text: #e8eaed; --soft: #b7bbc1; + --line: #454a50; --edge: #5c636a; + --accent: #8ab4f8; --good: #81c995; --warn: #f28b82; --amber: #f9ab00; + --kind-image: #81c995; --kind-video: #fcad7d; --kind-audio: #c58af9; --kind-doc: #8ab4f8; + --chev: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16'%3E%3Cpath d='M3.5 5.5 8 10.5 12.5 5.5' fill='none' stroke='%23e8eaed' stroke-width='2' stroke-linecap='round' stroke-linejoin='round'/%3E%3C/svg%3E"); + color-scheme: dark; + } +} + +:root { + /* Typography — the system stack the desktop UI uses; mono for paths/code/excerpts. */ + --font-ui: -apple-system, "Segoe UI", Roboto, "Helvetica Neue", sans-serif; + --font-mono: ui-monospace, SFMono-Regular, Menlo, monospace; + --text-ui: 15px; /* the body size the measured palettes were picked at */ + --leading-ui: 1.45; + --text-sm: 0.85rem; --text-xs: 0.78rem; --text-badge: 0.74rem; + + /* Spacing ladder, radii, the uniform gear row height, transitions, overlay layers. */ + --space-xs: 0.35rem; --space-sm: 0.5rem; --space-md: 0.7rem; --space-lg: 1.25rem; + --radius-sm: 6px; --radius-md: 8px; --radius-lg: 12px; --radius-pill: 999px; + --row-height: 2rem; /* every gear-panel row (round-3 styling ask) */ + --transition-fast: 0.15s; + --z-panel: 15; /* the settings panel */ + --z-big: 20; /* the big look over everything */ +} diff --git a/server/ui/vite.config.js b/server/ui/vite.config.js new file mode 100644 index 00000000..682346b4 --- /dev/null +++ b/server/ui/vite.config.js @@ -0,0 +1,57 @@ +// The DigUp server UI build (server/ui/AGENTS.md §Build). One rule above all: the built +// page must be served EXACTLY like the vanilla UI was — same URLs (`/`, `/embed.html`), +// same asset names the contract suite pins (`app.js`, `style.css`, `embed.js`, `embed.css`), +// same-origin only, no external URLs (README §10.4, test_media_ui pins). Node exists at +// BUILD time only (the Dockerfile's frontend-builder stage); the runtime image ships the +// static dist and FastAPI serves it. No server, no router library, no framework runtime. +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + root: ".", // server/ui: index.html and embed.html ARE the entries + base: "./", // relative asset URLs: the mount prefix never matters + build: { + outDir: "dist", + emptyOutDir: true, + target: "es2021", // modern browsers only — the server targets them + cssCodeSplit: true, // index graph → style.css, embed graph → embed.css + rollupOptions: { + input: { index: "index.html", embed: "embed.html" }, + output: { + // Stable names, the pinned contract: the entry chunks answer at /app.js and + // /embed.js; Vite names CSS assets after their html entry (index.css / embed.css) + // — map them to the pinned style.css / embed.css. Shared code lands in hashed + // chunks (imported from app.js in the html, so no pin depends on their names). + entryFileNames: ({ name }) => (name === "embed" ? "embed.js" : "app.js"), + chunkFileNames: "chunks/[name]-[hash].js", + assetFileNames: (a) => { + if (a.name === "index.css") return "style.css"; + if (a.name === "embed.css") return "embed.css"; + return "assets/[name][extname]"; + }, + manualChunks: (id) => (id.includes("node_modules") ? "vendor" : undefined), + }, + }, + }, + server: { + // Dev only: `npm run dev` (Vite on :5173) proxies the API to a running server + // (`deploy/` compose → 127.0.0.1:8787). Production needs no server: FastAPI serves dist. + // The default binds loopback (the server's own §7 discipline: nothing listens on a + // LAN address unless the operator says so). LAN dev access is env-forced: + // DIGUP_UI_DEV_HOST=0.0.0.0 DIGUP_UI_DEV_ALLOWED_HOSTS=, + // The proxy runs server-side, so the LAN browser talks only to the dev server, and + // `changeOrigin` hands the API its own loopback Host (the API's Host allowlist stays + // the API's, untouched by this dev shape). + host: process.env.DIGUP_UI_DEV_HOST || "127.0.0.1", + allowedHosts: (process.env.DIGUP_UI_DEV_ALLOWED_HOSTS || "").split(",").filter(Boolean), + // ENOSPC on a box whose inotify limit is reached (measured 2026-11-08: 65 536 watches, + // other work owns them): DIGUP_UI_DEV_POLL=1 switches the dev watcher to polling, + // scoped to the source tree. Hot reload stays live; only the mechanism changes. + watch: process.env.DIGUP_UI_DEV_POLL + ? { usePolling: true, interval: 300, ignored: ["**/node_modules/**", "**/dist/**", "**/data/**"] } + : undefined, + proxy: { "/api": { target: "http://127.0.0.1:8787", changeOrigin: true } }, + }, + test: { + environment: "jsdom", // unit tests: components in jsdom, fake timers, no real engine + }, +}); diff --git a/server/uv.lock b/server/uv.lock new file mode 100644 index 00000000..b7624e5f --- /dev/null +++ b/server/uv.lock @@ -0,0 +1,439 @@ +version = 1 +revision = 3 +requires-python = "==3.12.*" + +[[package]] +name = "annotated-doc" +version = "0.0.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/8e/38aa427ed5402449e226975b649c5dc73ccadfefeb95e6aecb8f8ea4b6b6/annotated_doc-0.0.5.tar.gz", hash = "sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb", size = 10758, upload-time = "2026-07-28T13:50:58.129Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/30/e900b21425a860e195f32e37657aa1f7c7f2b1bfb26f03ca209b90933c06/annotated_doc-0.0.5-py3-none-any.whl", hash = "sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101", size = 5302, upload-time = "2026-07-28T13:50:57.239Z" }, +] + +[[package]] +name = "annotated-types" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/56/a8120250d128bed162cd73c76d45f6ef9991f3e068f62a8ee060afa3104a/annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7", size = 15893, upload-time = "2026-07-23T20:16:13.995Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" }, +] + +[[package]] +name = "anyio" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a1/98/7e457a3b014976887a050561955c0b76e1e7e2d6e27b78ecef2ef80518f2/anyio-4.16.0.tar.gz", hash = "sha256:59bb03251907983d1603d27c3087ddb1e93299ea2163adcdba37899a4fb12ffb", size = 310971, upload-time = "2026-11-01T11:20:07.282Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/16/7757fb84c819accc7d7e9af5291b261e4abb8627ee040f99e7d35ef2457a/anyio-4.16.0-py3-none-any.whl", hash = "sha256:c66ae634e0b8228396b801741beb3294326d4b6102f7bba4041c0dfb58c29026", size = 134521, upload-time = "2026-11-01T11:20:05.59Z" }, +] + +[[package]] +name = "certifi" +version = "2026.10.23" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/11/b524cbd9bbdaca58608e4cf4053fd463417549e1959fa8d3aa18319f9555/certifi-2026.10.23.tar.gz", hash = "sha256:a9895e6e2758b371e0545e321a3a8fd1fcfd9829ada8824a936dfb16aaaf36cd", size = 136225, upload-time = "2026-10-23T11:36:16.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/d1/a3652fe2c19134d0d15922177821d863cbd7e842d53a21bbe7e8e3a1b5ac/certifi-2026.10.23-py3-none-any.whl", hash = "sha256:e9b54de76e530bb6cf0ccd036787103acb58fd42ba930ea8bf948ddbaa2f61d1", size = 135082, upload-time = "2026-10-23T11:36:14.763Z" }, +] + +[[package]] +name = "click" +version = "8.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/0e/7fa0ef50764b67090eca4114772a2abf8b6148198475e54c660b97caeee6/click-8.5.0.tar.gz", hash = "sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34", size = 382235, upload-time = "2026-08-26T13:33:14.56Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/50/6c0d534c5f134586a8e1ba4e330569e32f057e33372ae556463212fb4cd3/click-8.5.0-py3-none-any.whl", hash = "sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360", size = 125251, upload-time = "2026-08-26T13:33:12.928Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "digup-server" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "fastapi" }, + { name = "uvicorn", extra = ["standard"] }, +] + +[package.dev-dependencies] +dev = [ + { name = "httpx" }, + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [ + { name = "fastapi", specifier = ">=0.115" }, + { name = "uvicorn", extras = ["standard"], specifier = ">=0.30" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "httpx", specifier = ">=0.28" }, + { name = "pytest", specifier = ">=8.4" }, +] + +[[package]] +name = "fastapi" +version = "0.162.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "opentelemetry-api" }, + { name = "pydantic" }, + { name = "starlette" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/eb/2cf8c80abf09b2800aef0a83b41066fbfda8ee5495b7a08b9d1af40f7046/fastapi-0.162.0.tar.gz", hash = "sha256:6a22c1e2e6e32e0633ee7e170f00009a29b8228b16feb219ee42f32fd8d88a8f", size = 527867, upload-time = "2026-10-23T18:47:33.264Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/4f/3f2c25c9b43970b5e9a08beb5e04776247edea684ddc0930f4f02f0bb6cb/fastapi-0.162.0-py3-none-any.whl", hash = "sha256:3785963de305dd09912426cb81f2d230fce98ab3521cdfe97265c8d14983128c", size = 148912, upload-time = "2026-10-23T18:47:31.734Z" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httptools" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3a/ec/deed52912ab7ca6c0b12859330c571c60c61d7267b341b28951fcbf13694/httptools-0.9.0.tar.gz", hash = "sha256:d484ebb7e3a3f3597b0f645fbd1b85633674ca808c1f5ba11c2caf7c66f5c8b6", size = 282523, upload-time = "2026-10-09T19:57:04.301Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/ed/0916b8b7ebd1deeaf22acba71b68c57b4b6b69aa1918f3812dea208b4276/httptools-0.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f9ccc9884241efceb4547a92955d128574c864681f11b7ea3ecbde295fafbe8b", size = 119039, upload-time = "2026-10-09T19:54:32.556Z" }, + { url = "https://files.pythonhosted.org/packages/c2/0b/9b6de4a01a563a904d0826c9069c824b330e1816df26c9bdf93f60b50857/httptools-0.9.0-cp312-cp312-macosx_11_0_x86_64.whl", hash = "sha256:45b3002392948dcf578029c89f6318e1289a993a1a5ec38a4161560fab60f811", size = 115051, upload-time = "2026-10-09T19:54:33.908Z" }, + { url = "https://files.pythonhosted.org/packages/85/3f/642113e9882f53158ecddf58003d25f18ded2c210ed23bf6eb663d4d51c3/httptools-0.9.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3e3201fe4d46e0d15d7ff9fafc94a605da9eb82d2c5b9837f0368acb325481f1", size = 552848, upload-time = "2026-10-09T19:54:35.434Z" }, + { url = "https://files.pythonhosted.org/packages/95/4c/3ecc59c99c28652d8d08d9b5be65770a14d2cadc616dad94224cee2b0e7e/httptools-0.9.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:58a1b0ec4cbb930e69669f9771715b2c7898d3cdf064d9811f7a66afef96b544", size = 549181, upload-time = "2026-10-09T19:54:37.099Z" }, + { url = "https://files.pythonhosted.org/packages/43/ce/21f5b2759590b7054e38d3b704a3c6b853c3395c370b3d6f16c45aea0fc0/httptools-0.9.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4c58dc91aefb31adad500aa68054334f429b840b36dd29e34e834101044cb2ef", size = 569121, upload-time = "2026-10-09T19:54:38.772Z" }, + { url = "https://files.pythonhosted.org/packages/52/c3/7c523aa8d0fa7a57010a3e1bbdebc209585009076465f3d1ae6a3f54b814/httptools-0.9.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6b900073e7b8481ef1aaf4f6c1789d210a1db01a9da8789821578cfeb4c2d540", size = 487649, upload-time = "2026-10-09T19:54:40.398Z" }, + { url = "https://files.pythonhosted.org/packages/94/e2/d90d60002692b8afcbc06fb49ca3a4365b32abed6c40fb2b612c67721a00/httptools-0.9.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6c12d0393a903b58bc5f5a7406d6c5290acfb8284290d68547ce620c06f7d133", size = 529583, upload-time = "2026-10-09T19:54:42.296Z" }, + { url = "https://files.pythonhosted.org/packages/46/c0/19172874cde0344a20c85877a0b2d0dcfca31111729ad8a79e8b4ac4e207/httptools-0.9.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:29b0d823e3c1e7cd1093a5dc889245db693ef13ada624cd66e2262421ef38867", size = 552414, upload-time = "2026-10-09T19:54:44.19Z" }, + { url = "https://files.pythonhosted.org/packages/1b/b8/02ea7910f69e5371986b025fb3b410592106df54e977a5732fd1d95917b5/httptools-0.9.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:6ebd39ee26db460cfe5ab8b71a15d1149b289139a0d3981522757d6af620887e", size = 482400, upload-time = "2026-10-09T19:54:46.064Z" }, + { url = "https://files.pythonhosted.org/packages/de/97/f05eac916d44cbbfe43668a6a40ab93e7fd8f94d5120d1ce2d8e55c69871/httptools-0.9.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4efbee349138a3fee7a4cc3a95abd2d499fae70dd5bff9fed9138d6f570f4283", size = 536148, upload-time = "2026-10-09T19:54:47.695Z" }, + { url = "https://files.pythonhosted.org/packages/b6/e9/9435dfcb7f1a1d6ebdc79a902164dfca70e33774c80bb60d25c630c31ef1/httptools-0.9.0-cp312-cp312-win32.whl", hash = "sha256:36fac804b8cfd6b935ae64f71349f833d2b6298404626d017a2c57bb942bc643", size = 86274, upload-time = "2026-10-09T19:54:49.1Z" }, + { url = "https://files.pythonhosted.org/packages/8b/69/813f1bf90be507d4166c437be1a413574d0e0abf36e2fec10c266661b0ee/httptools-0.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:7e32b83bd8c2f8b6fa726ef34e63e21c4d7eddc277d40d4ef7245ea3ed28e5b6", size = 92446, upload-time = "2026-10-09T19:54:50.498Z" }, + { url = "https://files.pythonhosted.org/packages/ae/e0/1d29e328c4cafe843403341e1455e0aec18b0e6910fbb14f12b36b563f19/httptools-0.9.0-cp312-cp312-win_arm64.whl", hash = "sha256:813a32f94991b9627795528053c73a57d2ce3eb98ede89f0e1c7a31095938e81", size = 88676, upload-time = "2026-10-09T19:54:51.844Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + +[[package]] +name = "idna" +version = "3.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/08/8eea9d4b8302028f3abb2c0813953f7aec26d33b7a8960ed760e65ff29fa/idna-3.20.tar.gz", hash = "sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44", size = 216463, upload-time = "2026-09-17T14:11:04.752Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/a2/bb081bab032533a855d44de1d56f8e8426114ff1ba5d1f07a438a0a654f8/idna-3.20-py3-none-any.whl", hash = "sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c", size = 69583, upload-time = "2026-09-17T14:11:03.168Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/e1/2069291243c926a2ff1cd706c7f3eeb9b62144bf60f77c9fb9ff2fb26bd3/iniconfig-2.3.1.tar.gz", hash = "sha256:67f4b9c50da0dedf52af349e7749a80a9057a5031199791b906c3bb3ae878960", size = 21209, upload-time = "2026-10-06T22:48:38.076Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/43/4ca9e49d27a1fcf6bece6f6aec0ea46bb9112489b93d4b688fb415457bdb/iniconfig-2.3.1-py3-none-any.whl", hash = "sha256:9121e2c1fdb355232495be3194c8dfe87ccc2d5dee45947b78e68f499790d7a7", size = 7552, upload-time = "2026-10-06T22:48:36.959Z" }, +] + +[[package]] +name = "opentelemetry-api" +version = "1.45.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/14/cf3a6b6448849656e1e5f4e2101f3988dc6789724050096aba1bc5ccf592/opentelemetry_api-1.45.2.tar.gz", hash = "sha256:d7b0008e21e5fd61cbe7a7f34ca3b6905a78ab677d9bc395e299359c75cda6b4", size = 73277, upload-time = "2026-10-29T17:29:37.888Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/2d/610db864d7513158cebb8f36cc88e67e73bd6c7aa1905ef18d89e1281c81/opentelemetry_api-1.45.2-py3-none-any.whl", hash = "sha256:2cbd737377e710ee740d261c94e93fa0f08e6d037c18f17513cc0a2d94479a34", size = 60335, upload-time = "2026-10-29T17:29:06.955Z" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "pluggy" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/db/7fc19e6f2dc92a966727031389fc2e08b558f0f25eb7403c1119ad4713cd/pluggy-1.7.0.tar.gz", hash = "sha256:d1eaa46ebb595891b860ab086b4d09c8588af65ebd4361b8e8f4bb8920b90ba8", size = 123304, upload-time = "2026-10-15T09:50:58.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/9e/2b38731e0fc536806f16490e1a12d7f0dc2a1235aa8cc07bcc75416a7daa/pluggy-1.7.0-py3-none-any.whl", hash = "sha256:7dd7b0d8832ba3cb632c306926ded123429211b83641b35dc5c41ad2d34f9bec", size = 27082, upload-time = "2026-10-15T09:50:56.808Z" }, +] + +[[package]] +name = "pydantic" +version = "2.14.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9f/d5/b4867fa7c7f04c34997d7d8aad744f64d210bb99d67cf18bf3fb1eea1aed/pydantic-2.14.2.tar.gz", hash = "sha256:fb7b2db6cefe8414dd612edeede49c4c433acb0432b0f92b95024e45f66c505d", size = 911596, upload-time = "2026-10-29T16:37:12.476Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/97/01/c71a71258b58659c72504e5fd65b451f6e36e34dc931d0b237ded213c814/pydantic-2.14.2-py3-none-any.whl", hash = "sha256:7efbf7507b937931cd32a5755acb00ea1cf5ccb1d4e8067a5850d5069e382118", size = 490403, upload-time = "2026-10-29T16:37:10.181Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.50.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/af/8e/23ec1021834e35041fbb2f10946141e6acdfd39379c45cc4d65a4e8a1d98/pydantic_core-2.50.2.tar.gz", hash = "sha256:1b0e8dc3932cf4e9ed5727570325cacc716b9409ffc794a221c0ccc72e8eb775", size = 518711, upload-time = "2026-10-29T16:34:57.465Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/65/6609a66cda909d414c58438f9bd70b1e947fe12c25f6bd40c93121afd7ef/pydantic_core-2.50.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:e3d824b6152e1c0fd58dc6266f2de470eb7b2286135294f5f7cd26a0f0693738", size = 2220341, upload-time = "2026-10-29T16:32:10.193Z" }, + { url = "https://files.pythonhosted.org/packages/f9/0b/92463ae7be78d73885487f72c4599902c2cc190378bccaad5718b7495250/pydantic_core-2.50.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:596921489390a002d6a5e303384a0f1f98384f8bb550fbf38a1aafa6093da0f9", size = 2093701, upload-time = "2026-10-29T16:32:11.502Z" }, + { url = "https://files.pythonhosted.org/packages/62/d3/24147a1aa22f7eade546172750e1110fd7b2793998057809800d3ebeae4e/pydantic_core-2.50.2-cp312-cp312-manylinux1_i686.manylinux_2_5_i686.whl", hash = "sha256:36d8b9725684ca3bf658b90a101d335fc8cae405bb22196b7b5ef41344913756", size = 2253622, upload-time = "2026-10-29T16:32:13.051Z" }, + { url = "https://files.pythonhosted.org/packages/f6/50/2628ca3c83a0d9a6fda3305fe59b1b28e40c0221e8f234cd1ef86fb64745/pydantic_core-2.50.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:43a8fd56f3c681f3cda2d5c842ad28e88241df742dfcbe8fd1dd437a3056e1f6", size = 2114101, upload-time = "2026-10-29T16:32:14.702Z" }, + { url = "https://files.pythonhosted.org/packages/21/c8/cc6d7304a93f91f5e06260ccad71cc7d882489663a37dec009532379fc46/pydantic_core-2.50.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:6f5a432c6c1ef18a1a02a04b3616b8d3f2f88b13faa46caae1ac6559ab202234", size = 2199541, upload-time = "2026-10-29T16:32:16.153Z" }, + { url = "https://files.pythonhosted.org/packages/65/d9/18f44ac5696b3e2b2384cb89c9fbe6ed69860f528e232b9fb46cc5b45a17/pydantic_core-2.50.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:5d3cd8be71c4da3c9b42cc92f17e5e3036bdca98bf18555a0e072c3fe87abbb4", size = 2368597, upload-time = "2026-10-29T16:32:17.544Z" }, + { url = "https://files.pythonhosted.org/packages/1d/22/5f2a48ef819a7e1b74fb003a85d27576ad2f9ddf3901b88998ac63ce1e00/pydantic_core-2.50.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:6a438cb8a3f87f2c49e82e43fb19511d4d5043f53e0787dc4df937ccb904373a", size = 2451701, upload-time = "2026-10-29T16:32:18.956Z" }, + { url = "https://files.pythonhosted.org/packages/08/93/2a6556bbdb8e918175af9d849a5d0b367b5644e41939d040f3a7215b1339/pydantic_core-2.50.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9667637665acba166a7dbbc786f14c50512d3a01f7199e0e1a13b929e92d7c7a", size = 2242782, upload-time = "2026-10-29T16:32:20.381Z" }, + { url = "https://files.pythonhosted.org/packages/90/ed/dd9f29a92257f32eb5b99d5120826186ee0632ecc3aaf6c23e2fab89ed6d/pydantic_core-2.50.2-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:b3e681208e2d39fde13fbea449ef2736a85ec1ea2afa41e9193a8c536e71dafe", size = 2253706, upload-time = "2026-10-29T16:32:22.293Z" }, + { url = "https://files.pythonhosted.org/packages/11/19/1ccf3bf38eb4ed97d8aeefc558458834ae0ad53fdbb1edf61af7228ea500/pydantic_core-2.50.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:6eb6ff07964f39e33c6f57fefe939514e76b8c0d4a694e5663e2e04e80cc1798", size = 2288603, upload-time = "2026-10-29T16:32:23.465Z" }, + { url = "https://files.pythonhosted.org/packages/ea/cd/bae2320fc89a65bcd87bf66ec83aac51b3b43b970a2edece15feb8af64dd/pydantic_core-2.50.2-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:4eb54c25e3463d73e1e4ed439ffc45d1d7e590df40bc7bb06f11a906afd996c3", size = 2476597, upload-time = "2026-10-29T16:32:24.824Z" }, + { url = "https://files.pythonhosted.org/packages/db/35/1bdb8d52c8e562e95c5e6ebb5784792af568ac6a81470dd546d95f392b60/pydantic_core-2.50.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:91238117cd296352f54f24117ff5a227bdea428865dba1244aa0cdd1424c0bc2", size = 2478664, upload-time = "2026-10-29T16:32:26.197Z" }, + { url = "https://files.pythonhosted.org/packages/12/72/0179b40ac6a6b0899c61826f2e3e80f88063f6d821e0dd9641927de46700/pydantic_core-2.50.2-cp312-cp312-win32.whl", hash = "sha256:98722314e12a8e4fa346e0afd3fff2098b5732875ff45592b450a7b589b43c4f", size = 2108820, upload-time = "2026-10-29T16:32:27.736Z" }, + { url = "https://files.pythonhosted.org/packages/b4/fb/679144362d7e5162519d9f6631ec982521417de3196351c6c56485f5b29f/pydantic_core-2.50.2-cp312-cp312-win_amd64.whl", hash = "sha256:b381574614245f18fdb2d6d1aa8f149e225869c47b5ee7f81812de8977817314", size = 2252238, upload-time = "2026-10-29T16:32:29.572Z" }, + { url = "https://files.pythonhosted.org/packages/0f/68/e3ccc885c3e2086ff84d7ca496f45b974a9f0a49ce2da8973aaa3a369a4b/pydantic_core-2.50.2-cp312-cp312-win_arm64.whl", hash = "sha256:f07c965a8aa6d62145f122641f292129984f39a3e3697585b19298a1488287db", size = 2162565, upload-time = "2026-10-29T16:32:30.853Z" }, + { url = "https://files.pythonhosted.org/packages/64/4d/23560b11366b547e3e68a3b8f9c02557cee707effa54fd39dbb3c4fff1fa/pydantic_core-2.50.2-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:d18d56c1ec63b304f4bb3c7db7e0d311e61cca1b1c944590a4b13ac39b2ca92b", size = 2204868, upload-time = "2026-10-29T16:34:34.745Z" }, + { url = "https://files.pythonhosted.org/packages/06/58/b9ddfaad189143d6568627166a730b6624ca90bd9a9a4af4854df88d444b/pydantic_core-2.50.2-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:41ed8abeffda6a880207fdc0cd1343a448cec3cc6b38b53f730c0745f745292d", size = 2049162, upload-time = "2026-10-29T16:34:36.186Z" }, + { url = "https://files.pythonhosted.org/packages/8b/20/99d259bf7835b2f13c6d0b3eb7bcf669fb6349986736fa7a8308a9e620b0/pydantic_core-2.50.2-graalpy312-graalpy250_312_native-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b7abcfaf66c1dc14c5ed928c349614f223c972a8b021b4be9548dbea6c817371", size = 2112135, upload-time = "2026-10-29T16:34:37.777Z" }, + { url = "https://files.pythonhosted.org/packages/9b/ca/652c8fe599abacafd852479059d209e608596ad3de82f1bff22e9f0f02fe/pydantic_core-2.50.2-graalpy312-graalpy250_312_native-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9e7460abce637f90e37f8c443559fbd687c38884d05cb69ef265e634945f4324", size = 2270339, upload-time = "2026-10-29T16:34:39.039Z" }, +] + +[[package]] +name = "pygments" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, +] + +[[package]] +name = "pytest" +version = "9.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/79/42/bb4641652e58dedeac2ec51abc1f8cae4fee4ec63899a0999016229a0f2a/pytest-9.2.0.tar.gz", hash = "sha256:97c4f3b1f3c7414aa36465e09b8220f1d03627993bd51b1282e193753c80557b", size = 1797555, upload-time = "2026-10-27T21:30:20.728Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/39/1f5d8c8877a0fa327db959c19341d4be61660de1ab6b7391f9eb805bcd67/pytest-9.2.0-py3-none-any.whl", hash = "sha256:07acdcf88ec174f907dc01af70bf52d49b014bbab03faeeade31c96b18c2e588", size = 411776, upload-time = "2026-10-27T21:30:19.131Z" }, +] + +[[package]] +name = "python-dotenv" +version = "1.2.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/74/26/2fbeedb218a787a5eea551c7532cac4e009f83d689dd2faa0d0353473f86/python_dotenv-1.2.4.tar.gz", hash = "sha256:f0d53e69935a851c0dcc78f3ab7aaccd8cabef0b92382b576b824212902873c0", size = 60824, upload-time = "2026-10-01T05:36:10Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/60/d1/38f3a3405989a89ac18390803e70c6ad7c7760da4f9b83cbeca0c44a0c72/python_dotenv-1.2.4-py3-none-any.whl", hash = "sha256:42269a8a5b3fd54ffa6f3d84b18abed50064717576b4ecf03dc4a55d8aa04fdc", size = 23266, upload-time = "2026-10-01T05:36:08.633Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, +] + +[[package]] +name = "starlette" +version = "1.10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/48/bb/3d8757f21e9a53d9035c9eb039a777047139a16bfdbedfc7a58241c3d914/starlette-1.10.0.tar.gz", hash = "sha256:b8399098d4c1e3522734f7235dc79b1fc967f96cf139c81e0fa30479e0685f9b", size = 2734356, upload-time = "2026-10-24T15:04:12.342Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/2e/947cfe02def9371c07cd5bf5b6b61c829e9aecb9904db02a66cfafbc1eec/starlette-1.10.0-py3-none-any.whl", hash = "sha256:478c0ab3b4ced6e82719849c118fa406f0278a3499c1a898cd15cd31bdc38ce4", size = 80571, upload-time = "2026-10-24T15:04:10.371Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/26/b09b8010994eccc3c09092e6b34058f36a460eea2d4c3e8b910c695975a0/typing_inspection-0.4.4.tar.gz", hash = "sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47", size = 76928, upload-time = "2026-08-12T12:37:25.997Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147", size = 14750, upload-time = "2026-08-12T12:37:24.648Z" }, +] + +[[package]] +name = "uvicorn" +version = "0.54.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/da/34/30e9280707135d2cfc589dfff3cb796bd07a3aeb1a3e415ba09dd89d7bb4/uvicorn-0.54.0.tar.gz", hash = "sha256:a2e33cbfaa0306f8e6b0c13e0cb89d7d7a2da3e62b90c66e18c33d9807b28620", size = 112283, upload-time = "2026-09-25T06:52:37.601Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/0c/b54a4fdd7f90a3af8b02ebc9ce6712c2c208b7926a2f7bad95c33ebbe943/uvicorn-0.54.0-py3-none-any.whl", hash = "sha256:505bdb0f318731d45f1f712071fc781a8981f6847a31c902c9f5e652d4f67faf", size = 87427, upload-time = "2026-09-25T06:52:35.829Z" }, +] + +[package.optional-dependencies] +standard = [ + { name = "httptools" }, + { name = "python-dotenv" }, + { name = "pyyaml" }, + { name = "uvloop", marker = "platform_python_implementation != 'PyPy' and sys_platform != 'cygwin' and sys_platform != 'win32'" }, + { name = "watchfiles" }, + { name = "websockets" }, +] + +[[package]] +name = "uvloop" +version = "0.23.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fa/42/02c739ce85fb2ee8d99212c61417da8140c6b87e9d97c430bea520d76044/uvloop-0.23.0.tar.gz", hash = "sha256:28d160f51ab4da3b187063652e643dea6831072add4adc1e6d62afbe73b6be27", size = 2559185, upload-time = "2026-10-01T03:17:04.4Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/98/04e766a6de99e6f7f955ecb7829e8d5a557de3427cb85be2236de54dda0c/uvloop-0.23.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:93935ab27b6eaef4c3e5489aebc84284f0644592f7ab516df60ee1b27eaf5eb3", size = 1393055, upload-time = "2026-10-01T03:15:42.526Z" }, + { url = "https://files.pythonhosted.org/packages/33/8a/499e7b863a848ede009539bce39806b66205da5f8779354228e785601144/uvloop-0.23.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4448e9124537620f9c25d004c227bb5104440b58955c19bbd312d910af919a63", size = 768909, upload-time = "2026-10-01T03:15:43.974Z" }, + { url = "https://files.pythonhosted.org/packages/3d/95/a880f8ce3b87ac5b307c354e8ee480be4658d24bf01f87921d57e3530b4a/uvloop-0.23.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f7548ede3ee908cfabc0d068106e303a9a2d811af959cdf6ab85676344cedcda", size = 4419106, upload-time = "2026-10-01T03:15:45.551Z" }, + { url = "https://files.pythonhosted.org/packages/51/27/c1d2f9fa977f8f42ea294604166df10e0027e6dc6cd17f85ede386c9bf36/uvloop-0.23.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:090865d8ce7a03986755a3ce711b7dd0d4b44eb14ab74368b717f3fad1180208", size = 4532597, upload-time = "2026-10-01T03:15:47.258Z" }, + { url = "https://files.pythonhosted.org/packages/42/dd/2cb6a2c8a30ca55c07a882dd4ae4ceae0fa7d8c15b25b3b7cb9a4b6cf4ca/uvloop-0.23.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bd6f2f81c7b9da99d301c0b16b82044e76fe887086e42e1590ecf520b94dbdac", size = 4230048, upload-time = "2026-10-01T03:15:49.119Z" }, + { url = "https://files.pythonhosted.org/packages/f4/52/29989cbaa4022dc4ef35c1dd60a4ab989e4c2065f341ed483ae71d2bd950/uvloop-0.23.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a6ac96da66c35bf789bdcde78a88dc7d56b7907d8379648c54adc1c61594575d", size = 4394152, upload-time = "2026-10-01T03:15:50.829Z" }, +] + +[[package]] +name = "watchfiles" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/68/e6aa0b77d217b31f8f486ec0cdfe5e00e6e38dc0be657e7d85819b9faf0a/watchfiles-1.3.0.tar.gz", hash = "sha256:99aee4a07847c06820765fd7b1b49ceac4f3f711ccb7d104655a33231de1c207", size = 108119, upload-time = "2026-09-21T09:08:55.664Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/68/fa/c0b840d5d8bafe640925408ce11095948c8945bb61be1e67d0ab629b872a/watchfiles-1.3.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:000b9688fc8133037a8b075c8ebf98f32844ff8964dda61e85c1db547dafc441", size = 399703, upload-time = "2026-09-21T09:08:13.518Z" }, + { url = "https://files.pythonhosted.org/packages/c7/8a/894799b485fe9473ad10422a0d9668e53fb78e3a2b5cc6061159572844ad/watchfiles-1.3.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:bbc1198edfdc90fda0600f825aa94150f428dfcbf8138746f55998e0e660d64c", size = 397647, upload-time = "2026-09-21T09:08:15.16Z" }, + { url = "https://files.pythonhosted.org/packages/5a/61/f277cd5ead05f5d37b98b41fbdd0c943ee0c04d49738f62e902cb291cf43/watchfiles-1.3.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4af9c464d410e8c44b58ddf1cbec7c3a02c1cb2c6c80df9c74659ba2fbda0ab5", size = 454994, upload-time = "2026-09-21T09:08:16.587Z" }, + { url = "https://files.pythonhosted.org/packages/46/66/ea4c01382975e53b1ac1b0a0ea1b353416b8127fb6e04f5c1c313119aab6/watchfiles-1.3.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eb94e40b9a0db19636b5e3fed0a9803452ecc7381df13b385272abb779ee83ca", size = 460105, upload-time = "2026-09-21T09:08:17.961Z" }, + { url = "https://files.pythonhosted.org/packages/31/c3/1a501b597817096c9831c255f6270c124ce0e2faeb74124be24dcc93b86d/watchfiles-1.3.0-cp310-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7398930a2b76b9dc67a4e6b8bfc7baf30946892a86f417a3920c07bdd6febc30", size = 493761, upload-time = "2026-09-21T09:08:19.327Z" }, + { url = "https://files.pythonhosted.org/packages/5a/16/77352d05e152c7786a016e9f1decdb4507e8aa5bcf3343c72142a0413d14/watchfiles-1.3.0-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:025b108f2d5cc2799cb941f79380df3e8b22bfb589c218e801767fe78d598c95", size = 577500, upload-time = "2026-09-21T09:08:20.723Z" }, + { url = "https://files.pythonhosted.org/packages/95/fe/81001c55b24466fe9911135e50269fd3a29f43e446627a92adfab54553b6/watchfiles-1.3.0-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:be9ef3cd403d756a304a0a08a112163c63da7ec72c8c9d083c98818291ff8f29", size = 469209, upload-time = "2026-09-21T09:08:22.346Z" }, + { url = "https://files.pythonhosted.org/packages/8a/cf/e3ad894ed5909a7c44b0707e89bc25d19a3d78b3a7aec846119443b6ae56/watchfiles-1.3.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b5768b49e426fd5b550b012c866db347cdf15c398ef98dc557b6e6b72fa74cd1", size = 458353, upload-time = "2026-09-21T09:08:23.7Z" }, + { url = "https://files.pythonhosted.org/packages/7f/f5/2f106829e6e2bc91ad74614730f6df484202b9a4d20d927dbf2f08d37d44/watchfiles-1.3.0-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:d36c72fcdc08f143d87316a4c4c3b064e340da2c7383d39803950740ed418935", size = 468707, upload-time = "2026-09-21T09:08:25.167Z" }, + { url = "https://files.pythonhosted.org/packages/15/37/2741ab776b978669c1f3bba1e573f1a184f727ab8b4d3a6d9c61090800ed/watchfiles-1.3.0-cp310-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:f0c9865240e065a2247f4b5248528ae3d01e0e8ae071462c250c884dc16e3ee3", size = 632510, upload-time = "2026-09-21T09:08:26.532Z" }, + { url = "https://files.pythonhosted.org/packages/48/64/db4a4684275fe9b696ee5118c194705c1423b3d42181c3f8ee70194d5c73/watchfiles-1.3.0-cp310-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:380f513d26cc2e598266b88d66456e07d67ef4be8f0f435a1154d9c44b43d509", size = 661723, upload-time = "2026-09-21T09:08:28.004Z" }, + { url = "https://files.pythonhosted.org/packages/52/80/95ddb24f6a2f595c031c24432ab7b8aa7d6d1ada0c6d63a552166e06e8d1/watchfiles-1.3.0-cp310-abi3-win32.whl", hash = "sha256:509d9f74d2bec5c1f4cd868ddcfe0c9601f7ea53066dfcb19d8ede742c4de661", size = 277761, upload-time = "2026-09-21T09:08:29.335Z" }, + { url = "https://files.pythonhosted.org/packages/48/a1/1c4a1b3c8030cd03206232612be8a6e8799093490c220b0fcb935d02b75a/watchfiles-1.3.0-cp310-abi3-win_amd64.whl", hash = "sha256:1acabde19b67e673274e89a04d613b7fb1f122e2c4b8ec9a581e59c149264b61", size = 290643, upload-time = "2026-09-21T09:08:30.45Z" }, + { url = "https://files.pythonhosted.org/packages/82/e0/d2db577c632d26d08078c4390dd870b8555c4e558df25d22270f0166c71f/watchfiles-1.3.0-cp310-abi3-win_arm64.whl", hash = "sha256:d978cc1dd7ba44f5590d7de74f7a9c8be7262d9e469c2c0efb8bbfc2574321dc", size = 285510, upload-time = "2026-09-21T09:08:31.561Z" }, +] + +[[package]] +name = "websockets" +version = "17.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/75/79/590c5f0cc7d1661195aeefccb0e342b79f4c77d368d526d028821a390332/websockets-17.4.tar.gz", hash = "sha256:09d3939b70df0c8ad2da145ebd50978bccaee1d565e838b0e7921a34d38571eb", size = 191391, upload-time = "2026-10-30T15:10:35.873Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/af/71/b020221345ea6c012dcf7496827a48a744b58696470b2b07fef95a32905a/websockets-17.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:3d512e3a6c528c42ce220ed9c3caff83c6fdcd98a1b98249f08ccfc66ad111b1", size = 218779, upload-time = "2026-10-30T15:07:11.522Z" }, + { url = "https://files.pythonhosted.org/packages/8e/84/1c372ae7c032021a84e5c9e9b76b0dddefb354e9b4af783a97efd4d80ab3/websockets-17.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7390e060f830b3d132f484b21ac2783a2d227b682ef62bb8146e1c1b4ca0e45a", size = 216457, upload-time = "2026-10-30T15:07:13Z" }, + { url = "https://files.pythonhosted.org/packages/74/b6/cff249107473f149c6c916a49cba967379fa63b0a7d2dce6896d6848f0a3/websockets-17.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:bcad49104711f88b2b7c3ba166de172857d837248205e541b044a552558fecc3", size = 216726, upload-time = "2026-10-30T15:07:14.468Z" }, + { url = "https://files.pythonhosted.org/packages/77/5d/aff8ecbf1c55c4d96f6dacc46a52102895f58b85bb284c8d325aaf31606a/websockets-17.4-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:cf99debb17227a50620aee16647bb33a0387f5fa5418d628666c0f05745e0d2a", size = 226043, upload-time = "2026-10-30T15:07:15.937Z" }, + { url = "https://files.pythonhosted.org/packages/cb/18/6229d6583031ca81d092a0cd887a13a57b402fb555acc323d39b45967eb7/websockets-17.4-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9ae1429ee9e52b139167ed0169757806a27ed7307e942e8c62037a9184e9843b", size = 226321, upload-time = "2026-10-30T15:07:17.563Z" }, + { url = "https://files.pythonhosted.org/packages/b8/61/7aba018fd53873e792a25afe1f4c005518133477b727d7455f2f7550c425/websockets-17.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51adfc30f467077293a6695d8149bcef501c0728386f3a491054f835dadb60b2", size = 227559, upload-time = "2026-10-30T15:07:18.998Z" }, + { url = "https://files.pythonhosted.org/packages/fb/66/37b207df9cdacaa017aa4487f3a9f2fcd668f6f7cfbafdac5b95ea258ee0/websockets-17.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c564d1e6531611bf87078c28780456ecc75459dcca189629fff2fe7c020d8e67", size = 230386, upload-time = "2026-10-30T15:07:20.431Z" }, + { url = "https://files.pythonhosted.org/packages/ad/f6/87507e11fbae53817a435945273953b5b020589e309fac4fd60527ee5999/websockets-17.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4c792e6cce24d08f92199d907785c9e64f74c357deeed3dbaf7d699ed5930b64", size = 228193, upload-time = "2026-10-30T15:07:22.112Z" }, + { url = "https://files.pythonhosted.org/packages/2f/3d/90a04f0aa989b68fbd2efcd38237420afe2abe5a1cab0b5d7c2aa4f0dbb6/websockets-17.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c44776f82b04e623eb9e1b81fcd17ac6e9808ae1ba5cdf8edce1f7c869a95a9b", size = 226952, upload-time = "2026-10-30T15:07:23.643Z" }, + { url = "https://files.pythonhosted.org/packages/ad/3e/50952af22b8e3e2d03f7e83f0e335851c6b484c3fd88586626fb1c2a34b3/websockets-17.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4d2636a5b61c69e32db0e74b57073dd4a66f1ad195e9398c5f01bba41090deea", size = 224183, upload-time = "2026-10-30T15:07:25.507Z" }, + { url = "https://files.pythonhosted.org/packages/c4/73/2a4a5d9c6bca1127f24af7bf08f406a69052d7f341bc81f08d9e8eb446b4/websockets-17.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:cfbbba4597f19ad717f79fd3f2a67ae7efae647bd6a61e14554e1b5ad242a5bf", size = 226991, upload-time = "2026-10-30T15:07:27.355Z" }, + { url = "https://files.pythonhosted.org/packages/68/de/90d96a579839bfe541ef214315df81d2bd519d952847c303328518ffdd02/websockets-17.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:c7b996c02de5e032cb63dc65482b73693b7ecaa586654126be2749ddea239c40", size = 225600, upload-time = "2026-10-30T15:07:28.973Z" }, + { url = "https://files.pythonhosted.org/packages/91/17/12c15ed446d842e40918cd16e79b56ee4422d6fbf210258cb8882f50bbd5/websockets-17.4-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a920e99155d75d1b1c08172da8aaa7200e7b30871fb628d1631d7891a333e30e", size = 226366, upload-time = "2026-10-30T15:07:30.472Z" }, + { url = "https://files.pythonhosted.org/packages/9d/73/cb4c927e806a73a0377696e031ef3c0db409028cdb2373f96ee863e8df8e/websockets-17.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8c00c5c49f80a588a2c4f99d519d2abed4ce26cdca11273041bc8b7e60af60de", size = 227470, upload-time = "2026-10-30T15:07:32.122Z" }, + { url = "https://files.pythonhosted.org/packages/33/b3/e682b3ea069933044f4bf4809eea0cd83e3a5bdbdff601d1777a9ae90795/websockets-17.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:52c31d123f5adf14dec694c2a4036bf42c523df7cefc641d31cd05653de56e49", size = 225026, upload-time = "2026-10-30T15:07:33.544Z" }, + { url = "https://files.pythonhosted.org/packages/2d/e9/d039757260705e5a265a31c9fff36b0d4567b842ce2b54e5d14369f200bd/websockets-17.4-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:d0acd8191c0e5a61d1c27bfa33b6ec8431982299f5d71c02129aa4f16f182ad0", size = 226007, upload-time = "2026-10-30T15:07:34.954Z" }, + { url = "https://files.pythonhosted.org/packages/db/e7/cb9907fac6f9b28f93beded6efda2953960cd02b633d432acf3451590859/websockets-17.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2a9cb5bbfc33b4ebd3c8d3714d6e09805437f60a14977774e4bd11b5ace46c23", size = 226286, upload-time = "2026-10-30T15:07:36.292Z" }, + { url = "https://files.pythonhosted.org/packages/39/99/85f6f4dd104a5d9a33a1ef272f501d0f2e8527ddf66be1b383d51f90ce56/websockets-17.4-cp312-cp312-win32.whl", hash = "sha256:c63c0d7832956d5f3bb77888b93c854569d13f806289ff75c43926d25c6e5fd2", size = 219256, upload-time = "2026-10-30T15:07:37.686Z" }, + { url = "https://files.pythonhosted.org/packages/59/31/d1636ba67ee9232a60033d90483e48686aa3856014e59110f29015f97bd2/websockets-17.4-cp312-cp312-win_amd64.whl", hash = "sha256:dde9cd67f18803949ebf96de8aba12b8ec5ee3115e34c75a2db85a3b0a6c20d5", size = 219561, upload-time = "2026-10-30T15:07:39.19Z" }, + { url = "https://files.pythonhosted.org/packages/23/aa/e3859d52188f041a5a05c9f3ac005a9c829723f521f0fe60e2c5f9c93bb9/websockets-17.4-cp312-cp312-win_arm64.whl", hash = "sha256:c634053459278cf31fb404ddb5ecd38124e4609e8b609166da621edc2d5796db", size = 219487, upload-time = "2026-10-30T15:07:41.161Z" }, + { url = "https://files.pythonhosted.org/packages/e9/a3/d8fbc72c5c0e78905caf2d6dcf806e532451e3f48e2a09a8ebe1de9ad412/websockets-17.4-pp312-pypy312_pp80-macosx_10_15_x86_64.whl", hash = "sha256:c7f66a53e9b7ec831abb15e3d9b6cee8f52d6a24cd0a374786bf46c3ec218bc6", size = 216397, upload-time = "2026-10-30T15:10:25.255Z" }, + { url = "https://files.pythonhosted.org/packages/d3/de/1bf313cd0f4662df4884a7602932cd13347090e59dd6047865eee2bb5aa5/websockets-17.4-pp312-pypy312_pp80-macosx_11_0_arm64.whl", hash = "sha256:4fb2db1f649b6464711846b5bb9c1611bd906ce939d9460cbd1fa419552cd6c3", size = 216664, upload-time = "2026-10-30T15:10:27.016Z" }, + { url = "https://files.pythonhosted.org/packages/30/e2/7912bd417511b54907853bd495aef2daee111062dbafd43d133a3f7253a7/websockets-17.4-pp312-pypy312_pp80-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:32110d954d9d0e51f36edb0396673bf4d35193a0a80ba7013bf7f7d58a33ca07", size = 217603, upload-time = "2026-10-30T15:10:28.475Z" }, + { url = "https://files.pythonhosted.org/packages/56/6f/2e8d5610f6184a2e1720bf8ce256402ed99ad30ea87c461fba5bc4d651d9/websockets-17.4-pp312-pypy312_pp80-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:88ba8a439b2bad97bb69c38db5bf0ec16a061ee834cf0f244089186ee803e899", size = 217489, upload-time = "2026-10-30T15:10:29.999Z" }, + { url = "https://files.pythonhosted.org/packages/29/3a/a05bc57eadcd7e8e072d917888128e6d22e1203984afb21a8da3198982fe/websockets-17.4-pp312-pypy312_pp80-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:868744c68e9342768888c7326604da2228c19bb6946aa46b5f9e1ac32ea8d40b", size = 218356, upload-time = "2026-10-30T15:10:31.432Z" }, + { url = "https://files.pythonhosted.org/packages/b2/49/eed86bfb8863093a82cbaa52c9159be1ce2374d5ace6fe1f8e8031567e7d/websockets-17.4-pp312-pypy312_pp80-win_amd64.whl", hash = "sha256:829bcaa90be8737c132d9515d6a466e8528e3b3d050d3f3466f36ea94d3beb48", size = 306838, upload-time = "2026-10-30T15:10:32.954Z" }, + { url = "https://files.pythonhosted.org/packages/eb/27/9db924cb0dbf22eade78d98fbae77790f9bf6bc729e6ab29aef9d68e4156/websockets-17.4-py3-none-any.whl", hash = "sha256:c6bd2c9781915592b209160d0f017d4c7c82ad53075e6b0c24d2a89299e7f9e9", size = 212900, upload-time = "2026-10-30T15:10:34.482Z" }, +]