diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..d0e3a9f --- /dev/null +++ b/.dockerignore @@ -0,0 +1,10 @@ +.env +.env.* +!.env.example +data/ +secrets/ +.git/ +.github/ +tests/ +__pycache__/ +*.pyc diff --git a/.env.example b/.env.example index cd4b3e4..8ca59b5 100644 --- a/.env.example +++ b/.env.example @@ -1,33 +1,52 @@ -# Immich +# Immich (>= 3.2) -IMMICH_URL=http://localhost:2283 +IMMICH_URL=http://immich-server:2283 +# API key with exactly these permissions: album.read, asset.read, asset.download IMMICH_API_KEY=your_immich_api_key # Comma-separated album names (must match exactly as shown in Immich) IMMICH_ALBUMS=Family,Vacation 2024 -# Amazon Photos cookies -# -# These come from your browser after logging in to Amazon Photos. -# -# 1. Go to https://www.amazon.com/photos (or your regional TLD) -# 2. Open DevTools → Application → Cookies → https://www.amazon.com -# 3. Copy the values below. -# -# The cookie key names vary by region: -# US accounts: ubid-main, at-main -# Brazil accounts: ubid-acbbr, at-acbbr -# UK accounts: ubid-acbuk, at-acbuk -# etc. +# Amazon Photos target + +# Marketplace of your Amazon account: de (default), com, co.uk, fr, it, es, ... +AP_TLD=de +# Album in Amazon Photos that receives the photos (created if missing) +AP_ALBUM_NAME=Echo Show +# trash = remove from album and move to Amazon trash; album_only = only remove from album +AP_DELETE_MODE=trash + +# Amazon Photos cookies (from your browser, see README) # -# IMPORTANT: The at-* cookie is HttpOnly. You may need to grab it from -# DevTools → Network tab → any request to amazon.com → Cookie header. -# Cookies expire periodically (weeks to months) and must be refreshed manually. +# For amazon.de the cookie names are session-id, ubid-acbde and at-acbde. +# They are derived from AP_TLD; set AP_UBID_KEY / AP_AT_KEY only to override. +# Prefer the *_FILE variants (Docker secrets) so values don't show up in `docker inspect`. AP_SESSION_ID=your_session_id - -AP_UBID_KEY=ubid-main AP_UBID_VALUE=your_ubid_value - -AP_AT_KEY=at-main AP_AT_VALUE=your_at_value +# AP_SESSION_ID_FILE=/run/secrets/ap_session_id +# AP_UBID_VALUE_FILE=/run/secrets/ap_ubid_value +# AP_AT_VALUE_FILE=/run/secrets/ap_at_value +# IMMICH_API_KEY_FILE=/run/secrets/immich_api_key + +# Use the User-Agent of the browser you copied the cookies from +# AP_USER_AGENT=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36 + +# Safety + +# Start with a dry run: logs planned uploads/deletions, writes nothing. +DRY_RUN=true +# Refuse to delete more than this share / number of tracked photos in one run +MAX_DELETE_FRACTION=0.2 +MAX_DELETE_COUNT=50 + +# Misc + +# IMAGE (default) or IMAGE,VIDEO +SYNC_TYPES=IMAGE +# Files larger than this are skipped +MAX_FILE_MB=200 +SYNC_INTERVAL_HOURS=6 +# Pinged (GET) after every fully successful run, e.g. healthchecks.io +# HEALTHCHECK_URL=https://hc-ping.com/your-uuid diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..9ec8383 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +* text=auto eol=lf +*.sh text eol=lf +Dockerfile text eol=lf diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 6201224..0000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Build and push Docker image - -on: - push: - branches: [main] - tags: ["v*"] - -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - -jobs: - build: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - - steps: - - uses: actions/checkout@v4 - - - uses: docker/setup-qemu-action@v3 - - - uses: docker/setup-buildx-action@v3 - - - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tags: | - type=ref,event=branch - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=sha - - - uses: docker/build-push-action@v6 - with: - context: . - platforms: linux/amd64,linux/arm64 - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..14b6dc7 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,27 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-24.04 + steps: + # Actions are pinned by full commit SHA; update SHA and comment together. + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false + + - name: Unit tests + run: python3 -m unittest discover -s tests -v + + - name: Build image (no push) + run: docker build --pull -t immich-amazon-sync:ci . + + - name: Image runs as non-root + run: test "$(docker run --rm --entrypoint id immich-amazon-sync:ci -u)" = "10001" diff --git a/.github/workflows/codacy.yml b/.github/workflows/codacy.yml deleted file mode 100644 index ef569e9..0000000 --- a/.github/workflows/codacy.yml +++ /dev/null @@ -1,61 +0,0 @@ -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. - -# This workflow checks out code, performs a Codacy security scan -# and integrates the results with the -# GitHub Advanced Security code scanning feature. For more information on -# the Codacy security scan action usage and parameters, see -# https://github.com/codacy/codacy-analysis-cli-action. -# For more information on Codacy Analysis CLI in general, see -# https://github.com/codacy/codacy-analysis-cli. - -name: Codacy Security Scan - -on: - push: - branches: [ "main" ] - pull_request: - # The branches below must be a subset of the branches above - branches: [ "main" ] - schedule: - - cron: '33 2 * * 4' - -permissions: - contents: read - -jobs: - codacy-security-scan: - permissions: - contents: read # for actions/checkout to fetch code - security-events: write # for github/codeql-action/upload-sarif to upload SARIF results - actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status - name: Codacy Security Scan - runs-on: ubuntu-latest - steps: - # Checkout the repository to the GitHub Actions runner - - name: Checkout code - uses: actions/checkout@v4 - - # Execute Codacy Analysis CLI and generate a SARIF output with the security issues identified during the analysis - - name: Run Codacy Analysis CLI - uses: codacy/codacy-analysis-cli-action@d840f886c4bd4edc059706d09c6a1586111c540b - with: - # Check https://github.com/codacy/codacy-analysis-cli#project-token to get your project token from your Codacy repository - # You can also omit the token and run the tools that support default configurations - project-token: ${{ secrets.CODACY_PROJECT_TOKEN }} - verbose: true - output: results.sarif - format: sarif - # Adjust severity of non-security issues - gh-code-scanning-compat: true - # Force 0 exit code to allow SARIF file generation - # This will handover control about PR rejection to the GitHub side - max-allowed-issues: 2147483647 - - # Upload the SARIF file generated in the previous step - - name: Upload SARIF results file - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: results.sarif diff --git a/.gitignore b/.gitignore index b6bde21..7f02d8f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .env data/ +secrets/ __pycache__/ *.pyc diff --git a/Dockerfile b/Dockerfile index 2945731..e37cb48 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,22 @@ -FROM alpine:3.21 +# alpine:3.24, pinned by multi-arch index digest. Update both tag and digest together. +FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6 -RUN apk add --no-cache python3 +RUN apk add --no-cache python3 \ + && addgroup -S -g 10001 app \ + && adduser -S -D -H -u 10001 -G app -s /sbin/nologin app \ + && mkdir -p /data \ + && chown app:app /data \ + && chmod 700 /data WORKDIR /app COPY src/immich_sync.py entrypoint.sh ./ -RUN chmod +x entrypoint.sh +RUN chmod 0755 entrypoint.sh && chmod 0644 immich_sync.py +ENV STATE_DIR=/data \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +USER 10001:10001 VOLUME ["/data"] ENTRYPOINT ["./entrypoint.sh"] diff --git a/README.md b/README.md index e7269b1..45b65e2 100644 --- a/README.md +++ b/README.md @@ -1,77 +1,133 @@ # immich-amazon-sync -One-way sync from [Immich](https://immich.app) albums to [Amazon Photos](https://www.amazon.com/photos). +One-way sync from one or more [Immich](https://immich.app) albums into an **album** in [Amazon Photos](https://www.amazon.de/photos), with amazon.de as the default region. -Built to display self-hosted Immich photos on an **Amazon Echo Show** screensaver - since the Echo Show only supports Amazon Photos as a photo source, and Amazon Photos has no public API. +This is a fork of [vanessa/immich-amazon-sync](https://github.com/vanessa/immich-amazon-sync) (MIT, based on commit `6d95d8c`). It was built for the same purpose: show self-hosted Immich photos on an **Amazon Echo Show**, which only accepts Amazon Photos as a photo source. Changes compared to upstream: -> Important: This is a personal project that works on my specific setup. It's not a polished, general-purpose tool. You're welcome to use it, open a PR, fork it, or learn from it, but expect to read the code and adapt things to your environment. No guarantees it'll work out of the box for you. +- Works with **Immich ≥ 3.2**: album contents come from `POST /api/search/metadata` with cursor pagination. Upstream relies on `GET /api/albums/{id}` → `assets`, which Immich 3.x no longer returns. +- Configurable region (default **Germany / amazon.de**), with the upload host and cookie names derived from it. +- Targets an Amazon Photos **album** (created if missing), not just the library root. +- Safety: a deletion brake, dry-run mode, file name sanitising, secrets from files, no secrets in logs, non-root container. +- Robustness: streaming downloads and uploads, atomic and versioned state, clear handling of expired cookies, 409 conflict resolution by MD5. + +> [!WARNING] +> **Amazon Photos has no public API.** This tool uses the internal Amazon Drive v1 endpoints the web app uses, authenticated with your browser cookies. Amazon can change or block this at any time, and using it may conflict with Amazon's terms of use. Use at your own risk, and always start with a dry run. ## How it works ``` -Immich albums → (download originals) → Amazon Drive v1 API → Amazon Photos +Immich albums ──(search/metadata, download original)──▶ Amazon Drive v1 upload (root folder) + └─▶ add node to Amazon Photos album ``` -- Syncs photos from one or more Immich albums to your Amazon Photos root folder -- Tracks which Immich assets map to which Amazon Photos node IDs -- Photos removed from Immich albums are moved to Amazon Photos trash -- Runs on a configurable interval (default: every 6 hours) +- Every run collects all assets of the configured Immich albums (non-trashed, visibility `timeline` or `archive`, i.e. exactly what the album shows). +- New assets are downloaded, streamed to the Amazon Photos root folder and then added to the album `AP_ALBUM_NAME` in batches. +- Assets that are no longer in **any** of the configured albums are removed from the Amazon album and, with `AP_DELETE_MODE=trash` (the default), moved to the Amazon trash. Only nodes this tool uploaded or adopted (those listed in the state file) are ever touched. +- The state file maps Immich asset IDs to Amazon node IDs and is saved after every upload. ## Requirements -- An Immich instance with API access -- An Amazon Prime account (Amazon Photos requires Prime) -- Docker +- Immich **3.2 or newer** (cursor-based search). Older versions are detected, and the run aborts with an error. +- An Amazon account with Amazon Photos (Prime), tested for amazon.de. +- Docker. ## Setup -Create a directory and an `.env` file with your configuration (see [Configuration](#configuration) for all variables): +> 🇩🇪 Step-by-step guide in German (Immich + Echo Show, Docker Compose): [docs/anleitung-de.md](docs/anleitung-de.md) + +### 1. Build the image locally + +The image is built from this repository. Do not pull it from a third-party registry. ```bash -mkdir immich-amazon-sync && cd immich-amazon-sync +git clone immich-amazon-sync +cd immich-amazon-sync +docker build --pull -t immich-amazon-sync:local . ``` -### Getting Amazon Photos cookies +The base image is pinned by digest in the `Dockerfile`. To update it, change both the tag and the digest. + +### 2. Create a restricted Immich API key + +In Immich, go to **Account Settings → API Keys → New API Key**. Do not grant all permissions; select only: + +| Permission | Used for | +|------------------|-----------------------------------------------------------| +| `album.read` | `GET /api/albums`, `GET /api/albums/{id}` (`assetCount`) | +| `asset.read` | `POST /api/search/metadata` | +| `asset.download` | `GET /api/assets/{id}/original` | + +`GET /api/server/version` (the version check) is public. + +### 3. Get the Amazon Photos cookies (amazon.de) + +1. Log in to in your browser. +2. Open DevTools (F12) → **Application** (Chrome/Edge) / **Storage** (Firefox) → **Cookies** → `https://www.amazon.de`. This list includes `HttpOnly` cookies. +3. Copy these three values: + + | Cookie | Variable | + |--------------|-----------------| + | `session-id` | `AP_SESSION_ID` | + | `ubid-acbde` | `AP_UBID_VALUE` | + | `at-acbde` | `AP_AT_VALUE` | -Amazon Photos has no official API. This tool authenticates using browser cookies, which expire periodically (typically weeks to months) and must be refreshed manually. +4. Optionally copy your browser's User-Agent (DevTools → Network → any request → `User-Agent`) into `AP_USER_AGENT`. -1. Log in to [Amazon Photos](https://www.amazon.com/photos) in your browser -2. Open DevTools (F12) → **Application** tab → **Cookies** → `https://www.amazon.com` -3. Copy these three cookie values into `.env`: - - `session-id` → `AP_SESSION_ID` - - `ubid-main` → `AP_UBID_VALUE` (key name varies by region - see below) - - `at-main` → `AP_AT_VALUE` (key name varies by region - see below) +> **Note:** If `at-acbde` is missing from the cookie list, open **Network** → reload → any request to `www.amazon.de` → **Request Headers** → `cookie`, and copy the value after `at-acbde=` up to the next `;`. +> +> Do not log out of Amazon in that browser afterwards: logging out invalidates the cookies. Closing the browser is fine. -> **Note:** The `at-*` cookie is `HttpOnly` and may not appear in the Application tab. In that case, go to **Network** tab → click any request to `amazon.com` → look at the `Cookie` request header to find the value. +The cookies expire after a few weeks or months. When they do, every run stops at the first Amazon request with `cookies expired or invalid` (exit code 3). Copy fresh values and restart. -#### Regional cookie key names +Other regions: set `AP_TLD` (for example `com`, `co.uk`, `fr`). Cookie names follow the pattern `ubid-acb` / `at-acb`, and `ubid-main` / `at-main` for amazon.com. The upload host is known for North America (`com`, `ca`) and EU marketplaces. For anything else, set `AP_CONTENT_URL`. -The `ubid` and `at` cookie keys include a region suffix: +`IMMICH_URL` can point at the Immich container through a shared Docker network (`http://immich-server:2283`, see the German guide), or at a hostname or IP (`http://immich.local:2283`). A hostname that exists only in the host's `/etc/hosts` cannot be resolved by Docker. Use the IP in that case. -| Region | `AP_UBID_KEY` | `AP_AT_KEY` | -|--------|---------------|-------------| -| US | `ubid-main` | `at-main` | -| Brazil | `ubid-acbbr` | `at-acbbr` | -| UK | `ubid-acbuk` | `at-acbuk` | -| Japan | `ubid-acbjp` | `at-acbjp` | +#### Using an existing Amazon Photos album -Set `AP_UBID_KEY` and `AP_AT_KEY` in your `.env` to match your region. +`AP_ALBUM_NAME` can name an album that already exists. It is matched by exact name, case-sensitive. The dry run logs `Amazon Photos album '': `. The ID must match the end of the album URL `https://www.amazon./photos/album/`. If the log shows `Would create` instead, the album was not found. -### Getting an Immich API key +Photos that are already in the album are never touched. Only photos this tool added are removed again. -1. Open your Immich instance → User Settings → API Keys -2. Create a new key and copy it to `IMMICH_API_KEY` in `.env` +### 4. Configure + +```bash +cp .env.example .env +chmod 600 .env +# edit .env +``` + +Prefer Docker secrets (`*_FILE` variables) for the cookies and API key, so they don't show up in `docker inspect`. See the Compose example below. + +### 5. Prepare the data directory + +The container runs as UID/GID `10001`. The data directory must be writable by that user: + +```bash +mkdir -p data +sudo chown 10001:10001 data +chmod 700 data +``` -## Usage +### 6. Dry run first + +```bash +docker run --rm --env-file .env -e DRY_RUN=true -e RUN_ONCE=true \ + -v ./data:/data immich-amazon-sync:local +``` + +The dry run checks the Immich version, the album lookup, the cookies (read-only Amazon requests) and the album lookup on Amazon. It then logs every planned upload and deletion. It writes nothing to Amazon and nothing to the state file, and it does not download any photos. + +### 7. Run ```bash docker run -d \ --name immich-sync \ --restart unless-stopped \ --env-file .env \ - -e SYNC_INTERVAL_HOURS=6 \ + -e DRY_RUN=false \ -v ./data:/data \ - ghcr.io/vanessa/immich-amazon-sync:main + immich-amazon-sync:local ``` View logs: @@ -80,78 +136,156 @@ View logs: docker logs -f immich-sync ``` -Force a sync now: - -```bash -docker restart immich-sync -``` +With `--restart unless-stopped` (or `restart: unless-stopped` in Compose), the container comes back after a host reboot, as long as the Docker service starts at boot and the container wasn't stopped by hand. The first sync runs right after start. If Immich comes up later, that run fails without deleting anything, and the next attempt follows after `SYNC_INTERVAL_HOURS`. -Full re-sync (re-uploads everything): +Force a sync now: ```bash -rm data/state.json docker restart immich-sync ```
-Docker Compose +Docker Compose with secrets ```yaml services: immich-sync: - image: ghcr.io/vanessa/immich-amazon-sync:main + build: . + image: immich-amazon-sync:local container_name: immich-sync restart: unless-stopped - env_file: .env + env_file: .env # without the secret values environment: - - SYNC_INTERVAL_HOURS=6 + DRY_RUN: "false" + IMMICH_API_KEY_FILE: /run/secrets/immich_api_key + AP_SESSION_ID_FILE: /run/secrets/ap_session_id + AP_UBID_VALUE_FILE: /run/secrets/ap_ubid_value + AP_AT_VALUE_FILE: /run/secrets/ap_at_value + secrets: [immich_api_key, ap_session_id, ap_ubid_value, ap_at_value] volumes: - ./data:/data + read_only: true + tmpfs: [/tmp] + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + +secrets: + immich_api_key: { file: ./secrets/immich_api_key } + ap_session_id: { file: ./secrets/ap_session_id } + ap_ubid_value: { file: ./secrets/ap_ubid_value } + ap_at_value: { file: ./secrets/ap_at_value } ``` -
+Without Swarm, Compose bind-mounts the secret files, so they must be readable by UID 10001 (`sudo chown 10001 secrets/*; sudo chmod 400 secrets/*`). If both are set, a `*_FILE` variable takes precedence over the plain variable. -### Configuration +`/tmp` is used for the downloaded original while it is being uploaded. For videos, make sure the tmpfs is larger than `MAX_FILE_MB`. + + -All configuration is via environment variables in `.env`: +## Configuration | Variable | Description | Default | |----------|-------------|---------| -| `IMMICH_URL` | Immich server URL | - | -| `IMMICH_API_KEY` | Immich API key | - | -| `IMMICH_ALBUMS` | Comma-separated album names (exact match) | - | -| `AP_SESSION_ID` | Amazon `session-id` cookie | - | -| `AP_UBID_KEY` | Name of the `ubid` cookie (e.g. `ubid-main`) | - | -| `AP_UBID_VALUE` | Value of the `ubid` cookie | - | -| `AP_AT_KEY` | Name of the `at` cookie (e.g. `at-main`) | - | -| `AP_AT_VALUE` | Value of the `at` cookie | - | -| `SYNC_INTERVAL_HOURS` | Hours between sync runs | `6` | +| `IMMICH_URL` | Immich server URL (without `/api`) | - | +| `IMMICH_API_KEY` | Immich API key (`album.read`, `asset.read`, `asset.download`) | - | +| `IMMICH_ALBUMS` | Comma-separated Immich album names (exact match, must be unique) | - | +| `AP_ALBUM_NAME` | Target album in Amazon Photos (created if missing) | - | +| `AP_TLD` | Amazon marketplace TLD (`de`, `com`, `co.uk`, …) | `de` | +| `AP_SESSION_ID` | `session-id` cookie | - | +| `AP_UBID_VALUE` | `ubid-acbde` cookie (or the regional equivalent) | - | +| `AP_AT_VALUE` | `at-acbde` cookie (or the regional equivalent) | - | +| `AP_UBID_KEY` / `AP_AT_KEY` | Override the cookie names derived from `AP_TLD` | derived | +| `AP_CONTENT_URL` | Override the upload host, e.g. `https://content-eu.drive.amazonaws.com/cdproxy` | derived | +| `AP_USER_AGENT` | User-Agent sent to Amazon | recent Chrome | +| `AP_DELETE_MODE` | `trash` (remove from album and move to trash) or `album_only` | `trash` | +| `DRY_RUN` | Log the plan only; write nothing to Amazon or the state file | `false` | +| `MAX_DELETE_FRACTION` | Deletion brake: maximum share of tracked assets deleted per run | `0.2` | +| `MAX_DELETE_COUNT` | Deletion brake: maximum number of assets deleted per run | `50` | +| `SYNC_TYPES` | `IMAGE` or `IMAGE,VIDEO` | `IMAGE` | +| `MAX_FILE_MB` | Larger files are skipped (logged, retried next run) | `200` | +| `HEALTHCHECK_URL` | `GET` after every fully successful run (never logged) | - | +| `SYNC_INTERVAL_HOURS` | Hours between runs | `6` | +| `RUN_ONCE` | Run a single sync and exit (container entrypoint) | `false` | +| `STATE_DIR` | Directory for the state and log files | `/data` | + +Every variable can also be read from a file via `_FILE`. + +### Exit codes + +| Code | Meaning | +|------|---------| +| 0 | Success (the healthcheck is pinged) | +| 1 | Errors: config problem, Immich unreachable or too old, album not found, individual uploads failed, `assetCount` mismatch | +| 2 | Planned deletions were **not** executed (deletion brake or incomplete Immich data) | +| 3 | Amazon answered 401/403 or redirected to the sign-in page, so the cookies have expired. The run was aborted. | + +## Deletion safety + +Deletions only happen when **all** of the following are true: + +- Every album in `IMMICH_ALBUMS` was found, exactly once. +- Every Immich query succeeded. +- For every album, the number of assets found by the search equals `assetCount` from `GET /api/albums/{id}`. +- The run deletes at most `MAX_DELETE_COUNT` assets and at most `MAX_DELETE_FRACTION` of the assets tracked before the run (photos uploaded in the same run don't count). + +Otherwise nothing is deleted, an error is logged and the exit code is 2. An empty or broken Immich response therefore never causes a mass deletion. + +With small albums the fraction limit trips quickly (for example, removing 1 of 4 photos is 25%). To deliberately delete more, raise `MAX_DELETE_FRACTION` / `MAX_DELETE_COUNT` for one run. Ideally do a dry run first. + +Changing `SYNC_TYPES` does not delete anything: only assets that left all Immich albums are removed. + +**Albums that change completely and often** (for example refilled daily by a script) trip the brake on every run, and old photos stay in the Amazon album. Raise the limits permanently, e.g. `MAX_DELETE_FRACTION=1` and `MAX_DELETE_COUNT=500`. Incomplete Immich data still blocks deletions. However, if the automation briefly empties the album and a sync runs at exactly that moment, the Amazon album is emptied as well. Everything is uploaded again on the next run. ## State -Sync state is persisted in `data/state.json`, which maps Immich asset IDs to Amazon Photos node IDs: +`data/state.json` (mode `0600`, written atomically): ```json { - "synced": { - "immich-asset-uuid": "amazon-node-id", - ... - } + "version": 2, + "album_node_id": "amazon-album-node-id", + "synced": { "immich-asset-uuid": "amazon-node-id" }, + "pending_album_add": [], + "adopted_nodes": [] } ``` -- Delete `data/state.json` to force a full re-sync -- Logs are written to `data/immich-sync.log` and stdout +- `pending_album_add` holds uploaded nodes that are not yet linked to the album. They are linked on the next run, so an interrupted run loses nothing. +- `adopted_nodes` holds nodes that already existed on Amazon (same MD5, found via a 409 conflict), for example a copy from the phone's own Amazon Photos backup. They are linked to the album but, when the photo leaves Immich, they are **only removed from the album and never trashed**. +- The upstream format (`{"synced": {...}}`) is migrated automatically without re-uploading. On the first run, all previously synced photos are added to the album. +- If the album is deleted or renamed in Amazon Photos, it is looked up by name or recreated, and all tracked photos are linked again. +- If the state file is corrupt, the run aborts instead of silently starting from scratch. +- To start over, delete `data/state.json`. Existing uploads are matched again via the 409 conflict and MD5 comparison, not duplicated. +- Logs go to stdout and `data/immich-sync.log` (rotated at 5 MB, 3 backups, mode `0600`). ## Limitations -- **Cookie auth is fragile.** Amazon cookies expire and must be refreshed manually. There is no way around this - Amazon Photos has no OAuth or API key flow. When cookies expire, the container will log 401 errors until you update `.env` and restart. -- **Undocumented API.** This uses Amazon's internal Drive v1 endpoints. Amazon could change or remove them without notice. -- **Tested on a US account only.** Regional differences (content upload endpoints, cookie names, API behavior) may cause issues. The content upload URL is hardcoded to `content-na.drive.amazonaws.com` - other regions likely use different hostnames. -- **Uploads go to root folder.** Amazon Drive v1 API doesn't reliably support folder targeting for all account types. Photos land in your Amazon Photos root. -- **No content deduplication.** Dedup is by Immich asset ID, not file hash. Re-uploading the same photo with a different Immich ID will create a duplicate on Amazon. -- **Deletion moves to trash.** Removed photos are trashed, not permanently deleted. Empty your Amazon Photos trash manually if needed. +- **Cookie auth is fragile.** Cookies expire and must be refreshed by hand. There is no OAuth or API key flow. +- **Undocumented API.** Endpoints and request shapes come from the browser app and the [`amazon-photos`](https://github.com/trevorhobenshield/amazon_photos) library, not from official documentation. +- **Files land in the root folder.** They are then added to the album. An album is not a folder, so there is no way to upload into it directly. +- **Deduplication is by Immich asset ID.** On a 409 conflict, the existing Amazon node is adopted only if its MD5 matches. Otherwise the file is uploaded again under `_.`. +- **Deleted photos go to the trash**, not permanent deletion. Empty the Amazon trash yourself if needed. + +## Implementation notes and verification + +- **Immich search shape.** Immich 3.2 deprecated the flat fields of `MetadataSearchDto` (`albumIds`, `visibility`, `page`, …) in favour of `filter` / `orderBy` / `cursor`. A request that uses only flat fields takes the legacy path, which returns `nextPage` but **never `nextCursor`**. Combining flat fields with `cursor` is rejected with HTTP 400. This tool therefore sends: + + ```json + {"filter": {"albumIds": {"any": [""]}, + "visibility": {"in": ["archive", "timeline"]}, + "trashedAt": {"eq": null}}, + "size": 1000, "cursor": ""} + ``` + + Verified against `open-api/immich-openapi-specs.json` and the server source at tag `v3.3.1` (`search.service.ts`, `utils/database.ts`, `album.repository.ts`). +- **Visibility.** In the structured path, only `locked` is excluded by default. `hidden` (for example the video part of live photos) and trashed assets would be included. `assetCount` counts `archive` + `timeline`, without trashed assets. The filter above mirrors exactly that, so archived photos in the album are synced. +- **Amazon (from `amazon-photos`, `_api.py`).** + - EU upload host: `https://content-eu.drive.amazonaws.com/cdproxy/nodes`. North America: `content-na`. + - Album: `POST /drive/v1/nodes` `{"kind": "VISUAL_COLLECTION", "name": …}`. + - Add/remove: `PATCH /drive/v1/nodes/{albumId}/children` `{"op": "add" | "remove", "value": [nodeIds]}`. + - Trash: `PATCH /drive/v1/trash` `{"op": "add", "value": [nodeIds], …}`, batched 50 at a time. + - The library uploads to a folder via `parentNodeId`, with no way to upload into an album. Albums are collections, not parents. ## License -MIT +MIT, see [LICENSE](LICENSE) (unchanged from upstream [vanessa/immich-amazon-sync](https://github.com/vanessa/immich-amazon-sync)). diff --git a/docs/anleitung-de.md b/docs/anleitung-de.md new file mode 100644 index 0000000..fec5c8b --- /dev/null +++ b/docs/anleitung-de.md @@ -0,0 +1,411 @@ +# Anleitung: Immich-Album auf dem Echo Show anzeigen + +Ziel: Fotos aus einem Immich-Album landen automatisch in einem Amazon-Photos-Album, das der Echo Show als Diashow anzeigt. + +``` +Immich-Album „Echo Show“ ──▶ immich-amazon-sync (Docker) ──▶ Amazon Photos Album „Echo Show“ ──▶ Echo Show +``` + +Zeitaufwand beim ersten Mal: etwa 30 Minuten. + +## 0. Voraussetzungen + +- **Immich 3.2 oder neuer.** Version prüfen: + ```bash + curl -s http://:2283/api/server/version + ``` + Ausgabe z. B. `{"major":3,"minor":3,"patch":1,...}`. Bei älteren Versionen bricht das Tool mit einer Fehlermeldung ab. +- **Ein Docker-Host**, idealerweise derselbe Rechner, auf dem Immich läuft. +- **Echo Show und Amazon Photos im selben Amazon-Konto (amazon.de).** Der Echo zeigt nur Alben des Kontos, mit dem er registriert ist. +- Amazon Photos ist bei Prime unbegrenzt für Fotos. Videos zählen auf das Speicherkontingent, deshalb ist der Default `SYNC_TYPES=IMAGE`. + +## Schnellstart (Copy & Paste) + +Für einen Linux-Docker-Host mit `bash`, auf dem Immich per Docker Compose läuft. Die Blöcke nacheinander ins Terminal kopieren. Annahmen: +- Das Immich-Album und das Amazon-Album heißen beide **„Echo Show“**. +- Installiert wird nach `~/immich-amazon-sync`. + +Vorher erledigen: +- Album und API-Key in Immich anlegen (Abschnitt 2). +- Cookies und User-Agent aus dem Browser kopieren (Abschnitt 3). + +**1. Klonen und Image bauen** + +```bash +cd ~ && git clone https://github.com/airbone42/immich-amazon-sync.git && cd immich-amazon-sync && docker build --pull -t immich-amazon-sync:local . +``` + +**2. Immich-Netz finden und Verbindung testen.** Ausgabe muss `"major":3` und `"minor"` ≥ 2 enthalten. + +```bash +cd ~/immich-amazon-sync && NET=$(docker network ls --format '{{.Name}}' | grep -m1 immich) && echo "Immich-Netz: $NET" && docker run --rm --network "$NET" --entrypoint python3 immich-amazon-sync:local -c "import urllib.request; print(urllib.request.urlopen('http://immich-server:2283/api/server/version').read().decode())" +``` + +**3. Secrets eingeben.** Die Eingabe ist verdeckt; einfügen und Enter drücken. +- `immich_api_key`: der Immich-API-Key +- `ap_session_id`: Cookie `session-id` +- `ap_ubid_value`: Cookie `ubid-acbde` +- `ap_at_value`: Cookie `at-acbde` + +```bash +cd ~/immich-amazon-sync && mkdir -p data secrets && chmod 700 secrets && for s in immich_api_key ap_session_id ap_ubid_value ap_at_value; do read -rsp "$s: " v; echo; printf '%s' "$v" > "secrets/$s"; done; unset v +``` + +**4. Rechte für den Container-User (UID 10001)** + +```bash +cd ~/immich-amazon-sync && sudo chown 10001:10001 data secrets/* && sudo chmod 700 data && sudo chmod 400 secrets/* +``` + +**5. `.env` anlegen.** Fragt nach dem User-Agent deines Browsers. + +```bash +cd ~/immich-amazon-sync && read -rp "User-Agent aus dem Browser: " UA && cat > .env < compose.yml <': `. Die ID muss mit dem Ende der Album-Adresse übereinstimmen: `https://www.amazon.de/photos/album/`. Steht dort `Would create`, wurde das Album nicht gefunden. + +Fotos, die schon im Album liegen, fasst das Tool nicht an. Es entfernt nur Fotos, die es selbst hinzugefügt hat. + +## 1. Code holen und Image bauen + +Auf dem Docker-Host: + +```bash +git clone https://github.com/airbone42/immich-amazon-sync.git +cd immich-amazon-sync +docker build --pull -t immich-amazon-sync:local . +``` + +## 2. Immich vorbereiten + +### 2.1 Album anlegen + +In Immich ein Album anlegen, z. B. **„Echo Show“**, und die Fotos hinzufügen, die auf dem Echo erscheinen sollen. Der Name muss in Immich eindeutig sein. Mehrere Alben gehen auch (kommagetrennt, siehe unten). + +### 2.2 Eingeschränkten API-Key erstellen + +Immich → Profilbild → **Kontoeinstellungen → API-Schlüssel → Neuer API-Schlüssel**: + +- Name: `amazon-sync` +- Berechtigungen: **nur** diese drei anhaken + - `album.read` + - `asset.read` + - `asset.download` + +Den Schlüssel kopieren, er wird nur einmal angezeigt. + +## 3. Amazon-Cookies besorgen + +Das Tool meldet sich mit den Cookies deines Browsers bei Amazon an, ein offizielles API gibt es nicht. + +1. Am Desktop-Browser (Chrome/Edge/Firefox) auf einloggen. Dabei **„Angemeldet bleiben“** anhaken. +2. **F12** drücken. Reiter **Anwendung** (Chrome/Edge, engl. *Application*) bzw. **Web-Speicher** (Firefox, engl. *Storage*) öffnen. Falls der Reiter nicht zu sehen ist: `»`. +3. Links **Speicher → Cookies → `https://www.amazon.de`** anklicken. +4. Im Filterfeld nach dem Namen suchen und den Wert aus der Spalte **Wert** (*Value*) kopieren: Doppelklick, Strg+A, Strg+C. + + | Cookie | Wird abgefragt als | Sieht ungefähr so aus | + |---|---|---| + | `session-id` | `ap_session_id` | `262-1234567-1234567` | + | `ubid-acbde` | `ap_ubid_value` | `260-1234567-1234567` | + | `at-acbde` | `ap_at_value` | langer Text, beginnt meist mit `Atza|` (vollständig kopieren) | + +5. Den **User-Agent** findest du so: + 1. Reiter **Netzwerk** (*Network*) öffnen und die Seite mit F5 neu laden. + 2. Eine Anfrage an `www.amazon.de` anklicken → **Anforderungsheader** (*Request Headers*) → Zeile `user-agent`. + + Den Wert kopierst du für `AP_USER_AGENT`. + +Fehlt `at-acbde` in der Cookie-Tabelle: Im selben Anforderungsheader steht die Zeile `cookie`. Darin den Teil nach `at-acbde=` bis vor das nächste `;` kopieren. + +Wichtig: +- **Dich danach in diesem Browser nicht bei Amazon abmelden.** Abmelden macht die Cookies ungültig. Den Browser schließen ist okay. +- Die Cookies sind Zugangsdaten zu deinem Amazon-Konto. Behandle sie wie ein Passwort und schick sie nicht per Chat oder Mail. +- Sie laufen nach einigen Wochen bis Monaten ab. Dann diesen Schritt wiederholen (siehe Abschnitt 10). + +## 4. Konfiguration anlegen + +Im Ordner `immich-amazon-sync`: + +```bash +mkdir -p data secrets +chmod 700 secrets + +# Secrets je in eine eigene Datei, ohne Anführungszeichen +printf '%s' 'DEIN_IMMICH_API_KEY' > secrets/immich_api_key +printf '%s' 'DEINE_SESSION_ID' > secrets/ap_session_id +printf '%s' 'DEIN_UBID_ACBDE_WERT' > secrets/ap_ubid_value +printf '%s' 'DEIN_AT_ACBDE_WERT' > secrets/ap_at_value + +# Der Container läuft als UID 10001 und muss Daten und Secrets lesen können +sudo chown 10001:10001 data secrets/* +sudo chmod 700 data +sudo chmod 400 secrets/* +``` + +> Tipp: Ein Leerzeichen vor dem Befehl verhindert bei den meisten Shells, dass die Secrets in der Bash-History landen. Alternativ die Dateien mit einem Editor anlegen. + +Dann die Datei `.env` (ohne Secrets): + +```bash +cat > .env <<'EOF' +IMMICH_URL=http://immich-server:2283 +IMMICH_ALBUMS=Echo Show + +AP_TLD=de +AP_ALBUM_NAME=Echo Show +AP_DELETE_MODE=trash +AP_USER_AGENT=HIER_DEN_USER_AGENT_AUS_SCHRITT_3_EINFUEGEN + +SYNC_TYPES=IMAGE +MAX_FILE_MB=200 +SYNC_INTERVAL_HOURS=6 +MAX_DELETE_FRACTION=0.2 +MAX_DELETE_COUNT=50 +# HEALTHCHECK_URL=https://hc-ping.com/deine-uuid +EOF +chmod 600 .env +``` + +### Welche `IMMICH_URL`? + +- **Immich läuft per Docker Compose auf demselben Host** (Normalfall): Der Sync-Container wird ins Docker-Netz von Immich gehängt (siehe Schritt 5). Dann ist die URL `http://immich-server:2283`. Den Netzwerknamen findest du mit: + ```bash + docker network ls | grep immich + ``` + Bei der offiziellen Immich-Compose-Datei heißt das Netz meist `immich_default`. +- **Immich läuft auf einem anderen Rechner:** `http://:2283`. Dann in Schritt 5 den Block `networks` weglassen. +- Die URL ohne `/api` angeben. + +## 5. Docker Compose anlegen + +`compose.yml` im selben Ordner: + +```yaml +services: + immich-sync: + image: immich-amazon-sync:local + container_name: immich-sync + restart: unless-stopped + env_file: .env + environment: + IMMICH_API_KEY_FILE: /run/secrets/immich_api_key + AP_SESSION_ID_FILE: /run/secrets/ap_session_id + AP_UBID_VALUE_FILE: /run/secrets/ap_ubid_value + AP_AT_VALUE_FILE: /run/secrets/ap_at_value + secrets: [immich_api_key, ap_session_id, ap_ubid_value, ap_at_value] + volumes: + - ./data:/data + read_only: true + tmpfs: [/tmp] + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + networks: [immich] + +secrets: + immich_api_key: { file: ./secrets/immich_api_key } + ap_session_id: { file: ./secrets/ap_session_id } + ap_ubid_value: { file: ./secrets/ap_ubid_value } + ap_at_value: { file: ./secrets/ap_at_value } + +networks: + immich: + external: true + name: immich_default # Name aus `docker network ls` +``` + +## 6. Erster Test: Dry-Run + +Der Dry-Run schreibt nichts nach Amazon und nichts in den State. Er prüft aber alles: Immich-Version, Album, API-Key, Cookies und das Amazon-Album. + +```bash +docker compose run --rm -e DRY_RUN=true -e RUN_ONCE=true immich-sync +``` + +Erwartete Ausgabe (gekürzt): + +``` +Starting Immich → Amazon Photos sync [DRY RUN] +Immich server version 3.3.1 +Amazon Photos root node: ... +[dry-run] Would create Amazon Photos album 'Echo Show' +Album 'Echo Show': 42 assets, 42 of type IMAGE +42 assets to mirror, 42 new + [dry-run] Would upload IMG_1234.jpg (...) +Done. Uploaded: 0, Deleted: 0, Skipped: 0, Errors: 0, Tracked: 0 [DRY RUN - nothing written] +``` + +Typische Fehler: + +| Meldung | Ursache / Lösung | +|---|---| +| `Immich ... is too old` | Immich auf ≥ 3.2 aktualisieren | +| `Album not found in Immich: 'Echo Show'` | Name in `IMMICH_ALBUMS` exakt wie in Immich schreiben, Groß-/Kleinschreibung beachten | +| `HTTP Error 401` bei Immich | API-Key falsch oder ohne die drei Berechtigungen | +| `HTTP Error 403` bei Immich | API-Key fehlt eine der drei Berechtigungen | +| `Name or service not known` / `Connection refused` | `IMMICH_URL` oder Netzwerkname in `compose.yml` prüfen | +| `cookies expired or invalid` (Exit-Code 3) | Cookies falsch kopiert oder abgelaufen → Schritt 3 wiederholen | +| `Permission denied` auf `/data` oder `/run/secrets/...` | `chown 10001:10001` aus Schritt 4 vergessen | + +## 7. Erster echter Lauf + +```bash +docker compose run --rm -e RUN_ONCE=true immich-sync +``` + +Danach unter → **Alben** prüfen: Das Album **„Echo Show“** sollte existieren und alle Fotos enthalten. Die Dateien liegen zusätzlich in der Bibliothek; das ist normal, denn Amazon-Alben verweisen nur auf Dateien. + +Ein zweiter Aufruf desselben Befehls sollte `Uploaded: 0, Deleted: 0` melden. + +## 8. Dauerbetrieb starten + +```bash +docker compose up -d +docker logs -f immich-sync +``` + +Das Tool synchronisiert jetzt alle 6 Stunden (`SYNC_INTERVAL_HOURS`). Einen Sync sofort anstoßen: + +```bash +docker restart immich-sync +``` + +## 9. Echo Show einrichten + +Am Echo Show: + +1. Vom oberen Bildschirmrand nach unten wischen → **Einstellungen**. +2. **Startbildschirm & Uhr** → **Uhr und Fotoanzeige** (bei manchen Firmware-Versionen heißt es **Hintergrund** oder **Fotoanzeige**). +3. **Amazon Photos** wählen → Album **„Echo Show“** auswählen. + +Alternativ geht es in der **Alexa-App**: **Geräte** → **Echo & Alexa** → dein Echo Show → **Fotoanzeige** / **Startbildschirm**. + +Die Menübezeichnungen ändern sich gelegentlich mit Firmware-Updates. Gesucht ist die Einstellung für die Fotoquelle der Uhr bzw. des Startbildschirms. + +Neue Fotos erscheinen auf dem Echo, nachdem der nächste Sync gelaufen ist und der Echo das Album aktualisiert hat. Das kann ein paar Stunden dauern. + +## 10. Alltag + +- **Foto hinzufügen:** In Immich ins Album legen. Es erscheint nach dem nächsten Sync. +- **Foto entfernen:** In Immich aus dem Album nehmen. Beim nächsten Sync wird es aus dem Amazon-Album entfernt und in den Amazon-Papierkorb verschoben. Das Amazon-Album hat damit immer denselben Inhalt wie das Immich-Album, es sammelt sich nichts an. + - Mit `AP_DELETE_MODE=album_only` bleibt es stattdessen in der Amazon-Bibliothek. + - Fotos, die schon vorher in Amazon Photos lagen (z. B. aus der Handy-Sicherung), werden nie in den Papierkorb verschoben, nur aus dem Album genommen. + - Fotos, die schon vor dem ersten Sync im Amazon-Album waren, fasst das Tool nicht an. Die einmal von Hand entfernen. +- **Löschbremse:** Würden in einem Lauf mehr als 20 % oder mehr als 50 der Fotos gelöscht, löscht das Tool gar nichts und meldet `Deletion brake` (Exit-Code 2). Bei kleinen Alben passiert das schnell: 1 von 4 Fotos sind schon 25 %. Wenn das Absicht war, einmalig mit höherer Grenze laufen lassen: + ```bash + docker compose run --rm -e RUN_ONCE=true -e MAX_DELETE_FRACTION=1 immich-sync + ``` +- **Album, das sich oft komplett ändert** (z. B. täglich neu befüllt per Skript oder Automatisierung): Die Löschbremse würde dann jeden Tag greifen, und die alten Fotos blieben drin. Grenzen dauerhaft hochsetzen: + ```bash + cd ~/immich-amazon-sync && sed -i 's/^MAX_DELETE_FRACTION=.*/MAX_DELETE_FRACTION=1/; s/^MAX_DELETE_COUNT=.*/MAX_DELETE_COUNT=500/' .env && docker compose up -d --force-recreate + ``` + Ein unvollständiger Datenstand aus Immich löscht auch dann nichts: kein Album gefunden, Abfrage fehlgeschlagen oder Anzahl stimmt nicht. Leert die Automatisierung das Album aber kurz komplett und der Sync läuft genau dann, wird auch das Amazon-Album geleert. Beim nächsten Lauf wird alles wieder hochgeladen. +- **Neustart des Docker-Hosts:** Dank `restart: unless-stopped` startet der Container automatisch wieder, sofern Docker beim Booten startet (`systemctl is-enabled docker containerd` sollte zweimal `enabled` ausgeben) und du ihn nicht vorher selbst gestoppt hast. Nach dem Start synchronisiert er sofort. Startet Immich langsamer, schlägt dieser erste Lauf fehl, ohne etwas zu löschen. Der nächste Versuch kommt nach `SYNC_INTERVAL_HOURS`, oder sofort per `docker restart immich-sync`. +- **Cookies abgelaufen:** Im Log steht `cookies expired or invalid`, Exit-Code 3. Dann: + 1. Schritt 3 wiederholen. + 2. Die drei Dateien in `secrets/` überschreiben. Sie gehören jetzt UID 10001, also mit `sudo`, z. B.: + ```bash + sudo sh -c "printf '%s' 'NEUER_AT_WERT' > secrets/ap_at_value" + ``` + 3. `docker compose up -d --force-recreate` ausführen. +- **Überwachung (empfohlen):** Bei einen Check mit Periode 6 h anlegen und `HEALTHCHECK_URL` setzen. Bleibt der Ping aus (Cookies abgelaufen, Immich nicht erreichbar, Löschbremse), bekommst du eine Mail. +- **Updates des Tools:** + ```bash + git pull + docker build --pull -t immich-amazon-sync:local . + docker compose up -d --force-recreate + ``` +- **Logs:** `docker logs immich-sync` oder `data/immich-sync.log`. +- **Alles neu synchronisieren:** `data/state.json` löschen. Vorhandene Fotos werden über die MD5-Prüfung wiedererkannt und nicht doppelt hochgeladen. + +## Hinweis + +Das Tool nutzt die inoffiziellen Schnittstellen der Amazon-Photos-Webseite. Amazon kann sie jederzeit ändern. Die Nutzung erfolgt auf eigenes Risiko und kann gegen die Amazon-Nutzungsbedingungen verstoßen. diff --git a/entrypoint.sh b/entrypoint.sh old mode 100644 new mode 100755 index 0f595ee..d511c7b --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,14 +1,58 @@ #!/bin/sh -set -e +set -eu +umask 077 INTERVAL_HOURS="${SYNC_INTERVAL_HOURS:-6}" +case "$INTERVAL_HOURS" in + '' | *[!0-9]*) echo "SYNC_INTERVAL_HOURS must be a whole number of hours" >&2; exit 1 ;; +esac +# strip leading zeros so "08" is not read as octal +INTERVAL_HOURS=$(echo "$INTERVAL_HOURS" | sed 's/^0*//') +if [ -z "$INTERVAL_HOURS" ]; then + echo "SYNC_INTERVAL_HOURS must be at least 1" >&2 + exit 1 +fi INTERVAL_SECONDS=$((INTERVAL_HOURS * 3600)) +if [ "$(id -u)" = "0" ]; then + echo "Warning: running as root. The image is meant to run as UID 10001." >&2 +fi + +case "${RUN_ONCE:-false}" in + 1 | true | yes) + echo "Immich -> Amazon Photos sync: single run" + exec python3 /app/immich_sync.py + ;; +esac + +# PID 1 ignores SIGTERM by default: forward it so `docker stop` ends promptly +# instead of waiting for SIGKILL. State writes are atomic, so stopping mid-run is safe. +child="" +stop() { + echo "Stopping..." + if [ -n "$child" ]; then + kill -TERM "$child" 2>/dev/null || true + wait "$child" 2>/dev/null || true + fi + exit 0 +} +trap stop TERM INT + echo "Immich -> Amazon Photos sync starting. Interval: every ${INTERVAL_HOURS}h" while true; do echo "=== Sync run: $(date -Iseconds) ===" - python3 /app/immich_sync.py || echo "Sync finished with errors (will retry next run)" + python3 /app/immich_sync.py & + child=$! + rc=0 + wait "$child" || rc=$? + child="" + if [ "$rc" -ne 0 ]; then + echo "Sync finished with exit code $rc (will retry next run)" + fi echo "=== Next run in ${INTERVAL_HOURS}h ===" - sleep "$INTERVAL_SECONDS" + sleep "$INTERVAL_SECONDS" & + child=$! + wait "$child" || true + child="" done diff --git a/src/immich_sync.py b/src/immich_sync.py index 428e50d..ab2f49e 100644 --- a/src/immich_sync.py +++ b/src/immich_sync.py @@ -2,102 +2,338 @@ """ Immich → Amazon Photos sync. -Syncs photos from specified Immich albums to Amazon Photos using the -Amazon Drive v1 API directly. Zero external dependencies - stdlib only. +Syncs photos from specified Immich albums into an Amazon Photos album using +the (unofficial) Amazon Drive v1 API directly. Zero external dependencies - +stdlib only. Features: - - One-way sync: Immich albums → Amazon Photos root folder - - Deletion: photos removed from Immich albums are trashed in Amazon Photos + - One-way sync: Immich albums → Amazon Photos album (files live in root) + - Deletion: photos removed from all Immich albums are removed from the + Amazon album and (optionally) trashed - guarded by a deletion brake - State tracking: maps Immich asset IDs → Amazon node IDs in a JSON file - - Deduplication: skips assets already synced (by ID, not content hash) - - HEIC/JPEG/PNG/etc: uploads whatever Immich serves as the original file + - Deduplication: skips assets already synced (by ID); upload conflicts are + resolved by MD5 so existing Amazon nodes are adopted, not duplicated + - Region aware: amazon.de by default, any TLD via AP_TLD """ +import gzip +import hashlib import json import logging +import logging.handlers +import math +import mimetypes import os -import shutil +import re import sys import tempfile import uuid -from pathlib import Path +from pathlib import Path, PurePosixPath from urllib.error import HTTPError -from urllib.request import Request, urlopen +from urllib.parse import quote, urlsplit +from urllib.request import HTTPRedirectHandler, Request, build_opener, install_opener, urlopen # Constants -AMAZON_DRIVE_URL = "https://www.amazon.com/drive/v1" -AMAZON_CONTENT_URL = "https://content-na.drive.amazonaws.com/cdproxy" AMAZON_BASE_PARAMS = "asset=ALL&tempLink=false&resourceVersion=V2&ContentType=JSON" +DEFAULT_USER_AGENT = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36" +) + +# Upload hosts per region, taken from the amazon-photos library +# (trevorhobenshield/amazon_photos, AmazonPhotos.determine_cdproxy). +AMAZON_CONTENT_NA = "https://content-na.drive.amazonaws.com/cdproxy" +AMAZON_CONTENT_EU = "https://content-eu.drive.amazonaws.com/cdproxy" +NA_TLDS = {"com", "ca"} +EU_TLDS = { + "at", "be", "bg", "hr", "ch", "cy", "cz", "dk", "ee", "fi", "fr", "de", + "gr", "hu", "is", "ie", "it", "li", "lv", "lt", "lu", "mt", "no", "nl", + "pl", "pt", "ro", "sk", "si", "es", "se", "uk", "co.uk", +} + +MIN_IMMICH_VERSION = (3, 2) +IMMICH_PAGE_SIZE = 1000 # max allowed by POST /search/metadata +AMAZON_PAGE_SIZE = 200 # max nodes per GET /nodes request +AMAZON_MAX_OFFSET = 9999 # Amazon refuses offset + limit > 9999 +AMAZON_BATCH_SIZE = 50 # nodes per album/trash PATCH (amazon-photos uses 50) +DOWNLOAD_CHUNK = 1024 * 1024 +STATE_VERSION = 2 + +EXIT_OK = 0 +EXIT_ERROR = 1 +EXIT_DELETE_BLOCKED = 2 +EXIT_AUTH = 3 + +UUID_RE = re.compile( + r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}" +) # Runtime config (populated by _configure()) IMMICH_URL: str = "" -IMMICH_API_KEY: str = "" ALBUMS_TO_SYNC: list[str] = [] AP_HEADERS: dict[str, str] = {} IMMICH_HEADERS: dict[str, str] = {} +AMAZON_DRIVE_URL: str = "" +AMAZON_CONTENT_URL: str = "" +AP_ALBUM_NAME: str = "" +DELETE_MODE: str = "trash" +DRY_RUN: bool = False +MAX_DELETE_FRACTION: float = 0.2 +MAX_DELETE_COUNT: int = 50 +MAX_FILE_BYTES: int = 200 * 1024 * 1024 +SYNC_TYPES: set[str] = {"IMAGE"} +HEALTHCHECK_URL: str = "" +STATE_DIR: Path = Path("/data") STATE_FILE: Path = Path("/data/state.json") -log = logging.getLogger(__name__) +# Values that must never show up in logs (see _RedactingFormatter) +_SECRETS: set[str] = set() +log = logging.getLogger("immich_sync") -def _require_env(name: str) -> str: - """Get a required environment variable or exit with a clear message.""" + +class ConfigError(Exception): + """Invalid or missing configuration.""" + + +class FatalError(Exception): + """Unrecoverable problem - abort the run before touching anything.""" + + +class SkipAsset(Exception): + """Asset is deliberately not synced in this run (e.g. too large).""" + + +class AmazonError(Exception): + """Non-2xx response from Amazon. `body` holds (a prefix of) the response.""" + + def __init__(self, code: int, body: str = ""): + self.code = code + self.body = body + super().__init__(f"HTTP {code}: {body[:200]}") + + +class AmazonAuthError(AmazonError): + """401/403 from Amazon - cookies expired or invalid.""" + + +# Configuration + + +def _env(name: str, default: str | None = None) -> str | None: + """Read NAME, or the contents of the file named by NAME_FILE (Docker secrets).""" + file_path = os.getenv(f"{name}_FILE") + if file_path: + try: + return Path(file_path).read_text(encoding="utf-8").strip() + except OSError as e: + raise ConfigError(f"cannot read {name}_FILE: {e.strerror}") from None value = os.getenv(name) + return value if value else default + + +def _require_env(name: str) -> str: + """Get a required setting or raise with a clear message.""" + value = _env(name) if not value: - print( - f"Error: required environment variable {name} is not set", file=sys.stderr - ) - sys.exit(1) + raise ConfigError(f"required environment variable {name} (or {name}_FILE) is not set") return value +def _secret(name: str) -> str: + """Required setting that must never be logged and must be a safe header value.""" + value = _require_env(name) + if any(ch in value for ch in "\r\n;") or not value.isprintable(): + raise ConfigError(f"{name} contains characters that are not allowed in a cookie/header") + _SECRETS.add(value) + return value + + +def _env_bool(name: str, default: bool = False) -> bool: + value = _env(name) + if value is None: + return default + if value.lower() in ("1", "true", "yes", "on"): + return True + if value.lower() in ("0", "false", "no", "off"): + return False + raise ConfigError(f"{name} must be true or false") + + +def _env_number(name: str, default: float, cast=float): + value = _env(name) + if value is None: + return default + try: + number = cast(value) + except ValueError: + raise ConfigError(f"{name} must be a number") from None + if not math.isfinite(number) or number < 0: + raise ConfigError(f"{name} must be a finite, non-negative number") + return number + + +def _http_url(name: str, value: str, *, https_only: bool = False) -> str: + parts = urlsplit(value) + allowed = ("https",) if https_only else ("http", "https") + if parts.scheme not in allowed or not parts.netloc: + raise ConfigError(f"{name} must be an {' or '.join(allowed)} URL") + return value.rstrip("/") + + +def region_settings(tld: str) -> dict[str, str | None]: + """Derive Amazon hosts and cookie names from a TLD such as 'de' or 'co.uk'.""" + tld = tld.strip().lower().lstrip(".") + if not re.fullmatch(r"[a-z]{2,3}(\.[a-z]{2,3})?", tld): + raise ConfigError(f"AP_TLD '{tld}' is not a valid Amazon top-level domain") + # amazon.com uses "-main", every other marketplace "-acb" + # (e.g. amazon.de → ubid-acbde / at-acbde, amazon.co.uk → ubid-acbuk / at-acbuk) + suffix = "main" if tld == "com" else "acb" + tld.rsplit(".", 1)[-1] + if tld in NA_TLDS: + content_url = AMAZON_CONTENT_NA + elif tld in EU_TLDS: + content_url = AMAZON_CONTENT_EU + else: + content_url = None + return { + "drive_url": f"https://www.amazon.{tld}/drive/v1", + "content_url": content_url, + "ubid_key": f"ubid-{suffix}", + "at_key": f"at-{suffix}", + } + + def _configure() -> None: """Read env vars and set up module-level config. Called once from main().""" - global IMMICH_URL, IMMICH_API_KEY, ALBUMS_TO_SYNC - global AP_HEADERS, IMMICH_HEADERS, STATE_FILE + global IMMICH_URL, ALBUMS_TO_SYNC, AP_HEADERS, IMMICH_HEADERS + global AMAZON_DRIVE_URL, AMAZON_CONTENT_URL, AP_ALBUM_NAME, DELETE_MODE, DRY_RUN + global MAX_DELETE_FRACTION, MAX_DELETE_COUNT, MAX_FILE_BYTES, SYNC_TYPES + global HEALTHCHECK_URL, STATE_DIR, STATE_FILE - IMMICH_URL = _require_env("IMMICH_URL").rstrip("/") - IMMICH_API_KEY = _require_env("IMMICH_API_KEY") + _SECRETS.clear() + + IMMICH_URL = _http_url("IMMICH_URL", _require_env("IMMICH_URL")) + immich_api_key = _secret("IMMICH_API_KEY") ALBUMS_TO_SYNC = [ a.strip() for a in _require_env("IMMICH_ALBUMS").split(",") if a.strip() ] + if not ALBUMS_TO_SYNC: + raise ConfigError("IMMICH_ALBUMS does not contain any album name") + + region = region_settings(_env("AP_TLD", "de")) + AMAZON_DRIVE_URL = region["drive_url"] + content_override = _env("AP_CONTENT_URL") + if content_override: + content_override = _http_url("AP_CONTENT_URL", content_override, https_only=True) + AMAZON_CONTENT_URL = re.sub(r"/nodes$", "", content_override) + elif region["content_url"]: + AMAZON_CONTENT_URL = region["content_url"] + else: + raise ConfigError( + f"no known upload host for AP_TLD={_env('AP_TLD')}; set AP_CONTENT_URL" + ) - _ubid_key = _require_env("AP_UBID_KEY") - _at_key = _require_env("AP_AT_KEY") + ubid_key = _env("AP_UBID_KEY", region["ubid_key"]) + at_key = _env("AP_AT_KEY", region["at_key"]) + for key_name, key in (("AP_UBID_KEY", ubid_key), ("AP_AT_KEY", at_key)): + if not re.fullmatch(r"[A-Za-z0-9_-]+", key): + raise ConfigError(f"{key_name} is not a valid cookie name") + + session_id = _secret("AP_SESSION_ID") + ubid_value = _secret("AP_UBID_VALUE") + at_value = _secret("AP_AT_VALUE") + user_agent = _env("AP_USER_AGENT", DEFAULT_USER_AGENT) + if not user_agent.isprintable(): + raise ConfigError("AP_USER_AGENT contains control characters") - session_id = _require_env("AP_SESSION_ID") - ap_cookie_str = ( - f"session-id={session_id}; " - f"{_ubid_key}={_require_env('AP_UBID_VALUE')}; " - f"{_at_key}={_require_env('AP_AT_VALUE')}" - ) AP_HEADERS = { - "Cookie": ap_cookie_str, + "Cookie": f"session-id={session_id}; {ubid_key}={ubid_value}; {at_key}={at_value}", "x-amzn-sessionid": session_id, - "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " - "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "User-Agent": user_agent, } + IMMICH_HEADERS = {"x-api-key": immich_api_key, "Accept": "application/json"} + + AP_ALBUM_NAME = _require_env("AP_ALBUM_NAME").strip() + DELETE_MODE = _env("AP_DELETE_MODE", "trash").lower() + if DELETE_MODE not in ("trash", "album_only"): + raise ConfigError("AP_DELETE_MODE must be 'trash' or 'album_only'") + + DRY_RUN = _env_bool("DRY_RUN") + MAX_DELETE_FRACTION = _env_number("MAX_DELETE_FRACTION", 0.2) + MAX_DELETE_COUNT = _env_number("MAX_DELETE_COUNT", 50, int) + MAX_FILE_BYTES = int(_env_number("MAX_FILE_MB", 200) * 1024 * 1024) + + SYNC_TYPES = {t.strip().upper() for t in _env("SYNC_TYPES", "IMAGE").split(",") if t.strip()} + if not SYNC_TYPES or not SYNC_TYPES <= {"IMAGE", "VIDEO"}: + raise ConfigError("SYNC_TYPES must be IMAGE, VIDEO or IMAGE,VIDEO") + + HEALTHCHECK_URL = _env("HEALTHCHECK_URL", "") + if HEALTHCHECK_URL: + HEALTHCHECK_URL = _http_url("HEALTHCHECK_URL", HEALTHCHECK_URL) + _SECRETS.add(HEALTHCHECK_URL) # ping URLs usually embed a token + + STATE_DIR = Path(_env("STATE_DIR", "/data")) + STATE_FILE = STATE_DIR / "state.json" + + +# Logging + + +class _RedactingFormatter(logging.Formatter): + """Formatter that masks every configured secret, including in tracebacks.""" + + def format(self, record: logging.LogRecord) -> str: + text = super().format(record) + for secret in _SECRETS: + if len(secret) >= 4: + text = text.replace(secret, "***") + return text + + +def _setup_logging() -> None: + _teardown_logging() + STATE_DIR.mkdir(parents=True, exist_ok=True) + log_file = STATE_DIR / "immich-sync.log" + # create the log file 0600 before the handler opens it + os.close(os.open(log_file, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600)) + os.chmod(log_file, 0o600) + formatter = _RedactingFormatter("%(asctime)s [%(levelname)s] %(message)s") + handlers = [ + logging.StreamHandler(sys.stdout), + logging.handlers.RotatingFileHandler( + log_file, maxBytes=5 * 1024 * 1024, backupCount=3, encoding="utf-8" + ), + ] + for handler in handlers: + handler.setFormatter(formatter) + log.addHandler(handler) + log.setLevel(logging.INFO) + log.propagate = False - IMMICH_HEADERS = {"x-api-key": IMMICH_API_KEY} - - state_dir = Path(os.getenv("STATE_DIR", "/data")) - STATE_FILE = state_dir / "state.json" - logging.basicConfig( - level=logging.INFO, - format="%(asctime)s [%(levelname)s] %(message)s", - handlers=[ - logging.StreamHandler(sys.stdout), - logging.FileHandler(state_dir / "immich-sync.log"), - ], - ) +def _teardown_logging() -> None: + for handler in list(log.handlers): + if isinstance(handler.formatter, _RedactingFormatter): + log.removeHandler(handler) + handler.close() # HTTP helpers +class _NoRedirect(HTTPRedirectHandler): + """Never follow redirects: an expired Amazon session redirects to the sign-in + page, and following it would forward the Cookie header to another URL.""" + + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None + + +install_opener(build_opener(_NoRedirect)) + + def _url_with_params(base: str, extra_params: str = "") -> str: params = AMAZON_BASE_PARAMS if extra_params: @@ -109,93 +345,258 @@ def _request( url: str, *, headers: dict | None = None, - data: bytes | None = None, + data=None, method: str | None = None, timeout: int = 30, ) -> bytes: """Make an HTTP request and return the response body. Raises on non-2xx.""" req = Request(url, data=data, headers=headers or {}, method=method) with urlopen(req, timeout=timeout) as resp: - return resp.read() + body = resp.read() + encoding = (resp.headers.get("Content-Encoding") or "").strip().lower() + # amazon.de answers some writes (PATCH .../children) gzip-compressed without being asked to, + # and urllib does not decode that. The request itself succeeded; only reading the reply failed. + if encoding == "gzip" or body[:2] == b"\x1f\x8b": + body = gzip.decompress(body) + return body def _json_request( url: str, *, headers: dict | None = None, - data: bytes | None = None, + data=None, method: str | None = None, timeout: int = 30, ) -> dict: body = _request(url, headers=headers, data=data, method=method, timeout=timeout) - return json.loads(body) - - -# Multipart form-data encoder + return json.loads(body) if body.strip() else {} -def _encode_multipart( - fields: dict[str, tuple[str | None, bytes, str]], -) -> tuple[bytes, str]: - """Encode fields as multipart/form-data. - - Each field value is a tuple of (filename | None, data_bytes, content_type). - Returns (body_bytes, content_type_header). +def _error_body(e: HTTPError, limit: int = 4096) -> str: + try: + return e.read(limit).decode("utf-8", "replace") + except Exception: + return "" + + +# Streaming multipart form-data encoder + + +class _MultipartStream: + """File-like multipart body that streams file parts instead of loading them.""" + + def __init__(self, parts: list): + self._parts = parts + self._length = 0 + for part in parts: + if isinstance(part, bytes): + self._length += len(part) + else: + self._length += os.fstat(part.fileno()).st_size - part.tell() + + def __len__(self) -> int: + return self._length + + def read(self, size: int = -1) -> bytes: + out = bytearray() + while self._parts and (size < 0 or len(out) < size): + part = self._parts[0] + want = -1 if size < 0 else size - len(out) + if isinstance(part, bytes): + chunk = part if want < 0 else part[:want] + rest = part[len(chunk):] + if rest: + self._parts[0] = rest + else: + self._parts.pop(0) + else: + chunk = part.read(want) + if not chunk: + self._parts.pop(0) + continue + out += chunk + return bytes(out) + + +def _encode_multipart(fields: dict) -> tuple[_MultipartStream, str]: + """Encode fields as a streaming multipart/form-data body. + + Each field value is a tuple of (filename | None, bytes | binary file, content_type). + Returns (body_stream, content_type_header); len(body_stream) is the exact size. """ boundary = f"----FormBoundary{uuid.uuid4().hex[:16]}" - parts: list[bytes] = [] + parts: list = [] for field_name, (filename, data, content_type) in fields.items(): disposition = f'form-data; name="{field_name}"' if filename: disposition += f'; filename="{filename}"' - part = ( - f"--{boundary}\r\n" - f"Content-Disposition: {disposition}\r\n" - f"Content-Type: {content_type}\r\n" - f"\r\n" - ).encode() - parts.append(part + data + b"\r\n") + parts.append( + ( + f"--{boundary}\r\n" + f"Content-Disposition: {disposition}\r\n" + f"Content-Type: {content_type}\r\n" + f"\r\n" + ).encode() + ) + parts.append(data) + parts.append(b"\r\n") parts.append(f"--{boundary}--\r\n".encode()) - body = b"".join(parts) - return body, f"multipart/form-data; boundary={boundary}" + return _MultipartStream(parts), f"multipart/form-data; boundary={boundary}" + + +# File names + + +def safe_filename(raw: str | None, asset_id: str, mime_type: str | None = None) -> str: + """Turn an untrusted Immich file name into a harmless display name. + + Strips directories, control characters and quotes. Falls back to + . when nothing usable is left. + """ + name = PurePosixPath((raw or "").replace("\\", "/")).name + name = "".join(ch for ch in name if ch.isprintable() and ch != '"').strip() + if name in ("", ".", ".."): + ext = "" + if raw: + ext = PurePosixPath(raw.replace("\\", "/")).suffix + if not re.fullmatch(r"\.[A-Za-z0-9]{1,10}", ext or ""): + ext = mimetypes.guess_extension(mime_type or "") or ".jpg" + return f"{asset_id}{ext}" + if len(name) > 200: + stem, dot, ext = name.rpartition(".") + name = f"{stem[:190]}.{ext[:9]}" if dot and len(ext) <= 9 else name[:200] + return name + + +def _disambiguated_name(name: str, asset_id: str) -> str: + path = PurePosixPath(name) + return f"{path.stem}_{asset_id[:8]}{path.suffix}" # State -# state.json schema: {"synced": {"": "", ...}} +# state.json schema (v2): +# {"version": 2, "album_node_id": "" | null, +# "synced": {"": "", ...}, +# "pending_album_add": ["", ...]} + + +def _empty_state() -> dict: + return {"version": STATE_VERSION, "album_node_id": None, "synced": {}, + "pending_album_add": [], "adopted_nodes": []} def load_state() -> dict: - """Return dict of immich_asset_id → amazon_node_id.""" - if STATE_FILE.exists(): - try: - data = json.loads(STATE_FILE.read_text()) - # Migrate old format (flat list of IDs) → new format - if isinstance(data.get("synced_ids"), list): - log.info( - "Migrating state from old format (synced_ids list). " - "Previously synced photos will be re-uploaded once." - ) - return {} - return data.get("synced", {}) - except Exception: - log.warning("State file corrupt, starting fresh.") - return {} - - -def save_state(synced: dict) -> None: + """Load the state file, migrating older formats. Raises FatalError if unreadable.""" + if not STATE_FILE.exists(): + return _empty_state() + try: + data = json.loads(STATE_FILE.read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise ValueError("not a JSON object") + except (OSError, ValueError) as e: + raise FatalError( + f"State file {STATE_FILE} is unreadable ({e}). Fix or remove it - " + "refusing to continue so tracked photos are not lost." + ) from None + + version = data.get("version", 1) + if version == STATE_VERSION: + state = _empty_state() + state.update( + album_node_id=data.get("album_node_id"), + synced=dict(data.get("synced") or {}), + pending_album_add=list(data.get("pending_album_add") or []), + adopted_nodes=list(data.get("adopted_nodes") or []), + ) + return state + if version != 1: + raise FatalError(f"State file version {version} is newer than this tool supports.") + + state = _empty_state() + if isinstance(data.get("synced_ids"), list): + log.info( + "Migrating state from old format (synced_ids list). " + "Previously synced photos will be matched again by upload conflict." + ) + return state + state["synced"] = dict(data.get("synced") or {}) + log.info(f"Migrating state v1 → v2 ({len(state['synced'])} tracked assets, no re-upload)") + return state + + +def save_state(state: dict) -> None: + """Atomically write the state file (0600). No-op in dry-run mode.""" + if DRY_RUN: + return STATE_FILE.parent.mkdir(parents=True, exist_ok=True) - STATE_FILE.write_text(json.dumps({"synced": synced}, indent=2)) + fd, tmp_name = tempfile.mkstemp(dir=STATE_FILE.parent, prefix=".state-", suffix=".tmp") + try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + json.dump(state, f, indent=2) + f.flush() + os.fsync(f.fileno()) + os.chmod(tmp_name, 0o600) + os.replace(tmp_name, STATE_FILE) + except BaseException: + Path(tmp_name).unlink(missing_ok=True) + raise # Amazon Drive API +def _amazon_call(url: str, *, data=None, method: str | None = None, + headers: dict | None = None, timeout: int = 30) -> bytes: + try: + return _request(url, headers={**AP_HEADERS, **(headers or {})}, + data=data, method=method, timeout=timeout) + except HTTPError as e: + body = _error_body(e) + if e.code in (401, 403) or 300 <= e.code < 400: + # 3xx: Amazon sends expired sessions to the sign-in page + raise AmazonAuthError(e.code, body) from None + raise AmazonError(e.code, body) from None + + def ap_get(path: str, extra_params: str = "") -> dict: url = _url_with_params(f"{AMAZON_DRIVE_URL}/{path.lstrip('/')}", extra_params) - return _json_request(url, headers=AP_HEADERS) + raw = _amazon_call(url) + return json.loads(raw) if raw.strip() else {} + + +def ap_write(method: str, path: str, body: dict) -> dict: + """JSON write request, shaped like the amazon-photos library sends it.""" + payload = json.dumps({**body, "resourceVersion": "V2", "ContentType": "JSON"}).encode() + raw = _amazon_call( + f"{AMAZON_DRIVE_URL}/{path.lstrip('/')}", + data=payload, + method=method, + headers={"Content-Type": "application/json"}, + ) + return json.loads(raw) if raw.strip() else {} + + +def list_nodes(filters: str) -> list[dict]: + """GET /nodes with an Amazon filter expression, following offset pagination.""" + nodes: list[dict] = [] + offset = 0 + while offset < AMAZON_MAX_OFFSET: + limit = min(AMAZON_PAGE_SIZE, AMAZON_MAX_OFFSET - offset) + page = ap_get("nodes", f"filters={quote(filters, safe='')}&limit={limit}&offset={offset}") + data = page.get("data") or [] + nodes.extend(data) + offset += len(data) + if not data or offset >= int(page.get("count") or 0): + break + return nodes + + +def _escape_filter(value: str) -> str: + return "".join("\\" + ch if ch in '+-&|!(){}[]^"~*?:\\/\' ' else ch for ch in value) def get_root_id() -> str: @@ -203,176 +604,572 @@ def get_root_id() -> str: return data["data"][0]["id"] -def upload_to_amazon(local_path: Path, root_id: str) -> str | None: - """Upload a file to Amazon Photos root folder. +def find_or_create_album(name: str, known_id: str | None) -> str | None: + """Return the node ID of the Amazon Photos album called `name`, creating it if needed. - Returns the Amazon node ID on success, or None if the file already - exists (409 conflict) and the response doesn't include the existing ID. + Returns None only in dry-run mode when the album would have to be created. """ - metadata = json.dumps( - { - "name": local_path.name, - "kind": "FILE", - "parents": [root_id], - } - ).encode() - - file_data = local_path.read_bytes() - - body, content_type = _encode_multipart( - { - "metadata": (None, metadata, "application/json"), - "content": (local_path.name, file_data, "application/octet-stream"), - } - ) + if known_id: + try: + node = ap_get(f"nodes/{quote(known_id, safe='')}") + except AmazonAuthError: + raise + except AmazonError as e: + if e.code != 404: + raise + node = {} + if (node.get("kind") == "VISUAL_COLLECTION" and node.get("status", "AVAILABLE") == "AVAILABLE" + and node.get("name") == name): + return known_id + log.info(f"Stored album node {known_id} is gone or renamed - looking up '{name}'") + + matches = [ + n for n in list_nodes("kind:VISUAL_COLLECTION") + if n.get("name") == name and n.get("status", "AVAILABLE") == "AVAILABLE" + ] + if matches: + matches.sort(key=lambda n: n.get("createdDate", "")) + if len(matches) > 1: + log.warning(f"{len(matches)} Amazon albums are named '{name}' - using the oldest") + return matches[0]["id"] - url = _url_with_params(f"{AMAZON_CONTENT_URL}/nodes") - headers = {**AP_HEADERS, "Content-Type": content_type} + if DRY_RUN: + log.info(f"[dry-run] Would create Amazon Photos album '{name}'") + return None + created = ap_write("POST", "nodes", {"kind": "VISUAL_COLLECTION", "name": name}) + log.info(f"Created Amazon Photos album '{name}' → {created['id']}") + return created["id"] - try: - resp = _json_request( - url, headers=headers, data=body, method="POST", timeout=120 - ) - node_id = resp["id"] - log.info(f" Uploaded: {local_path.name} → {node_id}") - return node_id - except HTTPError as e: - if e.code == 409: - log.info(f" Already exists on Amazon: {local_path.name}") - try: - return json.loads(e.read()).get("id") - except Exception: - return None - error_body = "" + +def album_children(album_id: str, op: str, node_ids: list[str]) -> None: + ap_write("PATCH", f"nodes/{quote(album_id, safe='')}/children", {"op": op, "value": node_ids}) + + +def trash_nodes(node_ids: list[str]) -> None: + ap_write("PATCH", "trash", { + "recurse": "true", + "op": "add", + "filters": "", + "conflictResolution": "RENAME", + "value": node_ids, + }) + + +def _apply_in_batches(action, node_ids: list[str], what: str) -> tuple[set, set, set]: + """Run `action(batch)` in batches; on failure retry nodes one by one. + + Returns (done, gone, failed) node ID sets. "gone" means Amazon answered 404. + """ + done: set[str] = set() + gone: set[str] = set() + failed: set[str] = set() + for i in range(0, len(node_ids), AMAZON_BATCH_SIZE): + batch = node_ids[i:i + AMAZON_BATCH_SIZE] try: - error_body = e.read().decode()[:200] - except Exception: - pass - log.error(f" Amazon upload error: {e.code} {error_body}") - raise + action(batch) + done.update(batch) + continue + except AmazonAuthError: + raise + except Exception as e: + if len(batch) > 1: + log.warning(f" {what} failed for a batch of {len(batch)} ({e}); retrying one by one") + for node_id in batch: + try: + action([node_id]) + done.add(node_id) + except AmazonAuthError: + raise + except AmazonError as e: + if e.code == 404: + gone.add(node_id) + else: + log.error(f" {what} failed for node {node_id}: {e}") + failed.add(node_id) + except Exception as e: + log.error(f" {what} failed for node {node_id}: {e}") + failed.add(node_id) + return done, gone, failed + + +def upload_to_amazon(local_path: Path, name: str, root_id: str) -> str: + """Stream a file into the Amazon Photos root folder and return its node ID. + + Raises AmazonError (code 409 on name/content conflict). + """ + metadata = json.dumps({"name": name, "kind": "FILE", "parents": [root_id]}).encode() + with open(local_path, "rb") as f: + body, content_type = _encode_multipart( + { + "metadata": (None, metadata, "application/json"), + "content": (name, f, "application/octet-stream"), + } + ) + raw = _amazon_call( + _url_with_params(f"{AMAZON_CONTENT_URL}/nodes"), + data=body, + method="POST", + headers={"Content-Type": content_type, "Content-Length": str(len(body))}, + timeout=120, + ) + return json.loads(raw)["id"] -def delete_from_amazon(node_id: str, asset_id: str) -> None: - """Move an Amazon Photos node to trash (not permanent delete).""" - url = _url_with_params(f"{AMAZON_DRIVE_URL}/trash/{node_id}") +def _conflict_node_id(body: str) -> str | None: try: - _request(url, headers=AP_HEADERS, method="PUT", data=b"") - log.info(f" Trashed on Amazon: {asset_id} ({node_id})") - except HTTPError as e: - error_body = "" + data = json.loads(body) + except ValueError: + data = {} + if isinstance(data, dict): + info = data.get("info") if isinstance(data.get("info"), dict) else {} + for candidate in (data.get("id"), info.get("nodeId")): + if isinstance(candidate, str) and candidate: + return candidate + match = re.search(r"NodeId:?\s*([A-Za-z0-9_-]{8,})", body) + return match.group(1) if match else None + + +def find_existing_node(conflict_body: str, name: str, md5: str, root_id: str) -> str | None: + """After a 409, find an AVAILABLE node with identical content (MD5).""" + candidates: list[dict] = [] + node_id = _conflict_node_id(conflict_body) + if node_id: try: - error_body = e.read().decode()[:200] - except Exception: + candidates.append(ap_get(f"nodes/{quote(node_id, safe='')}")) + except AmazonAuthError: + raise + except AmazonError: pass - log.error(f" Amazon delete error: {e.code} {error_body}") - raise + for label, filters in ( + ("md5", f"kind:FILE AND contentProperties.md5:{md5}"), + ("name", f"kind:FILE AND parents:{root_id} AND name:{_escape_filter(name)}"), + ): + try: + candidates.extend(list_nodes(filters)) + except AmazonAuthError: + raise + except AmazonError as e: + log.info(f" Conflict lookup by {label} failed: {e}") + for node in candidates: + props = node.get("contentProperties") or {} + if (props.get("md5", "").lower() == md5 and node.get("status", "AVAILABLE") == "AVAILABLE" + and node.get("id")): + return node["id"] + return None # Immich API -def immich_get(path: str) -> dict | list: +def immich_request(path: str, body: dict | None = None) -> dict | list: url = f"{IMMICH_URL}/api/{path.lstrip('/')}" - return _json_request(url, headers=IMMICH_HEADERS) + if body is None: + return _json_request(url, headers=IMMICH_HEADERS) + return _json_request( + url, + headers={**IMMICH_HEADERS, "Content-Type": "application/json"}, + data=json.dumps(body).encode(), + method="POST", + ) -def get_album_assets(album_id: str) -> list: - return immich_get(f"albums/{album_id}").get("assets", []) +def immich_get(path: str) -> dict | list: + return immich_request(path) -def download_asset(asset_id: str, filename: str, dest_dir: str) -> Path: - dest = Path(dest_dir) / filename - url = f"{IMMICH_URL}/api/assets/{asset_id}/original" - req = Request(url, headers=IMMICH_HEADERS) - with urlopen(req, timeout=60) as resp, open(dest, "wb") as f: - shutil.copyfileobj(resp, f) - return dest +def check_immich_version() -> str: + v = immich_get("server/version") + version = (int(v.get("major", 0)), int(v.get("minor", 0))) + text = f"{v.get('major')}.{v.get('minor')}.{v.get('patch')}" + if version < MIN_IMMICH_VERSION: + raise FatalError( + f"Immich {text} is too old - this tool needs Immich >= " + f"{MIN_IMMICH_VERSION[0]}.{MIN_IMMICH_VERSION[1]} (cursor-based album search)." + ) + return text -# Main +def search_album_assets(album_id: str) -> list[dict]: + """All non-trashed, visible assets of an album via POST /search/metadata. + Uses the structured search shape (Immich >= 3.2): flat `albumIds`/`page` + are deprecated and cannot be combined with `cursor`. + `visibility in [archive, timeline]` mirrors the album's assetCount. + """ + body: dict = { + "filter": { + "albumIds": {"any": [album_id]}, + "visibility": {"in": ["archive", "timeline"]}, + "trashedAt": {"eq": None}, + }, + "size": IMMICH_PAGE_SIZE, + } + items: list[dict] = [] + seen_cursors: set[str] = set() + while True: + resp = immich_request("search/metadata", body) + assets = resp.get("assets") if isinstance(resp, dict) else None + if not isinstance(assets, dict) or "nextCursor" not in assets: + raise FatalError( + "Immich search response has no 'nextCursor' - Immich >= 3.2 is required." + ) + page = assets.get("items") + if not isinstance(page, list): + raise ValueError("Immich search response has no 'items' list") + items.extend(page) + cursor = assets["nextCursor"] + if not cursor: + return items + if cursor in seen_cursors: + raise ValueError("Immich returned the same search cursor twice") + seen_cursors.add(cursor) + body = {**body, "cursor": cursor} + + +def get_album_asset_count(album_id: str) -> int: + return int(immich_get(f"albums/{album_id}")["assetCount"]) + + +def download_asset(asset_id: str, dest: Path) -> tuple[str, int]: + """Stream an original into `dest`. Returns (md5 hex, size). Enforces MAX_FILE_MB.""" + url = f"{IMMICH_URL}/api/assets/{asset_id}/original" + req = Request(url, headers=IMMICH_HEADERS) + limit_mb = MAX_FILE_BYTES / 1024 / 1024 + md5 = hashlib.md5(usedforsecurity=False) + size = 0 + with urlopen(req, timeout=60) as resp: + length = resp.headers.get("Content-Length") + if length and length.isdigit() and int(length) > MAX_FILE_BYTES: + raise SkipAsset(f"{int(length) / 1024 / 1024:.0f} MB exceeds MAX_FILE_MB={limit_mb:g}") + fd = os.open(dest, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "wb") as f: + while chunk := resp.read(DOWNLOAD_CHUNK): + size += len(chunk) + if size > MAX_FILE_BYTES: + raise SkipAsset(f"exceeds MAX_FILE_MB={limit_mb:g}") + md5.update(chunk) + f.write(chunk) + return md5.hexdigest(), size + + +def sync_asset(asset: dict, name: str, tmp_dir: str, root_id: str) -> tuple[str, str]: + """Download one asset and upload it. Returns (node_id, how) with how in uploaded/adopted.""" + asset_id = asset["id"] + local_path = Path(tmp_dir) / asset_id # asset_id is a validated UUID + try: + md5, _ = download_asset(asset_id, local_path) + upload_name = name + for attempt in range(2): + try: + return upload_to_amazon(local_path, upload_name, root_id), "uploaded" + except AmazonError as e: + if isinstance(e, AmazonAuthError) or e.code != 409: + raise + existing = find_existing_node(e.body, upload_name, md5, root_id) + if existing: + return existing, "adopted" + if attempt == 0: + upload_name = _disambiguated_name(name, asset_id) + log.info(f" Name conflict with different content, retrying as {upload_name}") + raise AmazonError(409, "conflict persists after renaming") + finally: + local_path.unlink(missing_ok=True) + + +def ping_healthcheck() -> None: + if not HEALTHCHECK_URL: + return + try: + _request(HEALTHCHECK_URL, headers={"User-Agent": "immich-amazon-sync"}, timeout=10) + except Exception as e: + log.warning(f"Healthcheck ping failed: {type(e).__name__}") -def main() -> None: - _configure() - log.info("Starting Immich → Amazon Photos sync") - synced = load_state() +# Main - # Resolve album names → IDs - all_albums = immich_get("albums") - album_map = {a["albumName"]: a["id"] for a in all_albums} - target_albums = [] +def flush_pending_album_adds(state: dict) -> int: + """Add uploaded-but-not-yet-linked nodes to the album. Returns number of failures.""" + pending = list(dict.fromkeys(state["pending_album_add"])) # de-duplicate, keep order + album_id = state["album_node_id"] + if not pending: + return 0 + if DRY_RUN or not album_id: + log.info(f"[dry-run] Would add {len(pending)} node(s) to the Amazon album") + return 0 + done, gone, failed = _apply_in_batches( + lambda ids: album_children(album_id, "add", ids), pending, "Add to album" + ) + if gone: + # node no longer exists on Amazon → forget it so the asset is uploaded again + log.warning(f" {len(gone)} node(s) no longer exist on Amazon; will re-upload") + state["synced"] = {a: n for a, n in state["synced"].items() if n not in gone} + if done: + log.info(f" Added {len(done)} node(s) to album") + state["pending_album_add"] = [n for n in pending if n in failed] + save_state(state) + return len(failed) + + +def _resolve_albums() -> tuple[list[tuple[str, str]], bool]: + """Map IMMICH_ALBUMS names to IDs. Returns (albums, all_found).""" + by_name: dict[str, list[str]] = {} + for album in immich_get("albums"): + by_name.setdefault(album["albumName"], []).append(album["id"]) + + targets: list[tuple[str, str]] = [] + all_found = True for name in ALBUMS_TO_SYNC: - if name in album_map: - target_albums.append((name, album_map[name])) + ids = by_name.get(name, []) + if len(ids) == 1 and UUID_RE.fullmatch(ids[0]): + targets.append((name, ids[0])) + elif not ids: + log.error(f"Album not found in Immich: '{name}'") + all_found = False else: - log.warning(f"Album not found: {name} - skipping") + log.error(f"Album name '{name}' is ambiguous or invalid in Immich ({len(ids)} matches)") + all_found = False + return targets, all_found + +def run() -> int: + log.info("Starting Immich → Amazon Photos sync" + (" [DRY RUN]" if DRY_RUN else "")) + state = load_state() + synced: dict[str, str] = state["synced"] + + log.info(f"Immich server version {check_immich_version()}") + target_albums, immich_complete = _resolve_albums() if not target_albums: log.error("No matching albums found. Check IMMICH_ALBUMS.") - sys.exit(1) + return EXIT_ERROR root_id = get_root_id() log.info(f"Amazon Photos root node: {root_id}") + album_id = find_or_create_album(AP_ALBUM_NAME, state["album_node_id"]) + log.info(f"Amazon Photos album '{AP_ALBUM_NAME}': {album_id or '(to be created)'}") + if album_id != state["album_node_id"]: + if synced: + log.info(f"Album is new to this state - linking {len(synced)} tracked photo(s)") + pending = set(state["pending_album_add"]) + state["pending_album_add"] += [n for n in synced.values() if n not in pending] + state["album_node_id"] = album_id + save_state(state) + + errors = flush_pending_album_adds(state) + synced = state["synced"] + + # Collect all current assets across target albums + present_ids: set[str] = set() + wanted: dict[str, dict] = {} + for album_name, immich_album_id in target_albums: + try: + assets = search_album_assets(immich_album_id) + expected = get_album_asset_count(immich_album_id) + except FatalError: + raise + except Exception as e: + log.error(f"Immich query for album '{album_name}' failed: {e}") + immich_complete = False + errors += 1 + continue + + unique = {} + for a in assets: + if not isinstance(a.get("id"), str) or not UUID_RE.fullmatch(a["id"]): + log.error(f"Album '{album_name}': ignoring asset with invalid id") + immich_complete = False + continue + unique[a["id"]] = a + if len(unique) != expected: + log.warning( + f"Album '{album_name}': search returned {len(unique)} assets but album " + f"reports assetCount={expected} - no deletions this run" + ) + immich_complete = False + + album_wanted = 0 + for asset_id, a in unique.items(): + if a.get("isTrashed"): + continue + present_ids.add(asset_id) + if a.get("type") in SYNC_TYPES: + wanted[asset_id] = a + album_wanted += 1 + log.info(f"Album '{album_name}': {len(unique)} assets, {album_wanted} of type " + f"{'/'.join(sorted(SYNC_TYPES))}") + + new_assets = [a for asset_id, a in wanted.items() if asset_id not in synced] + log.info(f"{len(wanted)} assets to mirror, {len(new_assets)} new") + + tracked_before = len(synced) # deletion brake base, excluding this run's uploads total_uploaded = 0 + total_skipped = 0 + with tempfile.TemporaryDirectory(prefix="immich-sync-") as tmp: + for asset in new_assets: + asset_id = asset["id"] + name = safe_filename(asset.get("originalFileName"), asset_id, asset.get("originalMimeType")) + if DRY_RUN: + log.info(f" [dry-run] Would upload {name} ({asset_id})") + continue + try: + node_id, how = sync_asset(asset, name, tmp, root_id) + except AmazonAuthError: + raise + except SkipAsset as e: + log.warning(f" Skipped {name}: {e}") + total_skipped += 1 + continue + except Exception as e: + log.error(f" Failed {name}: {e}") + errors += 1 + continue + log.info(f" {how.capitalize()}: {name} → {node_id}") + synced[asset_id] = node_id + state["pending_album_add"].append(node_id) + if how == "adopted" and node_id not in state["adopted_nodes"]: + # not uploaded by us (could be the user's own backup): never trash it + state["adopted_nodes"].append(node_id) + save_state(state) + total_uploaded += 1 + if len(state["pending_album_add"]) >= AMAZON_BATCH_SIZE: + errors += flush_pending_album_adds(state) + synced = state["synced"] + errors += flush_pending_album_adds(state) + synced = state["synced"] + + # Delete assets that were removed from all target albums + removed = sorted(set(synced) - present_ids) total_deleted = 0 - total_errors = 0 - - # Collect all current asset IDs across target albums - current_asset_ids: set[str] = set() - - with tempfile.TemporaryDirectory() as tmp: - for album_name, album_id in target_albums: - assets = get_album_assets(album_id) - asset_ids = {a["id"] for a in assets} - current_asset_ids.update(asset_ids) - new_assets = [a for a in assets if a["id"] not in synced] - log.info( - f"Album '{album_name}': {len(assets)} total, {len(new_assets)} new" + delete_blocked = False + if removed: + fraction = len(removed) / max(tracked_before, 1) + if not immich_complete: + log.error(f"Not deleting {len(removed)} asset(s): Immich data incomplete in this run") + delete_blocked = True + elif len(removed) > MAX_DELETE_COUNT or fraction > MAX_DELETE_FRACTION: + log.error( + f"Deletion brake: {len(removed)} of {tracked_before} tracked assets " + f"({fraction:.0%}) would be deleted (limits: MAX_DELETE_COUNT={MAX_DELETE_COUNT}, " + f"MAX_DELETE_FRACTION={MAX_DELETE_FRACTION:g}). Nothing deleted." ) + delete_blocked = True + elif DRY_RUN: + for asset_id in removed: + log.info(f" [dry-run] Would remove {asset_id} ({synced[asset_id]}) " + f"from album{' and trash it' if DELETE_MODE == 'trash' else ''}") + else: + total_deleted, delete_errors = delete_assets(state, removed) + errors += delete_errors + synced = state["synced"] - for asset in new_assets: - asset_id = asset["id"] - filename = asset.get("originalFileName", f"{asset_id}.jpg") - try: - local_path = download_asset(asset_id, filename, tmp) - node_id = upload_to_amazon(local_path, root_id) - local_path.unlink(missing_ok=True) - if node_id: - synced[asset_id] = node_id - total_uploaded += 1 - except Exception as e: - log.error(f" Failed {filename}: {e}") - total_errors += 1 + log.info( + f"Done. Uploaded: {total_uploaded}, Deleted: {total_deleted}, Skipped: {total_skipped}, " + f"Errors: {errors}, Tracked: {len(synced)}" + (" [DRY RUN - nothing written]" if DRY_RUN else "") + ) + if delete_blocked: + return EXIT_DELETE_BLOCKED + if errors or not immich_complete: + return EXIT_ERROR + return EXIT_OK - # Delete assets that were removed from all target albums - removed_ids = set(synced.keys()) - current_asset_ids - if removed_ids: - log.info(f"Deleting {len(removed_ids)} asset(s) removed from albums") - for asset_id in removed_ids: - node_id = synced.get(asset_id) - if not node_id: - log.warning(f" No Amazon node ID for {asset_id}, removing from state") - del synced[asset_id] - continue - try: - delete_from_amazon(node_id, asset_id) + +def _is_trashed(node_id: str) -> bool: + try: + return ap_get(f"nodes/{quote(node_id, safe='')}").get("status") == "TRASH" + except AmazonAuthError: + raise + except Exception: + return False + + +def delete_assets(state: dict, asset_ids: list[str]) -> tuple[int, int]: + """Remove assets' nodes from the album (and trash them). Returns (deleted, errors). + + Nodes adopted from an existing Amazon file are only removed from the album, + never trashed. State is saved after every batch. + """ + synced = state["synced"] + album_id = state["album_node_id"] + leaving = set(asset_ids) + # identical content can map two Immich assets to one adopted node - keep shared nodes + still_used = {n for a, n in synced.items() if a not in leaving} + node_to_assets: dict[str, list[str]] = {} + for asset_id in asset_ids: + if synced[asset_id] in still_used: + log.info(f" Forgetting {asset_id}: its node is still used by another asset") del synced[asset_id] - total_deleted += 1 - except Exception as e: - log.error(f" Failed to delete {asset_id}: {e}") - total_errors += 1 + else: + node_to_assets.setdefault(synced[asset_id], []).append(asset_id) + node_ids = list(node_to_assets) + log.info(f"Removing {len(node_ids)} asset(s) that left all Immich albums") + + adopted = set(state["adopted_nodes"]) + total_finished = 0 + total_failed = 0 + for i in range(0, len(node_ids), AMAZON_BATCH_SIZE): + batch = node_ids[i:i + AMAZON_BATCH_SIZE] + # nodes still waiting for "add to album" are not in the album yet + pending = set(state["pending_album_add"]) + in_album = [n for n in batch if n not in pending] + done, gone, failed = _apply_in_batches( + lambda ids: album_children(album_id, "remove", ids), in_album, "Remove from album" + ) + removed_ok = [n for n in batch if n in done or n in gone or n in pending] + + finished = {n for n in removed_ok if DELETE_MODE != "trash" or n in adopted} + to_trash = [n for n in removed_ok if n not in finished] + if to_trash: + t_done, t_gone, t_failed = _apply_in_batches(trash_nodes, to_trash, "Trash") + already = {n for n in t_failed if _is_trashed(n)} # e.g. trashed by an interrupted run + finished |= t_done | t_gone | already + failed |= t_failed - already + + for node_id in batch: + if node_id not in finished: + continue + action = "Trashed" if node_id in to_trash else "Removed from album" + for asset_id in node_to_assets[node_id]: + del synced[asset_id] + log.info(f" {action}: {asset_id} ({node_id})") + state["pending_album_add"] = [n for n in state["pending_album_add"] if n not in finished] + state["adopted_nodes"] = [n for n in state["adopted_nodes"] if n not in finished] + save_state(state) + total_finished += len(finished) + total_failed += len(failed) + save_state(state) + return total_finished, total_failed + + +def main() -> int: + try: + _configure() + except ConfigError as e: + print(f"Error: {e}", file=sys.stderr) + return EXIT_ERROR - save_state(synced) - log.info( - f"Done. Uploaded: {total_uploaded}, Deleted: {total_deleted}, " - f"Errors: {total_errors}, Tracked: {len(synced)}" - ) - if total_errors: - sys.exit(1) + _setup_logging() + try: + code = run() + except AmazonAuthError as e: + log.error( + f"Amazon rejected the request (HTTP {e.code}): cookies expired or invalid. " + "Copy fresh cookies from the browser and restart. Run aborted." + ) + code = EXIT_AUTH + except FatalError as e: + log.error(str(e)) + code = EXIT_ERROR + except Exception as e: + log.error(f"Sync failed: {type(e).__name__}: {e}") + code = EXIT_ERROR + else: + if code == EXIT_OK and not DRY_RUN: + ping_healthcheck() + finally: + _teardown_logging() + return code if __name__ == "__main__": - main() + sys.exit(main()) diff --git a/tests/test_smoke.py b/tests/test_smoke.py index b4e9b32..442fe3b 100644 --- a/tests/test_smoke.py +++ b/tests/test_smoke.py @@ -1,5 +1,7 @@ """Smoke tests for pure-logic helpers in immich_sync.""" +import io +import json import sys import tempfile import unittest @@ -19,11 +21,14 @@ def test_round_trip(self): with patch("immich_sync.STATE_FILE", state_file): from immich_sync import load_state, save_state - self.assertEqual(load_state(), {}) + state = load_state() + self.assertEqual(state["synced"], {}) + self.assertEqual(state["version"], 2) - data = {"asset1": "node1", "asset2": "node2"} - save_state(data) - self.assertEqual(load_state(), data) + state["synced"] = {"asset1": "node1", "asset2": "node2"} + state["album_node_id"] = "album1" + save_state(state) + self.assertEqual(load_state(), state) def test_missing_file(self): with tempfile.TemporaryDirectory() as tmp: @@ -31,7 +36,48 @@ def test_missing_file(self): with patch("immich_sync.STATE_FILE", state_file): from immich_sync import load_state - self.assertEqual(load_state(), {}) + self.assertEqual(load_state()["synced"], {}) + + def test_migrates_original_format_without_reupload(self): + with tempfile.TemporaryDirectory() as tmp: + state_file = Path(tmp) / "state.json" + state_file.write_text(json.dumps({"synced": {"a1": "n1", "a2": "n2"}})) + with patch("immich_sync.STATE_FILE", state_file): + from immich_sync import load_state + + state = load_state() + self.assertEqual(state["version"], 2) + self.assertEqual(state["synced"], {"a1": "n1", "a2": "n2"}) + self.assertIsNone(state["album_node_id"]) + self.assertEqual(state["pending_album_add"], []) + + def test_corrupt_state_is_fatal(self): + with tempfile.TemporaryDirectory() as tmp: + state_file = Path(tmp) / "state.json" + state_file.write_text("{not json") + with patch("immich_sync.STATE_FILE", state_file): + from immich_sync import FatalError, load_state + + with self.assertRaises(FatalError): + load_state() + + def test_atomic_write_leaves_no_temp_files(self): + with tempfile.TemporaryDirectory() as tmp: + state_file = Path(tmp) / "state.json" + with patch("immich_sync.STATE_FILE", state_file), patch("immich_sync.DRY_RUN", False): + from immich_sync import load_state, save_state + + save_state(load_state()) + self.assertEqual([p.name for p in Path(tmp).iterdir()], ["state.json"]) + + def test_dry_run_does_not_write(self): + with tempfile.TemporaryDirectory() as tmp: + state_file = Path(tmp) / "state.json" + with patch("immich_sync.STATE_FILE", state_file), patch("immich_sync.DRY_RUN", True): + from immich_sync import load_state, save_state + + save_state(load_state()) + self.assertFalse(state_file.exists()) class TestEncodeMultipart(unittest.TestCase): @@ -46,6 +92,9 @@ def test_structure(self): "field2": ("file.txt", b"file content", "application/octet-stream"), } ) + length = len(body) + body = body.read() + self.assertEqual(length, len(body)) self.assertIn("multipart/form-data; boundary=", content_type) boundary = content_type.split("boundary=")[1] @@ -63,10 +112,27 @@ def test_no_filename(self): body, _ = _encode_multipart( {"meta": (None, b'{"key": "val"}', "application/json")} ) + body = body.read() self.assertNotIn(b"filename=", body) self.assertIn(b'name="meta"', body) + def test_streams_file_in_small_reads(self): + from immich_sync import _encode_multipart + + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "blob" + path.write_bytes(bytes(range(256)) * 100) + with open(path, "rb") as f: + stream, _ = _encode_multipart({"content": ("x.bin", f, "application/octet-stream")}) + expected_len = len(stream) + out = io.BytesIO() + while chunk := stream.read(1000): + self.assertLessEqual(len(chunk), 1000) + out.write(chunk) + self.assertEqual(len(out.getvalue()), expected_len) + self.assertIn(bytes(range(256)) * 100, out.getvalue()) + class TestUrlWithParams(unittest.TestCase): """Test _url_with_params URL construction.""" @@ -86,5 +152,96 @@ def test_with_extra_params(self): ) +class TestSafeFilename(unittest.TestCase): + """Untrusted originalFileName must never become a path or break headers.""" + + ASSET = "0b0e6d5a-1c1d-4b8e-9c3a-2f1e0d9c8b7a" + + def check(self, raw, mime=None): + from immich_sync import safe_filename + + name = safe_filename(raw, self.ASSET, mime) + self.assertNotIn("/", name) + self.assertNotIn("\\", name) + self.assertNotIn('"', name) + self.assertNotIn("\r", name) + self.assertNotIn("\n", name) + self.assertNotIn(name, ("", ".", "..")) + return name + + def test_plain(self): + self.assertEqual(self.check("IMG_0001.HEIC"), "IMG_0001.HEIC") + + def test_traversal(self): + self.assertEqual(self.check("../../etc/passwd"), "passwd") + self.assertEqual(self.check("..\\..\\win.ini"), "win.ini") + self.assertEqual(self.check(".."), f"{self.ASSET}.jpg") + + def test_absolute_and_slash(self): + self.assertEqual(self.check("/abs/path/photo.jpg"), "photo.jpg") + self.assertEqual(self.check("dir/"), "dir") + self.assertEqual(self.check("/"), f"{self.ASSET}.jpg") + + def test_quotes_and_crlf(self): + self.assertEqual(self.check('a"b.jpg'), "ab.jpg") + self.assertEqual( + self.check('x.jpg"\r\nX-Injected: 1\r\n\r\n'), + "x.jpgX-Injected: 1", + ) + self.assertEqual(self.check("tab\there\x00.png"), "tabhere.png") + + def test_empty_uses_asset_id_and_extension(self): + self.assertEqual(self.check(""), f"{self.ASSET}.jpg") + self.assertEqual(self.check(None), f"{self.ASSET}.jpg") + self.assertEqual(self.check("", "image/png"), f"{self.ASSET}.png") + self.assertEqual(self.check("\r\n", "video/mp4"), f"{self.ASSET}.mp4") + + def test_long_name_keeps_extension(self): + name = self.check("a" * 400 + ".jpeg") + self.assertLessEqual(len(name), 200) + self.assertTrue(name.endswith(".jpeg")) + + +class TestRegion(unittest.TestCase): + """Hosts and cookie names are derived from AP_TLD.""" + + def test_germany(self): + from immich_sync import region_settings + + r = region_settings("de") + self.assertEqual(r["drive_url"], "https://www.amazon.de/drive/v1") + self.assertEqual(r["content_url"], "https://content-eu.drive.amazonaws.com/cdproxy") + self.assertEqual(r["ubid_key"], "ubid-acbde") + self.assertEqual(r["at_key"], "at-acbde") + + def test_us(self): + from immich_sync import region_settings + + r = region_settings("com") + self.assertEqual(r["drive_url"], "https://www.amazon.com/drive/v1") + self.assertEqual(r["content_url"], "https://content-na.drive.amazonaws.com/cdproxy") + self.assertEqual((r["ubid_key"], r["at_key"]), ("ubid-main", "at-main")) + + def test_uk(self): + from immich_sync import region_settings + + r = region_settings("co.uk") + self.assertEqual(r["drive_url"], "https://www.amazon.co.uk/drive/v1") + self.assertEqual(r["content_url"], "https://content-eu.drive.amazonaws.com/cdproxy") + self.assertEqual((r["ubid_key"], r["at_key"]), ("ubid-acbuk", "at-acbuk")) + + def test_unknown_region_has_no_upload_host(self): + from immich_sync import region_settings + + self.assertIsNone(region_settings("co.jp")["content_url"]) + + def test_invalid_tld(self): + from immich_sync import ConfigError, region_settings + + for bad in ("evil.com/x", "", "d e"): + with self.assertRaises(ConfigError): + region_settings(bad) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_sync.py b/tests/test_sync.py new file mode 100644 index 0000000..5db9298 --- /dev/null +++ b/tests/test_sync.py @@ -0,0 +1,668 @@ +"""End-to-end tests for immich_sync.main() with all HTTP mocked at urlopen.""" + +import gzip +import io +import json +import os +import sys +import tempfile +import unittest +import uuid +from pathlib import Path +from unittest.mock import patch +from urllib.error import HTTPError +from urllib.parse import parse_qs, unquote, urlsplit + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src")) + +import immich_sync # noqa: E402 + +IMMICH = "http://immich.test:2283" +DRIVE = "https://www.amazon.de/drive/v1" +CONTENT = "https://content-eu.drive.amazonaws.com/cdproxy" + +SECRETS = { + "IMMICH_API_KEY": "immich-secret-key-123", + "AP_SESSION_ID": "262-1111111-2222222", + "AP_UBID_VALUE": "ubid-secret-333", + "AP_AT_VALUE": "Atza|at-secret-444", +} + + +def aid(n: int) -> str: + return str(uuid.UUID(int=n, version=4)) + + +class FakeResponse(io.BytesIO): + def __init__(self, payload, headers=None): + if not isinstance(payload, bytes): + payload = json.dumps(payload).encode() + super().__init__(payload) + self.headers = headers or {"Content-Length": str(len(payload))} + + +class Backend: + """In-memory Immich 3.x + Amazon Drive used as the urlopen replacement.""" + + def __init__(self): + self.immich_version = {"major": 3, "minor": 3, "patch": 1, "prerelease": None} + self.albums = {} # immich album id -> {"name", "assets": [asset dicts], "count": override} + self.page_size = None # force small search pages + self.legacy_search = False + self.search_fails = False + self.amazon_status = None # force every Amazon call to fail with this code + self.upload_conflict = {} # name -> existing node id + self.nodes = {"root": {"id": "root", "kind": "FOLDER", "isRoot": True, "status": "AVAILABLE"}} + self.album_children = {} # album node id -> set(node ids) + self.trashed = set() + self.trash_rejects = set() # nodes for which PATCH /trash fails with 400 + self.gzip_children = False # answer PATCH .../children gzip-compressed, as amazon.de does + self.calls = [] # (method, url, body) + self._n = 0 + + # Immich fixtures + + def add_album(self, name, asset_ids, **asset_overrides): + album_id = aid(10_000 + len(self.albums)) + assets = [self.make_asset(a, **asset_overrides) for a in asset_ids] + self.albums[album_id] = {"name": name, "assets": assets, "count": None} + return album_id + + @staticmethod + def make_asset(asset_id, **overrides): + asset = { + "id": asset_id, + "originalFileName": f"IMG_{asset_id[-4:]}.jpg", + "originalMimeType": "image/jpeg", + "type": "IMAGE", + "isTrashed": False, + "visibility": "timeline", + } + asset.update(overrides) + return asset + + # routing + + def __call__(self, req, timeout=None): + url = req.full_url + method = req.get_method() + data = req.data + if data is not None and not isinstance(data, bytes): + data = data.read() + self.calls.append((method, url, data)) + if url.startswith(IMMICH): + return self.immich(method, url, data) + if url.startswith("https://hc.test/"): + return FakeResponse(b"OK") + if self.amazon_status: + raise HTTPError(url, self.amazon_status, "err", {}, io.BytesIO(b'{"message":"denied"}')) + if url.startswith(CONTENT): + return self.upload(data) + if url.startswith(DRIVE): + return self.drive(method, url, data) + raise AssertionError(f"unexpected URL {url}") + + def immich(self, method, url, data): + path = urlsplit(url).path[len("/api/"):] + if path == "server/version": + return FakeResponse(self.immich_version) + if path == "albums": + return FakeResponse([{"id": k, "albumName": v["name"]} for k, v in self.albums.items()]) + if path.startswith("albums/"): + album = self.albums[path.split("/")[1]] + count = album["count"] if album["count"] is not None else len( + [a for a in album["assets"] if not a["isTrashed"]]) + return FakeResponse({"id": path.split("/")[1], "assetCount": count}) + if path == "search/metadata": + if self.search_fails: + raise HTTPError(url, 500, "boom", {}, io.BytesIO(b"{}")) + body = json.loads(data) + if self.legacy_search: + return FakeResponse({"albums": {}, "assets": {"items": [], "nextPage": None}}) + assert "albumIds" not in body and "page" not in body, "deprecated flat fields used" + album_id = body["filter"]["albumIds"]["any"][0] + assert body["filter"]["trashedAt"] == {"eq": None} + assets = [a for a in self.albums[album_id]["assets"] if not a["isTrashed"]] + size = self.page_size or body["size"] + offset = int(body.get("cursor", 0)) + page = assets[offset:offset + size] + nxt = str(offset + size) if offset + size < len(assets) else None + return FakeResponse({"albums": {}, "assets": { + "items": page, "count": len(page), "nextCursor": nxt, "nextPage": None}}) + if path.startswith("assets/") and path.endswith("/original"): + return FakeResponse(f"content-of-{path.split('/')[1]}".encode()) + raise AssertionError(f"unexpected Immich path {path}") + + def upload(self, body): + meta = json.loads(body.split(b"\r\n\r\n", 1)[1].split(b"\r\n", 1)[0]) + if meta["name"] in self.upload_conflict: + raise HTTPError("u", 409, "conflict", {}, io.BytesIO(json.dumps( + {"code": "NAME_ALREADY_EXISTS", "info": {"nodeId": self.upload_conflict[meta["name"]]}} + ).encode())) + self._n += 1 + node_id = f"node{self._n}" + self.nodes[node_id] = {"id": node_id, "name": meta["name"], "kind": "FILE", + "status": "AVAILABLE", "parents": meta["parents"]} + return FakeResponse({"id": node_id}) + + def drive(self, method, url, data): + parts = urlsplit(url) + path = parts.path[len("/drive/v1/"):] + query = parse_qs(parts.query) + body = json.loads(data) if data else None + if method == "GET" and path == "nodes": + filters = unquote(query["filters"][0]) + if filters == "isRoot:true": + return FakeResponse({"count": 1, "data": [self.nodes["root"]]}) + if filters == "kind:VISUAL_COLLECTION": + found = [n for n in self.nodes.values() if n["kind"] == "VISUAL_COLLECTION"] + return FakeResponse({"count": len(found), "data": found}) + return FakeResponse({"count": 0, "data": []}) + if method == "GET" and path.startswith("nodes/"): + node = self.nodes.get(path.split("/")[1]) + if not node: + raise HTTPError(url, 404, "nf", {}, io.BytesIO(b"{}")) + return FakeResponse(node) + if method == "POST" and path == "nodes": + assert body["kind"] == "VISUAL_COLLECTION" + node = {"id": f"album{len(self.album_children) + 1}", "name": body["name"], + "kind": "VISUAL_COLLECTION", "status": "AVAILABLE"} + self.nodes[node["id"]] = node + self.album_children[node["id"]] = set() + return FakeResponse(node) + if method == "PATCH" and path.endswith("/children"): + children = self.album_children[path.split("/")[1]] + if body["op"] == "add": + children.update(body["value"]) + else: + children.difference_update(body["value"]) + if self.gzip_children: + raw = gzip.compress(json.dumps({"op": body["op"]}).encode()) + return FakeResponse(raw, {"Content-Length": str(len(raw)), "Content-Encoding": "gzip"}) + return FakeResponse({}) + if method == "PATCH" and path == "trash": + assert body["op"] == "add" + if self.trash_rejects & set(body["value"]): + raise HTTPError(url, 400, "bad", {}, io.BytesIO(b'{"message":"already trashed"}')) + self.trashed.update(body["value"]) + return FakeResponse({}) + raise AssertionError(f"unexpected Amazon call {method} {path}") + + def amazon_writes(self): + return [(m, u) for m, u, _ in self.calls + if not u.startswith(IMMICH) and m in ("POST", "PATCH", "PUT", "DELETE")] + + +class SyncTestCase(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.state_dir = Path(self.tmp.name) + self.backend = Backend() + self.env = { + "IMMICH_URL": IMMICH, + "IMMICH_ALBUMS": "Echo", + "AP_ALBUM_NAME": "Echo Show", + "STATE_DIR": str(self.state_dir), + **SECRETS, + } + + def tearDown(self): + immich_sync._teardown_logging() + self.tmp.cleanup() + + def run_sync(self, **env): + with patch.dict(os.environ, {**self.env, **env}, clear=True), \ + patch("immich_sync.urlopen", self.backend), \ + patch("sys.stdout", new_callable=io.StringIO) as out, \ + patch("sys.stderr", new_callable=io.StringIO) as err: + code = immich_sync.main() + self.output = out.getvalue() + err.getvalue() + return code + + def state(self): + return json.loads((self.state_dir / "state.json").read_text()) + + def write_state(self, data): + (self.state_dir / "state.json").write_text(json.dumps(data)) + + +class TestImmichSearch(SyncTestCase): + def test_pagination_follows_next_cursor(self): + ids = [aid(i) for i in range(1, 8)] + album = self.backend.add_album("Echo", ids) + self.backend.page_size = 3 + with patch.dict(os.environ, self.env, clear=True): + immich_sync._configure() + with patch("immich_sync.urlopen", self.backend): + assets = immich_sync.search_album_assets(album) + self.assertEqual([a["id"] for a in assets], ids) + bodies = [json.loads(d) for m, u, d in self.backend.calls if u.endswith("/search/metadata")] + self.assertEqual(len(bodies), 3) + self.assertNotIn("cursor", bodies[0]) + self.assertEqual([b.get("cursor") for b in bodies[1:]], ["3", "6"]) + self.assertEqual(bodies[0]["filter"]["visibility"], {"in": ["archive", "timeline"]}) + + def test_old_immich_aborts(self): + self.backend.immich_version = {"major": 3, "minor": 1, "patch": 0} + self.backend.add_album("Echo", [aid(1)]) + self.assertEqual(self.run_sync(), immich_sync.EXIT_ERROR) + self.assertIn("too old", self.output) + self.assertEqual(self.backend.amazon_writes(), []) + + def test_missing_next_cursor_aborts(self): + self.backend.legacy_search = True + self.backend.add_album("Echo", [aid(1)]) + self.write_state({"synced": {aid(1): "nodeX"}}) + self.assertEqual(self.run_sync(), immich_sync.EXIT_ERROR) + self.assertIn("nextCursor", self.output) + self.assertEqual(self.backend.trashed, set()) + + +class TestFullSync(SyncTestCase): + def test_gzip_reply_to_album_add_counts_as_success(self): + ids = [aid(i) for i in range(1, 4)] + self.backend.add_album("Echo", ids) + self.backend.gzip_children = True + self.assertEqual(self.run_sync(), immich_sync.EXIT_OK) + state = self.state() + self.assertEqual(state["pending_album_add"], []) + self.assertEqual(self.backend.album_children[state["album_node_id"]], set(state["synced"].values())) + self.assertNotIn("codec", self.output) + + def test_upload_into_album_then_idempotent(self): + ids = [aid(i) for i in range(1, 4)] + self.backend.add_album("Echo", ids) + + self.assertEqual(self.run_sync(), immich_sync.EXIT_OK) + state = self.state() + self.assertEqual(state["version"], 2) + self.assertEqual(set(state["synced"]), set(ids)) + album_id = state["album_node_id"] + self.assertEqual(self.backend.nodes[album_id]["name"], "Echo Show") + self.assertEqual(self.backend.album_children[album_id], set(state["synced"].values())) + self.assertEqual(state["pending_album_add"], []) + # one batched "add" call for all three uploads + adds = [c for c in self.backend.calls if c[0] == "PATCH" and c[1].endswith("/children")] + self.assertEqual(len(adds), 1) + if os.name != "nt": + for name in ("state.json", "immich-sync.log"): + self.assertEqual(os.stat(self.state_dir / name).st_mode & 0o077, 0, name) + + # second run: nothing to do + self.backend.calls.clear() + self.assertEqual(self.run_sync(), immich_sync.EXIT_OK) + self.assertEqual(self.backend.amazon_writes(), []) + self.assertIn("Uploaded: 0, Deleted: 0", self.output) + + def test_removed_photo_leaves_album_and_is_trashed(self): + ids = [aid(i) for i in range(1, 7)] + album = self.backend.add_album("Echo", ids) + self.run_sync() + node = self.state()["synced"][ids[0]] + album_node = self.state()["album_node_id"] + + self.backend.albums[album]["assets"].pop(0) + self.assertEqual(self.run_sync(), immich_sync.EXIT_OK) + self.assertNotIn(ids[0], self.state()["synced"]) + self.assertNotIn(node, self.backend.album_children[album_node]) + self.assertEqual(self.backend.trashed, {node}) + + def test_album_only_mode_does_not_trash(self): + ids = [aid(i) for i in range(1, 7)] + album = self.backend.add_album("Echo", ids) + self.run_sync() + self.backend.albums[album]["assets"].pop(0) + self.assertEqual(self.run_sync(AP_DELETE_MODE="album_only"), immich_sync.EXIT_OK) + self.assertEqual(self.backend.trashed, set()) + self.assertNotIn(ids[0], self.state()["synced"]) + + def test_trashed_and_filtered_types_are_not_uploaded(self): + album = self.backend.add_album("Echo", [aid(1)]) + self.backend.albums[album]["assets"] += [ + Backend.make_asset(aid(2), isTrashed=True), + Backend.make_asset(aid(3), type="VIDEO", originalFileName="clip.mp4"), + ] + self.assertEqual(self.run_sync(), immich_sync.EXIT_OK) + self.assertEqual(set(self.state()["synced"]), {aid(1)}) + + self.assertEqual(self.run_sync(SYNC_TYPES="IMAGE,VIDEO"), immich_sync.EXIT_OK) + self.assertEqual(set(self.state()["synced"]), {aid(1), aid(3)}) + + def test_existing_album_is_reused_and_migrated_state_is_linked(self): + self.backend.nodes["albX"] = {"id": "albX", "name": "Echo Show", + "kind": "VISUAL_COLLECTION", "status": "AVAILABLE"} + self.backend.album_children["albX"] = set() + self.backend.nodes["old1"] = {"id": "old1", "kind": "FILE", "status": "AVAILABLE"} + self.backend.add_album("Echo", [aid(1), aid(2)]) + self.write_state({"synced": {aid(1): "old1"}}) # original upstream format + + self.assertEqual(self.run_sync(), immich_sync.EXIT_OK) + state = self.state() + self.assertEqual(state["album_node_id"], "albX") + self.assertEqual(state["synced"][aid(1)], "old1") # not re-uploaded + self.assertIn("old1", self.backend.album_children["albX"]) + uploads = [c for c in self.backend.calls if c[1].startswith(CONTENT)] + self.assertEqual(len(uploads), 1) + + def test_conflict_adopts_node_with_same_md5(self): + import hashlib + + self.backend.add_album("Echo", [aid(1)]) + name = Backend.make_asset(aid(1))["originalFileName"] + md5 = hashlib.md5(f"content-of-{aid(1)}".encode()).hexdigest() + self.backend.nodes["exist1"] = {"id": "exist1", "kind": "FILE", "status": "AVAILABLE", + "contentProperties": {"md5": md5}} + self.backend.upload_conflict[name] = "exist1" + self.assertEqual(self.run_sync(), immich_sync.EXIT_OK) + self.assertEqual(self.state()["synced"][aid(1)], "exist1") + + def test_adopted_node_is_never_trashed(self): + import hashlib + + album = self.backend.add_album("Echo", [aid(i) for i in range(1, 7)]) + name = Backend.make_asset(aid(1))["originalFileName"] + md5 = hashlib.md5(f"content-of-{aid(1)}".encode()).hexdigest() + # the user's own phone backup, somewhere else in the drive + self.backend.nodes["backup"] = {"id": "backup", "kind": "FILE", "status": "AVAILABLE", + "contentProperties": {"md5": md5}} + self.backend.upload_conflict[name] = "backup" + self.run_sync() + self.assertEqual(self.state()["adopted_nodes"], ["backup"]) + album_node = self.state()["album_node_id"] + + self.backend.albums[album]["assets"].pop(0) + self.assertEqual(self.run_sync(), immich_sync.EXIT_OK) + self.assertNotIn("backup", self.backend.trashed) + self.assertNotIn("backup", self.backend.album_children[album_node]) + self.assertNotIn(aid(1), self.state()["synced"]) + self.assertEqual(self.state()["adopted_nodes"], []) + + def test_already_trashed_node_counts_as_deleted(self): + album = self.backend.add_album("Echo", [aid(i) for i in range(1, 7)]) + self.run_sync() + node = self.state()["synced"][aid(1)] + # an interrupted earlier run already trashed it; Amazon now rejects the trash call + self.backend.nodes[node]["status"] = "TRASH" + self.backend.trash_rejects.add(node) + self.backend.albums[album]["assets"].pop(0) + self.assertEqual(self.run_sync(), immich_sync.EXIT_OK) + self.assertNotIn(aid(1), self.state()["synced"]) + + def test_shared_node_is_not_trashed_while_still_used(self): + album = self.backend.add_album("Echo", [aid(i) for i in range(1, 7)]) + self.backend.nodes["shared"] = {"id": "shared", "kind": "FILE", "status": "AVAILABLE"} + self.write_state({"version": 2, "album_node_id": None, "pending_album_add": [], + "synced": {aid(1): "shared", aid(2): "shared"}}) + self.run_sync() + self.backend.albums[album]["assets"].pop(0) # aid(1) leaves, aid(2) stays + self.assertEqual(self.run_sync(), immich_sync.EXIT_OK) + self.assertNotIn(aid(1), self.state()["synced"]) + self.assertEqual(self.state()["synced"][aid(2)], "shared") + self.assertEqual(self.backend.trashed, set()) + + def test_conflict_with_different_content_renames(self): + self.backend.add_album("Echo", [aid(1)]) + name = Backend.make_asset(aid(1))["originalFileName"] + self.backend.nodes["other"] = {"id": "other", "kind": "FILE", "status": "AVAILABLE", + "contentProperties": {"md5": "0" * 32}} + self.backend.upload_conflict[name] = "other" + self.assertEqual(self.run_sync(), immich_sync.EXIT_OK) + node = self.state()["synced"][aid(1)] + self.assertNotEqual(node, "other") + self.assertEqual(self.backend.nodes[node]["name"], f"IMG_{aid(1)[-4:]}_{aid(1)[:8]}.jpg") + + def test_file_size_limit_skips(self): + self.backend.add_album("Echo", [aid(1)]) + self.assertEqual(self.run_sync(MAX_FILE_MB="0.00001"), immich_sync.EXIT_OK) + self.assertIn("Skipped", self.output) + self.assertEqual(self.state()["synced"], {}) + + def test_healthcheck_ping_only_on_success(self): + self.backend.add_album("Echo", [aid(1)]) + self.run_sync(HEALTHCHECK_URL="https://hc.test/ping/abc") + self.assertTrue(any(u.startswith("https://hc.test/") for _, u, _ in self.backend.calls)) + self.backend.calls.clear() + self.backend.amazon_status = 401 + self.run_sync(HEALTHCHECK_URL="https://hc.test/ping/abc") + self.assertFalse(any(u.startswith("https://hc.test/") for _, u, _ in self.backend.calls)) + + +class TestDeletionSafety(SyncTestCase): + def setUp(self): + super().setUp() + self.ids = [aid(i) for i in range(1, 11)] + self.album = self.backend.add_album("Echo", self.ids) + self.assertEqual(self.run_sync(), immich_sync.EXIT_OK) + self.before = self.state() + self.backend.calls.clear() + + def assert_nothing_deleted(self): + self.assertEqual(self.backend.trashed, set()) + self.assertLessEqual(self.before["synced"].items(), self.state()["synced"].items()) + removes = [c for c in self.backend.calls if c[0] == "PATCH" and b'"remove"' in (c[2] or b"")] + self.assertEqual(removes, []) + + def test_asset_count_mismatch_blocks_deletions(self): + self.backend.albums[self.album]["assets"].pop(0) + self.backend.albums[self.album]["count"] = 10 # album claims more than search found + self.assertEqual(self.run_sync(), immich_sync.EXIT_DELETE_BLOCKED) + self.assertIn("assetCount=10", self.output) + self.assert_nothing_deleted() + + def test_deletion_brake_fraction(self): + del self.backend.albums[self.album]["assets"][:3] # 30% > 20% + self.assertEqual(self.run_sync(), immich_sync.EXIT_DELETE_BLOCKED) + self.assertIn("Deletion brake", self.output) + self.assert_nothing_deleted() + + def test_deletion_brake_count(self): + del self.backend.albums[self.album]["assets"][:2] + self.assertEqual(self.run_sync(MAX_DELETE_COUNT="1", MAX_DELETE_FRACTION="1"), + immich_sync.EXIT_DELETE_BLOCKED) + self.assert_nothing_deleted() + + def test_empty_album_never_mass_deletes(self): + self.backend.albums[self.album]["assets"] = [] + self.assertEqual(self.run_sync(), immich_sync.EXIT_DELETE_BLOCKED) + self.assert_nothing_deleted() + + def test_missing_album_blocks_deletions(self): + self.backend.add_album("Second", [aid(99)]) + self.backend.albums[self.album]["name"] = "Renamed" + code = self.run_sync(IMMICH_ALBUMS="Echo,Second", MAX_DELETE_FRACTION="1", + MAX_DELETE_COUNT="100") + self.assertEqual(code, immich_sync.EXIT_DELETE_BLOCKED) + self.assertIn("Album not found", self.output) + self.assert_nothing_deleted() + + def test_failed_immich_query_blocks_deletions(self): + self.backend.search_fails = True + self.backend.albums[self.album]["assets"] = [] + self.assertNotEqual(self.run_sync(MAX_DELETE_FRACTION="1", MAX_DELETE_COUNT="100"), + immich_sync.EXIT_OK) + self.assert_nothing_deleted() + + def test_new_uploads_do_not_dilute_the_brake(self): + # 3 of 10 tracked leave (30%) while 20 new photos arrive: 3/30 would pass 20% + del self.backend.albums[self.album]["assets"][:3] + self.backend.albums[self.album]["assets"] += [Backend.make_asset(aid(100 + i)) for i in range(20)] + self.assertEqual(self.run_sync(), immich_sync.EXIT_DELETE_BLOCKED) + self.assertIn("3 of 10 tracked", self.output) + self.assertEqual(self.backend.trashed, set()) + + def test_brake_can_be_lifted_explicitly(self): + del self.backend.albums[self.album]["assets"][:3] + self.assertEqual(self.run_sync(MAX_DELETE_FRACTION="0.5"), immich_sync.EXIT_OK) + self.assertEqual(len(self.backend.trashed), 3) + + +class TestStreamingUploadOverSocket(unittest.TestCase): + """The streamed multipart body must go over a real socket with Content-Length.""" + + def test_upload_reaches_server_intact(self): + import threading + from http.server import BaseHTTPRequestHandler, HTTPServer + + received = {} + + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + received["headers"] = dict(self.headers) + received["body"] = self.rfile.read(int(self.headers["Content-Length"])) + payload = b'{"id": "n1"}' + self.send_response(201) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *args): + pass + + server = HTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "f" + content = os.urandom(300_000) + path.write_bytes(content) + with patch("immich_sync.AMAZON_CONTENT_URL", f"http://127.0.0.1:{server.server_port}"), \ + patch("immich_sync.AP_HEADERS", {"Cookie": "x=y"}): + node = immich_sync.upload_to_amazon(path, "IMG.jpg", "root") + finally: + server.shutdown() + server.server_close() + self.assertEqual(node, "n1") + self.assertNotIn("Transfer-Encoding", received["headers"]) + self.assertIn(content, received["body"]) + self.assertIn(b'"parents": ["root"]', received["body"]) + self.assertIn(b'filename="IMG.jpg"', received["body"]) + + +class TestRedirectNotFollowed(unittest.TestCase): + def test_amazon_redirect_raises_auth_error_without_following(self): + import threading + from http.server import BaseHTTPRequestHandler, HTTPServer + + hits = [] + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + hits.append(self.path) + self.send_response(302) + self.send_header("Location", "/ap/signin") + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, *args): + pass + + server = HTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + with patch("immich_sync.AMAZON_DRIVE_URL", f"http://127.0.0.1:{server.server_port}/drive/v1"), patch("immich_sync.AP_HEADERS", {"Cookie": "x=y"}): + with self.assertRaises(immich_sync.AmazonAuthError): + immich_sync.get_root_id() + finally: + server.shutdown() + server.server_close() + self.assertEqual(len(hits), 1) + self.assertFalse(hits[0].startswith("/ap/signin")) + + +class TestDryRun(SyncTestCase): + def test_dry_run_writes_nothing(self): + self.backend.add_album("Echo", [aid(1), aid(2)]) + self.assertEqual(self.run_sync(DRY_RUN="true"), immich_sync.EXIT_OK) + self.assertEqual(self.backend.amazon_writes(), []) + self.assertFalse((self.state_dir / "state.json").exists()) + self.assertIn("[dry-run] Would upload", self.output) + self.assertIn("[dry-run] Would create Amazon Photos album", self.output) + downloads = [u for _, u, _ in self.backend.calls if u.endswith("/original")] + self.assertEqual(downloads, []) + + def test_dry_run_keeps_existing_state_and_reports_deletions(self): + album = self.backend.add_album("Echo", [aid(i) for i in range(1, 7)]) + self.run_sync() + before = (self.state_dir / "state.json").read_bytes() + self.backend.albums[album]["assets"].pop(0) + self.backend.calls.clear() + self.assertEqual(self.run_sync(DRY_RUN="true"), immich_sync.EXIT_OK) + self.assertEqual(self.backend.amazon_writes(), []) + self.assertEqual((self.state_dir / "state.json").read_bytes(), before) + self.assertIn("[dry-run] Would remove", self.output) + + +class TestAuthAndSecrets(SyncTestCase): + def test_expired_cookies_abort_immediately(self): + self.backend.add_album("Echo", [aid(i) for i in range(1, 6)]) + self.backend.amazon_status = 401 + self.assertEqual(self.run_sync(), immich_sync.EXIT_AUTH) + self.assertIn("cookies expired", self.output) + amazon_calls = [u for _, u, _ in self.backend.calls if not u.startswith(IMMICH)] + self.assertEqual(len(amazon_calls), 1) # aborted on the first request + + def test_forbidden_counts_as_auth_error(self): + self.backend.add_album("Echo", [aid(1)]) + self.backend.amazon_status = 403 + self.assertEqual(self.run_sync(), immich_sync.EXIT_AUTH) + + def test_secrets_never_logged(self): + self.backend.add_album("Echo", [aid(1)]) + self.backend.amazon_status = 401 + hc = "https://hc.test/ping/very-secret-token" + self.run_sync(HEALTHCHECK_URL=hc) + log_text = (self.state_dir / "immich-sync.log").read_text() + for secret in [*SECRETS.values(), hc]: + self.assertNotIn(secret, self.output) + self.assertNotIn(secret, log_text) + + def test_redacting_formatter_masks_secrets_in_exceptions(self): + import logging + + immich_sync._SECRETS.add("super-secret-value") + try: + raise RuntimeError("leaked super-secret-value here") + except RuntimeError: + record = logging.LogRecord("x", logging.ERROR, __file__, 1, "msg %s", ("super-secret-value",), + sys.exc_info()) + text = immich_sync._RedactingFormatter().format(record) + self.assertNotIn("super-secret-value", text) + self.assertIn("***", text) + + def test_secrets_from_files(self): + secret_file = self.state_dir / "at.txt" + secret_file.write_text("from-file-secret\n") + env = {k: v for k, v in self.env.items() if k != "AP_AT_VALUE"} + env["AP_AT_VALUE_FILE"] = str(secret_file) + with patch.dict(os.environ, env, clear=True): + immich_sync._configure() + self.assertIn("at-acbde=from-file-secret", immich_sync.AP_HEADERS["Cookie"]) + self.assertIn("ubid-acbde=", immich_sync.AP_HEADERS["Cookie"]) + + def test_redirect_to_sign_in_means_expired_cookies(self): + self.backend.add_album("Echo", [aid(1)]) + self.backend.amazon_status = 302 + self.assertEqual(self.run_sync(), immich_sync.EXIT_AUTH) + + def test_non_finite_numbers_rejected(self): + for env in ({"MAX_DELETE_FRACTION": "nan"}, {"MAX_FILE_MB": "inf"}, {"MAX_DELETE_COUNT": "-1"}): + self.assertEqual(self.run_sync(**env), immich_sync.EXIT_ERROR, env) + self.assertIn("finite", self.output) + + def test_cookie_value_with_header_injection_rejected(self): + self.assertEqual(self.run_sync(AP_AT_VALUE="abc\r\nX-Evil: 1"), immich_sync.EXIT_ERROR) + + def test_config_overrides(self): + env = {**self.env, "AP_TLD": "com", "AP_UBID_KEY": "ubid-custom", + "AP_CONTENT_URL": "https://upload.example/cdproxy/nodes", "AP_USER_AGENT": "UA/1"} + with patch.dict(os.environ, env, clear=True): + immich_sync._configure() + self.assertEqual(immich_sync.AMAZON_DRIVE_URL, "https://www.amazon.com/drive/v1") + self.assertEqual(immich_sync.AMAZON_CONTENT_URL, "https://upload.example/cdproxy") + self.assertIn("ubid-custom=", immich_sync.AP_HEADERS["Cookie"]) + self.assertIn("at-main=", immich_sync.AP_HEADERS["Cookie"]) + self.assertEqual(immich_sync.AP_HEADERS["User-Agent"], "UA/1") + + +if __name__ == "__main__": + unittest.main()