From ef7f04d3609da3efceb0cfe0a19baa0e4dd3ee71 Mon Sep 17 00:00:00 2001 From: Pavel Strunkin Date: Sun, 1 Nov 2026 21:20:31 +0200 Subject: [PATCH 1/8] =?UTF-8?q?refactor:=20backend=20review=20=E2=80=94=20?= =?UTF-8?q?security,=20correctness=20and=20performance=20fixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Security - Refuse to start with the default JWT_SECRET under NODE_ENV=production. - JWT carries `sub`; email-only tokens accepted while JWT_LEGACY_EMAIL_TOKENS is not false. - Global ValidationPipe with whitelist: unknown body fields are dropped. - PATCH /test-runs/update/:id changes only `comment`. - /users/all and PATCH /users/assignRole no longer return apiKey. - Image names validated (closes an open redirect); downloads are streamed. - Seed no longer logs users, never promotes an existing user, generates a first admin's password and key when env does not set them. - Container runs as uid 1000 when the image volume allows it (VRT_RUN_AS_ROOT to keep root); HTTPS_KEY_PATH / HTTPS_CERT_PATH are read. - REGISTRATION_ENABLED, STRICT_BUILD_PROJECT_CHECK; LDAP rejects empty passwords and times out. API (backward compatible) - Error body is {statusCode, path, name, message}; client errors that gave 500 now give 400/404/409/413/422; a full diff queue gives 503 with Retry-After; /health gives 503 without a database. - New: POST /test-runs/revert, GET /users/registration/enabled. - Websocket testRun_* events carry TestRunDto; empty events are not sent. Behaviour and performance - Ignore areas are rounded and clipped to the image. - Auto-approve compares each baseline once and skips size mismatches by the PNG header; VLM projects use the pixelmatch verdict there. - Build retention skips builds with recent uploads (BUILD_RETENTION_GRACE_MINUTES) and runs in the background. - Repeated approve no longer duplicates the baseline; deletes remove rows before files and surface errors. - Image work stays in the diff worker pool with job timeouts and heap limits; timeouts for S3, LDAP, Redis, Ollama and Gemini. Database - Three indexes via CREATE INDEX CONCURRENTLY and TestRun.statusBeforeDecision. Tests and build - Coverage thresholds raised and met; e2e and auth setup deduplicated. - Docker image gated on typecheck; compose binds Postgres to 127.0.0.1, api waits for the migration and has a healthcheck. --- .claude/skills/pre-pr-review/SKILL.md | 41 + .env | 31 +- .github/dependabot.yml | 13 +- .github/workflows/docker.yml | 15 +- .github/workflows/workflow.yml | 74 +- .gitignore | 4 + AGENTS.md | 10 +- Dockerfile | 20 +- README.md | 130 + docker-compose.ldap.yml | 3 - docker-compose.oidc.yml | 8 - docker-compose.yml | 29 +- docker-entrypoint.sh | 23 + eslint.config.mjs | 52 +- jest.config.ts | 12 - package-lock.json | 228 +- package.json | 29 +- prisma/Dockerfile | 1 - prisma/client.ts | 4 +- .../migration.sql | 9 + .../migration.sql | 9 + .../migration.sql | 9 + .../migration.sql | 2 + prisma/package-lock.json | 15 +- prisma/package.json | 8 +- prisma/pg-options.ts | 12 +- prisma/schema.prisma | 23 +- prisma/seed.ts | 143 +- prisma/test/seed.spec.ts | 223 +- src/_data_/index.ts | 9 +- src/app.module.spec.ts | 96 +- src/app.module.ts | 4 +- src/app.service.spec.ts | 44 +- src/app.service.ts | 20 +- src/auth/auth.service.spec.ts | 23 +- src/auth/auth.service.ts | 12 +- src/auth/guards/api.guard.spec.ts | 39 +- src/auth/guards/api.guard.ts | 20 +- src/auth/guards/mixed.guard.spec.ts | 21 +- src/auth/guards/mixed.guard.ts | 18 +- src/auth/guards/role.guard.spec.ts | 95 +- src/auth/guards/role.guard.ts | 28 +- src/auth/jwt-payload.model.ts | 4 +- src/auth/jwt.strategy.spec.ts | 106 +- src/auth/jwt.strategy.ts | 33 +- src/builds/build-stats.spec.ts | 41 +- src/builds/build-stats.ts | 31 +- src/builds/builds.controller.spec.ts | 186 +- src/builds/builds.controller.ts | 24 +- src/builds/builds.module.ts | 10 +- src/builds/builds.service.spec.ts | 1081 +++--- src/builds/builds.service.ts | 272 +- src/builds/dto/build-modify.dto.ts | 5 +- src/builds/dto/build.dto.spec.ts | 237 +- src/builds/dto/build.dto.ts | 2 - src/compare/compare.service.spec.ts | 527 ++- src/compare/compare.service.ts | 55 +- src/compare/diff-worker-pool.inline.spec.ts | 128 + src/compare/diff-worker-pool.spec.ts | 57 - src/compare/diff-worker-pool.ts | 190 +- src/compare/diff-worker-pool.workers.spec.ts | 363 +- src/compare/libs/ImageCompareInput.ts | 7 +- .../libs/blur-amplitude.config.spec.ts | 13 +- src/compare/libs/blur-amplitude.config.ts | 31 +- .../libs/dinov2/dinov2.controller.spec.ts | 1 - src/compare/libs/dinov2/dinov2.controller.ts | 1 - .../libs/dinov2/dinov2.service.spec.ts | 181 +- src/compare/libs/dinov2/dinov2.service.ts | 72 +- .../libs/image-comparator.interface.ts | 6 - .../libs/looks-same/looks-same.core.spec.ts | 177 + .../libs/looks-same/looks-same.core.ts | 92 + .../looks-same.service.boundaries.spec.ts | 58 - .../looks-same.service.real-images.spec.ts | 169 + .../looks-same/looks-same.service.spec.ts | 407 +- .../libs/looks-same/looks-same.service.ts | 113 +- .../libs/looks-same/looks-same.types.ts | 37 - src/compare/libs/odiff/odiff.service.spec.ts | 392 +- src/compare/libs/odiff/odiff.service.ts | 55 +- src/compare/libs/odiff/odiff.types.ts | 4 - .../pixelmatch/blur-amplitude.core.spec.ts | 134 +- .../libs/pixelmatch/blur-amplitude.core.ts | 104 +- .../libs/pixelmatch/patch-gate.core.spec.ts | 68 +- .../libs/pixelmatch/patch-gate.core.ts | 58 +- ...ts => pixelmatch.core.real-images.spec.ts} | 45 +- .../libs/pixelmatch/pixelmatch.core.spec.ts | 235 +- .../libs/pixelmatch/pixelmatch.core.ts | 98 +- ...=> pixelmatch.service.real-images.spec.ts} | 0 .../pixelmatch/pixelmatch.service.spec.ts | 708 ++-- .../libs/pixelmatch/pixelmatch.service.ts | 89 +- .../libs/pixelmatch/pixelmatch.worker.spec.ts | 38 +- .../libs/pixelmatch/pixelmatch.worker.ts | 8 +- .../libs/pixelmatch/signature.core.spec.ts | 67 +- src/compare/libs/pixelmatch/signature.core.ts | 70 +- .../libs/pixelmatch/thumbnail.core.spec.ts | 26 +- src/compare/libs/pixelmatch/thumbnail.core.ts | 14 - .../libs/pixelmatch/worker-job.spec.ts | 41 + src/compare/libs/pixelmatch/worker-job.ts | 45 +- .../libs/vlm/concurrency-limiter.spec.ts | 115 + src/compare/libs/vlm/concurrency-limiter.ts | 28 + .../providers/gemini/gemini.service.spec.ts | 47 +- .../vlm/providers/gemini/gemini.service.ts | 42 +- .../ollama/ollama.controller.spec.ts | 2 - .../providers/ollama/ollama.service.spec.ts | 162 +- .../vlm/providers/ollama/ollama.service.ts | 52 +- src/compare/libs/vlm/vlm-inputs.core.spec.ts | 94 + src/compare/libs/vlm/vlm-inputs.core.ts | 29 + .../libs/vlm/vlm-provider.interface.ts | 19 - src/compare/libs/vlm/vlm-result.schema.ts | 3 - src/compare/libs/vlm/vlm.service.spec.ts | 682 ++-- src/compare/libs/vlm/vlm.service.ts | 104 +- src/compare/libs/vlm/vlm.types.ts | 43 - src/compare/test-utils.ts | 33 + src/compare/utils/index.spec.ts | 129 +- src/compare/utils/index.ts | 55 +- src/config/env.validation.spec.ts | 111 + src/config/env.validation.ts | 52 + src/health/health.controller.spec.ts | 83 +- src/health/health.controller.ts | 10 +- src/http-exception.filter.spec.ts | 315 +- src/http-exception.filter.ts | 102 +- src/jest.config.ts | 13 +- src/jest.setup.ts | 4 + src/main.helpers.spec.ts | 101 + src/main.helpers.ts | 40 + src/main.ts | 46 +- src/prisma/prisma-errors.spec.ts | 20 + src/prisma/prisma-errors.ts | 4 + src/prisma/prisma.module.spec.ts | 2 - src/prisma/prisma.module.ts | 3 - src/prisma/prisma.service.spec.ts | 63 +- src/prisma/prisma.service.ts | 67 +- src/projects/dto/project.dto.spec.ts | 8 + src/projects/dto/project.dto.ts | 6 +- src/projects/projects.controller.spec.ts | 6 - src/projects/projects.controller.ts | 4 +- src/projects/projects.module.ts | 3 +- src/projects/projects.service.spec.ts | 456 ++- src/projects/projects.service.ts | 133 +- src/redis-io.adapter.spec.ts | 128 +- src/redis-io.adapter.ts | 52 +- src/shared/api-file.decorator.spec.ts | 4 +- src/shared/current-user.decorator.spec.ts | 1 - src/shared/dto/baseline.dto.ts | 1 - src/shared/events/debounce.spec.ts | 94 + src/shared/events/debounce.ts | 57 + src/shared/events/events.gateway.spec.ts | 198 +- src/shared/events/events.gateway.ts | 110 +- src/shared/fite-to-body.interceptor.spec.ts | 28 +- src/shared/tasks/tasks.service.spec.ts | 154 +- src/shared/tasks/tasks.service.ts | 90 +- src/shared/validation.audit.spec.ts | 69 + src/shared/validation.spec.ts | 57 + src/shared/validation.ts | 31 + src/shared/validation.whitelist.spec.ts | 158 + src/shared/wrapper-type.ts | 8 +- src/static/aws/s3.service.spec.ts | 224 +- src/static/aws/s3.service.ts | 66 +- src/static/hdd/hdd.service.spec.ts | 97 +- src/static/hdd/hdd.service.ts | 48 +- src/static/static.controller.spec.ts | 170 +- src/static/static.controller.ts | 37 +- src/static/static.interface.ts | 4 +- src/static/static.service.spec.ts | 91 +- src/static/static.service.ts | 27 +- src/static/utils.spec.ts | 295 +- src/static/utils.ts | 74 +- src/test-runs/diffResult.ts | 25 - .../dto/create-test-request-base64.dto.ts | 6 +- .../dto/create-test-request-multipart.dto.ts | 33 +- .../dto/create-test-request.dto.spec.ts | 149 +- src/test-runs/dto/create-test-request.dto.ts | 5 + src/test-runs/dto/ignore-area.dto.spec.ts | 38 + src/test-runs/dto/ignore-area.dto.ts | 14 +- .../dto/testRun-paginated.dto.spec.ts | 1 - src/test-runs/dto/testRun.dto.ts | 1 - src/test-runs/dto/testRunResult.dto.spec.ts | 3 +- src/test-runs/dto/testRunResult.dto.ts | 4 +- .../dto/update-ignore-area.dto.spec.ts | 6 + src/test-runs/dto/update-ignore-area.dto.ts | 4 +- src/test-runs/test-runs.controller.spec.ts | 256 +- src/test-runs/test-runs.controller.ts | 68 +- src/test-runs/test-runs.service.spec.ts | 3419 ++++++++++------- src/test-runs/test-runs.service.ts | 948 ++--- .../test-variations.controller.ts | 2 +- .../test-variations.service.spec.ts | 1200 +++--- .../test-variations.service.ts | 233 +- src/users/db/dbusers.service.spec.ts | 76 +- src/users/db/dbusers.service.ts | 23 +- src/users/dto/change-password.dto.spec.ts | 20 + src/users/dto/change-password.dto.ts | 9 + src/users/dto/user.dto.ts | 12 +- src/users/ldap/ldapusers.concurrency.spec.ts | 100 + src/users/ldap/ldapusers.service.spec.ts | 332 +- src/users/ldap/ldapusers.service.ts | 132 +- src/users/oidc/oidcusers.service.spec.ts | 391 +- src/users/oidc/oidcusers.service.ts | 61 +- src/users/users.controller.spec.ts | 263 +- src/users/users.controller.ts | 35 +- src/users/users.factory.spec.ts | 35 +- src/users/users.factory.ts | 14 +- src/users/users.interface.ts | 1 + src/users/users.service.spec.ts | 118 +- src/users/users.service.ts | 34 +- src/utils/bounded-cache.ts | 26 + src/utils/env.spec.ts | 44 + src/utils/env.ts | 10 + src/utils/index.spec.ts | 69 +- src/utils/index.ts | 30 + src/utils/map-with-concurrency.ts | 12 + src/utils/random-key.ts | 15 - test/api-helpers.ts | 104 +- test/body-limit.e2e-spec.ts | 36 + test/builds-api.e2e-spec.ts | 107 +- test/builds.e2e-spec.ts | 325 +- test/comparators.e2e-spec.ts | 690 +--- test/diff-worker-pool.e2e-spec.ts | 71 + test/jest.config.ts | 3 +- test/ollama.e2e-spec.ts | 19 +- test/preconditions.ts | 14 +- test/project-favorites.e2e-spec.ts | 24 +- test/projects-remove.e2e-spec.ts | 71 + test/projects.e2e-spec.ts | 115 +- test/static.e2e-spec.ts | 52 +- test/tasks.e2e-spec.ts | 59 +- test/test-runs-review.e2e-spec.ts | 358 +- test/test-runs-validation.e2e-spec.ts | 124 + test/test-runs.e2e-spec.ts | 457 +-- test/test-variations-api.e2e-spec.ts | 7 - test/test-variations.e2e-spec.ts | 69 - test/users-admin.e2e-spec.ts | 143 +- test/users.e2e-spec.ts | 146 +- test/validation-whitelist.e2e-spec.ts | 117 + test_acceptance/acceptance.spec.ts | 2 +- test_acceptance/jest.config.ts | 3 +- test_ldap/jest.config.ts | 3 +- test_ldap/ldap.spec.ts | 6 + test_oidc/jest.config.ts | 3 +- test_oidc/oidc.spec.ts | 3 - tsconfig.build.json | 2 +- tsconfig.jest.json | 9 - tsconfig.json | 4 + tslint.json | 22 - 242 files changed, 13907 insertions(+), 11827 deletions(-) create mode 100644 .claude/skills/pre-pr-review/SKILL.md create mode 100755 docker-entrypoint.sh delete mode 100644 jest.config.ts create mode 100644 prisma/migrations/20261024174953_add_test_run_test_variation_index/migration.sql create mode 100644 prisma/migrations/20261024175021_add_baseline_test_variation_index/migration.sql create mode 100644 prisma/migrations/20261024175038_add_test_run_name_baseline_name_index/migration.sql create mode 100644 prisma/migrations/20261101165840_add_test_run_status_before_decision/migration.sql create mode 100644 src/compare/diff-worker-pool.inline.spec.ts delete mode 100644 src/compare/diff-worker-pool.spec.ts create mode 100644 src/compare/libs/looks-same/looks-same.core.spec.ts create mode 100644 src/compare/libs/looks-same/looks-same.core.ts delete mode 100644 src/compare/libs/looks-same/looks-same.service.boundaries.spec.ts create mode 100644 src/compare/libs/looks-same/looks-same.service.real-images.spec.ts rename src/compare/libs/pixelmatch/{pixelmatch.core.boundaries.spec.ts => pixelmatch.core.real-images.spec.ts} (63%) rename src/compare/libs/pixelmatch/{pixelmatch.service.boundaries.spec.ts => pixelmatch.service.real-images.spec.ts} (100%) create mode 100644 src/compare/libs/pixelmatch/worker-job.spec.ts create mode 100644 src/compare/libs/vlm/concurrency-limiter.spec.ts create mode 100644 src/compare/libs/vlm/concurrency-limiter.ts create mode 100644 src/compare/libs/vlm/vlm-inputs.core.spec.ts create mode 100644 src/compare/libs/vlm/vlm-inputs.core.ts create mode 100644 src/compare/test-utils.ts create mode 100644 src/config/env.validation.spec.ts create mode 100644 src/config/env.validation.ts create mode 100644 src/jest.setup.ts create mode 100644 src/main.helpers.spec.ts create mode 100644 src/main.helpers.ts create mode 100644 src/prisma/prisma-errors.spec.ts create mode 100644 src/prisma/prisma-errors.ts create mode 100644 src/shared/events/debounce.spec.ts create mode 100644 src/shared/events/debounce.ts create mode 100644 src/shared/validation.audit.spec.ts create mode 100644 src/shared/validation.spec.ts create mode 100644 src/shared/validation.ts create mode 100644 src/shared/validation.whitelist.spec.ts create mode 100644 src/test-runs/dto/ignore-area.dto.spec.ts create mode 100644 src/users/dto/change-password.dto.spec.ts create mode 100644 src/users/dto/change-password.dto.ts create mode 100644 src/users/ldap/ldapusers.concurrency.spec.ts create mode 100644 src/utils/bounded-cache.ts create mode 100644 src/utils/env.spec.ts create mode 100644 src/utils/env.ts create mode 100644 src/utils/map-with-concurrency.ts create mode 100644 test/body-limit.e2e-spec.ts create mode 100644 test/diff-worker-pool.e2e-spec.ts create mode 100644 test/projects-remove.e2e-spec.ts create mode 100644 test/test-runs-validation.e2e-spec.ts delete mode 100644 test/test-variations.e2e-spec.ts create mode 100644 test/validation-whitelist.e2e-spec.ts delete mode 100644 tsconfig.jest.json delete mode 100644 tslint.json diff --git a/.claude/skills/pre-pr-review/SKILL.md b/.claude/skills/pre-pr-review/SKILL.md new file mode 100644 index 00000000..1b66066c --- /dev/null +++ b/.claude/skills/pre-pr-review/SKILL.md @@ -0,0 +1,41 @@ +--- +name: pre-pr-review +description: Review a branch before its pull request is opened or updated. Use whenever a PR or merge request is about to be created, pushed or marked ready, or the user asks "can we open the PR", "make a PR", "push this", even when they do not mention a review. +--- + +# Before a pull request + +No pull request is opened, and no branch is pushed for one, until these steps +pass. A step that fails is fixed on the branch, not noted for later. + +1. **Checks.** `npm run typecheck`, `npm run lint`, `npm run format:check`, + `npm run test:cov`, `npm run build`, and `npm run test:e2e` against a + database of its own (`docker run -d --name vrt-e2e-pr -e POSTGRES_PASSWORD=pg + -p 55499:5432 postgres:15.3-alpine3.18`, then with + `DATABASE_URL=postgresql://postgres:pg@localhost:55499/postgres`: + `npx prisma migrate deploy`, `npx prisma migrate diff --from-config-datasource + --to-schema prisma/schema.prisma --exit-code`, `npm run test:e2e`; remove it + with `docker rm -fv vrt-e2e-pr`). If the branch touches the Dockerfiles, + compose files, the seed or auth, also bring the stack up with + `docker compose up --build`, check `/health`, run `npm run test:acceptance` + (and `test:ldap` / `test:oidc` with their compose overlays when auth + changed), then `docker compose down --volumes`. If `dinov2/` changed, run + `python -m unittest test_patches` there. +2. **Code review.** Run the `code-review` skill on the branch against its base + (`master` unless the PR targets another branch), at `high` effort or above + for a branch of more than a few commits. Fix every confirmed finding on the + branch; for a finding left as is, say why in the PR description. +3. **Comments.** Go through the diff's comments with the rule in `AGENTS.md` + (Code conventions, "Comments"): delete any that restate the code, narrate + history, point at review notes or ids that are not in the repo, or would + not change what a reader does. Keep only non-obvious whys, one line each. + Test names follow the same rule. +4. **Plan status.** If the branch works through a plan in `docs/`, tick the + done tasks there and record follow-ups. +5. **Commits.** Author is the repo's configured git identity; no + `Co-Authored-By` or "Generated with" lines. + +Then open the PR. Its description lists what changed by area, how it was +verified (the commands above and their results), what needs attention when +deploying (migrations, new or changed environment variables, behaviour +changes), and what is deliberately left for later. diff --git a/.env b/.env index ef80ee54..51b88c22 100644 --- a/.env +++ b/.env @@ -1,4 +1,4 @@ -JWT_SECRET='jwtPrivateKey' +JWT_SECRET='vrt-local-development-only-jwt-secret' JWT_LIFE_TIME=1d BODY_PARSER_JSON_LIMIT="5mb" @@ -30,6 +30,22 @@ AWS_S3_BUCKET_NAME= # Multi-instance socket.io fan-out (optional). Unset = in-memory adapter. #REDIS_URL=redis://localhost:6379 +# How long the first Redis connection may take before the API exits instead of waiting (default 30000). +#REDIS_CONNECT_TIMEOUT_MS=30000 + +# Log levels, comma separated; unset logs everything. Read from the process environment, not from this file. +#LOG_LEVEL=log,warn,error,fatal + +# S3 request timeouts in milliseconds (STATIC_SERVICE=s3) +#S3_CONNECTION_TIMEOUT_MS=5000 +#S3_REQUEST_TIMEOUT_MS=60000 + +#BUILD_RETENTION_GRACE_MINUTES=60 # automatic build pruning spares builds active within this many minutes; 0 disables + +#REGISTRATION_ENABLED=true +#JWT_LEGACY_EMAIL_TOKENS=true +#LDAP_TIMEOUT_MS=10000 +#LDAP_CONNECT_TIMEOUT_MS=5000 #LDAP_ENABLED=true #LDAP_TLS_NO_VERIFY=false @@ -67,3 +83,16 @@ OLLAMA_BASE_URL=http://localhost:11434 # DINOv2 service for the rendering-noise filter's "dinov2Check" (see dinov2/README.md) #DINOV2_URL=http://localhost:8000 #DINOV2_TIMEOUT_MS=60000 +#DATABASE_URL=postgresql://postgres:postgres@localhost:5432/vrt_db_dev # required; docker-compose.yml builds it from POSTGRES_* +#NODE_ENV=production # production refuses to start with the sample JWT_SECRET +#VRT_RUN_AS_ROOT=false # docker entrypoint: true keeps the container on root +#MAX_IMAGE_PIXELS=100000000 # largest image in pixels (width x height); 0 turns the limit off +#MULTIPART_FILE_SIZE_LIMIT=104857600 # largest multipart upload in bytes (100 MiB) +#DIFF_WORKERS_COUNT= # diff worker threads; unset: CPUs minus 1, between 1 and 8 +#DIFF_QUEUE_LIMIT=256 # diff jobs waiting for a worker; when full the API answers 503 +#DIFF_JOB_TIMEOUT_MS=120000 # one diff job, ms; 0 turns the timeout off +#DIFF_WORKER_MAX_HEAP_MB=512 # heap limit of each diff worker, MB +#VLM_CONCURRENCY=4 # concurrent VLM requests per API instance +#OLLAMA_TIMEOUT_MS=300000 # Ollama request timeout, ms +#GEMINI_TIMEOUT_MS=300000 # Gemini request timeout, ms +#STRICT_BUILD_PROJECT_CHECK=false # true: POST /test-runs rejects a build of another project with 400 diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 6aa54ce8..0b352283 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -46,10 +46,11 @@ updates: - "*" - package-ecosystem: docker - # base images of the api (Dockerfile) and the migration image (prisma/Dockerfile) + # base images of the api (Dockerfile), the migration image (prisma/Dockerfile) and dinov2/Dockerfile directories: - "/" - "/prisma" + - "/dinov2" schedule: interval: weekly groups: @@ -57,6 +58,16 @@ updates: patterns: - "*" + - package-ecosystem: pip + # the DINOv2 service image installs dinov2/requirements.txt + directory: "/dinov2" + schedule: + interval: weekly + groups: + pip: + patterns: + - "*" + - package-ecosystem: github-actions directory: "/" schedule: diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 09837f5b..9e6d0e0c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,14 +1,5 @@ name: Docker images -# Builds visualregressiontracker/api, visualregressiontracker/migration and -# visualregressiontracker/dinov2 for linux/amd64 and linux/arm64 (Apple -# Silicon, Graviton), so docker compose runs them natively instead of under -# emulation. -# - pushing an api-vX.Y.Z, migration-vX.Y.Z or dinov2-vX.Y.Z tag publishes that image as X.Y.Z -# - "Run workflow" with an existing tag (re)publishes it, e.g. to add arm64 to a released version -# - pull requests that touch the images only build them, without pushing -# Publishing needs the DOCKERHUB_USERNAME and DOCKERHUB_TOKEN repository secrets. - on: push: tags: @@ -29,13 +20,12 @@ on: description: Existing tag to publish, e.g. api-v5.7.0, migration-v5.5.0 or dinov2-v1.0.0 required: true -# GITHUB_TOKEN is only used to check out the code; image pushes use the Docker Hub secrets permissions: contents: read jobs: docker: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: @@ -66,12 +56,11 @@ jobs: echo "build=false" >> "$GITHUB_OUTPUT" fi - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 if: steps.version.outputs.build == 'true' with: # a manual run checks out the tag itself, never a branch with the same name ref: ${{ inputs.tag && format('refs/tags/{0}', inputs.tag) || github.ref }} - # keep the token out of .git/config, which is part of the build context persist-credentials: false - name: Set up QEMU diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 185b7757..0040edce 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -6,7 +6,6 @@ on: pull_request: branches: [master] -# a new push to a pull request supersedes its running checks; master runs to completion concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} @@ -16,15 +15,14 @@ permissions: jobs: checks: - # https://github.com/actions/runner-images#available-images runs-on: ubuntu-24.04 timeout-minutes: 15 steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Setup Node.js environment - uses: actions/setup-node@v7 + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: '24' cache: npm @@ -55,7 +53,6 @@ jobs: fail-fast: false matrix: include: - # acceptance tests call the api container, e2e tests run the app in-process against its database - suite: e2e compose-file: docker-compose.yml test: npm run test:acceptance && npm run test:e2e @@ -66,14 +63,14 @@ jobs: compose-file: docker-compose.yml:docker-compose.oidc.yml test: npm run test:oidc env: - # read by every docker compose command below COMPOSE_FILE: ${{ matrix.compose-file }} + COMPOSE_PROJECT_NAME: vrt steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Setup Node.js environment - uses: actions/setup-node@v7 + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: '24' cache: npm @@ -84,12 +81,32 @@ jobs: - name: Install npm dependencies run: npm ci - - name: Build and run containers - run: docker compose up --build -d + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + + - name: Build api image + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: . + load: true + tags: vrt-api:latest + cache-from: type=gha,scope=ci-api + cache-to: type=gha,mode=max,scope=ci-api + + - name: Build migration image + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: prisma + load: true + tags: vrt-migration:latest + cache-from: type=gha,scope=ci-migration + cache-to: type=gha,mode=max,scope=ci-migration + + - name: Run containers + run: docker compose up --no-build -d - name: Wait for migrations and the api run: | - # the migration container applies the migrations, seeds the default user and exits status=$(docker wait vrt_migration) if [ "$status" != 0 ]; then echo "::error::migration container exited with $status" @@ -113,12 +130,28 @@ jobs: # See: https://github.com/Visual-Regression-Tracker/backend/issues/273 runs-on: ubuntu-24.04 timeout-minutes: 10 + services: + postgres: + image: postgres:15.3-alpine3.18 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: vrt_migrations + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres -d vrt_migrations" + --health-interval 5s + --health-timeout 5s + --health-retries 20 + env: + DATABASE_URL: postgresql://postgres:postgres@localhost:5432/vrt_migrations steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Setup Node.js environment - uses: actions/setup-node@v7 + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: '24' cache: npm @@ -130,19 +163,23 @@ jobs: working-directory: prisma run: npm ci - - name: Static analysis + - name: Unit tests working-directory: prisma run: npm run test + - name: Migrations match the schema + working-directory: prisma + run: | + npx prisma migrate deploy + npx prisma migrate diff --from-config-datasource --to-schema schema.prisma --exit-code + test-dinov2: runs-on: ubuntu-24.04 timeout-minutes: 10 steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - # the logic is tested with a stand-in encoder: numpy only, no model; - # the runner's own Python 3.12, in a venv (system pip is locked down) - name: Unit tests working-directory: dinov2 run: | @@ -150,8 +187,7 @@ jobs: .venv/bin/pip install numpy==2.5.4 .venv/bin/python -m unittest test_patches - # Branch protection requires a check named "build": it passes only when every job above did, - # so the required check keeps covering the whole pipeline after the split. + # branch protection requires a check named "build" build: if: ${{ !cancelled() }} needs: [checks, integration, test-migrations, test-dinov2] diff --git a/.gitignore b/.gitignore index 519d9953..912982c9 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,7 @@ lerna-debug.log* # TypeScript incremental build info *.tsbuildinfo + +# Claude Code +.claude/settings.local.json +.claude/worktrees/ diff --git a/AGENTS.md b/AGENTS.md index 966b429e..9163b84d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -55,7 +55,15 @@ Run typecheck, lint, format check and the unit tests before calling work done. Fields older clients do not send must be `@IsOptional()`, or saving breaks for them. - Format only the files you changed (`npx prettier --write `), not whole directories: prettier also rewrites untouched files, markdown included. -- Comments explain *why*, in the surrounding style; match existing naming. +- Comments: by default, none. Write one only for a non-obvious _why_ that a + reader would otherwise get wrong, and keep it to one line. Delete on sight: + comments that restate the code or its names, narrate past bugs or earlier + versions (that goes in the commit message), explain what a well-named + function already says, point at review notes or ticket ids that are not in + the repo, or run for several lines without changing what a reader would do. + No TODOs or "known issue" notes (open an issue). Tests follow the same rule: + the test name says what is checked. Match existing naming. Before a pull + request, see the `pre-pr-review` skill. ## Services beside the API diff --git a/Dockerfile b/Dockerfile index 093de53c..044c017b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,38 +1,30 @@ FROM node:24-bookworm-slim AS builder -# Create app directory WORKDIR /app COPY prisma.config.ts ./ COPY ./prisma/schema.prisma ./prisma/ -# A wildcard is used to ensure both package.json AND package-lock.json are copied COPY package*.json ./ -# Install app dependencies -ENV NODE_TLS_REJECT_UNAUTHORIZED='0' RUN npm ci COPY tsconfig*.json nest-cli.json .swcrc ./ COPY src ./src -# SWC compiles without checking types, so the image is gated on the type -# check here, as tsc's build used to gate it. Dev dependencies are only needed -# to build; the generated Prisma client under node_modules/.prisma survives -# the prune RUN npm run typecheck && npm run build && npm prune --omit=dev FROM node:24-bookworm-slim ENV NODE_ENV=production -# tini as PID 1: the kernel ignores a signal PID 1 has no handler for, so the -# SIGTERM Nest re-raises after its shutdown hooks would never end a node PID 1. -# It also reaps the processes odiff leaves behind. +# tini as PID 1: the SIGTERM Nest re-raises would never end a node PID 1 RUN apt-get update && apt-get install -y --no-install-recommends tini && rm -rf /var/lib/apt/lists/* -# No WORKDIR: the app resolves imageUploads/ against /, where deployments -# mount the image volume +# No WORKDIR: imageUploads/ resolves against /, where the image volume is mounted +RUN mkdir -p /imageUploads && chown node:node /imageUploads COPY --from=builder /app/node_modules ./node_modules COPY --from=builder /app/package*.json ./ COPY --from=builder /app/dist ./dist +COPY docker-entrypoint.sh /usr/local/bin/vrt-entrypoint.sh +RUN chmod 755 /usr/local/bin/vrt-entrypoint.sh EXPOSE 3000 # node without npm in between, so SIGTERM reaches the app's shutdown hooks -ENTRYPOINT [ "/usr/bin/tini", "--" ] +ENTRYPOINT [ "/usr/bin/tini", "--", "/usr/local/bin/vrt-entrypoint.sh" ] CMD [ "node", "dist/main" ] diff --git a/README.md b/README.md index f0e78732..c41fa223 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,136 @@ - Seed initial data `npx prisma db seed` - `npm run start:debug` +## Running the test suites + +| Suite | Command | Needs | Covers | +|---|---|---|---| +| unit | `npm test` | nothing | services with mocked dependencies (`src/**/*.spec.ts`) | +| e2e | `npm run test:e2e` | Postgres with the migrations applied (`DATABASE_URL`); run `npm run build` first | the app in-process against that database | +| acceptance | `npm run test:acceptance` | `docker compose up -d`; the API on `localhost:4200` | the running API over HTTP and socket.io | +| ldap | `npm run test:ldap` | `COMPOSE_FILE=docker-compose.yml:docker-compose.ldap.yml docker compose up -d` | LDAP login against the LDAP mock | +| oidc | `npm run test:oidc` | `COMPOSE_FILE=docker-compose.yml:docker-compose.oidc.yml docker compose up -d` | the OIDC endpoints against the OIDC mock | +| prisma package | `cd prisma && npm test` | `npm ci` in `prisma/` | the `prisma/` package's own tests | + +The e2e suite loads the diff worker threads from `dist/`. Without a build they run +inline, so `npm run build` makes the e2e run match production. + +Schema drift check (AGENTS.md: every schema change ships with its migration). Run it +against a database where the migrations are applied; exit code 2 means the schema has +changes without a migration: +``` +cd prisma && npx prisma migrate deploy && npx prisma migrate diff --from-config-datasource --to-schema schema.prisma --exit-code +``` + +## Environment variables + +Defaults are what the code uses when the variable is unset or empty. The commented +lines in `.env` (section `# optional`) show the same variables. + +### Core + +| Variable | Default | What it does | +|---|---|---| +| `DATABASE_URL` | none, required | Postgres connection string. `connection_limit` sets the pool size per instance (see Database connections). | +| `APP_PORT` | `3000` | Port the API listens on. | +| `APP_FRONTEND_URL` | none | Frontend base URL, used to build links in test run results. | +| `BODY_PARSER_JSON_LIMIT` | body-parser default | Maximum JSON request body size, e.g. `5mb`. | +| `JWT_SECRET` | none, required | Key that signs login tokens; startup fails when it is unset. The public sample `jwtPrivateKey` is refused with `NODE_ENV=production`, and the sample in `.env` only logs a warning. Generate one with `openssl rand -hex 32`. | +| `JWT_LIFE_TIME` | `1d` | Token lifetime as a timespan such as `1d`, `12h` or `3600s`. An invalid value falls back to `1d` with a warning. | +| `JWT_LEGACY_EMAIL_TOKENS` | `true` | Accept tokens issued before the `sub` claim existed (they carry only the email). Set `false` once those tokens have expired. | +| `SERVER_TIMEOUT` | Node.js default | HTTP `server.timeout`, in milliseconds. | +| `SERVER_HEADERS_TIMEOUT` | Node.js default | HTTP `headersTimeout`, in milliseconds. | +| `SERVER_KEEP_ALIVE_TIMEOUT` | Node.js default | HTTP `keepAliveTimeout`, in milliseconds. | +| `BUILD_RETENTION_GRACE_MINUTES` | `60` | Automatic build pruning keeps builds active within this many minutes. `0` turns that protection off. | +| `STRICT_BUILD_PROJECT_CHECK` | `false` | `true` (case-insensitive) makes `POST /test-runs` reject a build that belongs to another project with `400`. By default such a request is accepted with a warning, because older clients may send such pairs. | +| `LOG_LEVEL` | all levels | Comma-separated Nest log levels to keep, e.g. `log,warn,error,fatal`. | +| `NODE_ENV` | unset | `production` makes startup refuse the public sample `JWT_SECRET`. | +| `HTTPS_KEY_PATH` | `./secrets/ssl.key` | TLS private key. HTTPS is used only when both files exist; otherwise the API serves HTTP. | +| `HTTPS_CERT_PATH` | `./secrets/ssl.cert` | TLS certificate. | +| `VRT_RUN_AS_ROOT` | `false` | Read by `docker-entrypoint.sh`, not by the app. The container drops to the `node` user when the image volume and TLS files allow it; `true` keeps root. | + +### Storage + +| Variable | Default | What it does | +|---|---|---| +| `STATIC_SERVICE` | `hdd` | Where images are stored: `hdd` (local disk) or `s3`. Odiff works only with `hdd`. | +| `AWS_S3_BUCKET_NAME` | none | Bucket for `STATIC_SERVICE=s3`; required then. | +| `AWS_REGION` | from the AWS SDK | Read by the AWS SDK v3 default chain, not by VRT code. | +| `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` | from the AWS SDK | Same chain. An IAM role or shared AWS config works without them. | +| `S3_CONNECTION_TIMEOUT_MS` | `5000` | S3 connection timeout, in milliseconds. | +| `S3_REQUEST_TIMEOUT_MS` | `60000` | S3 request timeout, in milliseconds. A request that exceeds it fails. | + +### Image comparison + +| Variable | Default | What it does | +|---|---|---| +| `MAX_IMAGE_PIXELS` | `100000000` | Largest accepted image in pixels (width times height), checked from the PNG header before decoding. `0` turns the limit off. | +| `MULTIPART_FILE_SIZE_LIMIT` | `104857600` (100 MiB) | Largest accepted multipart file upload, in bytes. | +| `DIFF_WORKERS_COUNT` | CPUs minus 1, between 1 and 8 | Number of diff worker threads. | +| `DIFF_QUEUE_LIMIT` | `256` | Diff jobs allowed to wait for a worker. When the queue is full the API answers `503` with `Retry-After: 5`. | +| `DIFF_JOB_TIMEOUT_MS` | `120000` | Timeout for one diff job, in milliseconds. `0` turns it off. | +| `DIFF_WORKER_MAX_HEAP_MB` | `512` | Heap limit of each diff worker, in MB. | +| `VLM_CONCURRENCY` | `4` | Concurrent VLM requests per API instance (minimum 1). | +| `OLLAMA_BASE_URL` | none | Ollama server for VLM comparisons; required when the Ollama provider is used. | +| `OLLAMA_TIMEOUT_MS` | `300000` | Timeout for one Ollama request, in milliseconds. | +| `GEMINI_TIMEOUT_MS` | `300000` | Timeout for one Gemini request, in milliseconds. The API key comes from the project's VLM settings. | +| `DINOV2_URL` | none | DINOv2 service for the `dinov2Check` noise filter. With the filter on and this unset, runs go to review. | +| `DINOV2_TIMEOUT_MS` | `60000` | Timeout for one DINOv2 request, in milliseconds. | + +### Auth + +| Variable | Default | What it does | +|---|---|---| +| `REGISTRATION_ENABLED` | `true` | `false` (also `0`, `no`, `off`) disables self-registration. | +| `LDAP_ENABLED` | `false` | Enables LDAP login. When OIDC is enabled too, OIDC is used. | +| `LDAP_URL` | none | LDAP server URL, e.g. `ldaps://ldap.example.com:636/`. Required when LDAP is enabled. | +| `LDAP_BIND_USER` | none | DN of the service account used for the user search. Required when LDAP is enabled. | +| `LDAP_BIND_PASSWORD` | none | Password of the service account. Required when LDAP is enabled. | +| `LDAP_SEARCH_DN` | none | Base DN of the user search. Required when LDAP is enabled. | +| `LDAP_USERS_SEARCH_FILTER` | none | User search filter; `{{email}}` is replaced by the login email. Required when LDAP is enabled. | +| `LDAP_ATTRIBUTE_MAIL`, `LDAP_ATTRIBUTE_FIRST_NAME`, `LDAP_ATTRIBUTE_LAST_NAME` | none | Attributes that hold the email, first name and last name. Required when LDAP is enabled. | +| `LDAP_TLS_NO_VERIFY` | `false` | Skip TLS certificate verification for LDAP. Test servers only. | +| `LDAP_TIMEOUT_MS` | `10000` | LDAP operation timeout, in milliseconds. | +| `LDAP_CONNECT_TIMEOUT_MS` | `5000` | LDAP connection timeout, in milliseconds. | +| `OIDC_ENABLED` | `false` | Enables OIDC (SSO) login; see the OIDC section. | +| `OIDC_ISSUER_URL` | none | Issuer URL of the identity provider. Required when OIDC is enabled. | +| `OIDC_CLIENT_ID` | none | OIDC client ID. Required when OIDC is enabled. | +| `OIDC_CLIENT_SECRET` | none | OIDC client secret. Required when OIDC is enabled. | +| `OIDC_REDIRECT_URI` | `http://localhost:4200/users/oidc/callback` | Callback URL registered at the identity provider. | +| `OIDC_SCOPES` | `openid profile email` | Requested scopes. | +| `OIDC_ATTRIBUTE_MAIL` | `email` | Claim that holds the email. | +| `OIDC_ATTRIBUTE_FIRST_NAME` | `given_name` | Claim that holds the first name. | +| `OIDC_ATTRIBUTE_LAST_NAME` | `family_name` | Claim that holds the last name. | +| `OIDC_ROLE_CLAIM` | `groups` | Claim, or dotted path, that holds the roles. | +| `OIDC_ADMIN_VALUES` | `vrt-admin,admin` | Comma-separated claim values that give the admin role. | +| `OIDC_DEFAULT_ROLE` | `editor` | Role for every other user: `admin` or `editor`. | +| `OIDC_REQUIRE_VERIFIED_EMAIL` | `true` | Reject logins whose email is not verified. | +| `OIDC_TLS_NO_VERIFY` | `false` | Skip TLS verification for the identity provider. Test servers only. | + +### Service variables + +| Variable | Default | What it does | +|---|---|---| +| `REDIS_URL` | unset, in-memory adapter | Redis for socket.io fan-out between instances (see Scaling to multiple instances). | +| `REDIS_CONNECT_TIMEOUT_MS` | `30000` | How long the first Redis connection may take before the API exits instead of waiting. | + +### Seed + +Read by `prisma db seed`. + +| Variable | Default | What it does | +|---|---|---| +| `DEFAULT_USER_EMAIL` | `visual-regression-tracker@example.com` | Email of the default user. | +| `DEFAULT_USER_PASSWORD` | generated, printed once | Password of the default user. | +| `DEFAULT_USER_API_KEY` | generated, printed once | API key of the default user. | + +## Optional services + +- **Redis** runs several API instances behind one socket.io fan-out. Set `REDIS_URL`; see [Scaling to multiple instances](#scaling-to-multiple-instances-real-time-updates). +- **DINOv2** serves the `dinov2Check` rendering-noise filter (see `dinov2/README.md`). Start it with `docker compose --profile dinov2 up -d`. The service listens on port 8000 inside the compose network only, so the API container uses `DINOV2_URL=http://dinov2:8000`. +- **Ollama** runs the VLM comparison locally. Set `OLLAMA_BASE_URL`; from the API container, which is started by compose, the host is `http://host.docker.internal:11434`. +- **Gemini** runs the VLM comparison in the cloud. Its API key is set in the project's VLM settings, not in the environment; `GEMINI_TIMEOUT_MS` sets the request timeout. + ## Scaling to multiple instances (real-time updates) Real-time UI updates are delivered over socket.io. Running **more than one API diff --git a/docker-compose.ldap.yml b/docker-compose.ldap.yml index 83acfd7b..61f68e4d 100644 --- a/docker-compose.ldap.yml +++ b/docker-compose.ldap.yml @@ -18,9 +18,6 @@ services: condition: service_started ldapmock: container_name: ldapmock - # See https://github.com/docker-ThoTeam/slapd-server-mock/tree/main - # Default users in LDAP: https://github.com/docker-ThoTeam/slapd-server-mock/blob/main/bootstrap/data.ldif.TEMPLATE - # e.g.: developer.one@ldapmock.local:password image: thoteam/slapd-server-mock:latest restart: always ports: diff --git a/docker-compose.oidc.yml b/docker-compose.oidc.yml index 12c81f74..0cc4cd17 100644 --- a/docker-compose.oidc.yml +++ b/docker-compose.oidc.yml @@ -2,9 +2,6 @@ services: api: environment: OIDC_ENABLED: 'true' - # Issuer must be reachable both from the api container and the browser. - # On macOS/Windows Docker Desktop host.docker.internal works out of the box, - # on Linux add `127.0.0.1 host.docker.internal` to /etc/hosts. OIDC_ISSUER_URL: 'http://host.docker.internal:8090/default' OIDC_CLIENT_ID: 'vrt-client' OIDC_CLIENT_SECRET: 'vrt-secret' @@ -17,8 +14,6 @@ services: OIDC_ADMIN_VALUES: 'vrt-admin' OIDC_DEFAULT_ROLE: 'editor' OIDC_REQUIRE_VERIFIED_EMAIL: 'true' - # Docker Desktop provides host.docker.internal; on Linux map it to the host - # so the api container reaches the mock IdP through the published port. extra_hosts: - 'host.docker.internal:host-gateway' depends_on: @@ -28,9 +23,6 @@ services: condition: service_started oidcmock: container_name: oidcmock - # See https://github.com/navikt/mock-oauth2-server - # Interactive login: http://host.docker.internal:8090/default/debugger - # Default users: 'admin' (role admin) and any other username (role editor). image: ghcr.io/navikt/mock-oauth2-server:6.1.0 hostname: host.docker.internal restart: always diff --git a/docker-compose.yml b/docker-compose.yml index 32bbf9db..a1811b5c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,6 +19,7 @@ services: OLLAMA_BASE_URL: ${OLLAMA_BASE_URL} DINOV2_URL: ${DINOV2_URL} REDIS_URL: ${REDIS_URL:-redis://redis:6379} + BUILD_RETENTION_GRACE_MINUTES: ${BUILD_RETENTION_GRACE_MINUTES} ports: - "${APP_PORT}:3000" expose: @@ -30,6 +31,14 @@ services: condition: service_healthy redis: condition: service_healthy + migration: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:3000/health').then((r) => process.exit(r.ok ? 0 : 1), () => process.exit(1))"] + interval: 10s + timeout: 5s + retries: 12 + start_period: 30s migration: container_name: vrt_migration build: @@ -52,7 +61,7 @@ services: POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} POSTGRES_DB: ${POSTGRES_DB} ports: - - "${POSTGRES_PORT}:5432" + - "127.0.0.1:${POSTGRES_PORT}:5432" expose: - "${POSTGRES_PORT}" volumes: @@ -62,9 +71,6 @@ services: interval: 10s timeout: 120s retries: 10 - # The DINOv2 service behind an image-comparison config's "dinov2Check". Only - # with `docker compose --profile dinov2 up`, and the api reaches it once - # DINOV2_URL=http://dinov2:8000 is set. dinov2: container_name: vrt_dinov2 profiles: ["dinov2"] @@ -85,20 +91,5 @@ services: interval: 10s timeout: 5s retries: 10 - # Available at http://localhost:5050 - # pgadmin: - # container_name: pgadmin4 - # # https://hub.docker.com/r/dpage/pgadmin4 - # # https://www.pgadmin.org/docs/pgadmin4/latest/release_notes_7_4.html - # image: dpage/pgadmin4:7.4 - # restart: always - # environment: - # PGADMIN_DEFAULT_EMAIL: admin@admin.com - # PGADMIN_DEFAULT_PASSWORD: root - # ports: - # - "5050:80" - # depends_on: - # postgres: - # condition: service_healthy volumes: postgres: \ No newline at end of file diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100755 index 00000000..ca3ed1d7 --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,23 @@ +#!/bin/sh +set -e +# installations whose image volume or TLS files belong to root must keep running as root +as_node() { + setpriv --reuid=node --regid=node --init-groups "$@" +} + +can_drop_root() { + as_node test -w /imageUploads || return 1 + for file in "${HTTPS_KEY_PATH:-./secrets/ssl.key}" "${HTTPS_CERT_PATH:-./secrets/ssl.cert}"; do + if [ -e "$file" ] && ! as_node test -r "$file"; then + return 1 + fi + done +} + +if [ "$(id -u)" = "0" ] && [ "${VRT_RUN_AS_ROOT:-false}" != "true" ]; then + if can_drop_root; then + exec setpriv --reuid=node --regid=node --init-groups "$@" + fi + echo "vrt-entrypoint: /imageUploads or the TLS files are not accessible to uid 1000 (node); running as root" >&2 +fi +exec "$@" diff --git a/eslint.config.mjs b/eslint.config.mjs index b9fd1aee..f89d2f5d 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -1,33 +1,55 @@ -import typescriptParser from '@typescript-eslint/parser'; -import typescriptPlugin from '@typescript-eslint/eslint-plugin'; +import tsParser from '@typescript-eslint/parser'; +import tsPlugin from '@typescript-eslint/eslint-plugin'; + +const TESTS = [ + '**/*.spec.ts', + '**/*.e2e-spec.ts', + 'test/**/*.ts', + 'test_*/**/*.ts', + 'src/_data_/**/*.ts', + 'prisma/test/**/*.ts', +]; export default [ { - files: ["**/*.ts"], + ignores: [ + 'dist/**', + 'coverage/**', + 'node_modules/**', + 'dinov2/**', + 'prisma/node_modules/**', + 'prisma/.generate/**', + ], + }, + ...tsPlugin.configs['flat/recommended'].map((config) => ({ ...config, files: ['**/*.ts'] })), + { + files: ['**/*.ts'], languageOptions: { - parser: typescriptParser, + parser: tsParser, parserOptions: { project: ['tsconfig.json', 'prisma/tsconfig.json'], + tsconfigRootDir: import.meta.dirname, sourceType: 'module', }, - globals: { - node: true, - jest: true, - }, - }, - plugins: { - '@typescript-eslint': typescriptPlugin, }, rules: { - '@typescript-eslint/interface-name-prefix': 'off', - '@typescript-eslint/explicit-function-return-type': 'off', '@typescript-eslint/no-explicit-any': 'off', + '@typescript-eslint/no-unused-vars': [ + 'error', + { argsIgnorePattern: '^_', varsIgnorePattern: '^_', caughtErrors: 'none' }, + ], + '@typescript-eslint/no-floating-promises': 'error', + '@typescript-eslint/no-misused-promises': 'error', + '@typescript-eslint/await-thenable': 'error', + '@typescript-eslint/only-throw-error': 'error', + '@typescript-eslint/no-unnecessary-type-assertion': 'error', }, }, { - files: ["**/*.spec.ts"], + files: TESTS, rules: { - '@typescript-eslint/explicit-function-return-type': 'off', + '@typescript-eslint/no-require-imports': 'off', + '@typescript-eslint/no-misused-promises': ['error', { checksVoidReturn: false }], }, }, ]; diff --git a/jest.config.ts b/jest.config.ts deleted file mode 100644 index 53903ecd..00000000 --- a/jest.config.ts +++ /dev/null @@ -1,12 +0,0 @@ -/** @returns {Promise} */ -module.exports = async () => { - return { - projects: [ - './src/jest.config.ts', - './test/jest.config.ts', - './test_acceptance/jest.config.ts' - ], - roots: ['./'], - testTimeout: 30000, - }; -}; diff --git a/package-lock.json b/package-lock.json index 049e75d9..de23fc8d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28,25 +28,24 @@ "@prisma/client": "^7.10.1", "@socket.io/redis-adapter": "^8.3.0", "bcryptjs": "^2.4.3", - "body-parser": "^2.2.0", "class-transformer": "^0.5.1", "class-validator": "^0.15.1", "compression": "^1.8.2", + "express": "^5.3.0", "ldapts": "^9.2.0", - "lodash": "^4.17.21", "looks-same": "^10.0.1", "odiff-bin": "^4.5.0", "ollama": "^0.6.4", "openid-client": "^5.7.1", "passport": "^0.7.0", "passport-jwt": "^4.0.1", - "passport-local": "^1.0.0", "pg": "^8.16.3", "pixelmatch": "^5.3.0", "pngjs": "^7.0.0", "redis": "^4.7.0", "reflect-metadata": "^0.2.2", "rxjs": "^7.8.2", + "socket.io": "^4.8.4", "zod": "^4.6.7", "zod-to-json-schema": "^3.25.0" }, @@ -61,11 +60,9 @@ "@types/compression": "^1.8.1", "@types/express": "^5.0.6", "@types/jest": "^30.0.0", - "@types/lodash": "^4.17.26", "@types/multer": "^2.0.0", "@types/node": "^24.19.2", "@types/passport-jwt": "^3.0.9", - "@types/passport-local": "^1.0.38", "@types/pixelmatch": "^5.2.6", "@types/pngjs": "^6.0.5", "@types/supertest": "^7.2.1", @@ -74,16 +71,12 @@ "axios": "^1.20.0", "dotenv": "^17.2.3", "eslint": "^9.39.1", - "eslint-config-prettier": "^10.1.8", "jest": "^30.5.2", - "jest-mock-extended": "^4.0.2", "prettier": "^3.9.10", "prisma": "^7.10.1", "socket.io-client": "^4.8.4", "supertest": "^7.3.1", - "ts-jest": "^29.4.15", "ts-node": "^10.9.2", - "tsconfig-paths": "^4.2.0", "typescript": "~6.0.3" }, "engines": { @@ -5358,18 +5351,6 @@ "@types/passport-strategy": "*" } }, - "node_modules/@types/passport-local": { - "version": "1.0.38", - "resolved": "https://registry.npmjs.org/@types/passport-local/-/passport-local-1.0.38.tgz", - "integrity": "sha512-nsrW4A963lYE7lNTv9cr5WmiUD1ibYJvWrpE13oxApFsRt77b0RdtZvKbCdNIY4v/QZ6TRQWaDDEwV1kCTmcXg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/express": "*", - "@types/passport": "*", - "@types/passport-strategy": "*" - } - }, "node_modules/@types/passport-strategy": { "version": "0.2.38", "resolved": "https://registry.npmjs.org/@types/passport-strategy/-/passport-strategy-0.2.38.tgz", @@ -7507,19 +7488,6 @@ "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" } }, - "node_modules/bs-logger": { - "version": "0.2.6", - "resolved": "https://registry.npmjs.org/bs-logger/-/bs-logger-0.2.6.tgz", - "integrity": "sha512-pd8DCoxmbgc7hyPKOvxtqNcjYoOsABPQdcCUjGp3d42VR2CX1ORhk2A87oqqu5R1kk+76nsxZupkmyd+MVtCog==", - "dev": true, - "license": "MIT", - "dependencies": { - "fast-json-stable-stringify": "2.x" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/bser": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/bser/-/bser-2.1.1.tgz", @@ -8980,22 +8948,6 @@ } } }, - "node_modules/eslint-config-prettier": { - "version": "10.1.8", - "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-10.1.8.tgz", - "integrity": "sha512-82GZUjRS0p/jganf6q1rEO25VSoHH0hKPCTrgillPjdI/3bgBhAE1QzHrHTizjpRvy6pGAvKjDJtk2pF9NDq8w==", - "dev": true, - "license": "MIT", - "bin": { - "eslint-config-prettier": "bin/cli.js" - }, - "funding": { - "url": "https://opencollective.com/eslint-config-prettier" - }, - "peerDependencies": { - "eslint": ">=7.0.0" - } - }, "node_modules/eslint-scope": { "version": "8.4.0", "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.4.0.tgz", @@ -10213,38 +10165,6 @@ "devOptional": true, "license": "MIT" }, - "node_modules/handlebars": { - "version": "4.7.10", - "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.10.tgz", - "integrity": "sha512-P5VJMVM7qgBn6vjXMw8WG9uVI+ncf2pi72j4de4yz5ZULLj2RGqLYaKOYGsgyrViQ0tePOVlN1tDCCXXtFqXKg==", - "dev": true, - "license": "MIT", - "dependencies": { - "minimist": "^1.2.8", - "neo-async": "^2.6.2", - "source-map": "^0.6.1", - "wordwrap": "^1.0.0" - }, - "bin": { - "handlebars": "bin/handlebars" - }, - "engines": { - "node": ">=0.4.7" - }, - "optionalDependencies": { - "uglify-js": "^3.1.4" - } - }, - "node_modules/handlebars/node_modules/source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", @@ -11058,22 +10978,6 @@ "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, - "node_modules/jest-mock-extended": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/jest-mock-extended/-/jest-mock-extended-4.0.2.tgz", - "integrity": "sha512-7uSea/mNCcTozEaIHWGATiPXcUnHwx5pXuP87KrHgMKmZXXi9Qf3CywBh3ALXdvRrehd3ahP/BTl1M1FUSJkyg==", - "dev": true, - "license": "MIT", - "dependencies": { - "fast-deep-equal": "^3.1.3", - "ts-essentials": "^10.1.1" - }, - "peerDependencies": { - "@jest/globals": "^28.0.0 || ^29.0.0 || ^30.0.0", - "jest": "^24.0.0 || ^25.0.0 || ^26.0.0 || ^27.0.0 || ^28.0.0 || ^29.0.0 || ^30.0.0", - "typescript": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0" - } - }, "node_modules/jest-regex-util": { "version": "30.5.0", "resolved": "https://registry.npmjs.org/jest-regex-util/-/jest-regex-util-30.5.0.tgz", @@ -11609,6 +11513,7 @@ "version": "4.18.1", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", + "devOptional": true, "license": "MIT" }, "node_modules/lodash.includes": { @@ -11647,13 +11552,6 @@ "integrity": "sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==", "license": "MIT" }, - "node_modules/lodash.memoize": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", - "integrity": "sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag==", - "dev": true, - "license": "MIT" - }, "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -12131,13 +12029,6 @@ "url": "https://opencollective.com/express" } }, - "node_modules/neo-async": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", - "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", - "dev": true, - "license": "MIT" - }, "node_modules/nested-error-stacks": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/nested-error-stacks/-/nested-error-stacks-2.1.1.tgz", @@ -12709,17 +12600,6 @@ "passport-strategy": "^1.0.0" } }, - "node_modules/passport-local": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/passport-local/-/passport-local-1.0.0.tgz", - "integrity": "sha512-9wCE6qKznvf9mQYYbgJ3sVOHmCWoUNMVFoZzNoznmISbhnNNPhN9xfY3sLmScHMetEJeoY7CXwfhCe7argfQow==", - "dependencies": { - "passport-strategy": "1.x.x" - }, - "engines": { - "node": ">= 0.4.0" - } - }, "node_modules/passport-strategy": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/passport-strategy/-/passport-strategy-1.0.0.tgz", @@ -14604,87 +14484,6 @@ "url": "https://github.com/sponsors/Borewit" } }, - "node_modules/ts-essentials": { - "version": "10.2.1", - "resolved": "https://registry.npmjs.org/ts-essentials/-/ts-essentials-10.2.1.tgz", - "integrity": "sha512-+Id1fRkuir+CsgK2x04/icS2b4V1hQmq7ObzIrDjhN0ozfRYivnP7aaKMVJfLApQm0trjR39A6NIMVchiB9Erw==", - "dev": true, - "license": "MIT", - "peerDependencies": { - "typescript": ">=4.5.0" - }, - "peerDependenciesMeta": { - "typescript": { - "optional": true - } - } - }, - "node_modules/ts-jest": { - "version": "29.4.15", - "resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-29.4.15.tgz", - "integrity": "sha512-s2qmEUYFRqbjz1wv0EmccNPVOb45HOFpFuhzQJLKShYVisuTbCFPIIAIV2EuK5rIxb4BejLDbH7dnBZfYjybUQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "bs-logger": "^0.2.6", - "fast-json-stable-stringify": "^2.1.0", - "handlebars": "^4.7.10", - "json5": "^2.2.3", - "lodash.memoize": "^4.1.2", - "make-error": "^1.3.6", - "semver": "^7.8.5", - "type-fest": "^4.41.0", - "yargs-parser": "^21.1.1" - }, - "bin": { - "ts-jest": "cli.js" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || ^18.0.0 || >=20.0.0" - }, - "peerDependencies": { - "@babel/core": ">=7.0.0-beta.0 <9", - "@jest/transform": "^29.0.0 || ^30.0.0", - "@jest/types": "^29.0.0 || ^30.0.0", - "babel-jest": "^29.0.0 || ^30.0.0", - "jest": "^29.0.0 || ^30.0.0", - "jest-util": "^29.0.0 || ^30.0.0", - "typescript": ">=4.3 <7" - }, - "peerDependenciesMeta": { - "@babel/core": { - "optional": true - }, - "@jest/transform": { - "optional": true - }, - "@jest/types": { - "optional": true - }, - "babel-jest": { - "optional": true - }, - "esbuild": { - "optional": true - }, - "jest-util": { - "optional": true - } - } - }, - "node_modules/ts-jest/node_modules/type-fest": { - "version": "4.41.0", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.41.0.tgz", - "integrity": "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==", - "dev": true, - "license": "(MIT OR CC0-1.0)", - "engines": { - "node": ">=16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/ts-node": { "version": "10.9.2", "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", @@ -14828,20 +14627,6 @@ "node": ">=14.17" } }, - "node_modules/uglify-js": { - "version": "3.19.3", - "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.19.3.tgz", - "integrity": "sha512-v3Xu+yuwBXisp6QYTcH4UbH+xYJXqnq2m/LtQVWKWzYc1iehYnLixoQDN9FH6/j9/oybfd6W9Ghwkl8+UMKTKQ==", - "dev": true, - "license": "BSD-2-Clause", - "optional": true, - "bin": { - "uglifyjs": "bin/uglifyjs" - }, - "engines": { - "node": ">=0.8.0" - } - }, "node_modules/uid": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/uid/-/uid-2.0.2.tgz", @@ -15102,13 +14887,6 @@ "node": ">=0.10.0" } }, - "node_modules/wordwrap": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", - "integrity": "sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==", - "dev": true, - "license": "MIT" - }, "node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", diff --git a/package.json b/package.json index 4b3dcc20..5af92327 100644 --- a/package.json +++ b/package.json @@ -8,17 +8,17 @@ "scripts": { "postinstall": "prisma generate", "build": "nest build", - "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\" \"prisma/**/*.ts\"", - "format:check": "prettier --check \"src/**/*.ts\" \"test/**/*.ts\" \"prisma/**/*.ts\"", + "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\" \"test_*/**/*.ts\" \"prisma/**/*.ts\" \"*.ts\"", + "format:check": "prettier --check \"src/**/*.ts\" \"test/**/*.ts\" \"test_*/**/*.ts\" \"prisma/**/*.ts\" \"*.ts\"", "start": "nest start", "start:dev": "nest start --watch", "start:debug": "nest start --debug --watch", "start:prod": "node dist/main", - "lint": "eslint \"{src,apps,libs,test,prisma}/**/*.ts\"", + "lint": "eslint \"{src,test,test_acceptance,test_ldap,test_oidc,prisma}/**/*.ts\" \"*.ts\"", "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js --projects src", "test:watch": "node --experimental-vm-modules node_modules/jest/bin/jest.js --projects src --watch", "test:cov": "node --experimental-vm-modules node_modules/jest/bin/jest.js --projects src --coverage", - "test:debug": "node --experimental-vm-modules --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --projects src --runInBand", + "test:debug": "node --experimental-vm-modules --inspect-brk node_modules/jest/bin/jest.js --projects src --runInBand", "test:e2e": "node --experimental-vm-modules node_modules/jest/bin/jest.js --projects test", "test:acceptance": "node --experimental-vm-modules node_modules/jest/bin/jest.js --projects test_acceptance", "test:ldap": "node --experimental-vm-modules node_modules/jest/bin/jest.js --projects test_ldap", @@ -47,25 +47,24 @@ "@prisma/client": "^7.10.1", "@socket.io/redis-adapter": "^8.3.0", "bcryptjs": "^2.4.3", - "body-parser": "^2.2.0", "class-transformer": "^0.5.1", "class-validator": "^0.15.1", "compression": "^1.8.2", + "express": "^5.3.0", "ldapts": "^9.2.0", - "lodash": "^4.17.21", "looks-same": "^10.0.1", "odiff-bin": "^4.5.0", "ollama": "^0.6.4", "openid-client": "^5.7.1", "passport": "^0.7.0", "passport-jwt": "^4.0.1", - "passport-local": "^1.0.0", "pg": "^8.16.3", "pixelmatch": "^5.3.0", "pngjs": "^7.0.0", "redis": "^4.7.0", "reflect-metadata": "^0.2.2", "rxjs": "^7.8.2", + "socket.io": "^4.8.4", "zod": "^4.6.7", "zod-to-json-schema": "^3.25.0" }, @@ -80,11 +79,9 @@ "@types/compression": "^1.8.1", "@types/express": "^5.0.6", "@types/jest": "^30.0.0", - "@types/lodash": "^4.17.26", "@types/multer": "^2.0.0", "@types/node": "^24.19.2", "@types/passport-jwt": "^3.0.9", - "@types/passport-local": "^1.0.38", "@types/pixelmatch": "^5.2.6", "@types/pngjs": "^6.0.5", "@types/supertest": "^7.2.1", @@ -93,16 +90,22 @@ "axios": "^1.20.0", "dotenv": "^17.2.3", "eslint": "^9.39.1", - "eslint-config-prettier": "^10.1.8", "jest": "^30.5.2", - "jest-mock-extended": "^4.0.2", "prettier": "^3.9.10", "prisma": "^7.10.1", "socket.io-client": "^4.8.4", "supertest": "^7.3.1", - "ts-jest": "^29.4.15", "ts-node": "^10.9.2", - "tsconfig-paths": "^4.2.0", "typescript": "~6.0.3" + }, + "allowScripts": { + "prisma": true, + "@prisma/engines": true, + "@swc/core": true, + "@scarf/scarf": false, + "protobufjs": false, + "unrs-resolver": false, + "@google/genai": false, + "@parcel/watcher": false } } diff --git a/prisma/Dockerfile b/prisma/Dockerfile index a88ac27a..0ea1f3c5 100644 --- a/prisma/Dockerfile +++ b/prisma/Dockerfile @@ -4,7 +4,6 @@ WORKDIR /app COPY . . -ENV NODE_TLS_REJECT_UNAUTHORIZED='0' RUN npm ci RUN chmod +x /app/wait-for-it.sh diff --git a/prisma/client.ts b/prisma/client.ts index 6122d966..8a91a8c0 100644 --- a/prisma/client.ts +++ b/prisma/client.ts @@ -8,8 +8,8 @@ const prisma = new PrismaClient({ // the adapter builds its connection from this object when Prisma first // connects, so prepareConnection can still settle TLS on it adapter: new PrismaPg(config, schema ? { schema } : undefined), - // 'info' | 'query' | 'warn' | 'error' - log: ['query'], + // 'query' would print every statement, including the ones that carry password hashes and API keys + log: ['warn', 'error'], }); /** Settles sslmode=prefer; call before the first query. */ diff --git a/prisma/migrations/20261024174953_add_test_run_test_variation_index/migration.sql b/prisma/migrations/20261024174953_add_test_run_test_variation_index/migration.sql new file mode 100644 index 00000000..6311aa02 --- /dev/null +++ b/prisma/migrations/20261024174953_add_test_run_test_variation_index/migration.sql @@ -0,0 +1,9 @@ +-- CreateIndex +-- Single statement on purpose: Prisma runs one-statement migrations outside a +-- transaction, which CREATE INDEX CONCURRENTLY requires. Concurrent build +-- avoids write-locking TestRun on large production tables. +-- +-- Deliberately without IF NOT EXISTS: a concurrent build that fails leaves an +-- invalid index behind, and skipping it on a retry would let the migration +-- succeed while the index stays unusable. Drop the invalid index, then retry. +CREATE INDEX CONCURRENTLY "TestRun_testVariationId_idx" ON "TestRun"("testVariationId"); diff --git a/prisma/migrations/20261024175021_add_baseline_test_variation_index/migration.sql b/prisma/migrations/20261024175021_add_baseline_test_variation_index/migration.sql new file mode 100644 index 00000000..f3f49699 --- /dev/null +++ b/prisma/migrations/20261024175021_add_baseline_test_variation_index/migration.sql @@ -0,0 +1,9 @@ +-- CreateIndex +-- Single statement on purpose: Prisma runs one-statement migrations outside a +-- transaction, which CREATE INDEX CONCURRENTLY requires. Concurrent build +-- avoids write-locking Baseline on large production tables. +-- +-- Deliberately without IF NOT EXISTS: a concurrent build that fails leaves an +-- invalid index behind, and skipping it on a retry would let the migration +-- succeed while the index stays unusable. Drop the invalid index, then retry. +CREATE INDEX CONCURRENTLY "Baseline_testVariationId_createdAt_idx" ON "Baseline"("testVariationId", "createdAt"); diff --git a/prisma/migrations/20261024175038_add_test_run_name_baseline_name_index/migration.sql b/prisma/migrations/20261024175038_add_test_run_name_baseline_name_index/migration.sql new file mode 100644 index 00000000..435613ea --- /dev/null +++ b/prisma/migrations/20261024175038_add_test_run_name_baseline_name_index/migration.sql @@ -0,0 +1,9 @@ +-- CreateIndex +-- Single statement on purpose: Prisma runs one-statement migrations outside a +-- transaction, which CREATE INDEX CONCURRENTLY requires. Concurrent build +-- avoids write-locking TestRun on large production tables. +-- +-- Deliberately without IF NOT EXISTS: a concurrent build that fails leaves an +-- invalid index behind, and skipping it on a retry would let the migration +-- succeed while the index stays unusable. Drop the invalid index, then retry. +CREATE INDEX CONCURRENTLY "TestRun_name_baselineName_idx" ON "TestRun"("name", "baselineName"); diff --git a/prisma/migrations/20261101165840_add_test_run_status_before_decision/migration.sql b/prisma/migrations/20261101165840_add_test_run_status_before_decision/migration.sql new file mode 100644 index 00000000..a36feb92 --- /dev/null +++ b/prisma/migrations/20261101165840_add_test_run_status_before_decision/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "TestRun" ADD COLUMN "statusBeforeDecision" "TestStatus"; diff --git a/prisma/package-lock.json b/prisma/package-lock.json index 7daacb5e..c6ca7ac4 100644 --- a/prisma/package-lock.json +++ b/prisma/package-lock.json @@ -7,6 +7,7 @@ "": { "name": "vrt-migration", "version": "1.2.0", + "hasInstallScript": true, "license": "Apache-2.0", "dependencies": { "@prisma/adapter-pg": "^7.10.1", @@ -23,7 +24,7 @@ "prisma": "^7.10.1", "ts-jest": "^29.4.15", "ts-node": "^10.9.2", - "typescript": "^5.9.3" + "typescript": "~6.0.3" }, "engines": { "node": ">=24.15" @@ -6468,9 +6469,9 @@ } }, "node_modules/typescript": { - "version": "5.9.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", - "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", + "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", "devOptional": true, "license": "Apache-2.0", "bin": { @@ -11045,9 +11046,9 @@ "dev": true }, "typescript": { - "version": "5.9.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", - "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", + "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", "devOptional": true }, "uglify-js": { diff --git a/prisma/package.json b/prisma/package.json index f7797bd9..24af60bf 100644 --- a/prisma/package.json +++ b/prisma/package.json @@ -19,15 +19,19 @@ "devDependencies": { "@types/bcryptjs": "^2.4.2", "@types/jest": "^30.0.0", + "dotenv": "^17.2.3", "jest": "^30.5.2", "jest-mock-extended": "^4.0.2", "prisma": "^7.10.1", "ts-jest": "^29.4.15", "ts-node": "^10.9.2", - "typescript": "^5.9.3", - "dotenv": "^17.2.3" + "typescript": "~6.0.3" }, "engines": { "node": ">=24.15" + }, + "allowScripts": { + "prisma": true, + "@prisma/engines": true } } diff --git a/prisma/pg-options.ts b/prisma/pg-options.ts index 1bb78c61..b8356ce1 100644 --- a/prisma/pg-options.ts +++ b/prisma/pg-options.ts @@ -2,12 +2,7 @@ import { readFileSync } from 'fs'; import { isAbsolute, resolve } from 'path'; import { Client, ClientConfig } from 'pg'; -/** - * The TLS and schema handling of the backend's getPgOptions - * (src/prisma/prisma.service.ts), so the seed reaches the database the way the - * API does and Prisma 6 did. This package is built on its own and cannot - * import from src/. - */ +/** Mirrors getPgOptions (src/prisma/prisma.service.ts); this package cannot import from src/. */ const PRISMA_SSL_PARAMS = ['sslmode', 'sslaccept', 'sslcert']; const PG_ONLY_SSL_PARAMS = ['sslrootcert', 'sslkey', 'uselibpqcompat']; @@ -26,12 +21,8 @@ export function getSeedPgOptions(databaseUrl: string | undefined): { const schema = params.get('schema') ?? undefined; const sslmode = params.get('sslmode') ?? 'prefer'; if (!['prefer', 'require', 'disable'].includes(sslmode) || PG_ONLY_SSL_PARAMS.some((name) => params.has(name))) { - // written for pg: its TLS settings are pg's to read return { config: { connectionString: databaseUrl }, schema, preferSsl: false }; } - // Prisma 6: no certificate check unless sslaccept=strict, which checks - // against sslcert (relative to the prisma/ folder, this file's, wherever the - // seed is started from) const sslcert = params.get('sslcert'); const ssl = params.get('sslaccept') === 'strict' @@ -45,7 +36,6 @@ export function getSeedPgOptions(databaseUrl: string | undefined): { }; } -/** sslmode=prefer: plaintext from now on if the server has no TLS at all. */ export async function negotiatePreferredSsl(config: ClientConfig): Promise { const client = new Client(config); try { diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 8190ef04..40901248 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -1,12 +1,9 @@ generator client { provider = "prisma-client-js" - // Pinned beside the schema rather than "the nearest @prisma/client": with - // prisma/node_modules installed that would be the migration package's copy. - // Every @prisma/client below this directory finds it by walking up. + // Pinned beside the schema: the nearest @prisma/client would be the migration package's copy output = "../node_modules/.prisma/client" } -// the connection URL lives in prisma.config.ts datasource db { provider = "postgresql" } @@ -44,7 +41,7 @@ model Project { bulkApproveVariations Boolean @default(false) bulkApproveGroupBy String @default("customTags") builds Build[] - TestRun TestRun[] + testRuns TestRun[] testVariations TestVariation[] favoritedBy User[] @relation("UserFavoriteProjects") } @@ -57,6 +54,7 @@ model TestRun { diffTollerancePercent Float @default(0) pixelMisMatchCount Int? status TestStatus + statusBeforeDecision TestStatus? buildId String testVariationId String? projectId String? @@ -76,18 +74,7 @@ model TestRun { ignoreAreas String @default("[]") tempIgnoreAreas String @default("[]") vlmDescription String? - // Position-independent colour signature of this run's change, as JSON, so the - // variations dialog can group a screen's locales without fetching and - // decoding every sibling's screenshots at review time. Written once, next to - // the diff that already decoded them. Null on runs that predate this, that - // have no diff, or that were compared by something other than pixelmatch — - // those fall back to computing it on demand. changeSignature String? - // Small copies of the checkpoint and of the diff, made at ingest from the - // pixels the comparison had already decoded. The card grids draw pictures a - // hundred-odd pixels wide, and pulling the full-size files for that cost - // megabytes per screen. Null on runs ingested before this, which fall back to - // the full-size file. imageThumbnailName String? diffThumbnailName String? baseline Baseline? @@ -96,6 +83,8 @@ model TestRun { testVariation TestVariation? @relation(fields: [testVariationId], references: [id]) @@index([buildId, branchName, name]) + @@index([testVariationId]) + @@index([name, baselineName]) } model TestVariation { @@ -131,6 +120,8 @@ model Baseline { testRun TestRun? @relation(fields: [testRunId], references: [id]) testVariation TestVariation @relation(fields: [testVariationId], references: [id]) user User? @relation(fields: [userId], references: [id]) + + @@index([testVariationId, createdAt]) } model User { diff --git a/prisma/seed.ts b/prisma/seed.ts index 3c004195..86f26073 100644 --- a/prisma/seed.ts +++ b/prisma/seed.ts @@ -1,12 +1,14 @@ -/** - * @see https://www.prisma.io/docs/guides/migrate/seed-database - */ -import type { Prisma } from '@prisma/client'; +import { randomBytes } from 'crypto'; import { Role } from '@prisma/client'; import { genSalt, hash } from 'bcryptjs'; import prisma, { prepareConnection } from './client'; +const DEFAULT_EMAIL = 'visual-regression-tracker@example.com'; +const PUBLIC_DEFAULT_PASSWORD = '123456'; +const PUBLIC_DEFAULT_API_KEY = 'DEFAULTUSERAPIKEYTOBECHANGED'; +const CROCKFORD_DIGITS = '0123456789ABCDEFGHJKMNPQRSTVWXYZ'; + export async function seed() { await prepareConnection(); await prisma.$connect(); @@ -15,81 +17,76 @@ export async function seed() { await prisma.$disconnect(); } -seed() - .catch((e) => console.error('e', e)) - .finally(async () => await prisma.$disconnect()); +if (require.main === module) { + seed() + .catch((error) => { + console.error('Seeding failed:', error); + process.exitCode = 1; + }) + .finally(() => prisma.$disconnect()); +} -async function createDefaultUser() { - let userList: Prisma.UserGetPayload[] = []; - try { - userList = await prisma.user.findMany(); - console.log(userList); - } catch (error) { - // Expected to see that "user" table does not exist - console.log(error.message); +// same format as src/utils/random-key.ts; this package cannot import from src/ +export function generateApiKey(): string { + let key = ''; + for (const byte of randomBytes(28)) { + key += CROCKFORD_DIGITS[byte & 31]; } + return key; +} - const defaultEmail = process.env.DEFAULT_USER_EMAIL || 'visual-regression-tracker@example.com'; - const defaultPassword = process.env.DEFAULT_USER_PASSWORD || '123456'; - const salt = await genSalt(10); - - // Only create default user if the db has no admin user - if (!userList.some(({ role, isActive }) => role === Role.admin && isActive)) { - await prisma.user - .upsert({ - where: { - email: defaultEmail, - }, - update: { - role: Role.admin, - }, - create: { - email: defaultEmail, - firstName: 'fname', - lastName: 'lname', - role: Role.admin, - apiKey: process.env.DEFAULT_USER_API_KEY || 'DEFAULTUSERAPIKEYTOBECHANGED', - password: await hash(defaultPassword, salt), - }, - }) - .then((user) => { - console.log('###########################'); - console.log('####### DEFAULT USER ######'); - console.log('###########################'); - console.log(''); - console.log( - `The user with the email "${defaultEmail}" and password "${defaultPassword}" was created (if not changed before)` - ); - console.log(`The Api key is: ${user.apiKey}`); - }); +export async function createDefaultUser(): Promise { + const activeAdmin = await prisma.user.findFirst({ + where: { role: Role.admin, isActive: true }, + select: { id: true }, + }); + if (activeAdmin) { + return; } -} + const email = process.env.DEFAULT_USER_EMAIL?.trim() || DEFAULT_EMAIL; + const existing = await prisma.user.findUnique({ where: { email }, select: { role: true, isActive: true } }); + if (existing) { + // promoting it would hand admin to whoever registered this email + console.warn( + `No active admin exists, and ${email} is already a ${existing.isActive ? '' : 'deactivated '}${existing.role}; ` + + 'it is not promoted. Give a user the admin role in the database.' + ); + return; + } + // an empty variable (compose passes unset ones as '') counts as not set + const passwordFromEnv = process.env.DEFAULT_USER_PASSWORD || undefined; + const apiKeyFromEnv = process.env.DEFAULT_USER_API_KEY?.trim() || undefined; + const password = passwordFromEnv ?? randomBytes(18).toString('base64url'); + const apiKey = apiKeyFromEnv ?? generateApiKey(); -async function createDefaultProject() { - let projectList: Prisma.ProjectGetPayload[] = []; - try { - projectList = await prisma.project.findMany(); - console.log(projectList); - } catch (error) { - // Expected to see that "project" table does not exist - console.log(error.message); + await prisma.user.create({ + data: { + email, + firstName: 'fname', + lastName: 'lname', + role: Role.admin, + apiKey, + password: await hash(password, await genSalt(10)), + }, + }); + + console.log('###########################'); + console.log('####### DEFAULT USER ######'); + console.log('###########################'); + console.log(`Created admin ${email}`); + console.log( + passwordFromEnv ? 'Password: from DEFAULT_USER_PASSWORD' : `Password (generated, shown once): ${password}` + ); + console.log(apiKeyFromEnv ? 'API key: from DEFAULT_USER_API_KEY' : `API key (generated, shown once): ${apiKey}`); + if (passwordFromEnv === PUBLIC_DEFAULT_PASSWORD || apiKeyFromEnv === PUBLIC_DEFAULT_API_KEY) { + console.warn('The password or API key is the publicly known default: change both after the first login.'); } +} - if (projectList.length === 0) { - await prisma.project - .create({ - data: { - name: 'Default project', - }, - }) - .then((project) => { - console.log('##############################'); - console.log('## CREATING DEFAULT PROJECT ##'); - console.log('##############################'); - console.log(''); - console.log(`Project key: ${project.id}`); - console.log(`Project name ${project.name}`); - console.log(`Project name ${project.mainBranchName}`); - }); +export async function createDefaultProject(): Promise { + if ((await prisma.project.count()) > 0) { + return; } + const project = await prisma.project.create({ data: { name: 'Default project' } }); + console.log(`Created the default project "${project.name}" (${project.id}), main branch ${project.mainBranchName}`); } diff --git a/prisma/test/seed.spec.ts b/prisma/test/seed.spec.ts index e7031eb4..fb637d5c 100644 --- a/prisma/test/seed.spec.ts +++ b/prisma/test/seed.spec.ts @@ -1,7 +1,6 @@ -/** - * @see https://www.prisma.io/docs/orm/prisma-client/testing/unit-testing - */ -import { seed } from '../seed'; +import { compareSync } from 'bcryptjs'; + +import { createDefaultProject, createDefaultUser, generateApiKey, seed } from '../seed'; import { prismaMock } from './client.mock'; const mockDefaultProject = { @@ -33,74 +32,194 @@ const mockDefaultUser = { createdAt: new Date(), } as const; -describe('Defaults get created', () => { +const API_KEY_FORMAT = /^[0-9A-HJKMNP-TV-Z]{28}$/; + +const ENV_KEYS = ['DEFAULT_USER_EMAIL', 'DEFAULT_USER_PASSWORD', 'DEFAULT_USER_API_KEY'] as const; + +describe('seed', () => { + const savedEnv: Record = {}; + let log: jest.SpyInstance; + let warn: jest.SpyInstance; + beforeEach(() => { - prismaMock.user.upsert.mockResolvedValue(mockDefaultUser); + for (const key of ENV_KEYS) { + savedEnv[key] = process.env[key]; + delete process.env[key]; + } + log = jest.spyOn(console, 'log').mockImplementation(); + warn = jest.spyOn(console, 'warn').mockImplementation(); + prismaMock.user.create.mockResolvedValue(mockDefaultUser); prismaMock.project.create.mockResolvedValue(mockDefaultProject); }); - test('Default user and project are created for empty database', async () => { - prismaMock.user.findMany.mockResolvedValue([]); - prismaMock.project.findMany.mockResolvedValue([]); - await seed(); - - expect(prismaMock.user.upsert).toHaveBeenCalled(); - expect(prismaMock.project.create).toHaveBeenCalled(); + afterEach(() => { + for (const key of ENV_KEYS) { + if (savedEnv[key] === undefined) { + delete process.env[key]; + } else { + process.env[key] = savedEnv[key]; + } + } + log.mockRestore(); + warn.mockRestore(); }); - test('Default user and project are created on error', async () => { - prismaMock.user.findMany.mockRejectedValue(new Error('"user" table does not exist')); - prismaMock.project.findMany.mockRejectedValue(new Error('"project" table does not exist')); + const noAdmin = (existing: object | null = null) => { + prismaMock.user.findFirst.mockResolvedValue(null); + prismaMock.user.findUnique.mockResolvedValue(existing as never); + }; - await seed(); + const loggedText = () => JSON.stringify([...log.mock.calls, ...warn.mock.calls]); - expect(prismaMock.user.upsert).toHaveBeenCalled(); - expect(prismaMock.project.create).toHaveBeenCalled(); - }); + describe('seed()', () => { + test('creates the default user and project in an empty database', async () => { + noAdmin(); + prismaMock.project.count.mockResolvedValue(0); - test('Default user is create if NO admin user exists', async () => { - prismaMock.user.findMany.mockResolvedValue([{ ...mockDefaultUser, email: 'foo@bar.com', role: 'editor' }]); - prismaMock.project.findMany.mockResolvedValue([]); + await seed(); - await seed(); + expect(prismaMock.user.create).toHaveBeenCalledTimes(1); + expect(prismaMock.project.create).toHaveBeenCalledTimes(1); + }); - expect(prismaMock.user.upsert).toHaveBeenCalled(); - expect(prismaMock.project.create).toHaveBeenCalled(); + test('rejects when the database cannot be read', async () => { + prismaMock.user.findFirst.mockRejectedValue(new Error('"user" table does not exist')); + prismaMock.project.count.mockRejectedValue(new Error('"project" table does not exist')); + + await expect(seed()).rejects.toThrow(); + }); }); - test('Default user is create if the only admin user is inActive', async () => { - prismaMock.user.findMany.mockResolvedValue([{ ...mockDefaultUser, email: 'foo@bar.com', isActive: false }]); - prismaMock.project.findMany.mockResolvedValue([]); + describe('createDefaultUser', () => { + test('active admin exists: nothing is created or changed', async () => { + prismaMock.user.findFirst.mockResolvedValue({ id: 'admin' } as never); + + await createDefaultUser(); + + expect(prismaMock.user.findUnique).not.toHaveBeenCalled(); + expect(prismaMock.user.create).not.toHaveBeenCalled(); + expect(prismaMock.user.update).not.toHaveBeenCalled(); + expect(prismaMock.user.upsert).not.toHaveBeenCalled(); + }); + + test('never logs user rows, password hashes or api keys of existing users', async () => { + prismaMock.user.findFirst.mockResolvedValue({ + id: 'admin', + apiKey: 'EXISTINGKEY', + password: 'hash', + } as never); + + await createDefaultUser(); + + expect(loggedText()).not.toContain('EXISTINGKEY'); + expect(loggedText()).not.toContain('hash'); + }); + + test('fresh database with env values: uses them and does not print the password or key', async () => { + process.env.DEFAULT_USER_EMAIL = 'admin@corp.test'; + process.env.DEFAULT_USER_PASSWORD = 'env-password-value'; + process.env.DEFAULT_USER_API_KEY = 'ENVAPIKEYVALUE'; + noAdmin(); + + await createDefaultUser(); + + expect(prismaMock.user.create).toHaveBeenCalledTimes(1); + const { data } = prismaMock.user.create.mock.calls[0][0]; + expect(data.email).toBe('admin@corp.test'); + expect(data.role).toBe('admin'); + expect(data.apiKey).toBe('ENVAPIKEYVALUE'); + expect(compareSync('env-password-value', data.password)).toBe(true); + expect(loggedText()).not.toContain('env-password-value'); + expect(loggedText()).not.toContain('ENVAPIKEYVALUE'); + }); + + test('fresh database without env values: generates the secrets and prints each exactly once', async () => { + noAdmin(); + + await createDefaultUser(); + + const { data } = prismaMock.user.create.mock.calls[0][0]; + expect(data.email).toBe('visual-regression-tracker@example.com'); + expect(data.apiKey).toMatch(API_KEY_FORMAT); + const printed = log.mock.calls.map((args) => args.join(' ')); + const passwordLine = printed.find((line) => line.startsWith('Password (generated, shown once): ')); + const password = passwordLine?.replace('Password (generated, shown once): ', ''); + expect(password).toMatch(/^[A-Za-z0-9_-]{24}$/); + expect(compareSync(password as string, data.password)).toBe(true); + expect(printed.filter((line) => line.includes(password as string))).toHaveLength(1); + expect(printed.filter((line) => line.includes(data.apiKey))).toHaveLength(1); + }); + + test('empty env values count as unset', async () => { + process.env.DEFAULT_USER_EMAIL = ''; + process.env.DEFAULT_USER_PASSWORD = ''; + process.env.DEFAULT_USER_API_KEY = ''; + noAdmin(); + + await createDefaultUser(); + + const { data } = prismaMock.user.create.mock.calls[0][0]; + expect(data.email).toBe('visual-regression-tracker@example.com'); + expect(data.apiKey).toMatch(API_KEY_FORMAT); + expect(compareSync('', data.password)).toBe(false); + expect(loggedText()).toContain('generated, shown once'); + }); + + test('no active admin, email taken by an editor: not promoted', async () => { + noAdmin({ role: 'editor', isActive: true }); + + await createDefaultUser(); + + expect(prismaMock.user.create).not.toHaveBeenCalled(); + expect(prismaMock.user.update).not.toHaveBeenCalled(); + expect(prismaMock.user.upsert).not.toHaveBeenCalled(); + expect(warn).toHaveBeenCalled(); + }); + + test('warns about the public defaults', async () => { + process.env.DEFAULT_USER_PASSWORD = '123456'; + noAdmin(); + + await createDefaultUser(); + + expect(warn).toHaveBeenCalledWith(expect.stringContaining('publicly known default')); + }); + + test('does not warn about values that are not the public defaults', async () => { + process.env.DEFAULT_USER_PASSWORD = 'something-private'; + noAdmin(); + + await createDefaultUser(); + + expect(warn).not.toHaveBeenCalled(); + }); + }); - await seed(); + describe('createDefaultProject', () => { + test('creates it when there are no projects', async () => { + prismaMock.project.count.mockResolvedValue(0); - expect(prismaMock.user.upsert).toHaveBeenCalled(); - expect(prismaMock.project.create).toHaveBeenCalled(); - }); -}); + await createDefaultProject(); -describe('Defaults do NOT get created', () => { - beforeEach(() => { - prismaMock.user.upsert.mockResolvedValue(mockDefaultUser); - prismaMock.project.create.mockResolvedValue(mockDefaultProject); - }); - test('Default project is NOT create if any project exists', async () => { - prismaMock.user.findMany.mockResolvedValue([]); - prismaMock.project.findMany.mockResolvedValue([mockDefaultProject]); + expect(prismaMock.project.create).toHaveBeenCalledTimes(1); + }); - await seed(); + test('does nothing when a project exists', async () => { + prismaMock.project.count.mockResolvedValue(1); - expect(prismaMock.user.upsert).toHaveBeenCalled(); - expect(prismaMock.project.create).not.toHaveBeenCalled(); - }); + await createDefaultProject(); - test('Default user is NOT create if any active admin user exists', async () => { - prismaMock.user.findMany.mockResolvedValue([{ ...mockDefaultUser, email: 'foo@bar.com' }]); - prismaMock.project.findMany.mockResolvedValue([]); + expect(prismaMock.project.create).not.toHaveBeenCalled(); + }); + }); - await seed(); + describe('generateApiKey', () => { + test('has the format of the API keys generated by the backend', () => { + expect(generateApiKey()).toMatch(API_KEY_FORMAT); + }); - expect(prismaMock.user.upsert).not.toHaveBeenCalled(); - expect(prismaMock.project.create).toHaveBeenCalled(); + test('differs between calls', () => { + expect(generateApiKey()).not.toBe(generateApiKey()); + }); }); }); diff --git a/src/_data_/index.ts b/src/_data_/index.ts index 9b0c46ed..431accbb 100644 --- a/src/_data_/index.ts +++ b/src/_data_/index.ts @@ -80,6 +80,7 @@ export const generateTestRun = (testRun?: Partial): TestRun => { diffTollerancePercent: 1, pixelMisMatchCount: 123, status: 'new', + statusBeforeDecision: null, buildId: '146e7a8d-89f0-4565-aa2c-e61efabb0afd', testVariationId: '3bc4a5bc-006e-4d43-8e4e-eaa132627fca', projectId: '3bc4a5bc-006e-4d43-8e4e-eaa132627fcc', @@ -111,10 +112,10 @@ export const generatePng = (width: number, height: number): PNG => { for (let y = 0; y < png.height; y++) { for (let x = 0; x < png.width; x++) { const idx = (png.width * y + x) << 2; - png.data[idx] = 255; // red - png.data[idx + 1] = 255; // green - png.data[idx + 2] = 255; // blue - png.data[idx + 3] = 255; // alpha (0 is transparent) + png.data[idx] = 255; + png.data[idx + 1] = 255; + png.data[idx + 2] = 255; + png.data[idx + 3] = 255; } } return png.pack(); diff --git a/src/app.module.spec.ts b/src/app.module.spec.ts index c8497998..74afd8a3 100644 --- a/src/app.module.spec.ts +++ b/src/app.module.spec.ts @@ -7,6 +7,7 @@ import { HealthCheckService } from '@nestjs/terminus'; import { Test, TestingModule } from '@nestjs/testing'; import request from 'supertest'; import { AppService } from './app.service'; +import { validateEnv } from './config/env.validation'; import { AuthService } from './auth/auth.service'; import { ApiGuard } from './auth/guards/api.guard'; import { JwtAuthGuard } from './auth/guards/auth.guard'; @@ -41,38 +42,39 @@ import { UsersController } from './users/users.controller'; import { UsersFactoryService } from './users/users.factory'; import { UsersService } from './users/users.service'; -// The point of this spec is to catch dependency-injection breakage when the -// framework is upgraded: every module, provider and controller of the real -// AppModule has to resolve, with only the database client swapped out. +jest.mock('./config/env.validation', () => { + const actual = jest.requireActual('./config/env.validation'); + return { ...actual, validateEnv: jest.fn(actual.validateEnv) }; +}); type PrismaMock = { onModuleInit: jest.Mock; onApplicationShutdown: jest.Mock; $connect: jest.Mock; $disconnect: jest.Mock; - $queryRaw: jest.Mock; + $runCommandRaw: jest.Mock; + $queryRawUnsafe: jest.Mock; user: { findUnique: jest.Mock }; project: { findMany: jest.Mock }; }; -// a plain object rather than a PrismaService, so the lifecycle hooks are -// stand-ins: asserting they are called proves Nest still drives them const createPrismaMock = (): PrismaMock => ({ onModuleInit: jest.fn().mockResolvedValue(undefined), onApplicationShutdown: jest.fn().mockResolvedValue(undefined), $connect: jest.fn().mockResolvedValue(undefined), $disconnect: jest.fn().mockResolvedValue(undefined), - $queryRaw: jest.fn().mockResolvedValue([{ '?column?': 1 }]), + $runCommandRaw: jest + .fn() + .mockRejectedValue(new Error('The postgresql provider does not support $runCommandRaw. Use the mongodb provider.')), + $queryRawUnsafe: jest.fn().mockResolvedValue([{ '?column?': 1 }]), user: { findUnique: jest.fn().mockResolvedValue(null) }, project: { findMany: jest.fn().mockResolvedValue([]) }, }); -// Pinned so a developer .env (S3 storage, OIDC/LDAP login) cannot make the -// wiring reach out to the network. ConfigModule never overwrites variables that -// are already set, so these win over the .env file. const TEST_ENV = { JWT_SECRET: 'app-module-spec-secret', JWT_LIFE_TIME: '1h', + APP_FRONTEND_URL: 'http://localhost:8080', STATIC_SERVICE: 'hdd', OIDC_ENABLED: 'false', LDAP_ENABLED: 'false', @@ -86,13 +88,10 @@ describe('AppModule', () => { const get = (token: Type): T => moduleRef.get(token, { strict: false }); const prismaOf = (instance: unknown): PrismaMock => (instance as { prismaService: PrismaMock }).prismaService; + const inner = (token: Type) => get(token) as Record; beforeAll(async () => { Object.assign(process.env, TEST_ENV); - // imported only after the env is pinned: ConfigModule.forRoot() reads the - // .env file at import time of the module that calls it - // a plain require: under `module: nodenext` an import() stays a real ESM import - // eslint-disable-next-line @typescript-eslint/no-require-imports const { AppModule } = require('./app.module') as typeof import('./app.module'); moduleRef = await Test.createTestingModule({ imports: [AppModule] }) @@ -111,7 +110,6 @@ describe('AppModule', () => { }); afterAll(async () => { - // a no-op when the shutdown test below already closed the app await app?.close(); process.env = originalEnv; }); @@ -164,16 +162,16 @@ describe('AppModule', () => { }); it('closes the forwardRef cycles between builds, projects, test runs and test variations', () => { - const builds = get(BuildsService) as unknown as Record; - const projects = get(ProjectsService) as unknown as Record; - const testRuns = get(TestRunsService) as unknown as Record; - const testVariations = get(TestVariationsService) as unknown as Record; - const tasks = get(TasksService) as unknown as Record; + const builds = inner(BuildsService); + const projects = inner(ProjectsService); + const testRuns = inner(TestRunsService); + const testVariations = inner(TestVariationsService); + const tasks = inner(TasksService); expect(builds.testRunsService).toBe(testRuns); expect(builds.eventsGateway).toBe(get(EventsGateway)); - expect(projects.buildsService).toBe(builds); - expect(projects.testVariationsService).toBe(testVariations); + expect(projects.staticService).toBe(get(StaticService)); + expect(projects.eventsGateway).toBe(get(EventsGateway)); expect(testRuns.testVariationService).toBe(testVariations); expect(testRuns.compareService).toBe(get(CompareService)); expect(testRuns.staticService).toBe(get(StaticService)); @@ -183,20 +181,15 @@ describe('AppModule', () => { }); it('shares one StaticModule between every importer', () => { - const vlm = get(VlmService) as unknown as Record; - const testVariations = get(TestVariationsService) as unknown as Record; - - expect(vlm.staticService).toBe(get(StaticService)); - expect(testVariations.staticService).toBe(get(StaticService)); + expect(inner(VlmService).staticService).toBe(get(StaticService)); + expect(inner(TestVariationsService).staticService).toBe(get(StaticService)); }); it('picks the HDD storage and the DB users backend under the default configuration', () => { - expect((get(StaticService) as unknown as Record).staticService).toBeInstanceOf(HddService); - expect((get(UsersService) as unknown as Record).usersService).toBeInstanceOf(DbUsersService); + expect(inner(StaticService).staticService).toBeInstanceOf(HddService); + expect(inner(UsersService).usersService).toBeInstanceOf(DbUsersService); }); - // PrismaModule is global: every consumer shares one client and so one - // connection pool per instance. it('shares one PrismaService instance across the whole app', () => { expect(prismaInstances).toHaveLength(1); @@ -208,13 +201,20 @@ describe('AppModule', () => { prismaOf(get(ProjectsService)), prismaOf(get(TestRunsService)), prismaOf(get(TestVariationsService)), - (get(CompareService) as unknown as Record).prismaService, + inner(CompareService).prismaService, prismaOf(get(TasksService)), ]; consumers.forEach((prisma) => expect(prisma).toBe(prismaInstances[0])); }); }); + describe('configuration', () => { + it('exposes the validated config through ConfigService', () => { + expect(validateEnv).toHaveBeenCalledWith(expect.objectContaining({ JWT_SECRET: TEST_ENV.JWT_SECRET })); + expect(get(ConfigService).get('JWT_LIFE_TIME')).toBe('1h'); + }); + }); + describe('lifecycle', () => { it('runs onModuleInit of every PrismaService on init', () => { prismaInstances.forEach((prisma) => expect(prisma.onModuleInit).toHaveBeenCalledTimes(1)); @@ -244,12 +244,11 @@ describe('AppModule', () => { const response = await request(app.getHttpServer()).get('/does-not-exist'); expect(response.status).toBe(404); - expect(response.body).toMatchObject({ + expect(response.body).toEqual({ + statusCode: 404, path: '/does-not-exist', name: 'NotFoundException', message: 'Cannot GET /does-not-exist', - exception: expect.objectContaining({ status: 404 }), - stack: expect.stringContaining('NotFoundException'), }); }); @@ -276,24 +275,31 @@ describe('AppModule', () => { it('serves the health check through the AppModule PrismaService', async () => { const response = await request(app.getHttpServer()).get('/health'); - expect(prismaOf(get(HealthController)).$queryRaw).toHaveBeenCalled(); + expect(prismaOf(get(HealthController)).$queryRawUnsafe).toHaveBeenCalledWith('SELECT 1'); expect(response.status).toBe(200); - expect(response.body).toEqual({ status: 'ok', info: {}, error: {}, details: {} }); + expect(response.body).toEqual({ + status: 'ok', + info: { database: { status: 'up', responseTime: expect.any(Number) } }, + error: {}, + details: { database: { status: 'up', responseTime: expect.any(Number) } }, + }); }); - // NOTE: current behavior — a database failure surfaces as a 500 from the - // global filter, not as terminus' 503 health report. - it('answers 500 from the health check when the database query fails', async () => { - prismaOf(get(HealthController)).$queryRaw.mockRejectedValueOnce(new Error('Can not reach database server')); + it('answers 503 from the health check when the database query fails', async () => { + prismaOf(get(HealthController)).$queryRawUnsafe.mockRejectedValueOnce(new Error('Can not reach database server')); const response = await request(app.getHttpServer()).get('/health'); - expect(response.status).toBe(500); - expect(response.body).toMatchObject({ path: '/health', name: 'Error', message: 'Can not reach database server' }); + expect(response.status).toBe(503); + expect(response.body).toEqual({ + statusCode: 503, + path: '/health', + name: 'ServiceUnavailableException', + message: 'Service Unavailable Exception', + }); }); }); - // must stay the last block: it closes the application describe('shutdown', () => { it('stops the cron jobs, the diff worker pool and every PrismaService on close', async () => { const [job] = [...get(SchedulerRegistry).getCronJobs().values()]; @@ -303,8 +309,6 @@ describe('AppModule', () => { await app.close(); - // the database and the workers go only after the HTTP server has - // drained, so no request still running finds them closed const httpClosedAt = closeHttp.mock.invocationCallOrder[0]; expect(httpClosedAt).toBeDefined(); expect(stopWorkers.mock.invocationCallOrder[0]).toBeGreaterThan(httpClosedAt); diff --git a/src/app.module.ts b/src/app.module.ts index 7e537007..c6c2d6f6 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -8,6 +8,7 @@ import { TestRunsModule } from './test-runs/test-runs.module'; import { TestVariationsModule } from './test-variations/test-variations.module'; import { PrismaModule } from './prisma/prisma.module'; import { ConfigModule } from '@nestjs/config'; +import { validateEnv } from './config/env.validation'; import { APP_FILTER } from '@nestjs/core'; import { HttpExceptionFilter } from './http-exception.filter'; import { CompareModule } from './compare/compare.module'; @@ -19,9 +20,8 @@ import { HealthController } from './health/health.controller'; imports: [ ConfigModule.forRoot({ isGlobal: true, - // Prisma 6 read DATABASE_URL from prisma/.env on its own; Prisma 7 reads - // no env files, so load it here for local runs (.env wins on conflicts) envFilePath: ['.env', 'prisma/.env'], + validate: (config) => validateEnv(config), }), PrismaModule, ScheduleModule.forRoot(), diff --git a/src/app.service.spec.ts b/src/app.service.spec.ts index 13123fc1..8fe3861a 100644 --- a/src/app.service.spec.ts +++ b/src/app.service.spec.ts @@ -1,30 +1,9 @@ import { Logger } from '@nestjs/common'; -import { Test, TestingModule } from '@nestjs/testing'; import { HttpAdapterHost } from '@nestjs/core'; import { AppService } from './app.service'; import { ConfigService } from '@nestjs/config'; describe('AppService', () => { - let service: AppService; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [ - AppService, - { - provide: ConfigService, - useValue: {}, - }, - ], - }).compile(); - - service = module.get(AppService); - }); - - it('should be defined', () => { - expect(service).toBeDefined(); - }); - describe('onApplicationBootstrap', () => { const ENV_KEYS = ['SERVER_TIMEOUT', 'SERVER_HEADERS_TIMEOUT', 'SERVER_KEEP_ALIVE_TIMEOUT']; const originalEnv = { ...process.env }; @@ -79,10 +58,25 @@ describe('AppService', () => { expect(bootstrap({ SERVER_TIMEOUT: '1500.9' }).timeout).toBe(1500); }); - // NOTE: current behavior — the value is parsed with parseInt and never - // validated, so a typo in the env sets the timeout to NaN. - it('sets NaN for a non-numeric override', () => { - expect(bootstrap({ SERVER_TIMEOUT: 'abc' }).timeout).toBeNaN(); + it('keeps the server default and warns for a non-numeric override', () => { + const warnSpy = jest.spyOn(Logger.prototype, 'warn').mockImplementation(); + + const server = bootstrap({ SERVER_TIMEOUT: 'abc', SERVER_HEADERS_TIMEOUT: '-5' }); + + expect(server).toEqual({ timeout: 0, headersTimeout: 60000, keepAliveTimeout: 5000 }); + expect(warnSpy).toHaveBeenCalledWith('SERVER_TIMEOUT="abc" is not a number of milliseconds; keeping 0'); + expect(warnSpy).toHaveBeenCalledWith( + 'SERVER_HEADERS_TIMEOUT="-5" is not a number of milliseconds; keeping 60000' + ); + warnSpy.mockRestore(); + }); + + it('treats an empty value as not set, without a warning', () => { + const warnSpy = jest.spyOn(Logger.prototype, 'warn').mockImplementation(); + + expect(bootstrap({ SERVER_TIMEOUT: ' ' }).timeout).toBe(0); + expect(warnSpy).not.toHaveBeenCalled(); + warnSpy.mockRestore(); }); it('logs every resulting timeout', () => { diff --git a/src/app.service.ts b/src/app.service.ts index 3859ba98..04463fc8 100644 --- a/src/app.service.ts +++ b/src/app.service.ts @@ -14,14 +14,22 @@ export class AppService implements OnApplicationBootstrap { onApplicationBootstrap() { const server: Server = this.refHost.httpAdapter.getHttpServer(); - const { timeout, headersTimeout, keepAliveTimeout } = server; - server.timeout = parseInt(this.configService.get('SERVER_TIMEOUT', timeout.toString())); + server.timeout = this.readTimeout('SERVER_TIMEOUT', server.timeout); this.logger.log(`server.timeout is ${server.timeout}`); - server.headersTimeout = parseInt(this.configService.get('SERVER_HEADERS_TIMEOUT', headersTimeout.toString())); + server.headersTimeout = this.readTimeout('SERVER_HEADERS_TIMEOUT', server.headersTimeout); this.logger.log(`server.headersTimeout is ${server.headersTimeout}`); - server.keepAliveTimeout = parseInt( - this.configService.get('SERVER_KEEP_ALIVE_TIMEOUT', keepAliveTimeout.toString()) - ); + server.keepAliveTimeout = this.readTimeout('SERVER_KEEP_ALIVE_TIMEOUT', server.keepAliveTimeout); this.logger.log(`server.keepAliveTimeout is ${server.keepAliveTimeout}`); } + + private readTimeout(name: string, fallback: number): number { + const raw = this.configService.get(name); + if (raw === undefined || String(raw).trim() === '') return fallback; + const value = parseInt(String(raw), 10); + if (Number.isNaN(value) || value < 0) { + this.logger.warn(`${name}="${raw}" is not a number of milliseconds; keeping ${fallback}`); + return fallback; + } + return value; + } } diff --git a/src/auth/auth.service.spec.ts b/src/auth/auth.service.spec.ts index 8a5c729c..69fc16b3 100644 --- a/src/auth/auth.service.spec.ts +++ b/src/auth/auth.service.spec.ts @@ -18,10 +18,6 @@ describe('AuthService', () => { service = module.get(AuthService); }); - it('should be defined', () => { - expect(service).toBeDefined(); - }); - describe('encryptPassword', () => { it('produces a salted bcrypt hash with 10 rounds that matches the password', async () => { const hashed = await service.encryptPassword('secret'); @@ -55,7 +51,6 @@ describe('AuthService', () => { it('returns a key in the format keys were always issued in', () => { const apiKey = service.generateApiKey(); - // 28 Crockford base32 digits, upper case, no dashes expect(apiKey).toMatch(/^[0-9A-HJKMNP-TV-Z]{28}$/); }); @@ -65,11 +60,25 @@ describe('AuthService', () => { }); describe('signToken', () => { - it('signs a payload carrying only the email of the user', () => { + it('signs the user id as sub next to the email, so older API versions still read the token', () => { const user = { id: 'user-id', email: 'user@example.com', password: 'hash', apiKey: 'key' } as User; expect(service.signToken(user)).toBe('signed-token'); - expect(jwtService.sign).toHaveBeenCalledWith({ email: 'user@example.com' }); + expect(jwtService.sign).toHaveBeenCalledWith({ sub: 'user-id', email: 'user@example.com' }); + }); + + it('produces a token that decodes to sub, email, iat and exp', () => { + const real = new JwtService({ secret: 'secret', signOptions: { expiresIn: '1h' } }); + const user = { id: 'user-id', email: 'user@example.com' } as User; + + const token = new AuthService(real).signToken(user); + + expect(real.decode(token)).toEqual({ + sub: 'user-id', + email: 'user@example.com', + iat: expect.any(Number), + exp: expect.any(Number), + }); }); }); }); diff --git a/src/auth/auth.service.ts b/src/auth/auth.service.ts index 352d3e75..ce05fbbe 100644 --- a/src/auth/auth.service.ts +++ b/src/auth/auth.service.ts @@ -18,16 +18,12 @@ export class AuthService { return generateRandomKey(); } - async compare(password: string, hashedPassword: string): Promise { - return await compare(password, hashedPassword); + compare(password: string, hashedPassword: string): Promise { + return compare(password, hashedPassword); } signToken(user: User): string { - const payload: JwtPayload = { - email: user.email, - }; - - const token = this.jwtService.sign(payload); - return token; + const payload: JwtPayload = { sub: user.id, email: user.email }; + return this.jwtService.sign(payload); } } diff --git a/src/auth/guards/api.guard.spec.ts b/src/auth/guards/api.guard.spec.ts index 2b4065e4..ce0a9e72 100644 --- a/src/auth/guards/api.guard.spec.ts +++ b/src/auth/guards/api.guard.spec.ts @@ -21,7 +21,7 @@ describe('ApiGuard', () => { }); it('allows a request whose apiKey header belongs to a user', async () => { - prismaService.user.findUnique.mockResolvedValue({ id: '1' } as User); + prismaService.user.findUnique.mockResolvedValue({ id: '1' }); const { executionContext, request } = context('valid-key'); await expect(guard.canActivate(executionContext)).resolves.toBe(true); @@ -35,19 +35,40 @@ describe('ApiGuard', () => { await expect(guard.canActivate(context('unknown-key').executionContext)).resolves.toBe(false); }); - // NOTE: current behavior — a missing header is still sent to Prisma as - // `apiKey: undefined`; it is the lookup failing (or returning nothing) that - // rejects the request, not a check in the guard. - it('still queries the database when the apiKey header is missing', async () => { + it('answers 401 without a database query when the apiKey header is missing', async () => { + await expect(guard.canActivate(context(undefined).executionContext)).rejects.toThrow(UnauthorizedException); + + expect(prismaService.user.findUnique).not.toHaveBeenCalled(); + }); + + it('rejects an empty apiKey header with false, as before, without a database query', async () => { + await expect(guard.canActivate(context('').executionContext)).resolves.toBe(false); + + expect(prismaService.user.findUnique).not.toHaveBeenCalled(); + }); + + it('stores the user on the request', async () => { + const found = { id: '1' } as User; + prismaService.user.findUnique.mockResolvedValue(found); + const { executionContext, request } = context('valid-key'); + + await guard.canActivate(executionContext); + + expect((request as { user?: User }).user).toBe(found); + }); + + it('leaves the request without a user for an unknown key', async () => { prismaService.user.findUnique.mockResolvedValue(null); + const { executionContext, request } = context('unknown-key'); + + await guard.canActivate(executionContext); - await expect(guard.canActivate(context(undefined).executionContext)).resolves.toBe(false); - expect(prismaService.user.findUnique).toHaveBeenCalledWith({ where: { apiKey: undefined } }); + expect((request as { user?: User }).user).toBeUndefined(); }); it('turns a failing lookup into an UnauthorizedException', async () => { - prismaService.user.findUnique.mockRejectedValue(new Error('Argument where of type UserWhereUniqueInput needs')); + prismaService.user.findUnique.mockRejectedValue(new Error('connection lost')); - await expect(guard.canActivate(context(undefined).executionContext)).rejects.toThrow(UnauthorizedException); + await expect(guard.canActivate(context('some-key').executionContext)).rejects.toThrow(UnauthorizedException); }); }); diff --git a/src/auth/guards/api.guard.ts b/src/auth/guards/api.guard.ts index 045c04c7..52c660e0 100644 --- a/src/auth/guards/api.guard.ts +++ b/src/auth/guards/api.guard.ts @@ -1,4 +1,5 @@ import { ExecutionContext, Injectable, CanActivate, UnauthorizedException } from '@nestjs/common'; +import { User } from '@prisma/client'; import { Request } from 'express'; import { PrismaService } from '../../prisma/prisma.service'; @@ -8,13 +9,24 @@ export class ApiGuard implements CanActivate { async canActivate(context: ExecutionContext): Promise { const request: Request = context.switchToHttp().getRequest(); + const apiKey = request.header('apiKey'); + if (apiKey === undefined) { + throw new UnauthorizedException(); + } + // an empty header never matches a key: 403, not 401 + if (apiKey === '') { + return false; + } + let user: User | null; try { - const user = await this.prismaService.user.findUnique({ - where: { apiKey: request.header('apiKey') }, - }); - return !!user; + user = await this.prismaService.user.findUnique({ where: { apiKey } }); } catch { throw new UnauthorizedException(); } + if (!user) { + return false; + } + request.user = user; + return true; } } diff --git a/src/auth/guards/mixed.guard.spec.ts b/src/auth/guards/mixed.guard.spec.ts index 9804a9b1..c245ec70 100644 --- a/src/auth/guards/mixed.guard.spec.ts +++ b/src/auth/guards/mixed.guard.spec.ts @@ -42,15 +42,20 @@ describe('MixedGuard', () => { await expect(guard.canActivate(context)).resolves.toBe(expected); }); - // NOTE: current behavior — both guards always run, even when the JWT guard - // has already let the request through, so an extra api-key lookup hits the - // database on every JWT-authenticated request. - it('always consults both guards with the same context', async () => { - const { guard, authGuard, apiGuard } = createGuard('pass', 'fail'); + it('does not consult the api-key guard when the JWT is valid', async () => { + const { guard, authGuard, apiGuard } = createGuard('pass', 'pass'); await guard.canActivate(context); expect(authGuard.canActivate).toHaveBeenCalledWith(context); + expect(apiGuard.canActivate).not.toHaveBeenCalled(); + }); + + it('falls back to the api-key guard with the same context when the JWT fails', async () => { + const { guard, apiGuard } = createGuard('fail', 'pass'); + + await guard.canActivate(context); + expect(apiGuard.canActivate).toHaveBeenCalledWith(context); }); @@ -64,12 +69,10 @@ describe('MixedGuard', () => { ); }); - // NOTE: current behavior — the JWT guard's result is cast, not checked, so - // a truthy non-boolean value is passed through as-is. - it('passes a truthy non-boolean JWT result straight through', async () => { + it('returns true for a truthy non-boolean JWT result', async () => { const { guard, authGuard } = createGuard('fail', 'fail'); authGuard.canActivate.mockResolvedValue('yes'); - await expect(guard.canActivate(context)).resolves.toBe('yes'); + await expect(guard.canActivate(context)).resolves.toBe(true); }); }); diff --git a/src/auth/guards/mixed.guard.ts b/src/auth/guards/mixed.guard.ts index ffdb7bd9..04cef71f 100644 --- a/src/auth/guards/mixed.guard.ts +++ b/src/auth/guards/mixed.guard.ts @@ -10,15 +10,17 @@ export class MixedGuard implements CanActivate { ) {} async canActivate(context: ExecutionContext): Promise { - let jwtAuth = false; try { - jwtAuth = (await this.authGuard.canActivate(context)) as boolean; - } catch {} - - let apiAuth = false; + if (await this.authGuard.canActivate(context)) { + return true; + } + } catch { + // not a valid JWT: the api key is tried next + } try { - apiAuth = await this.apiGuard.canActivate(context); - } catch {} - return jwtAuth || apiAuth; + return await this.apiGuard.canActivate(context); + } catch { + return false; + } } } diff --git a/src/auth/guards/role.guard.spec.ts b/src/auth/guards/role.guard.spec.ts index c00fb5e8..a0d16616 100644 --- a/src/auth/guards/role.guard.spec.ts +++ b/src/auth/guards/role.guard.spec.ts @@ -3,6 +3,7 @@ import { Reflector } from '@nestjs/core'; import { Role, User } from '@prisma/client'; import { PrismaService } from '../../prisma/prisma.service'; import { Roles } from '../../shared/roles.decorator'; +import { ApiGuard } from './api.guard'; import { RoleGuard } from './role.guard'; const userWithRole = (role: Role): User => ({ id: '1', role }) as User; @@ -12,11 +13,8 @@ describe('RoleGuard', () => { let prismaService: { user: { findUnique: jest.Mock } }; let guard: RoleGuard; - const context = (user?: User, apiKey?: string): ExecutionContext => { - const request = { - user, - header: jest.fn().mockReturnValue(apiKey), - }; + const context = (user?: User): ExecutionContext => { + const request = { user }; return { switchToHttp: () => ({ getRequest: () => request }), getHandler: () => ({}), @@ -27,60 +25,27 @@ describe('RoleGuard', () => { beforeEach(() => { reflector = { getAllAndOverride: jest.fn() }; prismaService = { user: { findUnique: jest.fn() } }; - guard = new RoleGuard(prismaService as unknown as PrismaService, reflector as unknown as Reflector); + guard = new RoleGuard(reflector as unknown as Reflector); }); - it('allows the request when no roles are required', async () => { + it('allows the request when no roles are required', () => { reflector.getAllAndOverride.mockReturnValue(undefined); - await expect(guard.canActivate(context())).resolves.toBe(true); + expect(guard.canActivate(context())).toBe(true); }); - it('allows an admin on an admin-only endpoint', async () => { - reflector.getAllAndOverride.mockReturnValue([Role.admin]); - - await expect(guard.canActivate(context(userWithRole(Role.admin)))).resolves.toBe(true); - }); - - it('rejects an editor on an admin-only endpoint', async () => { - reflector.getAllAndOverride.mockReturnValue([Role.admin]); + it.each<[string, Role[], Role | undefined, boolean]>([ + ['allows an admin on an admin-only endpoint', [Role.admin], Role.admin, true], + ['rejects an editor on an admin-only endpoint', [Role.admin], Role.editor, false], + ['allows an editor where editor is permitted', [Role.admin, Role.editor], Role.editor, true], + ['rejects a guest where only admin and editor are permitted', [Role.admin, Role.editor], Role.guest, false], + ['rejects when no user was authenticated', [Role.admin], undefined, false], + ])('%s', (_name, required, role, allowed) => { + reflector.getAllAndOverride.mockReturnValue(required); - await expect(guard.canActivate(context(userWithRole(Role.editor)))).resolves.toBe(false); + expect(guard.canActivate(context(role && userWithRole(role)))).toBe(allowed); }); - it('allows an editor where editor is permitted', async () => { - reflector.getAllAndOverride.mockReturnValue([Role.admin, Role.editor]); - - await expect(guard.canActivate(context(userWithRole(Role.editor)))).resolves.toBe(true); - }); - - it('rejects a guest where only admin and editor are permitted', async () => { - reflector.getAllAndOverride.mockReturnValue([Role.admin, Role.editor]); - - await expect(guard.canActivate(context(userWithRole(Role.guest)))).resolves.toBe(false); - }); - - it('rejects when neither a user nor an api key is present', async () => { - reflector.getAllAndOverride.mockReturnValue([Role.admin]); - - await expect(guard.canActivate(context())).resolves.toBe(false); - expect(prismaService.user.findUnique).not.toHaveBeenCalled(); - }); - - it('resolves the user from the apiKey header when request.user is absent', async () => { - reflector.getAllAndOverride.mockReturnValue([Role.admin]); - prismaService.user.findUnique.mockResolvedValue(userWithRole(Role.admin)); - - await expect(guard.canActivate(context(undefined, 'api-key'))).resolves.toBe(true); - expect(prismaService.user.findUnique).toHaveBeenCalledWith({ where: { apiKey: 'api-key' } }); - }); - - it('rejects when the api key does not belong to a user', async () => { - reflector.getAllAndOverride.mockReturnValue([Role.admin]); - prismaService.user.findUnique.mockResolvedValue(null); - - await expect(guard.canActivate(context(undefined, 'unknown'))).resolves.toBe(false); - }); describe('with the @Roles metadata', () => { @Roles(Role.admin) class AdminController { @@ -98,21 +63,37 @@ describe('RoleGuard', () => { }) as unknown as ExecutionContext; beforeEach(() => { - guard = new RoleGuard(prismaService as unknown as PrismaService, new Reflector()); + guard = new RoleGuard(new Reflector()); }); - it('applies the roles declared on the controller class', async () => { + it('applies the roles declared on the controller class', () => { const handler = AdminController.prototype.inherited; - await expect(guard.canActivate(decoratedContext(handler, userWithRole(Role.editor)))).resolves.toBe(false); - await expect(guard.canActivate(decoratedContext(handler, userWithRole(Role.admin)))).resolves.toBe(true); + expect(guard.canActivate(decoratedContext(handler, userWithRole(Role.editor)))).toBe(false); + expect(guard.canActivate(decoratedContext(handler, userWithRole(Role.admin)))).toBe(true); }); - it('lets the roles on the handler override the class', async () => { + it('lets the roles on the handler override the class', () => { const handler = AdminController.prototype.overridden; - await expect(guard.canActivate(decoratedContext(handler, userWithRole(Role.editor)))).resolves.toBe(true); - await expect(guard.canActivate(decoratedContext(handler, userWithRole(Role.admin)))).resolves.toBe(false); + expect(guard.canActivate(decoratedContext(handler, userWithRole(Role.editor)))).toBe(true); + expect(guard.canActivate(decoratedContext(handler, userWithRole(Role.admin)))).toBe(false); }); }); + + it('uses the user stored by ApiGuard without a second lookup', async () => { + reflector.getAllAndOverride.mockReturnValue([Role.admin]); + prismaService.user.findUnique.mockResolvedValue(userWithRole(Role.admin)); + const request = { header: jest.fn().mockReturnValue('admin-key') }; + const shared = { + switchToHttp: () => ({ getRequest: () => request }), + getHandler: () => ({}), + getClass: () => ({}), + } as unknown as ExecutionContext; + + await expect(new ApiGuard(prismaService as unknown as PrismaService).canActivate(shared)).resolves.toBe(true); + expect(guard.canActivate(shared)).toBe(true); + + expect(prismaService.user.findUnique).toHaveBeenCalledTimes(1); + }); }); diff --git a/src/auth/guards/role.guard.ts b/src/auth/guards/role.guard.ts index d8cee282..9bf32784 100644 --- a/src/auth/guards/role.guard.ts +++ b/src/auth/guards/role.guard.ts @@ -1,41 +1,21 @@ import { ExecutionContext, Injectable, CanActivate } from '@nestjs/common'; import { Role, User } from '@prisma/client'; import { Request } from 'express'; -import { PrismaService } from '../../prisma/prisma.service'; import { Reflector } from '@nestjs/core'; @Injectable() export class RoleGuard implements CanActivate { - constructor( - private readonly prismaService: PrismaService, - private reflector: Reflector - ) {} + constructor(private reflector: Reflector) {} - async canActivate(context: ExecutionContext): Promise { + canActivate(context: ExecutionContext): boolean { const requiredRoles = this.reflector.getAllAndOverride('roles', [context.getHandler(), context.getClass()]); if (!requiredRoles || requiredRoles.length === 0) { return true; } - const user = await this.getUser(context); + const request: Request = context.switchToHttp().getRequest(); + const user = request.user as User | undefined; return !!user && requiredRoles.includes(user.role); } - - private getUser = async (context: ExecutionContext): Promise => { - const request: Request = context.switchToHttp().getRequest(); - - if (request.user) { - return request.user as User; - } - - const apiKey = request.header('apiKey'); - if (!apiKey) { - return null; - } - - return this.prismaService.user.findUnique({ - where: { apiKey }, - }); - }; } diff --git a/src/auth/jwt-payload.model.ts b/src/auth/jwt-payload.model.ts index 831b1c07..df1efa7d 100644 --- a/src/auth/jwt-payload.model.ts +++ b/src/auth/jwt-payload.model.ts @@ -1,4 +1,4 @@ export interface JwtPayload { - email: string; - expire?: Date; + sub?: string; + email?: string; } diff --git a/src/auth/jwt.strategy.spec.ts b/src/auth/jwt.strategy.spec.ts index 84ac966e..a7a9b388 100644 --- a/src/auth/jwt.strategy.spec.ts +++ b/src/auth/jwt.strategy.spec.ts @@ -1,4 +1,4 @@ -import { HttpException, HttpStatus } from '@nestjs/common'; +import { UnauthorizedException } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import { User } from '@prisma/client'; import { JwtService } from '@nestjs/jwt'; @@ -14,10 +14,17 @@ describe('JwtStrategy', () => { let configService: { get: jest.Mock }; let strategy: JwtStrategy; + const createStrategy = () => + new JwtStrategy(configService as unknown as ConfigService, prismaService as unknown as PrismaService); + const withLegacyTokens = (value: string) => { + configService.get.mockImplementation((key: string) => (key === 'JWT_LEGACY_EMAIL_TOKENS' ? value : SECRET)); + strategy = createStrategy(); + }; + beforeEach(() => { prismaService = { user: { findUnique: jest.fn() } }; configService = { get: jest.fn().mockReturnValue(SECRET) }; - strategy = new JwtStrategy(configService as unknown as ConfigService, prismaService as unknown as PrismaService); + strategy = createStrategy(); }); describe('constructor', () => { @@ -40,49 +47,32 @@ describe('JwtStrategy', () => { it('fails to construct without a secret', () => { configService.get.mockReturnValue(undefined); - expect( - () => new JwtStrategy(configService as unknown as ConfigService, prismaService as unknown as PrismaService) - ).toThrow('JwtStrategy requires a secret or key'); + expect(createStrategy).toThrow('JwtStrategy requires a secret or key'); }); }); describe('validate', () => { - it('looks the user up by the email in the payload and passes it to done', async () => { + it('returns the user the sub claim points to', async () => { prismaService.user.findUnique.mockResolvedValue(user); - const done = jest.fn().mockReturnValue('done-result'); - const result = await strategy.validate({ email: user.email }, done); + await expect(strategy.validate({ sub: 'user-id', email: 'old@example.com' })).resolves.toBe(user); - expect(prismaService.user.findUnique).toHaveBeenCalledWith({ where: { email: user.email } }); - expect(done).toHaveBeenCalledWith(null, user, undefined); - expect(result).toBe('done-result'); + expect(prismaService.user.findUnique).toHaveBeenCalledWith({ where: { id: 'user-id' } }); }); - it('passes payload.expire as the info argument', async () => { - prismaService.user.findUnique.mockResolvedValue(user); - const done = jest.fn(); - - await strategy.validate({ email: user.email, expire: 123 } as never, done); + it('throws UnauthorizedException when the user does not exist', async () => { + prismaService.user.findUnique.mockResolvedValue(null); - expect(done).toHaveBeenCalledWith(null, user, 123); + await expect(strategy.validate({ sub: 'ghost' })).rejects.toBeInstanceOf(UnauthorizedException); }); - it('passes a 401 HttpException and false to done when the user does not exist', async () => { - prismaService.user.findUnique.mockResolvedValue(null); - const done = jest.fn(); - - await strategy.validate({ email: 'ghost@example.com' }, done); + it('rejects a payload with neither sub nor email without a lookup', async () => { + await expect(strategy.validate({})).rejects.toBeInstanceOf(UnauthorizedException); - expect(done).toHaveBeenCalledTimes(1); - const [error, result] = done.mock.calls[0]; - expect(error).toBeInstanceOf(HttpException); - expect(error.getStatus()).toBe(HttpStatus.UNAUTHORIZED); - expect(error.getResponse()).toEqual({}); - expect(result).toBe(false); + expect(prismaService.user.findUnique).not.toHaveBeenCalled(); }); }); - // Drives the strategy the way passport does, with a real signed token. describe('authenticate', () => { const authenticate = (token?: string) => { const outcome = { success: jest.fn(), fail: jest.fn(), error: jest.fn() }; @@ -92,28 +82,66 @@ describe('JwtStrategy', () => { }; const settle = () => new Promise((resolve) => setTimeout(resolve, 20)); - // NOTE: current behavior — validate() calls `done` itself and the - // @nestjs/passport mixin calls it a second time with validate()'s return - // value (undefined), so passport sees success() followed by fail(). - it('authenticates a valid token, then also reports a failure from the second done call', async () => { + it('authenticates a valid token exactly once', async () => { prismaService.user.findUnique.mockResolvedValue(user); - const outcome = authenticate(sign({ email: user.email }, SECRET)); + const outcome = authenticate(sign({ sub: 'user-id', email: user.email }, SECRET)); await settle(); + expect(outcome.success).toHaveBeenCalledTimes(1); expect(outcome.success).toHaveBeenCalledWith(user, undefined); - expect(outcome.fail).toHaveBeenCalledTimes(1); - expect(outcome.success.mock.invocationCallOrder[0]).toBeLessThan(outcome.fail.mock.invocationCallOrder[0]); + expect(outcome.fail).not.toHaveBeenCalled(); expect(outcome.error).not.toHaveBeenCalled(); }); - it('errors with a 401 HttpException when the token belongs to an unknown user', async () => { + it('resolves a token with sub by user id', async () => { + prismaService.user.findUnique.mockResolvedValue(user); + + authenticate(sign({ sub: 'user-id', email: 'email-before-the-change@example.com' }, SECRET)); + await settle(); + + expect(prismaService.user.findUnique).toHaveBeenCalledTimes(1); + expect(prismaService.user.findUnique).toHaveBeenCalledWith({ where: { id: 'user-id' } }); + }); + + it('still accepts an email-only token issued before sub existed', async () => { + prismaService.user.findUnique.mockResolvedValue(user); + + const outcome = authenticate(sign({ email: user.email }, SECRET)); + await settle(); + + expect(prismaService.user.findUnique).toHaveBeenCalledWith({ where: { email: user.email } }); + expect(outcome.success).toHaveBeenCalledWith(user, undefined); + }); + + it.each(['false', ' FALSE '])('rejects an email-only token when JWT_LEGACY_EMAIL_TOKENS=%j', async (value) => { + withLegacyTokens(value); + + const outcome = authenticate(sign({ email: user.email }, SECRET)); + await settle(); + + expect(prismaService.user.findUnique).not.toHaveBeenCalled(); + expect(outcome.error).toHaveBeenCalledWith(expect.any(UnauthorizedException)); + expect(outcome.success).not.toHaveBeenCalled(); + }); + + it('keeps accepting tokens with sub when JWT_LEGACY_EMAIL_TOKENS=false', async () => { + withLegacyTokens('false'); + prismaService.user.findUnique.mockResolvedValue(user); + + const outcome = authenticate(sign({ sub: 'user-id', email: user.email }, SECRET)); + await settle(); + + expect(outcome.success).toHaveBeenCalledWith(user, undefined); + }); + + it('errors with a 401 when the token belongs to an unknown user', async () => { prismaService.user.findUnique.mockResolvedValue(null); - const outcome = authenticate(sign({ email: 'ghost@example.com' }, SECRET)); + const outcome = authenticate(sign({ sub: 'ghost' }, SECRET)); await settle(); - expect(outcome.error).toHaveBeenCalledWith(expect.any(HttpException)); + expect(outcome.error).toHaveBeenCalledWith(expect.any(UnauthorizedException)); expect(outcome.success).not.toHaveBeenCalled(); }); diff --git a/src/auth/jwt.strategy.ts b/src/auth/jwt.strategy.ts index 71fc617d..f4bf2b95 100644 --- a/src/auth/jwt.strategy.ts +++ b/src/auth/jwt.strategy.ts @@ -1,12 +1,15 @@ -import { ExtractJwt, Strategy, VerifiedCallback } from 'passport-jwt'; +import { ExtractJwt, Strategy } from 'passport-jwt'; import { PassportStrategy } from '@nestjs/passport'; -import { Injectable, HttpException, HttpStatus } from '@nestjs/common'; +import { Injectable, UnauthorizedException } from '@nestjs/common'; +import { User } from '@prisma/client'; import { JwtPayload } from './jwt-payload.model'; import { ConfigService } from '@nestjs/config'; import { PrismaService } from '../prisma/prisma.service'; @Injectable() export class JwtStrategy extends PassportStrategy(Strategy) { + private readonly acceptEmailOnlyTokens: boolean; + constructor( configService: ConfigService, private prismaService: PrismaService @@ -16,17 +19,29 @@ export class JwtStrategy extends PassportStrategy(Strategy) { ignoreExpiration: false, secretOrKey: configService.get('JWT_SECRET'), }); + // tokens issued before `sub` existed carry only the email; drop them once they have all expired + this.acceptEmailOnlyTokens = + String(configService.get('JWT_LEGACY_EMAIL_TOKENS') ?? 'true') + .trim() + .toLowerCase() !== 'false'; } - async validate(payload: JwtPayload, done: VerifiedCallback) { - const user = await this.prismaService.user.findUnique({ - where: { email: payload.email }, - }); - + // @nestjs/passport calls `done` with whatever this returns, so it must not call `done` itself + async validate(payload: JwtPayload): Promise { + const user = await this.findUser(payload); if (!user) { - return done(new HttpException({}, HttpStatus.UNAUTHORIZED), false); + throw new UnauthorizedException(); } + return user; + } - return done(null, user, payload.expire); + private findUser(payload: JwtPayload): Promise { + if (typeof payload.sub === 'string' && payload.sub) { + return this.prismaService.user.findUnique({ where: { id: payload.sub } }); + } + if (this.acceptEmailOnlyTokens && typeof payload.email === 'string' && payload.email) { + return this.prismaService.user.findUnique({ where: { email: payload.email } }); + } + return Promise.resolve(null); } } diff --git a/src/builds/build-stats.spec.ts b/src/builds/build-stats.spec.ts index 65b461d9..b42c1ff8 100644 --- a/src/builds/build-stats.spec.ts +++ b/src/builds/build-stats.spec.ts @@ -14,42 +14,36 @@ describe('getBuildsStats', () => { expect(groupByMock).not.toHaveBeenCalled(); }); - it('groups test runs by build and status, and by build for merge runs', async () => { + it('groups test runs by build, status and merge flag in one query', async () => { const groupByMock = jest.fn().mockResolvedValue([]); await getBuildsStats(prismaWith(groupByMock), ['b1', 'b2']); - expect(groupByMock).toHaveBeenNthCalledWith(1, { - by: ['buildId', 'status'], + expect(groupByMock).toHaveBeenCalledTimes(1); + expect(groupByMock).toHaveBeenCalledWith({ + by: ['buildId', 'status', 'merge'], where: { buildId: { in: ['b1', 'b2'] } }, _count: { _all: true }, }); - expect(groupByMock).toHaveBeenNthCalledWith(2, { - by: ['buildId'], - where: { buildId: { in: ['b1', 'b2'] }, merge: true }, - }); }); it('sums the status counts into passed, unresolved and failed per build', async () => { - const groupByMock = jest - .fn() - .mockResolvedValueOnce([ - { buildId: 'b1', status: TestStatus.ok, _count: { _all: 1 } }, - { buildId: 'b1', status: TestStatus.approved, _count: { _all: 2 } }, - { buildId: 'b1', status: TestStatus.autoApproved, _count: { _all: 4 } }, - { buildId: 'b1', status: TestStatus.new, _count: { _all: 8 } }, - { buildId: 'b1', status: TestStatus.unresolved, _count: { _all: 16 } }, - { buildId: 'b1', status: TestStatus.failed, _count: { _all: 32 } }, - { buildId: 'b2', status: TestStatus.failed, _count: { _all: 3 } }, - // a status outside the known groups is not counted anywhere - { buildId: 'b2', status: 'resolved' as TestStatus, _count: { _all: 100 } }, - ]) - .mockResolvedValueOnce([{ buildId: 'b2' }]); + const groupByMock = jest.fn().mockResolvedValueOnce([ + { buildId: 'b1', status: TestStatus.ok, merge: false, _count: { _all: 1 } }, + { buildId: 'b1', status: TestStatus.ok, merge: true, _count: { _all: 2 } }, + { buildId: 'b1', status: TestStatus.approved, merge: false, _count: { _all: 2 } }, + { buildId: 'b1', status: TestStatus.autoApproved, merge: false, _count: { _all: 4 } }, + { buildId: 'b1', status: TestStatus.new, merge: false, _count: { _all: 8 } }, + { buildId: 'b1', status: TestStatus.unresolved, merge: false, _count: { _all: 16 } }, + { buildId: 'b1', status: TestStatus.failed, merge: false, _count: { _all: 32 } }, + { buildId: 'b2', status: TestStatus.failed, merge: true, _count: { _all: 3 } }, + { buildId: 'b2', status: 'resolved' as TestStatus, merge: false, _count: { _all: 100 } }, + ]); const stats = await getBuildsStats(prismaWith(groupByMock), ['b1', 'b2', 'b3']); expect([...stats.entries()]).toEqual([ - ['b1', { passedCount: 7, unresolvedCount: 24, failedCount: 32, merge: false }], + ['b1', { passedCount: 9, unresolvedCount: 24, failedCount: 32, merge: true }], ['b2', { passedCount: 0, unresolvedCount: 0, failedCount: 3, merge: true }], ['b3', { passedCount: 0, unresolvedCount: 0, failedCount: 0, merge: false }], ]); @@ -58,8 +52,7 @@ describe('getBuildsStats', () => { it('keeps one entry per id when the same build id is asked for twice', async () => { const groupByMock = jest .fn() - .mockResolvedValueOnce([{ buildId: 'b1', status: TestStatus.ok, _count: { _all: 2 } }]) - .mockResolvedValueOnce([]); + .mockResolvedValueOnce([{ buildId: 'b1', status: TestStatus.ok, merge: false, _count: { _all: 2 } }]); const stats = await getBuildsStats(prismaWith(groupByMock), ['b1', 'b1']); diff --git a/src/builds/build-stats.ts b/src/builds/build-stats.ts index 5bf20fa6..8bbdde98 100644 --- a/src/builds/build-stats.ts +++ b/src/builds/build-stats.ts @@ -11,12 +11,6 @@ export interface BuildStats { const PASSED_STATUSES: TestStatus[] = [TestStatus.ok, TestStatus.approved, TestStatus.autoApproved]; const UNRESOLVED_STATUSES: TestStatus[] = [TestStatus.unresolved, TestStatus.new]; -/** - * Aggregates test-run statistics per build inside the database instead of - * loading every test-run row into the API process: builds can hold thousands - * of runs and these stats are recomputed on every build_updated event burst - * during ingestion. - */ export async function getBuildsStats(prisma: PrismaService, buildIds: string[]): Promise> { const stats = new Map( buildIds.map((id) => [id, { passedCount: 0, unresolvedCount: 0, failedCount: 0, merge: false }]) @@ -25,20 +19,13 @@ export async function getBuildsStats(prisma: PrismaService, buildIds: string[]): return stats; } - const [statusCounts, mergeBuilds] = await Promise.all([ - prisma.testRun.groupBy({ - by: ['buildId', 'status'], - where: { buildId: { in: buildIds } }, - _count: { _all: true }, - }), - prisma.testRun.groupBy({ - by: ['buildId'], - where: { buildId: { in: buildIds }, merge: true }, - }), - ]); + const rows = await prisma.testRun.groupBy({ + by: ['buildId', 'status', 'merge'], + where: { buildId: { in: buildIds } }, + _count: { _all: true }, + }); - for (const row of statusCounts) { - // every row belongs to one of buildIds, and the map holds an entry for each + for (const row of rows) { const buildStats = stats.get(row.buildId)!; const count = row._count._all; if (PASSED_STATUSES.includes(row.status)) { @@ -48,9 +35,9 @@ export async function getBuildsStats(prisma: PrismaService, buildIds: string[]): } else if (row.status === TestStatus.failed) { buildStats.failedCount += count; } - } - for (const row of mergeBuilds) { - stats.get(row.buildId)!.merge = true; + if (row.merge) { + buildStats.merge = true; + } } return stats; } diff --git a/src/builds/builds.controller.spec.ts b/src/builds/builds.controller.spec.ts index b7d05674..aa21b91d 100644 --- a/src/builds/builds.controller.spec.ts +++ b/src/builds/builds.controller.spec.ts @@ -1,5 +1,5 @@ -import { NotFoundException } from '@nestjs/common'; -import { Test, TestingModule } from '@nestjs/testing'; +import { BadRequestException, NotFoundException } from '@nestjs/common'; +import { Test } from '@nestjs/testing'; import { BuildsController } from './builds.controller'; import { BuildsService } from './builds.service'; import { PrismaService } from '../prisma/prisma.service'; @@ -7,130 +7,11 @@ import { ApiGuard } from '../auth/guards/api.guard'; import { JwtAuthGuard } from '../auth/guards/auth.guard'; import { ProjectsService } from '../projects/projects.service'; import { TEST_BUILD, TEST_PROJECT } from '../_data_'; -import { Build, User } from '@prisma/client'; +import { User } from '@prisma/client'; import { BuildDto } from './dto/build.dto'; -import { CreateBuildDto } from './dto/build-create.dto'; import { ModifyBuildDto } from './dto/build-modify.dto'; -import { EventsGateway } from '../shared/events/events.gateway'; - -const initController = async ({ - projectFindOneMock = jest.fn(), - buildFindOrCreateMock = jest.fn(), - buildIncrementBuildNumberMock = jest.fn(), - eventBuildCreatedMock = jest.fn(), - deleteOldBuilds = jest.fn(), - buildUpdateMock = jest.fn(), -}) => { - const module: TestingModule = await Test.createTestingModule({ - controllers: [BuildsController], - providers: [ - { - provide: ProjectsService, - useValue: { - findOne: projectFindOneMock, - }, - }, - { - provide: BuildsService, - useValue: { - findOrCreate: buildFindOrCreateMock, - incrementBuildNumber: buildIncrementBuildNumberMock, - deleteOldBuilds: deleteOldBuilds, - update: buildUpdateMock, - }, - }, - { provide: EventsGateway, useValue: { buildCreated: eventBuildCreatedMock } }, - { provide: PrismaService, useValue: {} }, - { provide: ApiGuard, useValue: {} }, - { provide: JwtAuthGuard, useValue: {} }, - ], - }).compile(); - - return module.get(BuildsController); -}; -describe('Builds Controller', () => { - let controller: BuildsController; - - const createBuildDto: CreateBuildDto = { - ciBuildId: 'ciBuildId', - branchName: 'branchName', - project: 'name', - }; - const project = TEST_PROJECT; - const buildWithNumber: Build = { - ...TEST_BUILD, - number: 12, - }; - - beforeEach(async () => {}); - - it('should be defined', async () => { - controller = await initController({}); - expect(controller).toBeDefined(); - }); - - it('should create new build', async () => { - const projectFindOneMock = jest.fn().mockResolvedValueOnce(project); - const buildFindOrCreateMock = jest.fn().mockResolvedValueOnce(buildWithNumber); - const deleteOldBuilds = jest.fn(); - controller = await initController({ - projectFindOneMock, - buildFindOrCreateMock, - deleteOldBuilds, - }); - - const result = await controller.create(createBuildDto); - - expect(result).toStrictEqual(new BuildDto(buildWithNumber)); - expect(deleteOldBuilds).toHaveBeenCalledWith(project.id, project.maxBuildAllowed, createBuildDto.ciBuildId); - }); - - it('should reuse build', async () => { - const eventBuildCreatedMock = jest.fn(); - const projectFindOneMock = jest.fn().mockResolvedValueOnce(project); - const buildFindOrCreateMock = jest.fn().mockResolvedValueOnce(buildWithNumber); - const deleteOldBuilds = jest.fn(); - controller = await initController({ - projectFindOneMock, - buildFindOrCreateMock, - eventBuildCreatedMock, - deleteOldBuilds, - }); - - const result = await controller.create(createBuildDto); - - expect(result).toStrictEqual(new BuildDto(buildWithNumber)); - }); - - describe('update', () => { - it.each([ - ['no body', undefined], - ['null body', null], - ['empty body', {}], - ])('should stop build when called with %s', async (_, body) => { - const buildUpdateMock = jest.fn().mockResolvedValueOnce(new BuildDto(TEST_BUILD)); - controller = await initController({ buildUpdateMock }); - - await controller.update(TEST_BUILD.id, body as ModifyBuildDto); - - expect(buildUpdateMock).toHaveBeenCalledWith(TEST_BUILD.id, { isRunning: false }); - }); - - it.each([[{ isRunning: false }], [{ isRunning: true }], [{ ciBuildId: 'ciBuildId' }]])( - 'should pass %o as is', - async (body) => { - const buildUpdateMock = jest.fn().mockResolvedValueOnce(new BuildDto(TEST_BUILD)); - controller = await initController({ buildUpdateMock }); - - await controller.update(TEST_BUILD.id, body); - expect(buildUpdateMock).toHaveBeenCalledWith(TEST_BUILD.id, body); - } - ); - }); -}); - -describe('Builds Controller handlers', () => { +describe('Builds Controller', () => { let controller: BuildsController; const buildsService = { findMany: jest.fn(), @@ -138,15 +19,16 @@ describe('Builds Controller handlers', () => { remove: jest.fn(), update: jest.fn(), approve: jest.fn(), - deleteOldBuilds: jest.fn(), + scheduleRetention: jest.fn(), findOrCreate: jest.fn(), }; const projectsService = { findOne: jest.fn() }; const buildDto = new BuildDto(TEST_BUILD); + const retentionArgs = [TEST_PROJECT.id, TEST_PROJECT.maxBuildAllowed, TEST_BUILD.id]; beforeEach(async () => { jest.resetAllMocks(); - const module: TestingModule = await Test.createTestingModule({ + const module = await Test.createTestingModule({ controllers: [BuildsController], providers: [ { provide: BuildsService, useValue: buildsService }, @@ -157,7 +39,7 @@ describe('Builds Controller handlers', () => { ], }).compile(); - controller = module.get(BuildsController); + controller = module.get(BuildsController); }); it('lists a page of builds with the filters it was given', async () => { @@ -178,6 +60,15 @@ describe('Builds Controller handlers', () => { expect(buildsService.findMany).toHaveBeenCalledWith('projectId', 10, 0, undefined, undefined); }); + it.each([ + [10, -1], + [-1, 0], + ])('answers 400 for take=%p skip=%p without querying', async (take, skip) => { + await expect(controller.get('projectId', take, skip)).rejects.toThrow(BadRequestException); + + expect(buildsService.findMany).not.toHaveBeenCalled(); + }); + it('returns the details of one build', async () => { buildsService.findOne.mockResolvedValue(buildDto); @@ -198,11 +89,29 @@ describe('Builds Controller handlers', () => { await expect(controller.remove('missing')).rejects.toThrow(new NotFoundException('Build not found: missing')); }); - it('updates a build with the body', async () => { - buildsService.update.mockResolvedValue(buildDto); + describe('update', () => { + it.each([ + ['no body', undefined], + ['null body', null], + ['empty body', {}], + ])('should stop build when called with %s', async (_, body) => { + buildsService.update.mockResolvedValue(buildDto); + + await expect(controller.update(TEST_BUILD.id, body as ModifyBuildDto)).resolves.toBe(buildDto); + + expect(buildsService.update).toHaveBeenCalledWith(TEST_BUILD.id, { isRunning: false }); + }); - await expect(controller.update(TEST_BUILD.id, { isRunning: false })).resolves.toBe(buildDto); - expect(buildsService.update).toHaveBeenCalledWith(TEST_BUILD.id, { isRunning: false }); + it.each([[{ isRunning: false }], [{ isRunning: true }], [{ ciBuildId: 'ciBuildId' }]])( + 'should pass %o as is', + async (body) => { + buildsService.update.mockResolvedValue(buildDto); + + await expect(controller.update(TEST_BUILD.id, body)).resolves.toBe(buildDto); + + expect(buildsService.update).toHaveBeenCalledWith(TEST_BUILD.id, body); + } + ); }); it.each([true, false])('approves a build with merge=%s', async (merge) => { @@ -214,32 +123,33 @@ describe('Builds Controller handlers', () => { it("creates the build on the project's main branch when no branch is given", async () => { projectsService.findOne.mockResolvedValue(TEST_PROJECT); - buildsService.findOrCreate.mockResolvedValue(TEST_BUILD); + buildsService.findOrCreate.mockResolvedValue({ ...TEST_BUILD, number: 12 }); - const result = await controller.create({ project: TEST_PROJECT.name, ciBuildId: 'ci-1' } as CreateBuildDto); + const result = await controller.create({ project: TEST_PROJECT.name, ciBuildId: 'ci-1' }); expect(projectsService.findOne).toHaveBeenCalledWith(TEST_PROJECT.name); - expect(buildsService.deleteOldBuilds).toHaveBeenCalledWith(TEST_PROJECT.id, TEST_PROJECT.maxBuildAllowed, 'ci-1'); + expect(buildsService.scheduleRetention).toHaveBeenCalledWith(...retentionArgs); expect(buildsService.findOrCreate).toHaveBeenCalledWith({ projectId: TEST_PROJECT.id, branchName: TEST_PROJECT.mainBranchName, ciBuildId: 'ci-1', }); - expect(result).toStrictEqual(new BuildDto(TEST_BUILD)); + expect(result).toStrictEqual(new BuildDto({ ...TEST_BUILD, number: 12 })); }); - it('deletes old builds before creating the new one', async () => { + it('schedules retention after creating the build, sparing it', async () => { const calls: string[] = []; projectsService.findOne.mockResolvedValue(TEST_PROJECT); - buildsService.deleteOldBuilds.mockImplementation(async () => calls.push('deleteOldBuilds')); + buildsService.scheduleRetention.mockImplementation(() => calls.push('scheduleRetention')); buildsService.findOrCreate.mockImplementation(async () => { calls.push('findOrCreate'); return TEST_BUILD; }); - await controller.create({ project: TEST_PROJECT.id, branchName: 'feature' } as CreateBuildDto); + await controller.create({ project: TEST_PROJECT.id, branchName: 'feature' }); - expect(calls).toEqual(['deleteOldBuilds', 'findOrCreate']); + expect(calls).toEqual(['findOrCreate', 'scheduleRetention']); + expect(buildsService.scheduleRetention).toHaveBeenCalledWith(...retentionArgs); expect(buildsService.findOrCreate).toHaveBeenCalledWith({ projectId: TEST_PROJECT.id, branchName: 'feature', diff --git a/src/builds/builds.controller.ts b/src/builds/builds.controller.ts index f455b953..f80aafbc 100644 --- a/src/builds/builds.controller.ts +++ b/src/builds/builds.controller.ts @@ -13,8 +13,8 @@ import { ParseBoolPipe, forwardRef, Inject, - ValidationPipe, NotFoundException, + BadRequestException, } from '@nestjs/common'; import { BuildsService } from './builds.service'; import { JwtAuthGuard } from '../auth/guards/auth.guard'; @@ -51,13 +51,16 @@ export class BuildsController { }) @ApiBearerAuth() @UseGuards(JwtAuthGuard) - get( + async get( @Query('projectId', new ParseUUIDPipe()) projectId: string, @Query('take', new ParseIntPipe()) take: number, @Query('skip', new ParseIntPipe()) skip: number, @Query('ciBuildId') ciBuildId?: string, @Query('aroundBuildId', new ParseUUIDPipe({ optional: true })) aroundBuildId?: string ): Promise { + if (skip < 0 || take < 0) { + throw new BadRequestException('take and skip must not be negative'); + } return this.buildsService.findMany(projectId, take, skip, ciBuildId, aroundBuildId); } @@ -75,9 +78,6 @@ export class BuildsController { @UseGuards(JwtAuthGuard, RoleGuard) @Roles(Role.admin, Role.editor) async remove(@Param('id', new ParseUUIDPipe()) id: string): Promise { - // remove() answers nothing for a build that is not there, and retention - // relies on that when two requests prune the same build; a client asking - // for it by id is told it does not exist const build = await this.buildsService.remove(id); if (!build) { throw new NotFoundException(`Build not found: ${id}`); @@ -91,13 +91,13 @@ export class BuildsController { @UseGuards(ApiGuard, RoleGuard) @Roles(Role.admin, Role.editor) async create(@Body() createBuildDto: CreateBuildDto): Promise { - const project = (await this.projectService.findOne(createBuildDto.project))!; - await this.buildsService.deleteOldBuilds(project.id, project.maxBuildAllowed, createBuildDto.ciBuildId); + const project = await this.projectService.findOne(createBuildDto.project); const build = await this.buildsService.findOrCreate({ projectId: project.id, branchName: createBuildDto.branchName ?? project.mainBranchName, ciBuildId: createBuildDto.ciBuildId, }); + this.buildsService.scheduleRetention(project.id, project.maxBuildAllowed, build.id); return new BuildDto(build); } @@ -107,14 +107,8 @@ export class BuildsController { @ApiBearerAuth() @UseGuards(MixedGuard, RoleGuard) @Roles(Role.admin, Role.editor) - update( - @Param('id', new ParseUUIDPipe()) id: string, - // forbidNonWhitelisted: a body naming any other column is refused, rather - // than written to the build or silently dropped (an emptied body would stop it) - @Body(new ValidationPipe({ whitelist: true, forbidNonWhitelisted: true })) modifyBuildDto?: ModifyBuildDto - ): Promise { - // Released SDKs (sdk-js, agent-cypress, sdk-python, sdk-java) stop a build with an empty PATCH body. - // Express 5 leaves req.body undefined when there is no body, so handle both cases. + update(@Param('id', new ParseUUIDPipe()) id: string, @Body() modifyBuildDto?: ModifyBuildDto): Promise { + // Released SDKs stop a build with an empty PATCH body; Express 5 leaves req.body undefined then. if (!modifyBuildDto || Object.keys(modifyBuildDto).length === 0) { modifyBuildDto = { isRunning: false }; } diff --git a/src/builds/builds.module.ts b/src/builds/builds.module.ts index 28853a1f..e8f4e057 100644 --- a/src/builds/builds.module.ts +++ b/src/builds/builds.module.ts @@ -6,9 +6,17 @@ import { TestRunsModule } from '../test-runs/test-runs.module'; import { SharedModule } from '../shared/shared.module'; import { AuthModule } from '../auth/auth.module'; import { ProjectsModule } from '../projects/projects.module'; +import { StaticModule } from '../static/static.module'; @Module({ - imports: [SharedModule, UsersModule, forwardRef(() => TestRunsModule), AuthModule, forwardRef(() => ProjectsModule)], + imports: [ + SharedModule, + UsersModule, + forwardRef(() => TestRunsModule), + AuthModule, + forwardRef(() => ProjectsModule), + StaticModule, + ], providers: [BuildsService], controllers: [BuildsController], exports: [BuildsService], diff --git a/src/builds/builds.service.spec.ts b/src/builds/builds.service.spec.ts index 83603c02..5e8d0a6e 100644 --- a/src/builds/builds.service.spec.ts +++ b/src/builds/builds.service.spec.ts @@ -1,5 +1,5 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { BuildsService } from './builds.service'; +import { Test } from '@nestjs/testing'; +import { BuildsService, retentionGraceMinutes } from './builds.service'; import { PrismaService } from '../prisma/prisma.service'; import { TestRunsService } from '../test-runs/test-runs.service'; import { EventsGateway } from '../shared/events/events.gateway'; @@ -7,92 +7,72 @@ import { Build, Prisma, TestRun, TestStatus } from '@prisma/client'; import { mocked, MockedObject } from 'jest-mock'; import { BuildDto } from './dto/build.dto'; import { ModifyBuildDto } from './dto/build-modify.dto'; -import { NotFoundException } from '@nestjs/common'; -import { ProjectsService } from '../projects/projects.service'; -import { generateTestRun, TEST_PROJECT } from '../_data_'; -import { Prisma as PrismaRuntime } from '@prisma/client'; -const { PrismaClientKnownRequestError } = PrismaRuntime; +import { ConflictException, NotFoundException } from '@nestjs/common'; +import { StaticService } from '../static/static.service'; +import { generateTestRun } from '../_data_'; + +const { PrismaClientKnownRequestError } = Prisma; jest.mock('./dto/build.dto'); -const initService = async ({ - buildFindManyMock = jest.fn(), - buildCreateMock = jest.fn(), - buildUpdateMock = jest.fn(), - // a row for whatever id is asked: findOne answers 404 for a missing build - buildFindUniqueMock = jest.fn().mockImplementation(async ({ where }) => ({ id: where.id })), - buildFindFirstMock = jest.fn(), - buildDeleteMock = jest.fn(), - buildUpsertMock = jest.fn(), - buildCountMock = jest.fn(), - testRunDeleteMock = jest.fn(), - testRunApproveMock = jest.fn(), - testRunFindManyMock = jest.fn(), - testRunGroupByMock = jest.fn().mockResolvedValue([]), - eventsBuildUpdatedMock = jest.fn(), - eventsBuildCreatedMock = jest.fn(), - eventBuildDeletedMock = jest.fn(), - projectFindOneMock = jest.fn(), - projectUpdateMock = jest.fn(), -}) => { - const module: TestingModule = await Test.createTestingModule({ +const prismaError = (code: string) => new PrismaClientKnownRequestError(code, { code, clientVersion: '5' }); +const tick = () => new Promise((resolve) => setImmediate(resolve)); +const silentLogger = () => ({ warn: jest.fn(), debug: jest.fn(), log: jest.fn() }); + +const setup = async () => { + const prisma = { + project: { update: jest.fn() }, + build: { + findMany: jest.fn(), + create: jest.fn(), + update: jest.fn(), + findUnique: jest.fn(async ({ where }): Promise => ({ id: where.id })), + findUniqueOrThrow: jest.fn(), + findFirst: jest.fn(), + delete: jest.fn(), + upsert: jest.fn(), + count: jest.fn(), + updateManyAndReturn: jest.fn(), + }, + testRun: { + groupBy: jest.fn().mockResolvedValue([]), + findMany: jest.fn().mockResolvedValue([]), + deleteMany: jest.fn(), + }, + $transaction: jest.fn(), + }; + prisma.$transaction.mockImplementation((fn: (tx: typeof prisma) => unknown) => fn(prisma)); + const events = { + buildUpdated: jest.fn(), + buildCreated: jest.fn(), + buildDeleted: jest.fn(), + testRunDeleted: jest.fn(), + }; + const testRuns = { approve: jest.fn(), delete: jest.fn(), findMany: jest.fn() }; + const deleteImage = jest.fn().mockResolvedValue(true); + + const module = await Test.createTestingModule({ providers: [ BuildsService, + { provide: PrismaService, useValue: prisma }, + { provide: TestRunsService, useValue: testRuns }, + { provide: EventsGateway, useValue: events }, { - provide: PrismaService, + provide: StaticService, useValue: { - project: { - update: projectUpdateMock, - }, - build: { - findMany: buildFindManyMock, - create: buildCreateMock, - update: buildUpdateMock, - findUnique: buildFindUniqueMock, - findFirst: buildFindFirstMock, - delete: buildDeleteMock, - upsert: buildUpsertMock, - count: buildCountMock, - }, - testRun: { - groupBy: testRunGroupByMock, + deleteImages: async (names: Iterable) => { + for (const name of names) if (name) await deleteImage(name); }, }, }, - { - provide: TestRunsService, - useValue: { - approve: testRunApproveMock, - delete: testRunDeleteMock, - findMany: testRunFindManyMock, - }, - }, - { - provide: EventsGateway, - useValue: { - buildUpdated: eventsBuildUpdatedMock, - buildCreated: eventsBuildCreatedMock, - buildDeleted: eventBuildDeletedMock, - }, - }, - { - provide: ProjectsService, - useValue: { - findOne: projectFindOneMock, - }, - }, ], }).compile(); - return module.get(BuildsService); + return { service: module.get(BuildsService), prisma, events, testRuns, deleteImage }; }; describe('BuildsService', () => { - let service: BuildsService; - - const build: Build & { - testRuns: TestRun[]; - } = { + const build: Build & { testRuns: TestRun[] } = { id: 'a9385fc1-884d-4f9f-915e-40da0e7773d5', ciBuildId: null, number: null, @@ -105,6 +85,8 @@ describe('BuildsService', () => { isRunning: true, testRuns: [generateTestRun()], }; + const { testRuns: buildRuns, ...buildRow } = build; + const [run] = buildRuns; const buildDto: BuildDto = { id: 'a9385fc1-884d-4f9f-915e-40da0e7773d5', @@ -122,18 +104,22 @@ describe('BuildsService', () => { isRunning: true, merge: false, }; + const returnDtoOnce = () => mocked(BuildDto).mockReturnValueOnce(buildDto as MockedObject); + const buildOrder = [{ createdAt: 'desc' }, { id: 'desc' }]; + const unhandledRejections = () => { + const unhandled = jest.fn(); + process.on('unhandledRejection', unhandled); + return { unhandled, stop: () => process.off('unhandledRejection', unhandled) }; + }; it('findOne', async () => { - const buildFindUniqueMock = jest.fn().mockResolvedValueOnce(build); - const testRunGroupByMock = jest - .fn() - .mockResolvedValueOnce([ - { buildId: 'someId', status: TestStatus.ok, _count: { _all: 2 } }, - { buildId: 'someId', status: TestStatus.unresolved, _count: { _all: 1 } }, - ]) - .mockResolvedValueOnce([{ buildId: 'someId' }]); - mocked(BuildDto).mockReturnValueOnce(buildDto as MockedObject); - service = await initService({ buildFindUniqueMock, testRunGroupByMock }); + const { service, prisma } = await setup(); + prisma.build.findUnique.mockResolvedValueOnce(build); + prisma.testRun.groupBy.mockResolvedValueOnce([ + { buildId: 'someId', status: TestStatus.ok, merge: false, _count: { _all: 2 } }, + { buildId: 'someId', status: TestStatus.unresolved, merge: true, _count: { _all: 1 } }, + ]); + returnDtoOnce(); const result = await service.findOne('someId'); @@ -147,309 +133,417 @@ describe('BuildsService', () => { }); it('findOne throws NotFoundException for an unknown id', async () => { - const buildFindUniqueMock = jest.fn().mockResolvedValueOnce(null); - const testRunGroupByMock = jest.fn().mockResolvedValue([]); - service = await initService({ buildFindUniqueMock, testRunGroupByMock }); + const { service, prisma } = await setup(); + prisma.build.findUnique.mockResolvedValueOnce(null); await expect(service.findOne('missing')).rejects.toThrow(new NotFoundException('Build not found: missing')); }); it('update writes only the columns a client may change', async () => { - const buildUpdateMock = jest.fn(); - service = await initService({ buildUpdateMock, eventsBuildUpdatedMock: jest.fn() }); + const { service, prisma } = await setup(); service.findOne = jest.fn(); await service.update('id', { isRunning: false, ciBuildId: 'ci', branchName: 'other' } as ModifyBuildDto); - expect(buildUpdateMock.mock.calls[0][0].data).toStrictEqual({ ciBuildId: 'ci', isRunning: false }); + expect(prisma.build.update.mock.calls[0][0].data).toStrictEqual({ ciBuildId: 'ci', isRunning: false }); }); - it('findMany', async () => { - const buildFindManyMock = jest.fn().mockResolvedValueOnce([build]); - const buildCountMock = jest.fn().mockResolvedValueOnce(33); - const projectId = 'someId'; - mocked(BuildDto).mockReturnValueOnce(buildDto as MockedObject); - service = await initService({ buildFindManyMock, buildCountMock }); + it('update announces the change and returns the build', async () => { + const { service, prisma, events } = await setup(); + returnDtoOnce(); - const result = await service.findMany(projectId, 10, 20); + const result = await service.update('some id', { isRunning: false }); - expect(buildCountMock).toHaveBeenCalledWith({ - where: { projectId }, - }); - expect(buildFindManyMock).toHaveBeenCalledWith({ - take: 10, - skip: 20, - orderBy: [{ createdAt: 'desc' }, { id: 'desc' }], - where: { projectId }, - }); - expect(result).toEqual({ - data: [buildDto], - total: 33, - take: 10, - skip: 20, - }); + expect(prisma.build.update).toHaveBeenCalledWith({ where: { id: 'some id' }, data: { isRunning: false } }); + expect(events.buildUpdated).toHaveBeenCalledWith('some id'); + expect(result).toBe(buildDto); }); - it('findMany filtered by ciBuildId', async () => { - const buildFindManyMock = jest.fn().mockResolvedValueOnce([build]); - const buildCountMock = jest.fn().mockResolvedValueOnce(1); + describe('findMany', () => { const projectId = 'someId'; - mocked(BuildDto).mockReturnValueOnce(buildDto as MockedObject); - service = await initService({ buildFindManyMock, buildCountMock }); + const ciBuildId = { contains: 'Onboarding', mode: Prisma.QueryMode.insensitive }; - await service.findMany(projectId, 10, 0, 'Onboarding'); + it('pages the builds of a project', async () => { + const { service, prisma } = await setup(); + prisma.build.findMany.mockResolvedValueOnce([build]); + prisma.build.count.mockResolvedValueOnce(33); + returnDtoOnce(); - const where = { - projectId, - ciBuildId: { contains: 'Onboarding', mode: Prisma.QueryMode.insensitive }, - }; - expect(buildCountMock).toHaveBeenCalledWith({ where }); - expect(buildFindManyMock).toHaveBeenCalledWith({ - take: 10, - skip: 0, - orderBy: [{ createdAt: 'desc' }, { id: 'desc' }], - where, + const result = await service.findMany(projectId, 10, 20); + + expect(prisma.build.count).toHaveBeenCalledWith({ where: { projectId } }); + expect(prisma.build.findMany).toHaveBeenCalledWith({ + take: 10, + skip: 20, + orderBy: buildOrder, + where: { projectId }, + }); + expect(result).toEqual({ data: [buildDto], total: 33, take: 10, skip: 20 }); }); - }); - it('findMany ignores an empty ciBuildId', async () => { - const buildFindManyMock = jest.fn().mockResolvedValueOnce([build]); - const buildCountMock = jest.fn().mockResolvedValueOnce(33); - const projectId = 'someId'; - mocked(BuildDto).mockReturnValueOnce(buildDto as MockedObject); - service = await initService({ buildFindManyMock, buildCountMock }); + it.each([ + ['ciBuildId', 'Onboarding', { projectId, ciBuildId }], + ['an empty ciBuildId', '', { projectId }], + ])('filters by %s', async (_, search, where) => { + const { service, prisma } = await setup(); + prisma.build.findMany.mockResolvedValueOnce([build]); + prisma.build.count.mockResolvedValueOnce(1); + returnDtoOnce(); + + await service.findMany(projectId, 10, 0, search); + + expect(prisma.build.count).toHaveBeenCalledWith({ where }); + expect(prisma.build.findMany).toHaveBeenCalledWith({ take: 10, skip: 0, orderBy: buildOrder, where }); + }); + + describe('around a build', () => { + const aroundBuildId = '7428548b-863f-40e8-b26d-2187cc98afa1'; + const createdAt = new Date('2026-09-01T10:00:00.000Z'); + const ahead = [{ createdAt: { gt: createdAt } }, { createdAt, id: { gt: aroundBuildId } }]; + + const around = async (found: unknown, counts: number[]) => { + const ctx = await setup(); + ctx.prisma.build.findFirst.mockResolvedValueOnce(found); + ctx.prisma.build.findMany.mockResolvedValueOnce([build]); + counts.forEach((count) => ctx.prisma.build.count.mockResolvedValueOnce(count)); + returnDtoOnce(); + return ctx; + }; + + it('returns the page that holds the build rather than the one asked for', async () => { + const { service, prisma } = await around({ id: aroundBuildId, createdAt }, [35, 50]); + + const result = await service.findMany(projectId, 10, 0, undefined, aroundBuildId); + + expect(prisma.build.findFirst).toHaveBeenCalledWith({ + where: { projectId, id: aroundBuildId }, + select: { id: true, createdAt: true }, + }); + expect(prisma.build.count).toHaveBeenNthCalledWith(1, { where: { projectId, OR: ahead } }); + expect(prisma.build.findMany).toHaveBeenCalledWith({ + take: 10, + skip: 30, + orderBy: buildOrder, + where: { projectId }, + }); + expect(result.skip).toBe(30); + }); - await service.findMany(projectId, 10, 0, ''); + it('counts only what the search left, so the page matches the list on screen', async () => { + const { service, prisma } = await around({ id: aroundBuildId, createdAt }, [2, 4]); - expect(buildCountMock).toHaveBeenCalledWith({ where: { projectId } }); - expect(buildFindManyMock).toHaveBeenCalledWith({ - take: 10, - skip: 0, - orderBy: [{ createdAt: 'desc' }, { id: 'desc' }], - where: { projectId }, - }); - }); + await service.findMany(projectId, 2, 0, 'Onboarding', aroundBuildId); - describe('findMany around a build', () => { - const aroundBuildId = '7428548b-863f-40e8-b26d-2187cc98afa1'; - const createdAt = new Date('2026-09-01T10:00:00.000Z'); - const projectId = 'someId'; + expect(prisma.build.findFirst).toHaveBeenCalledWith({ + where: { projectId, ciBuildId, id: aroundBuildId }, + select: { id: true, createdAt: true }, + }); + expect(prisma.build.count).toHaveBeenNthCalledWith(1, { where: { projectId, ciBuildId, OR: ahead } }); + expect(prisma.build.findMany).toHaveBeenCalledWith(expect.objectContaining({ skip: 2 })); + }); - it('returns the page that holds the build rather than the one asked for', async () => { - const buildFindFirstMock = jest.fn().mockResolvedValueOnce({ id: aroundBuildId, createdAt }); - const buildFindManyMock = jest.fn().mockResolvedValueOnce([build]); - // 35 builds sit ahead of it, then the total - const buildCountMock = jest.fn().mockResolvedValueOnce(35).mockResolvedValueOnce(50); - mocked(BuildDto).mockReturnValueOnce(buildDto as MockedObject); - service = await initService({ buildFindFirstMock, buildFindManyMock, buildCountMock }); + it('falls back to the page asked for when the build is not in the list', async () => { + const { service, prisma } = await around(null, [50]); - const result = await service.findMany(projectId, 10, 0, undefined, aroundBuildId); + const result = await service.findMany(projectId, 10, 20, undefined, aroundBuildId); - expect(buildFindFirstMock).toHaveBeenCalledWith({ - where: { projectId, id: aroundBuildId }, - select: { id: true, createdAt: true }, - }); - expect(buildCountMock).toHaveBeenNthCalledWith(1, { - where: { - projectId, - OR: [{ createdAt: { gt: createdAt } }, { createdAt, id: { gt: aroundBuildId } }], - }, - }); - // the fourth page of ten starts at 30, and 35 is on it - expect(buildFindManyMock).toHaveBeenCalledWith({ - take: 10, - skip: 30, - orderBy: [{ createdAt: 'desc' }, { id: 'desc' }], - where: { projectId }, + expect(prisma.build.count).toHaveBeenCalledTimes(1); + expect(prisma.build.findMany).toHaveBeenCalledWith(expect.objectContaining({ skip: 20 })); + expect(result.skip).toBe(20); }); - expect(result.skip).toBe(30); - }); - it('counts only what the search left, so the page matches the list on screen', async () => { - const buildFindFirstMock = jest.fn().mockResolvedValueOnce({ id: aroundBuildId, createdAt }); - const buildFindManyMock = jest.fn().mockResolvedValueOnce([build]); - const buildCountMock = jest.fn().mockResolvedValueOnce(2).mockResolvedValueOnce(4); - mocked(BuildDto).mockReturnValueOnce(buildDto as MockedObject); - service = await initService({ buildFindFirstMock, buildFindManyMock, buildCountMock }); + it('does not divide by a page of no size', async () => { + const { service, prisma } = await around({ id: aroundBuildId, createdAt }, [0]); - await service.findMany(projectId, 2, 0, 'Onboarding', aroundBuildId); + const result = await service.findMany(projectId, 0, 0, undefined, aroundBuildId); - const ciBuildId = { contains: 'Onboarding', mode: Prisma.QueryMode.insensitive }; - expect(buildFindFirstMock).toHaveBeenCalledWith({ - where: { projectId, ciBuildId, id: aroundBuildId }, - select: { id: true, createdAt: true }, - }); - expect(buildCountMock).toHaveBeenNthCalledWith(1, { - where: { - projectId, - ciBuildId, - OR: [{ createdAt: { gt: createdAt } }, { createdAt, id: { gt: aroundBuildId } }], - }, + expect(result.skip).toBe(0); + expect(prisma.build.findMany).toHaveBeenCalledWith(expect.objectContaining({ skip: 0 })); }); - expect(buildFindManyMock).toHaveBeenCalledWith(expect.objectContaining({ skip: 2 })); }); + }); - it('falls back to the page asked for when the build is not in the list', async () => { - // deleted, another project's, or filtered out by the search: a stale link - // should still open the list rather than fail - const buildFindFirstMock = jest.fn().mockResolvedValueOnce(null); - const buildFindManyMock = jest.fn().mockResolvedValueOnce([build]); - const buildCountMock = jest.fn().mockResolvedValueOnce(50); - mocked(BuildDto).mockReturnValueOnce(buildDto as MockedObject); - service = await initService({ buildFindFirstMock, buildFindManyMock, buildCountMock }); + describe('remove', () => { + const removedBuild = { ...buildRow, testRuns: [run] }; - const result = await service.findMany(projectId, 10, 20, undefined, aroundBuildId); + it('removes a build, its runs, their files and announces it', async () => { + const { service, prisma, events, testRuns, deleteImage } = await setup(); + prisma.build.findUnique.mockResolvedValueOnce(buildRow); + prisma.build.delete.mockResolvedValueOnce(buildRow); + prisma.testRun.findMany.mockResolvedValueOnce([run]); - expect(buildCountMock).toHaveBeenCalledTimes(1); - expect(buildFindManyMock).toHaveBeenCalledWith(expect.objectContaining({ skip: 20 })); - expect(result.skip).toBe(20); - }); + const result = await service.remove(build.id); - it('does not divide by a page of no size', async () => { - const buildFindFirstMock = jest.fn().mockResolvedValueOnce({ id: aroundBuildId, createdAt }); - const buildFindManyMock = jest.fn().mockResolvedValueOnce([build]); - const buildCountMock = jest.fn().mockResolvedValueOnce(0); - mocked(BuildDto).mockReturnValueOnce(buildDto as MockedObject); - service = await initService({ buildFindFirstMock, buildFindManyMock, buildCountMock }); + expect(prisma.build.findUnique).toHaveBeenCalledWith({ where: { id: build.id } }); + expect(prisma.testRun.findMany).toHaveBeenCalledWith({ where: { buildId: build.id }, take: 500 }); + expect(prisma.testRun.deleteMany).toHaveBeenCalledWith({ where: { id: { in: [run.id] } } }); + expect(deleteImage).toHaveBeenCalledTimes(2); + expect(deleteImage).toHaveBeenCalledWith(run.imageName); + expect(deleteImage).toHaveBeenCalledWith(run.diffName); + expect(events.testRunDeleted).toHaveBeenCalledWith(run); + expect(prisma.build.delete).toHaveBeenCalledWith({ where: { id: build.id } }); + expect(mocked(BuildDto).mock.calls.at(-1)).toEqual([removedBuild]); + expect(events.buildDeleted).toHaveBeenCalledTimes(1); + expect(result).toEqual(removedBuild); + expect(testRuns.delete).not.toHaveBeenCalled(); + }); + + it('deletes the rows of a chunk before its files', async () => { + const { service, prisma, deleteImage } = await setup(); + const calls: string[] = []; + prisma.build.findUnique.mockResolvedValueOnce(buildRow); + prisma.testRun.findMany.mockResolvedValueOnce([run]); + prisma.testRun.deleteMany.mockImplementation(async () => void calls.push('rows')); + deleteImage.mockImplementation(async () => void calls.push('file')); - const result = await service.findMany(projectId, 0, 0, undefined, aroundBuildId); + await service.remove(build.id); - expect(result.skip).toBe(0); - expect(buildFindManyMock).toHaveBeenCalledWith(expect.objectContaining({ skip: 0 })); + expect(calls).toEqual(['rows', 'file', 'file']); }); - }); - describe('remove', () => { - it('should remove a build successfully', async () => { - const buildFindUniqueMock = jest.fn().mockResolvedValueOnce(build); - const buildFindManyMock = jest.fn().mockImplementation(() => Promise.resolve(build)); - const buildDeleteMock = jest.fn().mockImplementation(() => Promise.resolve(build)); - const testRunDeleteMock = jest.fn(); - const eventBuildDeletedMock = jest.fn(); - service = await initService({ - buildFindUniqueMock, - buildDeleteMock, - testRunDeleteMock, - eventBuildDeletedMock, - buildFindManyMock, - }); + it('removes the runs chunk by chunk', async () => { + const { service, prisma } = await setup(); + const a = generateTestRun({ id: 'a' }); + const b = generateTestRun({ id: 'b' }); + prisma.build.findUnique.mockResolvedValueOnce(buildRow); + prisma.testRun.findMany.mockResolvedValueOnce([a]).mockResolvedValueOnce([b]); - await service.remove(build.id); + const result = await service.remove(build.id); - expect(buildFindUniqueMock).toHaveBeenCalledWith({ - where: { id: build.id }, - include: { - testRuns: true, - }, - }); - expect(testRunDeleteMock).toHaveBeenCalledWith(build.testRuns[0].id); - expect(eventBuildDeletedMock).toHaveBeenCalledWith(new BuildDto(build)); - expect(buildDeleteMock).toHaveBeenCalledWith({ - where: { id: build.id }, - }); + expect(prisma.testRun.deleteMany).toHaveBeenCalledTimes(2); + expect(prisma.testRun.deleteMany).toHaveBeenNthCalledWith(1, { where: { id: { in: ['a'] } } }); + expect(prisma.testRun.deleteMany).toHaveBeenNthCalledWith(2, { where: { id: { in: ['b'] } } }); + expect(result?.testRuns).toEqual([a, b]); }); it('should handle undefined ID gracefully', async () => { - const buildFindUniqueMock = jest.fn(); - const loggerWarnMock = jest.fn(); - service = await initService({ buildFindUniqueMock }); - (service as any).logger = { warn: loggerWarnMock }; // Mock the logger + const { service, prisma } = await setup(); + const warn = jest.fn(); + (service as any).logger = { warn }; await service.remove(undefined as unknown as string); - expect(loggerWarnMock).toHaveBeenCalledWith('Attempted to remove build with undefined ID.'); - expect(buildFindUniqueMock).not.toHaveBeenCalled(); + expect(warn).toHaveBeenCalledWith('Attempted to remove build with undefined ID.'); + expect(prisma.build.findUnique).not.toHaveBeenCalled(); }); - }); - describe('remove (characterization)', () => { it('does nothing when the build does not exist', async () => { - const buildFindUniqueMock = jest.fn().mockResolvedValueOnce(null); - const buildDeleteMock = jest.fn(); - const eventBuildDeletedMock = jest.fn(); - service = await initService({ buildFindUniqueMock, buildDeleteMock, eventBuildDeletedMock }); + const { service, prisma, events } = await setup(); + prisma.build.findUnique.mockResolvedValueOnce(null); await expect(service.remove('missing')).resolves.toBeUndefined(); - expect(buildDeleteMock).not.toHaveBeenCalled(); - expect(eventBuildDeletedMock).not.toHaveBeenCalled(); + expect(prisma.build.delete).not.toHaveBeenCalled(); + expect(events.buildDeleted).not.toHaveBeenCalled(); }); it('treats a build deleted in the meantime (P2025) as done, without an event', async () => { - const buildFindUniqueMock = jest.fn().mockResolvedValueOnce(build); - const buildDeleteMock = jest - .fn() - .mockRejectedValueOnce(new PrismaClientKnownRequestError('gone', { code: 'P2025', clientVersion: '5' })); - const testRunDeleteMock = jest.fn(); - const eventBuildDeletedMock = jest.fn(); - service = await initService({ buildFindUniqueMock, buildDeleteMock, testRunDeleteMock, eventBuildDeletedMock }); + const { service, prisma, events } = await setup(); + prisma.build.findUnique.mockResolvedValueOnce(buildRow); + prisma.build.delete.mockRejectedValueOnce(prismaError('P2025')); + prisma.testRun.findMany.mockResolvedValueOnce([run]); await expect(service.remove(build.id)).resolves.toBeUndefined(); - expect(testRunDeleteMock).toHaveBeenCalledWith(build.testRuns[0].id); - expect(eventBuildDeletedMock).not.toHaveBeenCalled(); + expect(prisma.testRun.deleteMany).toHaveBeenCalledWith({ where: { id: { in: [run.id] } } }); + expect(events.buildDeleted).not.toHaveBeenCalled(); }); it.each([ - ['another Prisma error', new PrismaClientKnownRequestError('fk', { code: 'P2003', clientVersion: '5' })], + ['a Prisma error other than P2003 and P2025', prismaError('P2034')], ['any other error', new Error('connection lost')], - ])('swallows %s on delete and still reports the build as deleted', async (_, error) => { - const buildFindUniqueMock = jest.fn().mockResolvedValueOnce(build); - const buildDeleteMock = jest.fn().mockRejectedValueOnce(error); - const eventBuildDeletedMock = jest.fn(); - service = await initService({ buildFindUniqueMock, buildDeleteMock, eventBuildDeletedMock }); + ])('rethrows %s from build.delete and announces nothing', async (_, error) => { + const { service, prisma, events } = await setup(); + prisma.build.findUnique.mockResolvedValueOnce(buildRow); + prisma.build.delete.mockRejectedValueOnce(error); + + await expect(service.remove(build.id)).rejects.toBe(error); + + expect(prisma.build.delete).toHaveBeenCalledTimes(1); + expect(events.buildDeleted).not.toHaveBeenCalled(); + }); + + it('deletes a run that arrived meanwhile and retries (P2003)', async () => { + const { service, prisma, events } = await setup(); + const r1 = generateTestRun({ id: 'r1' }); + const r2 = generateTestRun({ id: 'r2' }); + prisma.build.findUnique.mockResolvedValueOnce(buildRow); + prisma.build.delete.mockRejectedValueOnce(prismaError('P2003')).mockResolvedValueOnce(buildRow); + prisma.testRun.findMany.mockResolvedValueOnce([r1]).mockResolvedValueOnce([]).mockResolvedValueOnce([r2]); const result = await service.remove(build.id); - // NOTE: current behavior — only P2025 is handled; every other delete error - // is swallowed and the build is announced as deleted although it is not. - expect(result).toBe(build); - expect(eventBuildDeletedMock).toHaveBeenCalledTimes(1); + expect(prisma.build.delete).toHaveBeenCalledTimes(2); + expect(result?.testRuns).toEqual([r1, r2]); + expect(events.buildDeleted).toHaveBeenCalledTimes(1); }); - }); - describe('incrementBuildNumber', () => { - it("bumps the project's build counter and gives the build the new number", async () => { - const projectUpdateMock = jest.fn().mockResolvedValueOnce({ ...TEST_PROJECT, buildsCounter: 43 }); - const buildUpdateMock = jest.fn().mockResolvedValueOnce({ ...build, number: 43 }); - service = await initService({ projectUpdateMock, buildUpdateMock }); + it('answers 409 when runs keep arriving', async () => { + const { service, prisma, events } = await setup(); + prisma.build.findUnique.mockResolvedValueOnce(buildRow); + prisma.build.delete.mockRejectedValue(prismaError('P2003')); - const result = await service.incrementBuildNumber(build.id, 'projectId'); + await expect(service.remove(build.id)).rejects.toThrow(ConflictException); - expect(projectUpdateMock).toHaveBeenCalledWith({ - where: { id: 'projectId' }, - data: { buildsCounter: { increment: 1 } }, + expect(prisma.build.delete).toHaveBeenCalledTimes(3); + expect(events.buildDeleted).not.toHaveBeenCalled(); + }); + + it('skips the file names a run does not have', async () => { + const { service, prisma, deleteImage } = await setup(); + prisma.build.findUnique.mockResolvedValueOnce(buildRow); + prisma.testRun.findMany.mockResolvedValueOnce([ + generateTestRun({ imageName: null as unknown as string, diffName: null }), + ]); + + await service.remove(build.id); + + expect(deleteImage).not.toHaveBeenCalled(); + }); + + it('does not look at activity without a cutoff', async () => { + const { service, prisma } = await setup(); + prisma.build.findUnique.mockResolvedValueOnce(buildRow); + + await service.remove(build.id); + + expect(prisma.build.findFirst).not.toHaveBeenCalled(); + }); + + describe('only if the build is idle', () => { + const since = new Date('2026-01-01T00:00:00Z'); + + it('keeps a build in use: no rows, files or events', async () => { + const { service, prisma, events, deleteImage } = await setup(); + prisma.build.findUnique.mockResolvedValueOnce(buildRow); + prisma.build.findFirst.mockResolvedValue(null); + prisma.testRun.findMany.mockResolvedValueOnce([run]); + + await expect(service.remove(build.id, { onlyIfIdleSince: since })).resolves.toBeUndefined(); + + expect(prisma.testRun.deleteMany).not.toHaveBeenCalled(); + expect(deleteImage).not.toHaveBeenCalled(); + expect(prisma.build.delete).not.toHaveBeenCalled(); + expect(events.testRunDeleted).not.toHaveBeenCalled(); + expect(events.buildDeleted).not.toHaveBeenCalled(); + }); + + it('stops when the build becomes active between chunks', async () => { + const { service, prisma, events } = await setup(); + prisma.build.findUnique.mockResolvedValueOnce(buildRow); + prisma.build.findFirst.mockResolvedValueOnce({ id: build.id }).mockResolvedValue(null); + prisma.testRun.findMany.mockResolvedValueOnce([generateTestRun({ id: 'r1' })]); + + await expect(service.remove(build.id, { onlyIfIdleSince: since })).resolves.toBeUndefined(); + + expect(prisma.testRun.deleteMany).toHaveBeenCalledTimes(1); + expect(prisma.build.delete).not.toHaveBeenCalled(); + expect(events.buildDeleted).not.toHaveBeenCalled(); + }); + + it('removes an idle build', async () => { + const { service, prisma, events } = await setup(); + prisma.build.findUnique.mockResolvedValueOnce(buildRow); + prisma.build.findFirst.mockResolvedValue({ id: build.id }); + prisma.build.delete.mockResolvedValueOnce(buildRow); + prisma.testRun.findMany.mockResolvedValueOnce([run]); + + const result = await service.remove(build.id, { onlyIfIdleSince: since }); + + expect(prisma.build.findFirst).toHaveBeenCalledWith({ + where: { id: build.id, updatedAt: { lt: since }, testRuns: { none: { createdAt: { gte: since } } } }, + select: { id: true }, + }); + expect(prisma.build.delete).toHaveBeenCalledWith({ where: { id: build.id } }); + expect(result).toEqual(removedBuild); + expect(events.buildDeleted).toHaveBeenCalledTimes(1); + }); + + it('does not retry P2003 when pruning', async () => { + const { service, prisma, events } = await setup(); + prisma.build.findUnique.mockResolvedValueOnce(buildRow); + prisma.build.findFirst.mockResolvedValue({ id: build.id }); + prisma.build.delete.mockRejectedValue(prismaError('P2003')); + + await expect(service.remove(build.id, { onlyIfIdleSince: since })).resolves.toBeUndefined(); + + expect(prisma.build.delete).toHaveBeenCalledTimes(1); + expect(events.buildDeleted).not.toHaveBeenCalled(); }); - expect(buildUpdateMock).toHaveBeenCalledWith({ where: { id: build.id }, data: { number: 43 } }); - expect(result).toEqual({ ...build, number: 43 }); }); }); - describe('findOrCreate (characterization)', () => { + describe('findOrCreate', () => { + const params = { projectId: 'projectId', branchName: 'develop', ciBuildId: 'ci' }; + it('numbers a new build and announces it', async () => { + const { service, prisma, events } = await setup(); const numbered = { ...build, number: 3 }; - const buildUpsertMock = jest.fn().mockResolvedValueOnce(build); - const eventsBuildCreatedMock = jest.fn(); - service = await initService({ buildUpsertMock, eventsBuildCreatedMock }); - service.incrementBuildNumber = jest.fn().mockResolvedValueOnce(numbered); + prisma.build.upsert.mockResolvedValueOnce(build); + prisma.project.update.mockResolvedValueOnce({ buildsCounter: 3 }); + prisma.build.updateManyAndReturn.mockResolvedValueOnce([numbered]); - const result = await service.findOrCreate({ projectId: 'projectId', branchName: 'develop', ciBuildId: 'ci' }); + const result = await service.findOrCreate(params); - expect(service.incrementBuildNumber).toHaveBeenCalledWith(build.id, 'projectId'); - expect(eventsBuildCreatedMock).toHaveBeenCalledWith(new BuildDto(numbered)); + expect(prisma.project.update).toHaveBeenCalledWith({ + where: { id: 'projectId' }, + data: { buildsCounter: { increment: 1 } }, + select: { buildsCounter: true }, + }); + expect(prisma.build.updateManyAndReturn).toHaveBeenCalledWith({ + where: { id: build.id, number: null }, + data: { number: 3 }, + }); + expect(mocked(BuildDto).mock.calls.at(-1)).toEqual([numbered]); + expect(events.buildCreated).toHaveBeenCalledTimes(1); expect(result).toBe(numbered); }); + it('numbers a new build once when parallel callers join it', async () => { + const { service, prisma, events } = await setup(); + prisma.build.upsert.mockResolvedValue({ ...build, number: null }); + prisma.project.update.mockResolvedValueOnce({ buildsCounter: 42 }).mockResolvedValueOnce({ buildsCounter: 43 }); + prisma.build.updateManyAndReturn.mockResolvedValueOnce([{ ...build, number: 42 }]).mockResolvedValueOnce([]); + prisma.build.findUniqueOrThrow.mockResolvedValue({ ...build, number: 42 }); + + const results = await Promise.all([service.findOrCreate(params), service.findOrCreate(params)]); + + expect(events.buildCreated).toHaveBeenCalledTimes(1); + expect(results.map((result) => result.number)).toEqual([42, 42]); + }); + + it('returns the build another caller numbered, without announcing it', async () => { + const { service, prisma, events } = await setup(); + prisma.build.upsert.mockResolvedValueOnce(build); + prisma.project.update.mockResolvedValueOnce({ buildsCounter: 8 }); + prisma.build.updateManyAndReturn.mockResolvedValueOnce([]); + prisma.build.findUniqueOrThrow.mockResolvedValueOnce({ ...build, number: 7 }); + + const result = await service.findOrCreate(params); + + expect(prisma.build.findUniqueOrThrow).toHaveBeenCalledWith({ where: { id: build.id } }); + expect(events.buildCreated).not.toHaveBeenCalled(); + expect(result).toEqual({ ...build, number: 7 }); + }); + + it('rethrows a failure of the numbering transaction', async () => { + const { service, prisma, events } = await setup(); + const error = new Error('connection lost'); + prisma.build.upsert.mockResolvedValueOnce(build); + prisma.project.update.mockRejectedValueOnce(error); + + await expect(service.findOrCreate(params)).rejects.toBe(error); + expect(events.buildCreated).not.toHaveBeenCalled(); + }); + it('retries a concurrent upsert (P2002) as an update with the same unique key', async () => { - const buildUpsertMock = jest - .fn() - .mockRejectedValueOnce(new PrismaClientKnownRequestError('dup', { code: 'P2002', clientVersion: '5' })); - const buildUpdateMock = jest.fn().mockResolvedValueOnce({ ...build, number: 9 }); - service = await initService({ buildUpsertMock, buildUpdateMock }); + const { service, prisma } = await setup(); + prisma.build.upsert.mockRejectedValueOnce(prismaError('P2002')); + prisma.build.update.mockResolvedValueOnce({ ...build, number: 9 }); - const result = await service.findOrCreate({ projectId: 'projectId', branchName: 'develop', ciBuildId: 'ci' }); + const result = await service.findOrCreate(params); - expect(buildUpdateMock).toHaveBeenCalledWith({ + expect(prisma.build.update).toHaveBeenCalledWith({ where: { projectId_ciBuildId: { projectId: 'projectId', ciBuildId: 'ci' } }, data: { isRunning: true }, }); @@ -457,254 +551,283 @@ describe('BuildsService', () => { }); it.each([ - ['another Prisma error', new PrismaClientKnownRequestError('x', { code: 'P2003', clientVersion: '5' })], + ['another Prisma error', prismaError('P2003')], ['any other error', new Error('connection lost')], - ])('turns %s on upsert into a TypeError', async (_, error) => { - const buildUpsertMock = jest.fn().mockRejectedValueOnce(error); - const buildUpdateMock = jest.fn(); - service = await initService({ buildUpsertMock, buildUpdateMock }); - - // NOTE: current behavior — the original error is swallowed and the - // undefined build is dereferenced afterwards - await expect( - service.findOrCreate({ projectId: 'projectId', branchName: 'develop', ciBuildId: 'ci' }) - ).rejects.toThrow(TypeError); - expect(buildUpdateMock).not.toHaveBeenCalled(); + ])('rethrows %s from upsert unchanged', async (_, error) => { + const { service, prisma } = await setup(); + prisma.build.upsert.mockRejectedValueOnce(error); + + await expect(service.findOrCreate(params)).rejects.toBe(error); + expect(prisma.build.update).not.toHaveBeenCalled(); + }); + + it.each([ + ['without ciBuildId', undefined, { id: '111' }], + ['with ciBuildId', '222', { projectId_ciBuildId: { projectId: '111', ciBuildId: '222' } }], + ])('upserts %s', async (_, ciBuildId, where) => { + const { service, prisma } = await setup(); + prisma.build.upsert.mockResolvedValueOnce({ ...build, number: 1 }); + + await service.findOrCreate({ projectId: '111', branchName: 'develop', ciBuildId }); + + expect(prisma.build.upsert).toHaveBeenCalledWith({ + where, + create: { + branchName: 'develop', + ...(ciBuildId && { ciBuildId }), + isRunning: true, + project: { connect: { id: '111' } }, + }, + update: { isRunning: true }, + }); + }); + + it('leaves an already numbered build alone', async () => { + const { service, prisma } = await setup(); + prisma.build.upsert.mockResolvedValueOnce({ ...build, number: 100 }); + + const result = await service.findOrCreate({ projectId: '111', branchName: 'develop' }); + + expect(prisma.project.update).not.toHaveBeenCalled(); + expect(result).toEqual({ ...build, number: 100 }); }); }); describe('deleteOldBuilds', () => { const projectId = 'someProjectId'; - const orderBy = [{ createdAt: 'desc' }, { id: 'desc' }]; + const ids = (...names: string[]) => names.map((id) => ({ id })); const init = async (oldBuilds: { id: string }[]) => { - const buildFindManyMock = jest.fn().mockResolvedValueOnce(oldBuilds); - service = await initService({ buildFindManyMock }); - service.remove = jest.fn().mockResolvedValue(undefined); - return buildFindManyMock; + const ctx = await setup(); + ctx.prisma.build.findMany.mockResolvedValueOnce(oldBuilds); + ctx.service.remove = jest.fn().mockResolvedValue(undefined); + return ctx; }; + afterEach(() => { + delete process.env.BUILD_RETENTION_GRACE_MINUTES; + jest.useRealTimers(); + }); + it('removes every build past the newest maxBuildAllowed - 1, making room for the new one', async () => { - const buildFindManyMock = await init([{ id: 'build1' }, { id: 'build2' }]); + const { service, prisma } = await init(ids('build1', 'build2')); - await service.deleteOldBuilds(projectId, 3); + await service.deleteOldBuilds(projectId, 3, 'b-new'); - expect(buildFindManyMock).toHaveBeenCalledWith({ where: { projectId }, orderBy, skip: 2, select: { id: true } }); + expect(prisma.build.findMany).toHaveBeenCalledWith({ + where: { projectId, id: { not: 'b-new' } }, + orderBy: buildOrder, + skip: 2, + select: { id: true }, + }); expect(service.remove).toHaveBeenCalledTimes(2); - expect(service.remove).toHaveBeenCalledWith('build1'); - expect(service.remove).toHaveBeenCalledWith('build2'); + expect(service.remove).toHaveBeenCalledWith('build1', expect.anything()); + expect(service.remove).toHaveBeenCalledWith('build2', expect.anything()); + }); + + it.each([[1], [0]])('with maxBuildAllowed=%p keeps no old build at all', async (max) => { + const { service, prisma } = await init([]); + + await service.deleteOldBuilds(projectId, max, 'b-new'); + + expect(prisma.build.findMany).toHaveBeenCalledWith(expect.objectContaining({ skip: 0 })); }); it.each([ - [1, 0], - [0, 0], - ])('with maxBuildAllowed=%p keeps no old build at all', async (max, skip) => { - const buildFindManyMock = await init([]); + [undefined, 60], + ['', 60], + ['15', 15], + ['0', 0], + ['-5', 60], + ['abc', 60], + ])('retentionGraceMinutes with %p is %p', (raw, expected) => { + if (raw === undefined) { + delete process.env.BUILD_RETENTION_GRACE_MINUTES; + } else { + process.env.BUILD_RETENTION_GRACE_MINUTES = raw; + } + + expect(retentionGraceMinutes()).toBe(expected); + }); + + it('passes the grace cutoff to remove', async () => { + jest.useFakeTimers({ now: new Date('2026-05-01T12:00:00Z'), doNotFake: ['setImmediate', 'nextTick'] }); + const { service } = await init(ids('build1')); + + await service.deleteOldBuilds(projectId, 1, 'b-new'); + + expect(service.remove).toHaveBeenCalledWith('build1', { onlyIfIdleSince: new Date('2026-05-01T11:00:00Z') }); + }); + + it('with grace 0 removes without the idle check', async () => { + process.env.BUILD_RETENTION_GRACE_MINUTES = '0'; + const { service } = await init(ids('build1')); - await service.deleteOldBuilds(projectId, max); + await service.deleteOldBuilds(projectId, 1, 'b-new'); - expect(buildFindManyMock).toHaveBeenCalledWith(expect.objectContaining({ skip })); + expect(service.remove).toHaveBeenCalledWith('build1', { onlyIfIdleSince: undefined }); }); - it('never prunes the build a CI build id is about to reuse', async () => { - const buildFindManyMock = await init([]); + it('runs a separate pruning for another kept build of the same project', async () => { + const { service, prisma } = await setup(); + prisma.build.findMany.mockResolvedValue([]); - await service.deleteOldBuilds(projectId, 1, 'ci-1'); + await Promise.all([service.deleteOldBuilds(projectId, 2, 'b1'), service.deleteOldBuilds(projectId, 2, 'b2')]); - expect(buildFindManyMock).toHaveBeenCalledWith( - expect.objectContaining({ - where: { projectId, OR: [{ ciBuildId: null }, { ciBuildId: { not: 'ci-1' } }] }, - }) + expect(prisma.build.findMany).toHaveBeenCalledTimes(2); + expect(prisma.build.findMany).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ where: { projectId, id: { not: 'b1' } } }) + ); + expect(prisma.build.findMany).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ where: { projectId, id: { not: 'b2' } } }) ); }); - it('should handle concurrent calls for the same project by returning the existing promise', async () => { - const buildFindManyMock = await init([{ id: 'build1' }]); + it('should handle concurrent calls for the same kept build by returning the existing promise', async () => { + const { service, prisma } = await init(ids('build1')); - const promise1 = service.deleteOldBuilds(projectId, 2); - const promise2 = service.deleteOldBuilds(projectId, 2); + const promise1 = service.deleteOldBuilds(projectId, 2, 'b-new'); + const promise2 = service.deleteOldBuilds(projectId, 2, 'b-new'); expect(promise1).toStrictEqual(promise2); await promise1; - expect(buildFindManyMock).toHaveBeenCalledTimes(1); + expect(prisma.build.findMany).toHaveBeenCalledTimes(1); expect(service.remove).toHaveBeenCalledTimes(1); }); it('should remove the promise from the map after completion (success)', async () => { - await init([{ id: 'build1' }]); + const { service } = await init(ids('build1')); + + await service.deleteOldBuilds(projectId, 1, 'b-new'); + + expect(service['ongoingDeletions'].has(`${projectId}|b-new`)).toBe(false); + }); + + it('one failing old build does not stop pruning the others', async () => { + const { service } = await init(ids('old1', 'old2')); + jest.mocked(service.remove).mockRejectedValueOnce(new Error('boom')).mockResolvedValueOnce(undefined); - await service.deleteOldBuilds(projectId, 1); + await expect(service.deleteOldBuilds(projectId, 1, 'b-new')).resolves.toBeUndefined(); - expect(service['ongoingDeletions'].has(projectId)).toBe(false); + expect(service.remove).toHaveBeenCalledWith('old2', expect.anything()); }); it('hands a failure to the caller and leaves no unhandled rejection behind', async () => { - const unhandled = jest.fn(); - process.on('unhandledRejection', unhandled); + const { unhandled, stop } = unhandledRejections(); try { - const buildFindManyMock = jest.fn().mockRejectedValueOnce(new Error('db down')); - service = await initService({ buildFindManyMock }); + const { service, prisma } = await setup(); + prisma.build.findMany.mockRejectedValueOnce(new Error('db down')); - await expect(service.deleteOldBuilds(projectId, 1)).rejects.toThrow('db down'); - await new Promise((resolve) => setImmediate(resolve)); + await expect(service.deleteOldBuilds(projectId, 1, 'b-new')).rejects.toThrow('db down'); + await tick(); expect(unhandled).not.toHaveBeenCalled(); - expect(service['ongoingDeletions'].has(projectId)).toBe(false); + expect(service['ongoingDeletions'].has(`${projectId}|b-new`)).toBe(false); } finally { - process.off('unhandledRejection', unhandled); + stop(); } }); }); - it('should stop', async () => { - const id = 'some id'; - const buildUpdateMock = jest.fn(); - const eventsBuildUpdatedMock = jest.fn(); - mocked(BuildDto).mockReturnValueOnce(buildDto as MockedObject); - service = await initService({ buildUpdateMock, eventsBuildUpdatedMock }); + describe('background retention', () => { + const projectId = 'someProjectId'; - const result = await service.update(id, { isRunning: false }); + it('scheduleRetention logs a failure instead of leaving it unhandled', async () => { + const { unhandled, stop } = unhandledRejections(); + try { + const { service, prisma } = await setup(); + prisma.build.findMany.mockRejectedValueOnce(new Error('db down')); + const logger = silentLogger(); + (service as any).logger = logger; - expect(buildUpdateMock).toHaveBeenCalledWith({ - where: { id }, - data: { isRunning: false }, - }); - expect(eventsBuildUpdatedMock).toHaveBeenCalledWith(id); - expect(result).toBe(buildDto); - }); + service.scheduleRetention(projectId, 1, 'b-new'); + await service.whenRetentionIdle(); + await tick(); - it('approve', async () => { - const buildFindUniqueMock = jest.fn().mockResolvedValueOnce(build); - const testRunApproveMock = jest.fn().mockResolvedValueOnce({ - ...build.testRuns[0], - status: TestStatus.approved, + expect(unhandled).not.toHaveBeenCalled(); + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('db down')); + } finally { + stop(); + } }); - service = await initService({ buildFindUniqueMock, testRunApproveMock }); - service.findOne = jest.fn(); - await service.approve('someId', true, 'user id'); + it('whenRetentionIdle resolves when nothing runs', async () => { + const { service } = await setup(); - expect(buildFindUniqueMock).toHaveBeenCalledWith({ - where: { id: 'someId' }, - include: { - testRuns: { - where: { - status: { - in: [TestStatus.new, TestStatus.unresolved], - }, - }, - }, - }, + await expect(service.whenRetentionIdle()).resolves.toBeUndefined(); }); - expect(testRunApproveMock).toHaveBeenCalledWith(build.testRuns[0].id, true, false, 'user id'); - }); - it('approve throws NotFoundException for an unknown build', async () => { - const buildFindUniqueMock = jest.fn().mockResolvedValueOnce(null); - service = await initService({ buildFindUniqueMock }); + it('whenRetentionIdle waits for a running pruning, even a failing one', async () => { + const { service, prisma } = await setup(); + let fail: (e: Error) => void = () => undefined; + prisma.build.findMany.mockReturnValueOnce(new Promise((_, reject) => (fail = reject))); + (service as any).logger = silentLogger(); + service.scheduleRetention(projectId, 1, 'b-new'); + let idle = false; + const waiting = service.whenRetentionIdle().then(() => (idle = true)); - await expect(service.approve('missing', false)).rejects.toThrow(new NotFoundException('Build not found: missing')); - }); - - describe('findOsCreate', () => { - it('create without ciBuildId', async () => { - const buildUpsertMock = jest.fn().mockResolvedValueOnce(build); - - service = await initService({ buildUpsertMock }); - service.incrementBuildNumber = jest.fn().mockResolvedValueOnce(build); + await tick(); + expect(idle).toBe(false); + fail(new Error('boom')); + await waiting; - await service.findOrCreate({ - projectId: '111', - branchName: 'develop', - }); + expect(idle).toBe(true); + }); - expect(buildUpsertMock).toHaveBeenCalledWith({ - where: { - id: '111', - }, - create: { - branchName: 'develop', - isRunning: true, - project: { - connect: { - id: '111', - }, - }, - }, - update: { - isRunning: true, - }, - }); + it('beforeApplicationShutdown stops between builds and waits', async () => { + const { service, prisma } = await setup(); + prisma.build.findMany.mockResolvedValueOnce([{ id: 'o1' }, { id: 'o2' }]); + let finishFirst: () => void = () => undefined; + service.remove = jest + .fn() + .mockImplementationOnce(() => new Promise((resolve) => (finishFirst = () => resolve(undefined)))) + .mockResolvedValue(undefined); + service.scheduleRetention(projectId, 1, 'b-new'); + await tick(); + expect(service.remove).toHaveBeenCalledWith('o1', expect.anything()); + + let stopped = false; + const shutdown = service.beforeApplicationShutdown().then(() => (stopped = true)); + await tick(); + expect(stopped).toBe(false); + finishFirst(); + await shutdown; + + expect(stopped).toBe(true); + expect(service.remove).toHaveBeenCalledTimes(1); + expect(service.remove).not.toHaveBeenCalledWith('o2', expect.anything()); }); + }); - it('create with ciBuildId', async () => { - const buildUpsertMock = jest.fn().mockResolvedValueOnce(build); - service = await initService({ buildUpsertMock }); - service.incrementBuildNumber = jest.fn().mockResolvedValueOnce(build); + describe('approve', () => { + it('approves the new and unresolved runs of the build', async () => { + const { service, prisma, testRuns } = await setup(); + prisma.build.findUnique.mockResolvedValueOnce(build); + testRuns.approve.mockResolvedValueOnce({ ...run, status: TestStatus.approved }); - await service.findOrCreate({ - projectId: '111', - branchName: 'develop', - ciBuildId: '222', - }); + await service.approve('someId', true, 'user id'); - expect(buildUpsertMock).toHaveBeenCalledWith({ - where: { - projectId_ciBuildId: { - projectId: '111', - ciBuildId: '222', - }, - }, - create: { - branchName: 'develop', - ciBuildId: '222', - isRunning: true, - project: { - connect: { - id: '111', - }, + expect(prisma.build.findUnique).toHaveBeenCalledWith({ + where: { id: 'someId' }, + include: { + testRuns: { + where: { status: { in: [TestStatus.new, TestStatus.unresolved] } }, + select: { id: true }, }, }, - update: { - isRunning: true, - }, - }); - }); - - it('create with retry', async () => { - const buildUpsertMock = jest - .fn() - .mockRejectedValueOnce(new PrismaClientKnownRequestError('mock error', { code: 'P2002', clientVersion: '5' })); - const buildUpdateMock = jest.fn().mockResolvedValueOnce(build); - service = await initService({ buildUpsertMock, buildUpdateMock }); - service.incrementBuildNumber = jest.fn().mockResolvedValueOnce(build); - - const result = await service.findOrCreate({ - projectId: '111', - branchName: 'develop', }); - - expect(result).toEqual(build); + expect(testRuns.approve).toHaveBeenCalledWith(run.id, true, false, 'user id'); }); - it('update already created', async () => { - const buildUpsertMock = jest.fn().mockResolvedValueOnce({ - ...build, - number: 100, - }); - service = await initService({ buildUpsertMock }); - service.incrementBuildNumber = jest.fn(); - - const result = await service.findOrCreate({ - projectId: '111', - branchName: 'develop', - }); + it('throws NotFoundException for an unknown build', async () => { + const { service, prisma } = await setup(); + prisma.build.findUnique.mockResolvedValueOnce(null); - expect(service.incrementBuildNumber).toHaveBeenCalledTimes(0); - expect(result).toEqual({ - ...build, - number: 100, - }); + await expect(service.approve('missing', false)).rejects.toThrow( + new NotFoundException('Build not found: missing') + ); }); }); }); diff --git a/src/builds/builds.service.ts b/src/builds/builds.service.ts index b8273e70..ec51995a 100644 --- a/src/builds/builds.service.ts +++ b/src/builds/builds.service.ts @@ -1,30 +1,50 @@ -import { forwardRef, Inject, Injectable, Logger, NotFoundException } from '@nestjs/common'; +import { + BeforeApplicationShutdown, + ConflictException, + forwardRef, + Inject, + Injectable, + Logger, + NotFoundException, +} from '@nestjs/common'; import { PrismaService } from '../prisma/prisma.service'; -import { Build, Prisma, TestStatus } from '@prisma/client'; +import { isPrismaError } from '../prisma/prisma-errors'; +import { Build, Prisma, TestRun, TestStatus } from '@prisma/client'; import { TestRunsService } from '../test-runs/test-runs.service'; import { EventsGateway } from '../shared/events/events.gateway'; import { BuildDto } from './dto/build.dto'; import { getBuildsStats } from './build-stats'; +import { testRunImageNames } from '../utils'; +import { nonNegativeNumber } from '../utils/env'; import { PaginatedBuildDto } from './dto/build-paginated.dto'; import { ModifyBuildDto } from './dto/build-modify.dto'; import type { WrapperType } from '../shared/wrapper-type'; +import { StaticService } from '../static/static.service'; -// Newest first, with the id to break a tie: without a total order two builds -// made in the same instant have no fixed place, so an offset page could show -// one of them twice or not at all, and the page a build sits on would be a -// matter of chance. +const TEST_RUN_CHUNK = 500; +const MAX_REMOVE_ATTEMPTS = 3; const BUILD_ORDER: Prisma.BuildOrderByWithRelationInput[] = [{ createdAt: 'desc' }, { id: 'desc' }]; +const DEFAULT_RETENTION_GRACE_MINUTES = 60; + +export function retentionGraceMinutes(): number { + return nonNegativeNumber(process.env.BUILD_RETENTION_GRACE_MINUTES, DEFAULT_RETENTION_GRACE_MINUTES); +} + +class BuildAlreadyNumbered extends Error {} + @Injectable() -export class BuildsService { +export class BuildsService implements BeforeApplicationShutdown { private readonly logger: Logger = new Logger(BuildsService.name); private readonly ongoingDeletions = new Map>(); + private shuttingDown = false; constructor( private prismaService: PrismaService, private eventsGateway: EventsGateway, @Inject(forwardRef(() => TestRunsService)) - private testRunsService: WrapperType + private testRunsService: WrapperType, + private staticService: StaticService ) {} async findOne(id: string): Promise { @@ -42,8 +62,7 @@ export class BuildsService { async findMany( projectId: string, - // undefined reads every build, which is how deleteOldBuilds finds the ones past the limit - take: number | undefined, + take: number, skip: number, ciBuildId?: string, aroundBuildId?: string @@ -55,10 +74,7 @@ export class BuildsService { }), }; - // A link carries a build, not a page, so the page has to be worked out from - // the build before anything is read. - // the controller always sends take with aroundBuildId; deleteOldBuilds sends neither - const pageSkip = aroundBuildId ? ((await this.skipToPageWith(where, take!, aroundBuildId)) ?? skip) : skip; + const pageSkip = aroundBuildId ? ((await this.skipToPageWith(where, take, aroundBuildId)) ?? skip) : skip; const [total, buildList] = await Promise.all([ this.prismaService.build.count({ where }), @@ -70,21 +86,18 @@ export class BuildsService { }), ]); + const stats = await getBuildsStats( + this.prismaService, + buildList.map((build) => build.id) + ); return { - data: await Promise.all(buildList.map((build) => this.findOne(build.id))), + data: buildList.map((build) => new BuildDto(build, stats.get(build.id))), total, - // only deleteOldBuilds leaves take unset, and it reads just the data - take: take!, + take, skip: pageSkip, }; } - /** - * Where the page holding this build starts, or null when there is no such - * page: the build has been deleted, belongs elsewhere, or the search in force - * leaves it out. A stale link should then open the list it asked for rather - * than fail. - */ private async skipToPageWith(where: Prisma.BuildWhereInput, take: number, buildId: string): Promise { if (take < 1) { return null; @@ -98,9 +111,6 @@ export class BuildsService { return null; } - // BUILD_ORDER puts a build ahead of this one when it is newer, or was made - // in the same instant and carries the larger id. Counting those gives the - // build's place in the list, and the page follows from the page size. const ahead = await this.prismaService.build.count({ where: { ...where, @@ -114,97 +124,150 @@ export class BuildsService { async update(id: string, modifyBuildDto: ModifyBuildDto): Promise { await this.prismaService.build.update({ where: { id }, - // named one by one so nothing else a caller passes can reach the row data: { ciBuildId: modifyBuildDto.ciBuildId, isRunning: modifyBuildDto.isRunning }, }); this.eventsGateway.buildUpdated(id); return this.findOne(id); } - async remove(id: string): Promise { + async remove( + id: string, + { onlyIfIdleSince }: { onlyIfIdleSince?: Date } = {} + ): Promise<(Build & { testRuns: TestRun[] }) | undefined> { if (!id) { this.logger.warn(`Attempted to remove build with undefined ID.`); return; } this.logger.debug(`Going to remove Build ${id}`); - const build = await this.prismaService.build.findUnique({ - where: { id }, - include: { - testRuns: true, - }, - }); - + const build = await this.prismaService.build.findUnique({ where: { id } }); if (!build) { this.logger.warn(`Build not found ${id}`); return; } - await Promise.all(build.testRuns.map((testRun) => this.testRunsService.delete(testRun.id))); - - try { - await this.prismaService.build.delete({ where: { id } }); - } catch (e) { - if (e instanceof Prisma.PrismaClientKnownRequestError) { + const testRuns: TestRun[] = []; + for (let attempt = 1; ; attempt++) { + const removedRuns = await this.removeTestRuns(id, onlyIfIdleSince); + if (!removedRuns || (onlyIfIdleSince && !(await this.isIdle(id, onlyIfIdleSince)))) { + this.logger.log(`Keeping build ${id}: it is in use`); + return; + } + testRuns.push(...removedRuns); + try { + await this.prismaService.build.delete({ where: { id } }); + break; + } catch (e) { // workaround https://github.com/Visual-Regression-Tracker/Visual-Regression-Tracker/issues/435 - if (e.code === 'P2025') { + if (isPrismaError(e, 'P2025')) { this.logger.warn(`Build already deleted ${id}`); return; } + if (!isPrismaError(e, 'P2003')) { + throw e; + } + if (onlyIfIdleSince) { + this.logger.log(`Keeping build ${id}: it received new test runs`); + return; + } + if (attempt === MAX_REMOVE_ATTEMPTS) { + throw new ConflictException(`Build ${id} keeps receiving test runs, try again later`); + } } } this.logger.log(`Build deleted ${id}`); - this.eventsGateway.buildDeleted(new BuildDto({ ...build })); - return build; + const removed = { ...build, testRuns }; + this.eventsGateway.buildDeleted(new BuildDto(removed)); + return removed; + } + + private async isIdle(buildId: string, since: Date): Promise { + const idle = await this.prismaService.build.findFirst({ + where: { id: buildId, updatedAt: { lt: since }, testRuns: { none: { createdAt: { gte: since } } } }, + select: { id: true }, + }); + return idle !== null; } - /** - * Makes room for the build about to be started: keeps the newest - * `keepBuilds - 1` builds, so that with it the project holds `keepBuilds`. - * The build that a POST with `ciBuildId` is about to reuse (parallel CI jobs - * share one) is never pruned, and does not count among the ones kept. - */ - async deleteOldBuilds(projectId: string, keepBuilds: number, ciBuildId?: string): Promise { - if (this.ongoingDeletions.has(projectId)) { - this.logger.debug(`Deletion for project ${projectId} is already in progress. Returning existing promise.`); - return this.ongoingDeletions.get(projectId); + // rows before files: a failure leaks bytes instead of leaving runs that point at deleted images + private async removeTestRuns(buildId: string, onlyIfIdleSince?: Date): Promise { + const removed: TestRun[] = []; + for (;;) { + if (onlyIfIdleSince && !(await this.isIdle(buildId, onlyIfIdleSince))) { + return null; + } + const chunk = await this.prismaService.testRun.findMany({ where: { buildId }, take: TEST_RUN_CHUNK }); + if (chunk.length === 0) { + return removed; + } + await this.prismaService.testRun.deleteMany({ where: { id: { in: chunk.map((testRun) => testRun.id) } } }); + await this.staticService.deleteImages(chunk.flatMap(testRunImageNames)); + chunk.forEach((testRun) => this.eventsGateway.testRunDeleted(testRun)); + removed.push(...chunk); + } + } + + async deleteOldBuilds(projectId: string, keepBuilds: number, keepBuildId: string): Promise { + const key = `${projectId}|${keepBuildId}`; + const ongoing = this.ongoingDeletions.get(key); + if (ongoing) { + this.logger.debug(`Deletion for ${key} is already in progress. Returning existing promise.`); + return ongoing; } const deletionPromise = (async () => { this.logger.debug(`Starting to delete old builds for project ${projectId}, keeping ${keepBuilds} builds.`); + const graceMinutes = retentionGraceMinutes(); + const onlyIfIdleSince = graceMinutes > 0 ? new Date(Date.now() - graceMinutes * 60_000) : undefined; const oldBuilds = await this.prismaService.build.findMany({ - where: { - projectId, - // spelled out with the null case: NOT { ciBuildId } alone would also - // leave out every build without a CI build id - ...(ciBuildId && { OR: [{ ciBuildId: null }, { ciBuildId: { not: ciBuildId } }] }), - }, + where: { projectId, id: { not: keepBuildId } }, orderBy: BUILD_ORDER, skip: Math.max(keepBuilds - 1, 0), select: { id: true }, }); - for (const eachBuild of oldBuilds) { - await this.remove(eachBuild.id); + for (const { id } of oldBuilds) { + if (this.shuttingDown) { + break; + } + try { + await this.remove(id, { onlyIfIdleSince }); + } catch (e) { + this.logger.warn(`Cannot remove old build ${id}: ${e}`); + } } this.logger.debug(`Finished deleting old builds for project ${projectId}.`); })(); - this.ongoingDeletions.set(projectId, deletionPromise); + this.ongoingDeletions.set(key, deletionPromise); deletionPromise .finally(() => { - this.ongoingDeletions.delete(projectId); - this.logger.debug(`Deletion promise for project ${projectId} resolved/rejected and removed from map.`); + this.ongoingDeletions.delete(key); + this.logger.debug(`Deletion promise for ${key} resolved/rejected and removed from map.`); }) - // finally() hands on the rejection in a promise nobody awaits: unhandled, - // it would take the whole process down. The caller gets the error from - // deletionPromise itself. + // finally() would pass the rejection to a promise nobody awaits and crash the process .catch((): void => undefined); return deletionPromise; } + scheduleRetention(projectId: string, keepBuilds: number, keepBuildId: string): void { + this.deleteOldBuilds(projectId, keepBuilds, keepBuildId).catch((e) => + this.logger.warn(`Cannot remove old builds of project ${projectId}: ${e}`) + ); + } + + async whenRetentionIdle(): Promise { + await Promise.allSettled([...this.ongoingDeletions.values()]); + } + + // Runs before Prisma disconnects in onApplicationShutdown + async beforeApplicationShutdown(): Promise { + this.shuttingDown = true; + await this.whenRetentionIdle(); + } + async approve(id: string, merge: boolean, userId?: string): Promise { const build = await this.prismaService.build.findUnique({ where: { id }, @@ -215,6 +278,7 @@ export class BuildsService { in: [TestStatus.new, TestStatus.unresolved], }, }, + select: { id: true }, }, }, }); @@ -224,7 +288,6 @@ export class BuildsService { } for (const testRun of build.testRuns) { - // recorded on each new baseline, as POST /test-runs/approve does await this.testRunsService.approve(testRun.id, merge, false, userId); } } @@ -247,8 +310,7 @@ export class BuildsService { } : { id: projectId }; - // left unset when the upsert fails with anything but P2002, and then reading it throws - let build!: Build; + let build: Build; try { build = await this.prismaService.build.upsert({ where, @@ -267,42 +329,52 @@ export class BuildsService { }, }); } catch (e) { - if (e instanceof Prisma.PrismaClientKnownRequestError) { - if (e.code === 'P2002') { - // cuncurent upsert workaround https://www.prisma.io/docs/reference/api-reference/prisma-client-reference#unique-key-constraint-errors-on-upserts - build = await this.prismaService.build.update({ - where, - data: { - isRunning: true, - }, - }); - } + if (!isPrismaError(e, 'P2002')) { + throw e; } + // concurrent upsert workaround https://www.prisma.io/docs/reference/api-reference/prisma-client-reference#unique-key-constraint-errors-on-upserts + build = await this.prismaService.build.update({ + where, + data: { + isRunning: true, + }, + }); } - // assigne build number - if (!build.number) { - build = await this.incrementBuildNumber(build.id, projectId); - this.eventsGateway.buildCreated(new BuildDto(build)); + if (build.number === null) { + const numbered = await this.assignNumber(build.id, projectId); + if (numbered) { + this.eventsGateway.buildCreated(new BuildDto(numbered)); + return numbered; + } + return this.prismaService.build.findUniqueOrThrow({ where: { id: build.id } }); } return build; } - async incrementBuildNumber(buildId: string, projectId: string): Promise { - const project = await this.prismaService.project.update({ - where: { - id: projectId, - }, - data: { - buildsCounter: { - increment: 1, - }, - }, - }); - return this.prismaService.build.update({ - where: { id: buildId }, - data: { number: project.buildsCounter }, - }); + private async assignNumber(buildId: string, projectId: string): Promise { + try { + return await this.prismaService.$transaction(async (tx) => { + const { buildsCounter } = await tx.project.update({ + where: { id: projectId }, + data: { buildsCounter: { increment: 1 } }, + select: { buildsCounter: true }, + }); + const [numbered] = await tx.build.updateManyAndReturn({ + where: { id: buildId, number: null }, + data: { number: buildsCounter }, + }); + if (!numbered) { + throw new BuildAlreadyNumbered(); + } + return numbered; + }); + } catch (e) { + if (e instanceof BuildAlreadyNumbered) { + return null; + } + throw e; + } } } diff --git a/src/builds/dto/build-modify.dto.ts b/src/builds/dto/build-modify.dto.ts index 2777dd07..d84fc70e 100644 --- a/src/builds/dto/build-modify.dto.ts +++ b/src/builds/dto/build-modify.dto.ts @@ -1,9 +1,8 @@ import { ApiPropertyOptional } from '@nestjs/swagger'; import { IsBoolean, IsNotEmpty, IsOptional, IsString } from 'class-validator'; +import { ValidationOptions } from '../../shared/validation'; -// The only build columns a client may change: the SDKs stop a build (an empty -// body, see BuildsController.update) and a CI may attach its own build id. -// Everything else — branch, number, status — belongs to the server. +@ValidationOptions({ forbidNonWhitelisted: true }) export class ModifyBuildDto { @ApiPropertyOptional() @IsOptional() diff --git a/src/builds/dto/build.dto.spec.ts b/src/builds/dto/build.dto.spec.ts index 8ba5ead4..23c420c4 100644 --- a/src/builds/dto/build.dto.spec.ts +++ b/src/builds/dto/build.dto.spec.ts @@ -2,220 +2,84 @@ import { Build, TestRun, TestStatus } from '@prisma/client'; import { generateTestRun, TEST_BUILD } from '../../_data_'; import { BuildDto } from './build.dto'; +const makeBuild = (testRuns?: TestRun[]): Build & { testRuns?: TestRun[] } => ({ + id: 'a9385fc1-884d-4f9f-915e-40da0e7773d5', + ciBuildId: 'ciBuildId', + number: null, + branchName: 'develop', + status: null, + projectId: 'e0a37894-6f29-478d-b13e-6182fecc715e', + updatedAt: new Date(), + createdAt: new Date(), + userId: null, + isRunning: true, + testRuns, +}); + describe('BuildDto', () => { it.each([ + ['new', 'undefined testRuns', undefined, { passedCount: 0, unresolvedCount: 0, failedCount: 0, merge: false }], + ['new', 'empty testRuns', [], { passedCount: 0, unresolvedCount: 0, failedCount: 0, merge: false }], [ - 'undefined testRuns', - { - id: 'a9385fc1-884d-4f9f-915e-40da0e7773d5', - ciBuildId: 'ciBuildId', - number: null, - branchName: 'develop', - status: null, - projectId: 'e0a37894-6f29-478d-b13e-6182fecc715e', - updatedAt: new Date(), - createdAt: new Date(), - userId: null, - isRunning: true, - testRuns: undefined, - merge: false, - }, - ], - [ - 'empty testRuns', - { - id: 'a9385fc1-884d-4f9f-915e-40da0e7773d5', - ciBuildId: 'ciBuildId', - number: null, - branchName: 'develop', - status: null, - projectId: 'e0a37894-6f29-478d-b13e-6182fecc715e', - updatedAt: new Date(), - createdAt: new Date(), - userId: null, - isRunning: true, - testRuns: [], - merge: false, - }, - ], - ])('new with %s', (_, build) => { - const buildDto: BuildDto = { - id: build.id, - ciBuildId: build.ciBuildId, - number: build.number, - branchName: build.branchName, - status: 'new', - projectId: build.projectId, - updatedAt: build.updatedAt, - createdAt: build.createdAt, - userId: null, - passedCount: 0, - unresolvedCount: 0, - failedCount: 0, - isRunning: true, - merge: false, - }; - - const result = new BuildDto(build); - - expect(result).toEqual(buildDto); - }); - - it('passed', () => { - const build: Build & { - testRuns: TestRun[]; - } = { - id: 'a9385fc1-884d-4f9f-915e-40da0e7773d5', - ciBuildId: 'ciBuildId', - number: null, - branchName: 'develop', - status: null, - projectId: 'e0a37894-6f29-478d-b13e-6182fecc715e', - updatedAt: new Date(), - createdAt: new Date(), - userId: null, - isRunning: true, - testRuns: [ + 'passed', + 'passed and approved merge runs', + [ generateTestRun({ status: TestStatus.ok, merge: true }), generateTestRun({ status: TestStatus.approved, merge: true }), ], - }; - const buildDto: BuildDto = { - id: build.id, - ciBuildId: build.ciBuildId, - number: build.number, - branchName: build.branchName, - status: 'passed', - projectId: build.projectId, - updatedAt: build.updatedAt, - createdAt: build.createdAt, - userId: null, - passedCount: 2, - unresolvedCount: 0, - failedCount: 0, - isRunning: true, - merge: true, - }; - - const result = new BuildDto(build); - - expect(result).toEqual(buildDto); - }); - - it('failed', () => { - const build: Build & { testRuns: TestRun[] } = { - id: 'a9385fc1-884d-4f9f-915e-40da0e7773d5', - ciBuildId: 'ciBuildId', - number: null, - branchName: 'develop', - status: null, - projectId: 'e0a37894-6f29-478d-b13e-6182fecc715e', - updatedAt: new Date(), - createdAt: new Date(), - userId: null, - isRunning: true, - testRuns: [ + { passedCount: 2, unresolvedCount: 0, failedCount: 0, merge: true }, + ], + [ + 'failed', + 'a failed run', + [ generateTestRun({ status: TestStatus.ok }), generateTestRun({ status: TestStatus.approved }), generateTestRun({ status: TestStatus.failed }), ], - }; - const buildDto: BuildDto = { - id: build.id, - ciBuildId: build.ciBuildId, - number: build.number, - branchName: build.branchName, - status: 'failed', - projectId: build.projectId, - updatedAt: build.updatedAt, - createdAt: build.createdAt, - userId: null, - passedCount: 2, - unresolvedCount: 0, - failedCount: 1, - isRunning: true, - merge: false, - }; - - const result = new BuildDto(build); - - expect(result).toEqual(buildDto); - }); - - it('unresolved', () => { - const build: Build & { testRuns: TestRun[] } = { - id: 'a9385fc1-884d-4f9f-915e-40da0e7773d5', - ciBuildId: 'ciBuildId', - number: null, - branchName: 'develop', - status: null, - projectId: 'e0a37894-6f29-478d-b13e-6182fecc715e', - updatedAt: new Date(), - createdAt: new Date(), - userId: null, - isRunning: true, - testRuns: [ + { passedCount: 2, unresolvedCount: 0, failedCount: 1, merge: false }, + ], + [ + 'unresolved', + 'new and unresolved runs', + [ generateTestRun({ status: TestStatus.ok }), generateTestRun({ status: TestStatus.approved }), generateTestRun({ status: TestStatus.failed }), generateTestRun({ status: TestStatus.new }), generateTestRun({ status: TestStatus.unresolved }), ], - }; - const buildDto: BuildDto = { + { passedCount: 2, unresolvedCount: 2, failedCount: 1, merge: false }, + ], + ])('is %s with %s', (status, _, testRuns, counts) => { + const build = makeBuild(testRuns); + + expect(new BuildDto(build)).toEqual({ id: build.id, ciBuildId: build.ciBuildId, - number: build.number, - branchName: build.branchName, - status: 'unresolved', + number: null, + branchName: 'develop', + status, projectId: build.projectId, updatedAt: build.updatedAt, createdAt: build.createdAt, userId: null, - passedCount: 2, - unresolvedCount: 2, - failedCount: 1, isRunning: true, - merge: false, - }; - - const result = new BuildDto(build); - - expect(result).toEqual(buildDto); - }); - - it('counts auto-approved test runs as passed', () => { - const result = new BuildDto({ - ...TEST_BUILD, - testRuns: [generateTestRun({ status: TestStatus.autoApproved })], - } as Build & { testRuns: TestRun[] }); - - expect(result).toMatchObject({ status: 'passed', passedCount: 1, unresolvedCount: 0, failedCount: 0 }); + ...counts, + }); }); - it('is passed when its loaded test runs have a status outside the known groups', () => { - const result = new BuildDto({ - ...TEST_BUILD, - testRuns: [generateTestRun({ status: 'resolved' as TestStatus })], - } as Build & { testRuns: TestRun[] }); + it.each([ + ['counts auto-approved test runs as passed', TestStatus.autoApproved, 1], + ['is passed when it has runs but none in a known status group', 'resolved' as TestStatus, 0], + ])('%s', (_, status, passedCount) => { + const result = new BuildDto({ ...TEST_BUILD, testRuns: [generateTestRun({ status })] }); - // NOTE: current behavior — the run counts nowhere, but having runs at all makes it "passed" - expect(result).toMatchObject({ status: 'passed', passedCount: 0, unresolvedCount: 0, failedCount: 0 }); + expect(result).toMatchObject({ status: 'passed', passedCount, unresolvedCount: 0, failedCount: 0 }); }); describe('with stats aggregated by the database', () => { - const build: Build & { testRuns?: TestRun[] } = { - id: 'a9385fc1-884d-4f9f-915e-40da0e7773d5', - ciBuildId: 'ciBuildId', - number: 7, - branchName: 'develop', - status: 'stored status', - projectId: 'e0a37894-6f29-478d-b13e-6182fecc715e', - updatedAt: new Date(), - createdAt: new Date(), - userId: 'userId', - isRunning: false, - }; + const build: Build = { ...makeBuild(), number: 7, status: 'stored status', userId: 'userId', isRunning: false }; it.each([ ['new', { passedCount: 0, unresolvedCount: 0, failedCount: 0, merge: false }], @@ -249,8 +113,7 @@ describe('BuildDto', () => { expect(result).toMatchObject({ status: 'new', passedCount: 0, failedCount: 0, merge: false }); }); - it('never keeps the status stored on the build', () => { - // NOTE: current behavior — build.status is copied and then always overwritten + it('derives the status from the runs, never from the stored column', () => { expect(new BuildDto(build).status).toBe('new'); }); }); diff --git a/src/builds/dto/build.dto.ts b/src/builds/dto/build.dto.ts index af97eb7a..51d74d90 100644 --- a/src/builds/dto/build.dto.ts +++ b/src/builds/dto/build.dto.ts @@ -64,10 +64,8 @@ export class BuildDto { this.failedCount = stats.failedCount; this.merge = stats.merge; } else if (build.testRuns) { - // determine if merge this.merge = build.testRuns.some((testRun) => testRun.merge); - // calculate statistics build.testRuns.forEach((testRun) => { switch (testRun.status) { case TestStatus.autoApproved: diff --git a/src/compare/compare.service.spec.ts b/src/compare/compare.service.spec.ts index aab21bb6..aa993390 100644 --- a/src/compare/compare.service.spec.ts +++ b/src/compare/compare.service.spec.ts @@ -1,4 +1,4 @@ -import { Test, TestingModule } from '@nestjs/testing'; +import { Test } from '@nestjs/testing'; import { ConfigService } from '@nestjs/config'; import { PrismaService } from '../prisma/prisma.service'; import { CompareService } from './compare.service'; @@ -12,79 +12,61 @@ import { GeminiService } from './libs/vlm/providers/gemini/gemini.service'; import { StaticModule } from '../static/static.module'; import { StaticService } from '../static/static.service'; import { Dinov2Service } from './libs/dinov2/dinov2.service'; -import { ImageComparison, TestStatus } from '@prisma/client'; +import { ImageComparison, Project, TestStatus } from '@prisma/client'; import { isHddStaticServiceConfigured } from '../static/utils'; import { ImageCompareInput } from './libs/ImageCompareInput'; -import { BlurredDifference, DEFAULT_BLUR_SIGMA, MAX_SOLID_SHARE } from './libs/pixelmatch/blur-amplitude.core'; +import { BlurredDifference, MAX_SOLID_SHARE } from './libs/pixelmatch/blur-amplitude.core'; -// a mock function in place of the export rather than a spy on it: SWC's -// CommonJS exports are getters, which jest.spyOn cannot replace jest.mock('../static/utils', () => { const actual = jest.requireActual('../static/utils'); return { ...actual, isHddStaticServiceConfigured: jest.fn(actual.isHddStaticServiceConfigured) }; }); -jest.mock('zod/v3', () => { - const actualZod = jest.requireActual('zod'); - return actualZod; -}); - describe('CompareService', () => { - let service: CompareService; - let pixelmatchService: PixelmatchService; - let lookSameService: LookSameService; - let odiffService: OdiffService; - let vlmService: VlmService; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [ - CompareService, - DiffWorkerPool, - Dinov2Service, - OdiffService, - PixelmatchService, - LookSameService, - VlmService, - OllamaService, - GeminiService, - PrismaService, - { - provide: ConfigService, - useValue: { - getOrThrow: jest.fn(), - }, - }, - ], - imports: [StaticModule], - }).compile(); - - service = module.get(CompareService); - pixelmatchService = module.get(PixelmatchService); - lookSameService = module.get(LookSameService); - odiffService = module.get(OdiffService); - vlmService = module.get(VlmService); - }); - - it('should be defined', () => { - expect(service).toBeDefined(); - }); - describe('getComparator', () => { - it('should return pixelmatchService', () => { - const result = service.getComparator(ImageComparison.pixelmatch); - expect(result).toBe(pixelmatchService); + let service: CompareService; + let comparators: Record; + + beforeEach(async () => { + const module = await Test.createTestingModule({ + providers: [ + CompareService, + DiffWorkerPool, + Dinov2Service, + OdiffService, + PixelmatchService, + LookSameService, + VlmService, + OllamaService, + GeminiService, + PrismaService, + { provide: ConfigService, useValue: { getOrThrow: jest.fn() } }, + ], + imports: [StaticModule], + }).compile(); + + service = module.get(CompareService); + comparators = { + pixelmatch: module.get(PixelmatchService), + lookSame: module.get(LookSameService), + odiff: module.get(OdiffService), + vlm: module.get(VlmService), + }; }); - it('should return lookSameService', () => { - const result = service.getComparator(ImageComparison.lookSame); - expect(result).toBe(lookSameService); + it.each([ + [ImageComparison.pixelmatch, 'pixelmatch'], + [ImageComparison.lookSame, 'lookSame'], + [ImageComparison.vlm, 'vlm'], + ['unknown' as ImageComparison, 'pixelmatch'], + ])('should return the comparator for %s', (imageComparison, name) => { + expect(service.getComparator(imageComparison)).toBe(comparators[name]); }); it('should return odiffService', () => { jest.mocked(isHddStaticServiceConfigured).mockReturnValue(true); - expect(service.getComparator(ImageComparison.odiff)).toBe(odiffService); + expect(service.getComparator(ImageComparison.odiff)).toBe(comparators.odiff); }); it('should throw if not HDD for Odiff', () => { @@ -94,52 +76,86 @@ describe('CompareService', () => { 'Odiff can only be used with HDD static service. Please use another image comparison lib in project settings or switch STATIC_SERVICE envitonmental variable to HDD.' ); }); - - it('should return vlmService', () => { - expect(service.getComparator(ImageComparison.vlm)).toBe(vlmService); - }); - - it('should return pixelmatchService for unknown value', () => { - const result = service.getComparator('unknown' as ImageComparison); - expect(result).toBe(pixelmatchService); - }); }); }); describe('CompareService with mocked collaborators', () => { - // pixelmatch applies the rendering-noise filter in its own worker pass, and - // VLM through the pixelmatch it runs first; the others leave it to the service - const comparator = (name: string, appliesBlurAmplitudeFilter = false) => ({ - parseConfig: jest.fn().mockReturnValue({ parsedBy: name }), - getDiff: jest.fn().mockResolvedValue({ comparedBy: name }), + const comparator = (appliesBlurAmplitudeFilter = false) => ({ + parseConfig: jest.fn().mockImplementation((json: string) => JSON.parse(json)), + getDiff: jest.fn(), appliesBlurAmplitudeFilter, }); - const initService = async ({ - projectFindUniqueMock = jest.fn(), - diffWorkerPoolRunMock = jest.fn(), - staticServiceMock = { getImageBuffer: jest.fn(), deleteImage: jest.fn() }, - confirmsNoiseMock = jest.fn(), - }) => { - const pixelmatch = comparator('pixelmatch', true); - const lookSame = comparator('lookSame'); - const odiff = comparator('odiff'); - const vlm = comparator('vlm', true); - const module: TestingModule = await Test.createTestingModule({ + const unresolved = { + status: TestStatus.unresolved, + diffName: 'diff.png', + pixelMisMatchCount: 10, + diffPercent: 5, + isSameDimension: true, + }; + + interface Options { + imageComparison?: ImageComparison; + config?: Record; + project?: Project | null; + result?: object; + measure?: BlurredDifference | null; + gate?: object; + run?: jest.Mock; + confirmsNoiseForGate?: jest.Mock; + } + + const setup = async ({ + imageComparison = ImageComparison.lookSame, + config = {}, + project, + result = unresolved, + measure = { amplitude: 40, solidPixels: 0, solidShare: 0 }, + gate, + run = jest.fn().mockResolvedValue({ measure, gate }), + confirmsNoiseForGate = jest.fn(), + }: Options = {}) => { + const staticService = { + getImageBuffer: jest.fn().mockImplementation(async (name: string) => Buffer.from(name)), + deleteImage: jest.fn(), + }; + const findUnique = jest + .fn() + .mockResolvedValue( + project === undefined + ? { id: 'projectId', imageComparison, imageComparisonConfig: JSON.stringify(config) } + : project + ); + const pixelmatch = comparator(true); + const lookSame = comparator(); + const odiff = comparator(); + const vlm = comparator(true); + [pixelmatch, lookSame, odiff, vlm].forEach((c) => c.getDiff.mockResolvedValue(result)); + const module = await Test.createTestingModule({ providers: [ CompareService, { provide: PixelmatchService, useValue: pixelmatch }, { provide: LookSameService, useValue: lookSame }, { provide: OdiffService, useValue: odiff }, { provide: VlmService, useValue: vlm }, - { provide: PrismaService, useValue: { project: { findUnique: projectFindUniqueMock } } }, - { provide: DiffWorkerPool, useValue: { run: diffWorkerPoolRunMock } }, - { provide: StaticService, useValue: staticServiceMock }, - { provide: Dinov2Service, useValue: { confirmsNoise: confirmsNoiseMock } }, + { provide: PrismaService, useValue: { project: { findUnique } } }, + { provide: DiffWorkerPool, useValue: { run } }, + { provide: StaticService, useValue: staticService }, + { provide: Dinov2Service, useValue: { confirmsNoiseForGate } }, ], }).compile(); - return { service: module.get(CompareService), pixelmatch, lookSame, odiff, vlm }; + return { + service: module.get(CompareService), + pixelmatch, + lookSame, + odiff, + vlm, + run, + staticService, + findUnique, + confirmsNoiseForGate, + }; }; const data: ImageCompareInput = { @@ -150,18 +166,30 @@ describe('CompareService with mocked collaborators', () => { saveDiffAsFile: true, }; + const diffProject = (service: CompareService, input: ImageCompareInput = data) => + service.getDiff({ projectId: 'projectId', data: input }); + describe('getDiff', () => { - it("diffs with the project's comparator and the config it parses", async () => { - const projectFindUniqueMock = jest.fn().mockResolvedValue({ + it.each([ + ['reads the project and', false], + ['uses the project it is given without reading it again, and', true], + ])("%s diffs with the project's comparator and the config it parses", async (_, given) => { + const { service, lookSame, pixelmatch, findUnique } = await setup({ config: { strict: true } }); + lookSame.parseConfig.mockReturnValue({ parsedBy: 'lookSame' }); + lookSame.getDiff.mockResolvedValue({ comparedBy: 'lookSame' }); + const project = { id: 'projectId', imageComparison: ImageComparison.lookSame, imageComparisonConfig: '{"strict":true}', - }); - const { service, lookSame, pixelmatch } = await initService({ projectFindUniqueMock }); + } as Project; - const result = await service.getDiff({ projectId: 'projectId', data }); + const result = await service.getDiff({ projectId: 'projectId', ...(given ? { project } : {}), data }); - expect(projectFindUniqueMock).toHaveBeenCalledWith({ where: { id: 'projectId' } }); + if (given) { + expect(findUnique).not.toHaveBeenCalled(); + } else { + expect(findUnique).toHaveBeenCalledWith({ where: { id: 'projectId' } }); + } expect(lookSame.parseConfig).toHaveBeenCalledWith('{"strict":true}'); expect(lookSame.getDiff).toHaveBeenCalledWith(data, { parsedBy: 'lookSame' }); expect(pixelmatch.getDiff).not.toHaveBeenCalled(); @@ -169,310 +197,249 @@ describe('CompareService with mocked collaborators', () => { }); it('throws a TypeError when the project does not exist', async () => { - const projectFindUniqueMock = jest.fn().mockResolvedValue(null); - const { service } = await initService({ projectFindUniqueMock }); + const { service } = await setup({ project: null }); - // NOTE: current behavior — no not-found handling, the null project is dereferenced await expect(service.getDiff({ projectId: 'missing', data })).rejects.toThrow(TypeError); }); }); describe('rendering-noise filter', () => { - const unresolved = { - status: TestStatus.unresolved, - diffName: 'diff.png', - pixelMisMatchCount: 10, - diffPercent: 5, - isSameDimension: true, - }; - - // the filter's settings ride in the library's config, which CompareService - // reads and strips before the comparator sees it const filterOn = { blurAmplitudeFilter: true, blurAmplitudeThreshold: 80 }; - - const setup = async ({ - imageComparison = ImageComparison.lookSame as ImageComparison, - config = filterOn as Record, - result = unresolved as object, - measure = { amplitude: 40, solidPixels: 0, solidShare: 0 } as BlurredDifference | null, - run = jest.fn().mockResolvedValue({ measure }), - confirmsNoiseMock = jest.fn(), - } = {}) => { - const staticServiceMock = { - getImageBuffer: jest.fn().mockImplementation(async (name: string) => Buffer.from(name)), - deleteImage: jest.fn(), - }; - const projectFindUniqueMock = jest.fn().mockResolvedValue({ - id: 'projectId', - imageComparison, - imageComparisonConfig: JSON.stringify({ libraryOption: 1, ...config }), - }); - const context = await initService({ - projectFindUniqueMock, - diffWorkerPoolRunMock: run, - staticServiceMock, - confirmsNoiseMock, - }); - [context.pixelmatch, context.lookSame, context.odiff, context.vlm].forEach((c) => { - c.parseConfig.mockImplementation((json: string) => JSON.parse(json)); - c.getDiff.mockResolvedValue(result); - }); - return { ...context, run, staticServiceMock, confirmsNoiseMock }; - }; - - const filter = { sigma: DEFAULT_BLUR_SIGMA, threshold: 80 }; + const filtered = (options: Options = {}) => + setup({ ...options, config: { libraryOption: 1, ...(options.config ?? filterOn) } }); it('leaves a config that does not turn it on alone', async () => { - const { service, lookSame, run } = await setup({ config: {} }); + const { service, lookSame, run } = await filtered({ config: {} }); - const result = await service.getDiff({ projectId: 'projectId', data }); + const result = await diffProject(service); expect(lookSame.getDiff).toHaveBeenCalledWith(data, { libraryOption: 1 }); expect(run).not.toHaveBeenCalled(); expect(result).toEqual(unresolved); }); - // decoded already inside its own worker pass, where a run that is about to - // pass also never writes a diff - it('hands it to pixelmatch rather than measuring a second time', async () => { - const { service, pixelmatch, run } = await setup({ imageComparison: ImageComparison.pixelmatch }); - - await service.getDiff({ projectId: 'projectId', data }); - - expect(pixelmatch.getDiff).toHaveBeenCalledWith({ ...data, blurAmplitude: filter }, { libraryOption: 1 }); - expect(run).not.toHaveBeenCalled(); - }); - - // VLM runs pixelmatch first and stops when it passes, so noise is settled - // before a model is ever asked about it - it('hands it to VLM as a prefilter', async () => { - const { service, vlm, run } = await setup({ imageComparison: ImageComparison.vlm }); + it.each([ + ['pixelmatch rather than measuring a second time', ImageComparison.pixelmatch, 'pixelmatch'], + ['VLM as a prefilter', ImageComparison.vlm, 'vlm'], + ] as const)('hands it to %s', async (_, imageComparison, key) => { + const context = await filtered({ imageComparison }); - await service.getDiff({ projectId: 'projectId', data }); + await diffProject(context.service); - expect(vlm.getDiff).toHaveBeenCalledWith({ ...data, blurAmplitude: filter }, { libraryOption: 1 }); - expect(run).not.toHaveBeenCalled(); + expect(context[key].getDiff).toHaveBeenCalledWith( + { ...data, blurAmplitude: { threshold: 80 } }, + { libraryOption: 1 } + ); + expect(context.run).not.toHaveBeenCalled(); }); it('passes a looks-same failure that is only rendering noise', async () => { - const { service, lookSame, run, staticServiceMock } = await setup(); + const { service, lookSame, run, staticService } = await filtered(); - const result = await service.getDiff({ projectId: 'projectId', data }); + const result = await diffProject(service); - // looks-same gets its own options, not the filter's expect(lookSame.getDiff).toHaveBeenCalledWith(data, { libraryOption: 1 }); - expect(run).toHaveBeenCalledWith({ kind: 'blur-amplitude', baseline: Buffer.from('baseline.png'), image: Buffer.from('image.png'), ignoreAreas: data.ignoreAreas, - sigma: DEFAULT_BLUR_SIGMA, }); - // the measured mismatch stays, only the verdict changes expect(result).toEqual({ ...unresolved, status: TestStatus.ok, diffName: null }); - // nothing will ever point at the diff of a run that passed - expect(staticServiceMock.deleteImage).toHaveBeenCalledWith('diff.png'); + expect(staticService.deleteImage).toHaveBeenCalledWith('diff.png'); + }); + + it('measures with the image it is given', async () => { + const { service, run, staticService } = await filtered(); + const imageBuffer = Buffer.from('already read'); + + await diffProject(service, { ...data, imageBuffer }); + + expect(staticService.getImageBuffer).toHaveBeenCalledTimes(1); + expect(staticService.getImageBuffer).toHaveBeenCalledWith('baseline.png'); + expect(run).toHaveBeenCalledWith(expect.objectContaining({ kind: 'blur-amplitude', image: imageBuffer })); + expect(run.mock.calls[0][0].image).toBe(imageBuffer); }); it('passes an odiff failure that is only rendering noise', async () => { jest.mocked(isHddStaticServiceConfigured).mockReturnValue(true); - const { service } = await setup({ imageComparison: ImageComparison.odiff }); - - const result = await service.getDiff({ projectId: 'projectId', data }); + const { service } = await filtered({ imageComparison: ImageComparison.odiff }); - expect(result.status).toBe(TestStatus.ok); + expect((await diffProject(service)).status).toBe(TestStatus.ok); }); it('keeps a change stronger than rendering noise unresolved', async () => { - const { service, staticServiceMock } = await setup({ + const { service, staticService } = await filtered({ measure: { amplitude: 200, solidPixels: 0, solidShare: 0 }, }); - const result = await service.getDiff({ projectId: 'projectId', data }); - - expect(result).toEqual(unresolved); - expect(staticServiceMock.deleteImage).not.toHaveBeenCalled(); + expect(await diffProject(service)).toEqual(unresolved); + expect(staticService.deleteImage).not.toHaveBeenCalled(); }); - // a panel turned light grey is faint, but too solid to be noise it('keeps a faint but solid change unresolved', async () => { - const { service } = await setup({ + const { service } = await filtered({ measure: { amplitude: 31, solidPixels: 9300, solidShare: MAX_SOLID_SHARE + 0.001 }, }); - expect(await service.getDiff({ projectId: 'projectId', data })).toEqual(unresolved); + expect(await diffProject(service)).toEqual(unresolved); }); it("measures against the config's own threshold", async () => { - const { service } = await setup({ config: { ...filterOn, blurAmplitudeThreshold: 30 } }); + const { service } = await filtered({ config: { ...filterOn, blurAmplitudeThreshold: 30 } }); - expect((await service.getDiff({ projectId: 'projectId', data })).status).toBe(TestStatus.unresolved); + expect((await diffProject(service)).status).toBe(TestStatus.unresolved); }); it('does not second-guess a run that already passed', async () => { const ok = { ...unresolved, status: TestStatus.ok, diffName: null as string | null }; - const { service, run } = await setup({ result: ok }); + const { service, run } = await filtered({ result: ok }); - expect(await service.getDiff({ projectId: 'projectId', data })).toEqual(ok); + expect(await diffProject(service)).toEqual(ok); expect(run).not.toHaveBeenCalled(); }); - // nothing to line up, and the size change is a finding of its own it('leaves screenshots of different sizes to the comparator', async () => { const resized = { ...unresolved, isSameDimension: false }; - const { service, run } = await setup({ result: resized }); + const { service, run } = await filtered({ result: resized }); - expect(await service.getDiff({ projectId: 'projectId', data })).toEqual(resized); + expect(await diffProject(service)).toEqual(resized); expect(run).not.toHaveBeenCalled(); }); it('keeps the verdict when the change cannot be measured', async () => { - const { service } = await setup({ measure: null }); + const { service } = await filtered({ measure: null }); - expect(await service.getDiff({ projectId: 'projectId', data })).toEqual(unresolved); + expect(await diffProject(service)).toEqual(unresolved); }); - // a filter that can only ever pass runs must never be the reason one fails - it('keeps the verdict when measuring fails', async () => { - const run = jest.fn().mockRejectedValue(new Error('Image diff queue is full (256 jobs)')); - const { service } = await setup({ run }); - - expect(await service.getDiff({ projectId: 'projectId', data })).toEqual(unresolved); - }); - - it('keeps the verdict when measuring fails with something that is not an Error', async () => { - const { service } = await setup({ run: jest.fn().mockRejectedValue('worker gone') }); + it.each([ + ['an Error', new Error('Image diff queue is full (256 jobs)')], + ['something that is not an Error', 'worker gone'], + ])('keeps the verdict when measuring fails with %s', async (_, failure) => { + const { service } = await filtered({ run: jest.fn().mockRejectedValue(failure) }); - expect(await service.getDiff({ projectId: 'projectId', data })).toEqual(unresolved); + expect(await diffProject(service)).toEqual(unresolved); }); it('keeps the verdict when a screenshot cannot be read', async () => { - const { service, staticServiceMock, run } = await setup(); - staticServiceMock.getImageBuffer.mockResolvedValueOnce(undefined); + const { service, staticService, run } = await filtered(); + staticService.getImageBuffer.mockResolvedValueOnce(undefined); - expect(await service.getDiff({ projectId: 'projectId', data })).toEqual(unresolved); + expect(await diffProject(service)).toEqual(unresolved); expect(run).not.toHaveBeenCalled(); }); it('has no diff to delete when none was saved', async () => { - const { service, staticServiceMock } = await setup({ result: { ...unresolved, diffName: null } }); + const { service, staticService } = await filtered({ result: { ...unresolved, diffName: null } }); - expect((await service.getDiff({ projectId: 'projectId', data })).status).toBe(TestStatus.ok); - expect(staticServiceMock.deleteImage).not.toHaveBeenCalled(); + expect((await diffProject(service)).status).toBe(TestStatus.ok); + expect(staticService.deleteImage).not.toHaveBeenCalled(); }); }); describe('rendering-noise filter with the DINOv2 check', () => { - const unresolved = { - status: TestStatus.unresolved, - diffName: 'diff.png', - pixelMisMatchCount: 10, - diffPercent: 5, - isSameDimension: true, - }; - const checked = { blurAmplitudeFilter: true, dinov2Check: true }; - - const setup = async (imageComparison: ImageComparison, confirmsNoiseMock = jest.fn()) => { - const staticServiceMock = { - getImageBuffer: jest.fn().mockImplementation(async (name: string) => Buffer.from(name)), - deleteImage: jest.fn(), - }; - const context = await initService({ - projectFindUniqueMock: jest.fn().mockResolvedValue({ - id: 'projectId', - imageComparison, - imageComparisonConfig: JSON.stringify(checked), - }), - diffWorkerPoolRunMock: jest - .fn() - .mockResolvedValue({ measure: { amplitude: 40, solidPixels: 0, solidShare: 0 } }), - staticServiceMock, - confirmsNoiseMock, - }); - [context.pixelmatch, context.lookSame, context.odiff, context.vlm].forEach((c) => { - c.parseConfig.mockImplementation((json: string) => JSON.parse(json)); - c.getDiff.mockResolvedValue(unresolved); - }); - return { ...context, staticServiceMock }; - }; - + const gate = { width: 5, height: 3, cells: [7] }; + const checked = (options: Options = {}) => + setup({ gate, config: { blurAmplitudeFilter: true, dinov2Check: true }, ...options }); const dinov2 = { threshold: 0.15 }; - it('passes a looks-same failure DINOv2 confirms is only noise', async () => { - const confirmsNoise = jest.fn().mockResolvedValue({ noise: true, score: 0.04 }); - const { service, staticServiceMock } = await setup(ImageComparison.lookSame, confirmsNoise); + it('passes a looks-same failure DINOv2 confirms is only noise, with the gate the blur job returned', async () => { + const confirmsNoiseForGate = jest.fn().mockResolvedValue({ noise: true, score: 0.04 }); + const { service, staticService, run } = await checked({ confirmsNoiseForGate }); - const result = await service.getDiff({ projectId: 'projectId', data }); + const result = await diffProject(service); - expect(confirmsNoise).toHaveBeenCalledWith( + expect(run).toHaveBeenCalledTimes(1); + expect(run).toHaveBeenCalledWith({ + kind: 'blur-amplitude', + baseline: Buffer.from('baseline.png'), + image: Buffer.from('image.png'), + ignoreAreas: data.ignoreAreas, + gateIfNoise: { threshold: 80 }, + }); + expect(confirmsNoiseForGate).toHaveBeenCalledWith( + gate, Buffer.from('baseline.png'), Buffer.from('image.png'), - data.ignoreAreas, 0.15 ); expect(result.status).toBe(TestStatus.ok); - expect(staticServiceMock.deleteImage).toHaveBeenCalledWith('diff.png'); + expect(staticService.deleteImage).toHaveBeenCalledWith('diff.png'); }); it('keeps a looks-same failure in which DINOv2 sees a change of content', async () => { - const confirmsNoise = jest.fn().mockResolvedValue({ noise: false, score: 0.56 }); - const { service, staticServiceMock } = await setup(ImageComparison.lookSame, confirmsNoise); + const { service, staticService } = await checked({ + confirmsNoiseForGate: jest.fn().mockResolvedValue({ noise: false, score: 0.56 }), + }); + + expect(await diffProject(service)).toEqual(unresolved); + expect(staticService.deleteImage).not.toHaveBeenCalled(); + }); + + it('keeps the failure and does not ask DINOv2 when the blur job returned no gate', async () => { + const confirmsNoiseForGate = jest.fn(); + const { service, staticService } = await checked({ + gate: undefined, + confirmsNoiseForGate, + }); + + expect(await diffProject(service)).toEqual(unresolved); + expect(confirmsNoiseForGate).not.toHaveBeenCalled(); + expect(staticService.deleteImage).not.toHaveBeenCalled(); + }); - expect(await service.getDiff({ projectId: 'projectId', data })).toEqual(unresolved); - expect(staticServiceMock.deleteImage).not.toHaveBeenCalled(); + it.each([ + [ + 'a change the blur already fails', + { run: jest.fn().mockResolvedValue({ measure: { amplitude: 200, solidPixels: 0, solidShare: 0 } }) }, + ], + ['a pair when the blur job fails', { run: jest.fn().mockRejectedValue(new Error('worker crashed')) }], + ])('does not ask DINOv2 about %s', async (_, options) => { + const confirmsNoiseForGate = jest.fn(); + const { service } = await checked({ ...options, confirmsNoiseForGate }); + + expect(await diffProject(service)).toEqual(unresolved); + expect(confirmsNoiseForGate).not.toHaveBeenCalled(); }); - // pixelmatch — and VLM through it — ask DINOv2 themselves, inside their - // own pass, where the diff is still at hand it.each([ [ImageComparison.pixelmatch, 'pixelmatch'], [ImageComparison.vlm, 'vlm'], ] as const)('hands the check to %s', async (imageComparison, key) => { - const confirmsNoise = jest.fn(); - const context = await setup(imageComparison, confirmsNoise); + const context = await checked({ imageComparison }); - await context.service.getDiff({ projectId: 'projectId', data }); + await diffProject(context.service); - expect(context[key].getDiff).toHaveBeenCalledWith( - { ...data, blurAmplitude: { sigma: DEFAULT_BLUR_SIGMA, threshold: 80, dinov2 } }, - {} - ); - expect(confirmsNoise).not.toHaveBeenCalled(); + expect(context[key].getDiff).toHaveBeenCalledWith({ ...data, blurAmplitude: { threshold: 80, dinov2 } }, {}); + expect(context.confirmsNoiseForGate).not.toHaveBeenCalled(); }); }); describe('getChangeSignature', () => { + const input = { + baseline: Buffer.from('a'), + image: Buffer.from('b'), + ignoreAreas: [{ x: 1, y: 2, width: 3, height: 4 }], + threshold: 0.2, + includeAA: true, + }; + it('runs a signature job on the diff worker pool and returns its answer', async () => { - const diffWorkerPoolRunMock = jest.fn().mockResolvedValue({ signature: [0.5, 0.5] }); - const { service } = await initService({ diffWorkerPoolRunMock }); - const input = { - baseline: Buffer.from('a'), - image: Buffer.from('b'), - ignoreAreas: [{ x: 1, y: 2, width: 3, height: 4 }], - threshold: 0.2, - includeAA: true, - }; + const { service, run } = await setup({ run: jest.fn().mockResolvedValue({ signature: [0.5, 0.5] }) }); const result = await service.getChangeSignature(input); - expect(diffWorkerPoolRunMock).toHaveBeenCalledWith({ kind: 'signature', ...input }); + expect(run).toHaveBeenCalledWith({ kind: 'signature', ...input }); expect(result).toEqual({ signature: [0.5, 0.5] }); }); it('passes a pool rejection through', async () => { - const diffWorkerPoolRunMock = jest.fn().mockRejectedValue(new Error('Image diff queue is full (256 jobs)')); - const { service } = await initService({ diffWorkerPoolRunMock }); - - await expect( - service.getChangeSignature({ - baseline: Buffer.from('a'), - image: Buffer.from('b'), - ignoreAreas: [], - threshold: 0.1, - includeAA: false, - }) - ).rejects.toThrow('Image diff queue is full (256 jobs)'); + const { service } = await setup({ + run: jest.fn().mockRejectedValue(new Error('Image diff queue is full (256 jobs)')), + }); + + await expect(service.getChangeSignature(input)).rejects.toThrow('Image diff queue is full (256 jobs)'); }); }); }); diff --git a/src/compare/compare.service.ts b/src/compare/compare.service.ts index 20b661f8..c8b5fba1 100644 --- a/src/compare/compare.service.ts +++ b/src/compare/compare.service.ts @@ -31,11 +31,18 @@ export class CompareService { private readonly dinov2Service: Dinov2Service ) {} - async getDiff({ projectId, data }: { projectId: string; data: ImageCompareInput }): Promise { - const project: Project = (await this.prismaService.project.findUnique({ where: { id: projectId } }))!; + async getDiff({ + projectId, + project: loadedProject, + data, + }: { + projectId: string; + project?: Project; + data: ImageCompareInput; + }): Promise { + const project: Project = + loadedProject ?? (await this.prismaService.project.findUnique({ where: { id: projectId } }))!; const comparator = this.getComparator(project.imageComparison); - // the rendering-noise filter is switched on in the library's config but is - // applied here, for every library alike; the library gets the rest const { blurAmplitude, comparatorConfig } = splitBlurAmplitudeConfig( comparator.parseConfig(project.imageComparisonConfig), this.logger @@ -51,68 +58,54 @@ export class CompareService { return this.filterRenderingNoise(data, result, blurAmplitude); } - /** - * Overturns a failing verdict whose change, blurred, is no stronger than - * anti-aliasing and sub-pixel rendering noise. The filter can only ever pass - * a run, never fail one, so anything that keeps it from measuring — a missing - * screenshot, a full queue, different sizes — leaves the verdict as it was. - */ private async filterRenderingNoise( data: ImageCompareInput, result: DiffResult, - { sigma, threshold, dinov2 }: BlurAmplitudeOptions + { threshold, dinov2 }: BlurAmplitudeOptions ): Promise { - // a size change is a finding of its own, and there is nothing to line up if (result.status !== TestStatus.unresolved || !result.isSameDimension) { return result; } try { const [baseline, image] = await Promise.all([ this.staticService.getImageBuffer(data.baseline), - this.staticService.getImageBuffer(data.image), + data.imageBuffer ?? this.staticService.getImageBuffer(data.image), ]); if (!baseline || !image) { return result; } - const { measure } = await this.diffWorkerPool.run({ + const { measure, gate } = await this.diffWorkerPool.run({ kind: 'blur-amplitude', baseline, image, ignoreAreas: data.ignoreAreas, - sigma, + ...(dinov2 ? { gateIfNoise: { threshold } } : {}), }); if (!measure || !isRenderingNoise(measure, threshold)) { return result; } - // the project's DINOv2 check, when it has one, gets the last word - if ( - dinov2 && - !(await this.dinov2Service.confirmsNoise(baseline, image, data.ignoreAreas, dinov2.threshold)).noise - ) { - return result; + if (dinov2) { + if (!gate) { + return result; + } + if (!(await this.dinov2Service.confirmsNoiseForGate(gate, baseline, image, dinov2.threshold)).noise) { + return result; + } } this.logger.debug( `Diff of ${result.diffPercent}% passed as rendering noise (blur amplitude ${measure.amplitude.toFixed(1)}, ${measure.solidPixels} solid pixels)` ); - // nothing will ever point at the diff of a run that passed if (result.diffName) { - this.staticService.deleteImage(result.diffName); + void this.staticService.deleteImage(result.diffName); } return { ...result, status: TestStatus.ok, diffName: null }; } catch (error) { - // read as an Error, not checked with instanceof: fetch errors come from - // another realm under Jest + // a cast, not instanceof: fetch errors come from another realm under Jest this.logger.warn(`Rendering-noise filter skipped: ${(error as Error)?.message ?? error}`); return result; } } - /** - * Position-independent signature of what changed between a baseline and a - * screenshot, used to tell whether two screens carry the same change. Runs on - * the diff worker pool: decoding a pair of full-size screenshots takes long - * enough that doing it on the event loop stalls every other request. - */ async getChangeSignature(input: Omit): Promise { return this.diffWorkerPool.run({ kind: 'signature', ...input }); } diff --git a/src/compare/diff-worker-pool.inline.spec.ts b/src/compare/diff-worker-pool.inline.spec.ts new file mode 100644 index 00000000..e5788e50 --- /dev/null +++ b/src/compare/diff-worker-pool.inline.spec.ts @@ -0,0 +1,128 @@ +import { readFileSync } from 'fs'; +import { join } from 'path'; +import { PNG } from 'pngjs'; +import { DiffWorkerPool } from './diff-worker-pool'; +import { ImageTooLargeException } from '../static/utils'; +import { PixelmatchJobInput } from './libs/pixelmatch/pixelmatch.core'; +import { SignatureJobInput } from './libs/pixelmatch/signature.core'; +import { pngBytes, solidPng } from './test-utils'; + +const png = (width: number, height: number, level: number): Buffer => + pngBytes(solidPng(width, height, [level, level, level])); + +const diffJob = (baseline: Buffer, image: Buffer, allowDiffDimensions = false): PixelmatchJobInput => ({ + kind: 'diff', + baseline, + image, + ignoreAreas: [], + threshold: 0.1, + includeAA: false, + allowDiffDimensions, + diffTolerancePercent: 0, + saveDiff: false, +}); + +const signatureJob = (baseline: Buffer, image: Buffer): SignatureJobInput => ({ + kind: 'signature', + baseline, + image, + ignoreAreas: [], + threshold: 0.1, + includeAA: false, +}); + +describe('DiffWorkerPool', () => { + let pool: DiffWorkerPool; + + beforeEach(() => { + pool = new DiffWorkerPool(); + }); + + afterEach(async () => { + await pool.onApplicationShutdown(); + }); + + it('answers a diff job with the pixel mismatch it found', async () => { + const output = await pool.run(diffJob(png(10, 10, 0), png(10, 10, 255))); + + expect(output).toMatchObject({ equal: false, pixelMisMatchCount: 100 }); + }); + + it('answers a signature job with a change signature', async () => { + const output = await pool.run(signatureJob(png(10, 10, 0), png(10, 10, 255))); + + expect(output.signature).toHaveLength(64); + expect(output.signature!.reduce((sum, value) => sum + value, 0)).toBeCloseTo(1); + }); + + it('answers a looks-same job', async () => { + const fixture = (name: string) => readFileSync(join(__dirname, '../../test', name)); + + const output = await pool.run({ + kind: 'looks-same', + baseline: fixture('image.png'), + image: fixture('image_edited.png'), + ignoreAreas: [], + config: { strict: false, ignoreAntialiasing: true, ignoreCaret: true }, + saveDiff: false, + }); + + expect(output).toMatchObject({ outcome: 'compared', result: { equal: false, differentPixels: 31068 } }); + }); + + describe('MAX_IMAGE_PIXELS', () => { + const originalLimit = process.env.MAX_IMAGE_PIXELS; + + beforeEach(() => { + process.env.MAX_IMAGE_PIXELS = '50'; + }); + + afterEach(() => { + if (originalLimit === undefined) { + delete process.env.MAX_IMAGE_PIXELS; + } else { + process.env.MAX_IMAGE_PIXELS = originalLimit; + } + jest.restoreAllMocks(); + }); + + it('rejects a job whose image is above MAX_IMAGE_PIXELS with 422, before decoding', async () => { + const read = jest.spyOn(PNG.sync, 'read'); + + const rejection = pool.run(diffJob(png(10, 10, 0), png(5, 5, 0))); + + await expect(rejection).rejects.toBeInstanceOf(ImageTooLargeException); + await expect(rejection).rejects.toMatchObject({ status: 422 }); + expect(read).not.toHaveBeenCalled(); + }); + + it('checks the baseline as well as the new image', async () => { + await expect(pool.run(diffJob(png(5, 5, 0), png(10, 10, 0)))).rejects.toBeInstanceOf(ImageTooLargeException); + }); + + it('checks every job kind', async () => { + await expect(pool.run(signatureJob(png(10, 10, 0), png(10, 10, 255)))).rejects.toBeInstanceOf( + ImageTooLargeException + ); + }); + + it('rejects a diff whose canvas would exceed the limit when other sizes are allowed', async () => { + process.env.MAX_IMAGE_PIXELS = '60'; + const wide = png(10, 5, 0); + const tall = png(5, 10, 0); + + await expect(pool.run(diffJob(wide, tall, true))).rejects.toBeInstanceOf(ImageTooLargeException); + await expect(pool.run(diffJob(wide, tall, false))).resolves.toMatchObject({ + equal: false, + isSameDimension: false, + }); + }); + + it('leaves a malformed header to the decoder', async () => { + const rejection = pool.run(diffJob(Buffer.from('png'), png(5, 5, 0))); + + await expect(rejection).rejects.toThrow(); + await expect(rejection).rejects.not.toBeInstanceOf(ImageTooLargeException); + }); + }); +}); diff --git a/src/compare/diff-worker-pool.spec.ts b/src/compare/diff-worker-pool.spec.ts deleted file mode 100644 index aa3c6e7c..00000000 --- a/src/compare/diff-worker-pool.spec.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { PNG } from 'pngjs'; -import { DiffWorkerPool } from './diff-worker-pool'; - -// Opaque throughout: pixelmatch composites transparent pixels onto white, so a -// fully transparent image would read as equal to a white one. -const png = (width: number, height: number, rgb: number): Buffer => { - const image = new PNG({ width, height }); - for (let i = 0; i < width * height; i++) { - image.data[i * 4] = rgb; - image.data[i * 4 + 1] = rgb; - image.data[i * 4 + 2] = rgb; - image.data[i * 4 + 3] = 255; - } - return PNG.sync.write(image); -}; - -describe('DiffWorkerPool', () => { - let pool: DiffWorkerPool; - - beforeEach(() => { - pool = new DiffWorkerPool(); - }); - - afterEach(async () => { - await pool.onApplicationShutdown(); - }); - - it('answers a diff job with the pixel mismatch it found', async () => { - const output = await pool.run({ - kind: 'diff', - baseline: png(10, 10, 0), - image: png(10, 10, 255), - ignoreAreas: [], - threshold: 0.1, - includeAA: false, - allowDiffDimensions: false, - diffTolerancePercent: 0, - saveDiff: false, - }); - - expect(output).toMatchObject({ equal: false, pixelMisMatchCount: 100 }); - }); - - it('answers a signature job with a change signature', async () => { - const output = await pool.run({ - kind: 'signature', - baseline: png(10, 10, 0), - image: png(10, 10, 255), - ignoreAreas: [], - threshold: 0.1, - includeAA: false, - }); - - expect(output.signature).toHaveLength(64); - expect(output.signature!.reduce((sum, value) => sum + value, 0)).toBeCloseTo(1); - }); -}); diff --git a/src/compare/diff-worker-pool.ts b/src/compare/diff-worker-pool.ts index f6562c71..6a6cfa5f 100644 --- a/src/compare/diff-worker-pool.ts +++ b/src/compare/diff-worker-pool.ts @@ -1,35 +1,55 @@ -import { Injectable, Logger, OnApplicationShutdown } from '@nestjs/common'; +import { Injectable, Logger, OnApplicationShutdown, ServiceUnavailableException } from '@nestjs/common'; import { Worker } from 'worker_threads'; import { availableParallelism } from 'os'; import { existsSync } from 'fs'; import { join } from 'path'; -import { PixelmatchJobInput, PixelmatchJobOutput } from './libs/pixelmatch/pixelmatch.core'; -import { SignatureJobInput, SignatureJobOutput } from './libs/pixelmatch/signature.core'; -import { BlurAmplitudeJobInput, BlurAmplitudeJobOutput } from './libs/pixelmatch/blur-amplitude.core'; -import { PatchGateJobInput, PatchGateJobOutput } from './libs/pixelmatch/patch-gate.core'; -import { runWorkerJob, WorkerJobInput, WorkerJobOutput } from './libs/pixelmatch/worker-job'; +import { nonNegativeNumber } from '../utils/env'; +import { assertWithinPixelLimit, readPngDimensions } from '../static/utils'; +import { + runWorkerJob, + WorkerJobInput, + WorkerJobInputOf, + WorkerJobKind, + WorkerJobOutput, + WorkerJobResult, +} from './libs/pixelmatch/worker-job'; const WORKER_FILE = join(__dirname, 'libs', 'pixelmatch', 'pixelmatch.worker.js'); -// consecutive worker failures after which the pool stops respawning const MAX_SPAWN_FAILURES = 3; -// how many times a job may be handed to a worker before it is given up on -const MAX_JOB_ATTEMPTS = 2; +const MAX_JOB_CRASHES = 2; + +export const QUEUE_FULL_RETRY_AFTER_SECONDS = 5; + +export class ImageDiffQueueFullException extends ServiceUnavailableException { + readonly retryAfterSeconds = QUEUE_FULL_RETRY_AFTER_SECONDS; + constructor(queueLimit: number) { + super(`Image diff queue is full (${queueLimit} jobs)`); + } +} + +export class ImageDiffPoolStoppedException extends ServiceUnavailableException { + readonly retryAfterSeconds = QUEUE_FULL_RETRY_AFTER_SECONDS; +} + +export class ImageDiffJobTimeoutError extends Error { + constructor(timeoutMs: number) { + super(`Image diff job timed out after ${timeoutMs} ms`); + this.name = 'ImageDiffJobTimeoutError'; + } +} + +const DEFAULT_JOB_TIMEOUT_MS = 120_000; +const DEFAULT_WORKER_MAX_HEAP_MB = 512; interface Job { input: WorkerJobInput; resolve: (output: WorkerJobOutput) => void; reject: (error: Error) => void; - attempts: number; + crashes: number; + timer?: NodeJS.Timeout; } -/** - * Fixed pool of worker threads for CPU-bound image work — diffing a screenshot - * against its baseline, the change signatures the variations dialog compares, - * and the rendering-noise measurement behind a project's blur-amplitude filter. Keeps the event loop free so the API stays responsive while - * screenshots are decoded and compared. Pool size: DIFF_WORKERS_COUNT env var, - * defaulting to cores - 1 (capped) so the main thread always has a core left. - */ @Injectable() export class DiffWorkerPool implements OnApplicationShutdown { private readonly logger: Logger = new Logger(DiffWorkerPool.name); @@ -37,41 +57,49 @@ export class DiffWorkerPool implements OnApplicationShutdown { 1, Math.min(Number(process.env.DIFF_WORKERS_COUNT) || availableParallelism() - 1, 8) ); - // Queued jobs hold both full image buffers, so an unbounded queue could - // exhaust memory under a flood of concurrent uploads. private readonly queueLimit = Number(process.env.DIFF_QUEUE_LIMIT) || 256; - // The compiled worker file only exists in the built app (dist). Under - // ts-jest / ts-node run the job inline instead. + private readonly jobTimeoutMs = nonNegativeNumber(process.env.DIFF_JOB_TIMEOUT_MS, DEFAULT_JOB_TIMEOUT_MS); + private readonly maxHeapMb = nonNegativeNumber(process.env.DIFF_WORKER_MAX_HEAP_MB, DEFAULT_WORKER_MAX_HEAP_MB); private inline = !existsSync(WORKER_FILE); private workers: Worker[] = []; private idle: Worker[] = []; private queue: Job[] = []; + private ready = new Set(); private inFlight = new Map(); private started = false; private destroyed = false; private spawnFailures = 0; - async run(input: PixelmatchJobInput): Promise; - async run(input: SignatureJobInput): Promise; - async run(input: BlurAmplitudeJobInput): Promise; - async run(input: PatchGateJobInput): Promise; - async run(input: WorkerJobInput): Promise { + async run(input: WorkerJobInputOf): Promise> { + this.assertWithinPixelLimit(input); if (this.inline) { - return runWorkerJob(input); + return (await runWorkerJob(input)) as WorkerJobResult; } if (this.destroyed) { - throw new Error('Image diff worker pool is shut down'); + throw new ImageDiffPoolStoppedException('Image diff worker pool is shut down'); } if (this.queue.length >= this.queueLimit) { - throw new Error(`Image diff queue is full (${this.queueLimit} jobs)`); + throw new ImageDiffQueueFullException(this.queueLimit); } this.start(); - return new Promise((resolve, reject) => { - this.queue.push({ input, resolve, reject, attempts: 0 }); + return new Promise>((resolve, reject) => { + this.queue.push({ input, resolve: resolve as Job['resolve'], reject, crashes: 0 }); this.dispatch(); }); } + // a 69-byte PNG can claim 30000x30000: check IHDR before decoding + private assertWithinPixelLimit(input: WorkerJobInput): void { + const sizes = [input.baseline, input.image].map((bytes) => readPngDimensions(bytes)); + for (const size of sizes) { + if (size) assertWithinPixelLimit(size.width, size.height); + } + const [baseline, image] = sizes; + if (input.kind === 'diff' && input.allowDiffDimensions && baseline && image) { + assertWithinPixelLimit(Math.max(baseline.width, image.width), Math.max(baseline.height, image.height)); + } + } + private start(): void { if (this.started) return; this.started = true; @@ -82,18 +110,31 @@ export class DiffWorkerPool implements OnApplicationShutdown { } private spawn(): void { - const worker = new Worker(WORKER_FILE); + const worker = new Worker( + WORKER_FILE, + this.maxHeapMb > 0 ? { resourceLimits: { maxOldGenerationSizeMb: this.maxHeapMb } } : {} + ); worker.on('message', (output: WorkerJobOutput & { error?: string }) => { + if ((output as { ready?: unknown }).ready === true) { + if (this.workers.includes(worker)) { + this.ready.add(worker); + this.spawnFailures = 0; + } + return; + } const job = this.inFlight.get(worker); this.inFlight.delete(worker); - // a message can arrive after the worker was dropped or the pool shut - // down; taking it back would hand it a job it will never answer + clearTimeout(job?.timer); if (this.workers.includes(worker)) { this.idle.push(worker); this.spawnFailures = 0; } if (job) { - output.error ? job.reject(new Error(output.error)) : job.resolve(output); + if (output.error) { + job.reject(new Error(output.error)); + } else { + job.resolve(output); + } } this.dispatch(); }); @@ -101,26 +142,18 @@ export class DiffWorkerPool implements OnApplicationShutdown { this.logger.error(`Image diff worker crashed: ${error}`); this.replace(worker, error); }); - worker.on('exit', (code) => { - if (code !== 0) { - this.replace(worker, new Error(`Image diff worker exited with code ${code}`)); - } - }); + worker.on('exit', (code) => this.replace(worker, new Error(`Image diff worker exited with code ${code}`))); this.workers.push(worker); this.idle.push(worker); } private replace(worker: Worker, error: Error): void { if (!this.workers.includes(worker)) return; - this.workers = this.workers.filter((w) => w !== worker); - this.idle = this.idle.filter((w) => w !== worker); - const job = this.inFlight.get(worker); - this.inFlight.delete(worker); + const loaded = this.ready.has(worker); + const job = this.forget(worker); if (job) { - // a worker dying takes its job down with it; give the job one more run - // rather than failing the upload waiting behind it - if (!this.destroyed && job.attempts < MAX_JOB_ATTEMPTS) { - job.attempts += 1; + if (loaded) job.crashes += 1; + if (!this.destroyed && job.crashes < MAX_JOB_CRASHES) { this.queue.unshift(job); } else { job.reject(error); @@ -130,62 +163,81 @@ export class DiffWorkerPool implements OnApplicationShutdown { return; } - // a worker script that fails to load fails the same way every time, so - // respawning on each error would spin. Fall back to comparing on this - // thread instead: slower, but the uploads still go through. - this.spawnFailures += 1; - if (this.spawnFailures >= MAX_SPAWN_FAILURES) { - this.inline = true; - this.logger.error( - `Image diff workers failed to run ${this.spawnFailures} times, comparing on the main thread from now on` - ); - this.drainQueueInline(); - return; + if (!loaded) { + this.spawnFailures += 1; + if (this.spawnFailures >= MAX_SPAWN_FAILURES) { + this.inline = true; + this.logger.error( + `Image diff workers failed to load ${this.spawnFailures} times, comparing on the main thread from now on` + ); + this.drainQueueInline(); + return; + } } this.spawn(); this.dispatch(); } - // the pool gave up on workers: answer what is already queued here + private forget(worker: Worker): Job | undefined { + this.ready.delete(worker); + this.workers = this.workers.filter((w) => w !== worker); + this.idle = this.idle.filter((w) => w !== worker); + const job = this.inFlight.get(worker); + this.inFlight.delete(worker); + clearTimeout(job?.timer); + return job; + } + private drainQueueInline(): void { const queued = this.queue; this.queue = []; for (const job of queued) { - try { - job.resolve(runWorkerJob(job.input)); - } catch (error) { - job.reject(error instanceof Error ? error : new Error(String(error))); - } + Promise.resolve() + .then(() => runWorkerJob(job.input)) + .then(job.resolve, (error) => job.reject(error instanceof Error ? error : new Error(String(error)))); } } private dispatch(): void { while (this.idle.length > 0 && this.queue.length > 0) { - // the loop condition guarantees both are there const worker = this.idle.pop()!; const job = this.queue.shift()!; this.inFlight.set(worker, job); worker.postMessage(job.input); + if (this.jobTimeoutMs > 0) { + job.timer = setTimeout(() => this.timeOut(worker, job), this.jobTimeoutMs); + job.timer.unref(); + } } } - // After the HTTP server has stopped and drained its requests, so an upload - // still being compared at shutdown gets its result instead of an error + private timeOut(worker: Worker, job: Job): void { + if (this.inFlight.get(worker) !== job) return; + this.forget(worker); + this.logger.error(`Image diff job ran over ${this.jobTimeoutMs} ms, replacing its worker`); + job.reject(new ImageDiffJobTimeoutError(this.jobTimeoutMs)); + worker.terminate().catch(() => undefined); + this.spawn(); + this.dispatch(); + } + async onApplicationShutdown(): Promise { this.destroyed = true; - const shutdownError = new Error('Image diff worker pool is shutting down'); + const shutdownError = new ImageDiffPoolStoppedException('Image diff worker pool is shutting down'); for (const job of this.queue) { job.reject(shutdownError); } this.queue = []; for (const job of this.inFlight.values()) { + clearTimeout(job.timer); job.reject(shutdownError); } this.inFlight.clear(); const workers = this.workers; this.workers = []; this.idle = []; + this.ready.clear(); await Promise.all(workers.map((worker) => worker.terminate())); } } diff --git a/src/compare/diff-worker-pool.workers.spec.ts b/src/compare/diff-worker-pool.workers.spec.ts index f2526c39..37bf6dcf 100644 --- a/src/compare/diff-worker-pool.workers.spec.ts +++ b/src/compare/diff-worker-pool.workers.spec.ts @@ -4,13 +4,17 @@ import { Logger } from '@nestjs/common'; import { Worker } from 'worker_threads'; import { availableParallelism } from 'os'; import { existsSync } from 'fs'; -import { DiffWorkerPool } from './diff-worker-pool'; +import { + DiffWorkerPool, + ImageDiffJobTimeoutError, + ImageDiffPoolStoppedException, + ImageDiffQueueFullException, +} from './diff-worker-pool'; +import { PNG } from 'pngjs'; +import { ImageTooLargeException } from '../static/utils'; import { runWorkerJob } from './libs/pixelmatch/worker-job'; import { PixelmatchJobInput } from './libs/pixelmatch/pixelmatch.core'; -// The pool only talks to real worker threads once the compiled worker file -// exists (dist). These tests stand in for both: the file "exists", and every -// Worker is an in-process fake the test drives by emitting its events. class FakeWorker extends EventEmitter { postMessage = jest.fn(); terminate = jest.fn().mockResolvedValue(0); @@ -67,7 +71,8 @@ const output = (pixelMisMatchCount: number) => ({ diffPercent: pixelMisMatchCount, }); -// settles a promise into a value the test can inspect without an unhandled rejection +const ready = (worker: FakeWorker) => worker.emit('message', { ready: true }); + const settle = (promise: Promise) => promise.then( (value) => ({ value, error: undefined as Error | undefined }), @@ -80,12 +85,21 @@ describe('DiffWorkerPool with worker threads', () => { let loggerErrorSpy: jest.SpyInstance; let loggerLogSpy: jest.SpyInstance; - const createPool = ({ workers = '1', queueLimit }: { workers?: string; queueLimit?: string } = {}) => { - process.env.DIFF_WORKERS_COUNT = workers; - if (queueLimit === undefined) { - delete process.env.DIFF_QUEUE_LIMIT; - } else { - process.env.DIFF_QUEUE_LIMIT = queueLimit; + const createPool = ({ + workers = '1', + queueLimit, + jobTimeout, + maxHeap, + }: { workers?: string; queueLimit?: string; jobTimeout?: string; maxHeap?: string } = {}) => { + const env = { + DIFF_WORKER_MAX_HEAP_MB: maxHeap, + DIFF_WORKERS_COUNT: workers, + DIFF_JOB_TIMEOUT_MS: jobTimeout, + DIFF_QUEUE_LIMIT: queueLimit, + }; + for (const [name, value] of Object.entries(env)) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; } pool = new DiffWorkerPool(); return pool; @@ -103,7 +117,6 @@ describe('DiffWorkerPool with worker threads', () => { afterEach(async () => { await pool?.onApplicationShutdown(); - // each test makes its own pool; the shutdown above skips a test that made none pool = undefined as unknown as DiffWorkerPool; process.env = { ...originalEnv }; jest.restoreAllMocks(); @@ -114,15 +127,14 @@ describe('DiffWorkerPool with worker threads', () => { ['DIFF_WORKERS_COUNT when set', '3', 4, 3], ['DIFF_WORKERS_COUNT capped at 8', '20', 4, 8], ['cores - 1 when DIFF_WORKERS_COUNT is not a number', 'abc', 4, 3], - // NOTE: current behavior — an explicit "0" is falsy and treated as unset - ['cores - 1 when DIFF_WORKERS_COUNT is 0', '0', 6, 5], + ['cores - 1 when DIFF_WORKERS_COUNT is 0 (no workers is not an option)', '0', 6, 5], ['cores - 1 capped at 8', '', 32, 8], ['at least one worker on a single core', '', 1, 1], ])('uses %s', async (_, workers, cores, expected) => { availableParallelismMock.mockReturnValue(cores); createPool({ workers }); - settle(pool.run(job(1))); + void settle(pool.run(job(1))); expect(WorkerMock).toHaveBeenCalledTimes(expected); }); @@ -132,15 +144,31 @@ describe('DiffWorkerPool with worker threads', () => { expect(WorkerMock).not.toHaveBeenCalled(); - settle(pool.run(job(1))); - settle(pool.run(job(2))); + void settle(pool.run(job(1))); + void settle(pool.run(job(2))); expect(WorkerMock).toHaveBeenCalledTimes(2); - expect(WorkerMock).toHaveBeenCalledWith(join(__dirname, 'libs', 'pixelmatch', 'pixelmatch.worker.js')); + expect(WorkerMock).toHaveBeenCalledWith(join(__dirname, 'libs', 'pixelmatch', 'pixelmatch.worker.js'), { + resourceLimits: { maxOldGenerationSizeMb: 512 }, + }); expect(loggerLogSpy).toHaveBeenCalledWith('Started 2 image diff workers'); }); }); + describe('worker heap limit', () => { + it.each([ + ['takes the limit from DIFF_WORKER_MAX_HEAP_MB', '256', { resourceLimits: { maxOldGenerationSizeMb: 256 } }], + ['drops the limit for 0', '0', {}], + ['falls back to 512 for garbage', 'lots', { resourceLimits: { maxOldGenerationSizeMb: 512 } }], + ])('%s', async (_, maxHeap, options) => { + createPool({ maxHeap }); + + void settle(pool.run(job(1))); + + expect(WorkerMock).toHaveBeenCalledWith(expect.any(String), options); + }); + }); + describe('inline mode', () => { it('runs the job on the calling thread when the compiled worker file is missing', async () => { existsSyncMock.mockReturnValue(false); @@ -170,8 +198,8 @@ describe('DiffWorkerPool with worker threads', () => { it('hands the first job to the most recently spawned idle worker', async () => { createPool({ workers: '2' }); - settle(pool.run(job(1))); - settle(pool.run(job(2))); + void settle(pool.run(job(1))); + void settle(pool.run(job(2))); expect(mockWorkers[1].postMessage).toHaveBeenCalledWith(job(1)); expect(mockWorkers[0].postMessage).toHaveBeenCalledWith(job(2)); @@ -215,13 +243,17 @@ describe('DiffWorkerPool with worker threads', () => { }); describe('queue limit', () => { - it('rejects a job once the waiting queue holds DIFF_QUEUE_LIMIT jobs, not counting the ones in flight', async () => { + it('rejects a job once the waiting queue holds DIFF_QUEUE_LIMIT jobs, not counting the ones in flight, with 503 and a retry hint', async () => { createPool({ queueLimit: '1' }); const inFlight = settle(pool.run(job(1))); const queued = settle(pool.run(job(2))); - await expect(pool.run(job(3))).rejects.toThrow('Image diff queue is full (1 jobs)'); + const { error } = await settle(pool.run(job(3))); + expect(error).toBeInstanceOf(ImageDiffQueueFullException); + expect((error as ImageDiffQueueFullException).getStatus()).toBe(503); + expect((error as ImageDiffQueueFullException).retryAfterSeconds).toBe(5); + expect(error?.message).toBe('Image diff queue is full (1 jobs)'); expect(mockWorkers[0].postMessage).toHaveBeenCalledTimes(1); mockWorkers[0].emit('message', output(1)); @@ -230,6 +262,17 @@ describe('DiffWorkerPool with worker threads', () => { expect((await queued).value).toEqual(output(2)); }); + it('does not queue an oversized job', async () => { + process.env.MAX_IMAGE_PIXELS = '50'; + createPool(); + const big = PNG.sync.write(new PNG({ width: 10, height: 10 })); + + await expect(pool.run({ ...job(1), baseline: big, image: big })).rejects.toBeInstanceOf(ImageTooLargeException); + + expect(WorkerMock).not.toHaveBeenCalled(); + expect(runWorkerJobMock).not.toHaveBeenCalled(); + }); + it('defaults to a queue of 256 waiting jobs', async () => { createPool(); @@ -282,50 +325,107 @@ describe('DiffWorkerPool with worker threads', () => { await expect(result).resolves.toEqual(output(1)); }); - it('keeps a worker that exited with code 0 in the pool', async () => { + it('replaces a worker that exited with code 0', async () => { createPool(); - pool.run(job(1)); + const first = pool.run(job(1)); + ready(mockWorkers[0]); mockWorkers[0].emit('message', output(1)); + await expect(first).resolves.toEqual(output(1)); mockWorkers[0].emit('exit', 0); - const pending = settle(pool.run(job(2))); - - // NOTE: current behavior — a clean exit is ignored, so the exited worker - // stays idle and is handed the next job, which would never be answered. - expect(mockWorkers).toHaveLength(1); - expect(mockWorkers[0].postMessage).toHaveBeenLastCalledWith(job(2)); + const second = pool.run(job(2)); - await pool.onApplicationShutdown(); - expect((await pending).error).toEqual(new Error('Image diff worker pool is shutting down')); + expect(mockWorkers).toHaveLength(2); + expect(mockWorkers[1].postMessage).toHaveBeenCalledWith(job(2)); + expect(mockWorkers[0].postMessage).toHaveBeenCalledTimes(1); + mockWorkers[1].emit('message', output(2)); + await expect(second).resolves.toEqual(output(2)); }); - it('gives a job up with the error of the last worker after it was handed out three times', async () => { + it('runs a job at most twice on loaded workers', async () => { createPool({ workers: '2' }); const target = settle(pool.run(job(1))); const other = pool.run(job(2)); - const [, first] = mockWorkers; + const [first0, first] = mockWorkers; + ready(first0); + ready(first); expect(first.postMessage).toHaveBeenCalledWith(job(1)); first.emit('error', new Error('first')); const second = mockWorkers[2]; + ready(second); expect(second.postMessage).toHaveBeenCalledWith(job(1)); - // a healthy answer in between resets the consecutive failure count - mockWorkers[0].emit('message', output(2)); + first0.emit('message', output(2)); await expect(other).resolves.toEqual(output(2)); second.emit('error', new Error('second')); - const third = mockWorkers[3]; - expect(third.postMessage).toHaveBeenCalledWith(job(1)); - third.emit('error', new Error('third')); + expect((await target).error).toEqual(new Error('second')); + expect(mockWorkers).toHaveLength(4); + expect(mockWorkers[3].postMessage).not.toHaveBeenCalled(); + }); + + it('does not switch to inline mode when a job keeps crashing loaded workers', async () => { + createPool(); + + const poison = settle(pool.run(job(1))); + ready(mockWorkers[0]); + mockWorkers[0].emit('error', new Error('oom 1')); + ready(mockWorkers[1]); + mockWorkers[1].emit('error', new Error('oom 2')); - // NOTE: current behavior — MAX_JOB_ATTEMPTS = 2 reads as "handed to a worker - // at most twice", but the job runs on three workers before it is rejected. - expect((await target).error).toEqual(new Error('third')); - expect(mockWorkers).toHaveLength(5); - expect(mockWorkers[4].postMessage).not.toHaveBeenCalled(); + expect((await poison).error).toEqual(new Error('oom 2')); + + const next = settle(pool.run(job(2))); + expect(runWorkerJobMock).not.toHaveBeenCalled(); + expect(mockWorkers[2].postMessage).toHaveBeenCalledWith(job(2)); + ready(mockWorkers[2]); + mockWorkers[2].emit('message', output(2)); + expect((await next).value).toEqual(output(2)); + }); + + it('does not count a load failure against the job', async () => { + createPool(); + + const result = pool.run(job(1)); + mockWorkers[0].emit('error', new Error('cannot load')); + + expect(mockWorkers[1].postMessage).toHaveBeenCalledWith(job(1)); + ready(mockWorkers[1]); + mockWorkers[1].emit('message', output(1)); + await expect(result).resolves.toEqual(output(1)); + }); + + it('resets the load-failure count once a worker has loaded', async () => { + createPool(); + + void settle(pool.run(job(1))); + mockWorkers[0].emit('error', new Error('load 1')); + mockWorkers[1].emit('error', new Error('load 2')); + ready(mockWorkers[2]); + mockWorkers[2].emit('message', output(1)); + mockWorkers[2].emit('error', new Error('idle crash')); + mockWorkers[3].emit('error', new Error('load 3')); + mockWorkers[4].emit('error', new Error('load 4')); + + expect(runWorkerJobMock).not.toHaveBeenCalled(); + expect(loggerErrorSpy).not.toHaveBeenCalledWith(expect.stringContaining('comparing on the main thread')); + }); + + it('ignores a ready message from a dropped worker', async () => { + createPool(); + + void settle(pool.run(job(1))); + const dropped = mockWorkers[0]; + dropped.emit('error', new Error('load 1')); + dropped.emit('message', { ready: true }); + mockWorkers[1].emit('error', new Error('load 2')); + dropped.emit('message', { ready: true }); + mockWorkers[2].emit('error', new Error('load 3')); + + expect(loggerErrorSpy).toHaveBeenCalledWith(expect.stringContaining('comparing on the main thread')); }); it('rejects with the exit code when the last failure was a non-zero exit', async () => { @@ -333,28 +433,33 @@ describe('DiffWorkerPool with worker threads', () => { const target = settle(pool.run(job(1))); const other = pool.run(job(2)); + ready(mockWorkers[0]); + ready(mockWorkers[1]); mockWorkers[1].emit('exit', 2); mockWorkers[0].emit('message', output(2)); await other; + ready(mockWorkers[2]); mockWorkers[2].emit('exit', 3); - mockWorkers[3].emit('exit', 4); - expect((await target).error).toEqual(new Error('Image diff worker exited with code 4')); + expect((await target).error).toEqual(new Error('Image diff worker exited with code 3')); + expect(mockWorkers[3].postMessage).not.toHaveBeenCalled(); }); - it('falls back to inline after three consecutive failures and answers the queued jobs on this thread', async () => { + it('falls back to inline after three workers fail to load and answers every job on this thread', async () => { createPool(); runWorkerJobMock + .mockReturnValueOnce(output(1)) .mockReturnValueOnce(output(2)) .mockImplementationOnce(() => { throw new Error('inline failed'); }) .mockImplementationOnce(() => { + // eslint-disable-next-line @typescript-eslint/only-throw-error -- pool must survive non-Error throws throw 'not an error'; }) .mockReturnValueOnce(output(5)); - const inFlight = settle(pool.run(job(1))); + const inFlight = pool.run(job(1)); const queued2 = pool.run(job(2)); const queued3 = settle(pool.run(job(3))); const queued4 = settle(pool.run(job(4))); @@ -365,22 +470,40 @@ describe('DiffWorkerPool with worker threads', () => { mockWorkers[2].emit('error', new Error('load failed 3')); expect(loggerErrorSpy).toHaveBeenCalledWith( - 'Image diff workers failed to run 3 times, comparing on the main thread from now on' + 'Image diff workers failed to load 3 times, comparing on the main thread from now on' ); - // the job that was on the failing worker is rejected, the waiting ones run inline - expect((await inFlight).error).toEqual(new Error('load failed 3')); + await expect(inFlight).resolves.toEqual(output(1)); await expect(queued2).resolves.toEqual(output(2)); expect((await queued3).error).toEqual(new Error('inline failed')); expect((await queued4).error).toEqual(new Error('not an error')); - expect(runWorkerJobMock.mock.calls).toEqual([[job(2)], [job(3)], [job(4)]]); + expect(runWorkerJobMock.mock.calls).toEqual([[job(1)], [job(2)], [job(3)], [job(4)]]); expect(mockWorkers).toHaveLength(3); - // and every later job runs inline without spawning anything await expect(pool.run(job(5))).resolves.toEqual(output(5)); expect(runWorkerJobMock).toHaveBeenLastCalledWith(job(5)); expect(mockWorkers).toHaveLength(3); }); + it('answers jobs that finish asynchronously once it runs inline', async () => { + createPool(); + runWorkerJobMock + .mockReturnValueOnce(Promise.reject(new Error('async failed'))) + .mockReturnValueOnce(Promise.resolve(output(9))) + .mockReturnValueOnce(Promise.reject('not an error')); + + const failing = settle(pool.run(job(1))); + const succeeding = pool.run(job(2)); + const rejectingNonError = settle(pool.run(job(3))); + + mockWorkers[0].emit('error', new Error('load failed 1')); + mockWorkers[1].emit('error', new Error('load failed 2')); + mockWorkers[2].emit('error', new Error('load failed 3')); + + expect((await failing).error).toEqual(new Error('async failed')); + await expect(succeeding).resolves.toEqual(output(9)); + expect((await rejectingNonError).error).toEqual(new Error('not an error')); + }); + it('does not run a job again when the worker that already answered it crashes later', async () => { createPool(); @@ -397,19 +520,18 @@ describe('DiffWorkerPool with worker threads', () => { it('takes only the crashed worker out of the idle ones', async () => { createPool({ workers: '3' }); - settle(pool.run(job(1))); + void settle(pool.run(job(1))); const [crashed, idle, busy] = mockWorkers; expect(busy.postMessage).toHaveBeenCalledWith(job(1)); crashed.emit('error', new Error('boom')); const replacement = mockWorkers[3]; - settle(pool.run(job(2))); - settle(pool.run(job(3))); - settle(pool.run(job(4))); + void settle(pool.run(job(2))); + void settle(pool.run(job(3))); + void settle(pool.run(job(4))); expect(replacement.postMessage).toHaveBeenCalledWith(job(2)); expect(idle.postMessage).toHaveBeenCalledWith(job(3)); - // job 4 waits: the crashed worker is never handed anything expect(crashed.postMessage).not.toHaveBeenCalled(); }); @@ -424,7 +546,6 @@ describe('DiffWorkerPool with worker threads', () => { dropped.emit('message', output(99)); const next = pool.run(job(2)); - // the late answer neither settles the retried job nor brings the worker back expect(dropped.postMessage).toHaveBeenCalledTimes(1); expect(replacement.postMessage).toHaveBeenCalledTimes(1); @@ -437,6 +558,117 @@ describe('DiffWorkerPool with worker threads', () => { }); }); + describe('job timeout', () => { + beforeEach(() => { + jest.useFakeTimers(); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + it('rejects a job that runs longer than DIFF_JOB_TIMEOUT_MS and replaces its worker', async () => { + createPool({ jobTimeout: '1000' }); + + const hung = settle(pool.run(job(1))); + ready(mockWorkers[0]); + jest.advanceTimersByTime(1000); + + expect((await hung).error).toBeInstanceOf(ImageDiffJobTimeoutError); + expect((await hung).error?.message).toBe('Image diff job timed out after 1000 ms'); + expect(mockWorkers[0].terminate).toHaveBeenCalledTimes(1); + expect(mockWorkers).toHaveLength(2); + expect(loggerErrorSpy).toHaveBeenCalledWith('Image diff job ran over 1000 ms, replacing its worker'); + + void settle(pool.run(job(2))); + expect(mockWorkers[1].postMessage).toHaveBeenCalledWith(job(2)); + }); + + it('does not time out a job answered in time', async () => { + createPool({ jobTimeout: '1000' }); + + const result = pool.run(job(1)); + ready(mockWorkers[0]); + jest.advanceTimersByTime(999); + mockWorkers[0].emit('message', output(1)); + jest.advanceTimersByTime(5000); + + await expect(result).resolves.toEqual(output(1)); + expect(mockWorkers[0].terminate).not.toHaveBeenCalled(); + expect(mockWorkers).toHaveLength(1); + }); + + it('defaults to 120 s', async () => { + createPool(); + + const result = settle(pool.run(job(1))); + jest.advanceTimersByTime(119_999); + expect(mockWorkers[0].terminate).not.toHaveBeenCalled(); + jest.advanceTimersByTime(1); + + expect((await result).error).toBeInstanceOf(ImageDiffJobTimeoutError); + }); + + it.each([ + ['turns off with 0', '0'], + ['falls back to the default for garbage', 'abc'], + ['falls back to the default for a negative number', '-5'], + ['falls back to the default for an empty value', ''], + ])('%s', async (title, jobTimeout) => { + createPool({ jobTimeout }); + + const result = settle(pool.run(job(1))); + jest.advanceTimersByTime(600_000); + + if (jobTimeout === '0') { + expect(mockWorkers[0].terminate).not.toHaveBeenCalled(); + mockWorkers[0].emit('message', output(1)); + expect((await result).value).toEqual(output(1)); + } else { + expect((await result).error).toBeInstanceOf(ImageDiffJobTimeoutError); + expect((await result).error?.message).toBe('Image diff job timed out after 120000 ms'); + } + }); + + it('ignores the exit of the worker it terminated', async () => { + createPool({ jobTimeout: '1000' }); + + const hung = settle(pool.run(job(1))); + jest.advanceTimersByTime(1000); + await hung; + mockWorkers[0].emit('exit', 1); + mockWorkers[0].emit('error', new Error('terminated')); + + expect(mockWorkers).toHaveLength(2); + }); + + it('does not retry the timed-out job and does not count it as a load failure', async () => { + createPool({ jobTimeout: '1000' }); + + for (const id of [1, 2, 3]) { + const hung = settle(pool.run(job(id))); + jest.advanceTimersByTime(1000); + expect((await hung).error).toBeInstanceOf(ImageDiffJobTimeoutError); + } + + expect(runWorkerJobMock).not.toHaveBeenCalled(); + expect(mockWorkers).toHaveLength(4); + expect(mockWorkers.map((worker) => worker.postMessage.mock.calls.length)).toEqual([1, 1, 1, 0]); + }); + + it('does not spawn a replacement for a pool that is shut down while the job runs', async () => { + createPool({ jobTimeout: '1000' }); + + const pending = settle(pool.run(job(1))); + await pool.onApplicationShutdown(); + jest.advanceTimersByTime(5000); + + expect((await pending).error).toBeInstanceOf(ImageDiffPoolStoppedException); + expect(mockWorkers).toHaveLength(1); + expect(mockWorkers[0].terminate).toHaveBeenCalledTimes(1); + }); + }); + describe('onApplicationShutdown', () => { it('rejects queued and in-flight jobs and terminates every worker', async () => { createPool({ workers: '2' }); @@ -455,13 +687,20 @@ describe('DiffWorkerPool with worker threads', () => { expect(mockWorkers.map((worker) => worker.terminate.mock.calls.length)).toEqual([1, 1]); }); - it('refuses new jobs once shut down', async () => { + it('rejects with 503 once shut down', async () => { createPool(); - settle(pool.run(job(1))); + const queued = [1, 2].map((id) => settle(pool.run(job(id)))); await pool.onApplicationShutdown(); - await expect(pool.run(job(2))).rejects.toThrow('Image diff worker pool is shut down'); + for (const { error } of await Promise.all(queued)) { + expect(error).toBeInstanceOf(ImageDiffPoolStoppedException); + } + const { error } = await settle(pool.run(job(3))); + expect(error).toBeInstanceOf(ImageDiffPoolStoppedException); + expect(error?.message).toBe('Image diff worker pool is shut down'); + expect((error as ImageDiffPoolStoppedException).getStatus()).toBe(503); + expect((error as ImageDiffPoolStoppedException).retryAfterSeconds).toBe(5); expect(mockWorkers).toHaveLength(1); }); diff --git a/src/compare/libs/ImageCompareInput.ts b/src/compare/libs/ImageCompareInput.ts index 48dee42a..1f8a7c0e 100644 --- a/src/compare/libs/ImageCompareInput.ts +++ b/src/compare/libs/ImageCompareInput.ts @@ -2,16 +2,11 @@ import { IgnoreAreaDto } from '../../test-runs/dto/ignore-area.dto'; import { BlurAmplitudeOptions } from './pixelmatch/blur-amplitude.core'; export interface ImageCompareInput { - // null when the variation has no baseline yet; comparators then report NO_BASELINE_RESULT baseline: string | null; image: string; + imageBuffer?: Buffer; diffTollerancePercent: number; ignoreAreas: IgnoreAreaDto[]; saveDiffAsFile: boolean; - /** - * The project's rendering-noise filter, when it has one on. Read only by a - * comparator that applies it itself (see ImageComparator); for the others - * CompareService applies it to their verdict. - */ blurAmplitude?: BlurAmplitudeOptions; } diff --git a/src/compare/libs/blur-amplitude.config.spec.ts b/src/compare/libs/blur-amplitude.config.spec.ts index f5e41c8b..3a49fa39 100644 --- a/src/compare/libs/blur-amplitude.config.spec.ts +++ b/src/compare/libs/blur-amplitude.config.spec.ts @@ -1,5 +1,5 @@ import { Logger } from '@nestjs/common'; -import { DEFAULT_BLUR_AMPLITUDE_THRESHOLD, DEFAULT_BLUR_SIGMA } from './pixelmatch/blur-amplitude.core'; +import { DEFAULT_BLUR_AMPLITUDE_THRESHOLD } from './pixelmatch/blur-amplitude.core'; import { DEFAULT_DINOV2_THRESHOLD, MAX_BLUR_AMPLITUDE_THRESHOLD, @@ -24,10 +24,9 @@ describe('splitBlurAmplitudeConfig', () => { it('turns it on at the default threshold', () => { const { blurAmplitude } = splitBlurAmplitudeConfig({ blurAmplitudeFilter: true }, logger); - expect(blurAmplitude).toEqual({ sigma: DEFAULT_BLUR_SIGMA, threshold: DEFAULT_BLUR_AMPLITUDE_THRESHOLD }); + expect(blurAmplitude).toEqual({ threshold: DEFAULT_BLUR_AMPLITUDE_THRESHOLD }); }); - // anything near 255 would pass almost every change of one size it('accepts a threshold up to half the contrast range', () => { const { blurAmplitude } = splitBlurAmplitudeConfig( { blurAmplitudeFilter: true, blurAmplitudeThreshold: MAX_BLUR_AMPLITUDE_THRESHOLD }, @@ -44,11 +43,9 @@ describe('splitBlurAmplitudeConfig', () => { logger ); - expect(blurAmplitude).toEqual({ sigma: DEFAULT_BLUR_SIGMA, threshold: 60.5 }); + expect(blurAmplitude).toEqual({ threshold: 60.5 }); }); - // the config is a JSON string nothing validates on the way in; only a real - // `true` turns on something that can pass runs it.each([['true'], [1], [false]])('stays off for blurAmplitudeFilter %p', (value) => { expect(splitBlurAmplitudeConfig({ blurAmplitudeFilter: value as boolean }, logger).blurAmplitude).toBeUndefined(); }); @@ -66,8 +63,6 @@ describe('splitBlurAmplitudeConfig', () => { } ); - // the keys are CompareService's, not the library's: looks-same, odiff and the - // VLM providers get their options passed straight through it("keeps the filter's keys out of the comparator's config", () => { const { comparatorConfig } = splitBlurAmplitudeConfig( { strict: false, blurAmplitudeFilter: false, blurAmplitudeThreshold: 70 }, @@ -98,8 +93,6 @@ describe('splitBlurAmplitudeConfig with the DINOv2 check', () => { expect(blurAmplitude?.dinov2).toEqual({ threshold: 0.3 }); }); - // it double-checks what the blur would pass; with no blur filter there is - // nothing for it to check it('has no effect without the blur filter', () => { const { blurAmplitude } = splitBlurAmplitudeConfig({ dinov2Check: true }, logger); diff --git a/src/compare/libs/blur-amplitude.config.ts b/src/compare/libs/blur-amplitude.config.ts index 612c3659..aa34d443 100644 --- a/src/compare/libs/blur-amplitude.config.ts +++ b/src/compare/libs/blur-amplitude.config.ts @@ -1,42 +1,19 @@ import { Logger } from '@nestjs/common'; -import { - BlurAmplitudeOptions, - DEFAULT_BLUR_AMPLITUDE_THRESHOLD, - DEFAULT_BLUR_SIGMA, -} from './pixelmatch/blur-amplitude.core'; +import { BlurAmplitudeOptions, DEFAULT_BLUR_AMPLITUDE_THRESHOLD } from './pixelmatch/blur-amplitude.core'; -/** - * The keys an image-comparison config carries to switch on the rendering-noise - * filter (see pixelmatch/blur-amplitude.core.ts). A library that supports the - * filter extends its config type with these; CompareService reads them and - * strips them, so the library itself never sees them. - */ export interface BlurAmplitudeConfig { - // pass a failing comparison whose blurred change is no stronger than noise blurAmplitudeFilter?: boolean; - // strongest blurred difference (0-128) still treated as noise blurAmplitudeThreshold?: number; - // ask the DINOv2 service before passing a change the blur calls noise dinov2Check?: boolean; - // largest content distance (0-2, cosine) still treated as noise dinov2Threshold?: number; } -// The blurred difference never exceeds 255, so a threshold near it would pass -// almost any change of one size; half the contrast range is the useful ceiling. export const MAX_BLUR_AMPLITUDE_THRESHOLD = 128; -// Above the strongest of 57 anti-aliasing-only pairs of real VRT pages (0.133) -// in the research dataset, measured the way Dinov2Service measures. +// strongest of 57 anti-aliasing-only real pairs: 0.133 export const DEFAULT_DINOV2_THRESHOLD = 0.15; const MAX_DINOV2_THRESHOLD = 2; -/** - * Splits a parsed config into the filter's settings — absent when it is off — - * and the config the comparator gets. The config is a JSON string nothing - * validates on the way in, so a threshold that is not a number in range falls - * back to the default rather than letting a typo pass everything. - */ export function splitBlurAmplitudeConfig( config: T, logger: Logger @@ -60,8 +37,7 @@ export function splitBlurAmplitudeConfig( ); } } - // the threshold was calibrated at this sigma; it is not a setting of its own - const blurAmplitude: BlurAmplitudeOptions = { sigma: DEFAULT_BLUR_SIGMA, threshold }; + const blurAmplitude: BlurAmplitudeOptions = { threshold }; if (dinov2Check === true) { blurAmplitude.dinov2 = { threshold: dinov2ThresholdOf(dinov2Threshold, logger) }; } @@ -72,7 +48,6 @@ function dinov2ThresholdOf(value: unknown, logger: Logger): number { if (value === undefined) { return DEFAULT_DINOV2_THRESHOLD; } - // a distance of 0 would treat every change as content and pass nothing if (typeof value === 'number' && value > 0 && value <= MAX_DINOV2_THRESHOLD) { return value; } diff --git a/src/compare/libs/dinov2/dinov2.controller.spec.ts b/src/compare/libs/dinov2/dinov2.controller.spec.ts index 73dcba80..12aca2ef 100644 --- a/src/compare/libs/dinov2/dinov2.controller.spec.ts +++ b/src/compare/libs/dinov2/dinov2.controller.spec.ts @@ -35,7 +35,6 @@ describe('Dinov2Controller', () => { expect(guards()).toContain(JwtAuthGuard); }); - // the people who can edit a project's config it('is limited to admins and editors', () => { expect(guards()).toContain(RoleGuard); expect(new Reflector().getAllAndOverride('roles', [Dinov2Controller.prototype.status, Dinov2Controller])).toEqual([ diff --git a/src/compare/libs/dinov2/dinov2.controller.ts b/src/compare/libs/dinov2/dinov2.controller.ts index 470399e4..1884d8a4 100644 --- a/src/compare/libs/dinov2/dinov2.controller.ts +++ b/src/compare/libs/dinov2/dinov2.controller.ts @@ -14,7 +14,6 @@ import { Dinov2Service, Dinov2Status } from './dinov2.service'; export class Dinov2Controller { constructor(private readonly dinov2Service: Dinov2Service) {} - // what the project form offers: the check only where a service is set up @Get('status') status(): Promise { return this.dinov2Service.status(); diff --git a/src/compare/libs/dinov2/dinov2.service.spec.ts b/src/compare/libs/dinov2/dinov2.service.spec.ts index c510ba32..e860641e 100644 --- a/src/compare/libs/dinov2/dinov2.service.spec.ts +++ b/src/compare/libs/dinov2/dinov2.service.spec.ts @@ -1,9 +1,7 @@ import { Test, TestingModule } from '@nestjs/testing'; import { ConfigService } from '@nestjs/config'; -import { DiffWorkerPool } from '../../diff-worker-pool'; import { blobScore, Dinov2Service, tilesOf } from './dinov2.service'; -// a 32 x 20 patch grid: two tile columns, two tile rows (16 x 16 patches each) const WIDTH = 32; const HEIGHT = 20; const at = (row: number, col: number) => row * WIDTH + col; @@ -11,7 +9,6 @@ const at = (row: number, col: number) => row * WIDTH + col; const grid = (values: Record) => Array.from({ length: WIDTH * HEIGHT }, (_, i) => values[i] ?? 0); describe('blobScore', () => { - // a lone patch is how noise looks: one changed patch, nothing beside it it('ignores a single changed patch', () => { const cells = [at(3, 3)]; @@ -24,8 +21,6 @@ describe('blobScore', () => { expect(blobScore(grid({ [at(3, 3)]: 0.9, [at(3, 4)]: 0.4 }), WIDTH, HEIGHT, cells)).toBeCloseTo(0.4); }); - // only where pixelmatch saw a change: elsewhere the distance is context - // bleeding across a tile, not a change it('counts only the patches pixelmatch marked', () => { const distances = grid({ [at(3, 3)]: 0.9, [at(3, 4)]: 0.9 }); @@ -69,53 +64,47 @@ describe('Dinov2Service', () => { json: async () => ({ patch: 14, width: WIDTH, height: HEIGHT, distances: grid(values) }), }); - const init = async ({ url = 'http://dinov2:8000', poolOutput = { gate } as object } = {}) => { - const run = jest.fn().mockResolvedValue(poolOutput); + const init = async (env: Record = { DINOV2_URL: 'http://dinov2:8000' }) => { const module: TestingModule = await Test.createTestingModule({ - providers: [ - Dinov2Service, - { provide: ConfigService, useValue: { get: jest.fn().mockReturnValue(url) } }, - { provide: DiffWorkerPool, useValue: { run } }, - ], + providers: [Dinov2Service, { provide: ConfigService, useValue: { get: (key: string) => env[key] } }], }).compile(); - return { service: module.get(Dinov2Service), run }; + return { service: module.get(Dinov2Service) }; }; const fetchMock = jest.spyOn(global, 'fetch'); afterEach(() => fetchMock.mockReset()); - const confirm = (service: Dinov2Service) => - service.confirmsNoise(Buffer.from('baseline'), Buffer.from('image'), [{ x: 1, y: 2, width: 3, height: 4 }], 0.15); + const confirm = (service: Dinov2Service, given = gate) => + service.confirmsNoiseForGate(given, Buffer.from('baseline'), Buffer.from('image'), 0.15); - it('confirms noise when the content did not change', async () => { - const { service, run } = await init(); - fetchMock.mockResolvedValue(answer({ [at(3, 3)]: 0.05, [at(3, 4)]: 0.04 }) as Response); - - expect(await confirm(service)).toEqual({ noise: true, score: 0.04 }); - expect(run).toHaveBeenCalledWith({ - kind: 'patch-gate', - baseline: Buffer.from('baseline'), - image: Buffer.from('image'), - ignoreAreas: [{ x: 1, y: 2, width: 3, height: 4 }], - patch: 14, - }); - }); + it('takes the service address with or without a trailing slash, and its own timeout', async () => { + const { service } = await init({ DINOV2_URL: 'http://dinov2:8000/', DINOV2_TIMEOUT_MS: '5000' }); + const timeout = jest.spyOn(AbortSignal, 'timeout'); + fetchMock.mockResolvedValue(answer({}) as Response); - it('sees a change in content', async () => { - const { service } = await init(); - fetchMock.mockResolvedValue(answer({ [at(3, 3)]: 0.6, [at(3, 4)]: 0.5 }) as Response); + await confirm(service); - expect(await confirm(service)).toEqual({ noise: false, score: 0.5 }); + expect(fetchMock.mock.calls[0][0]).toBe('http://dinov2:8000/v1/patch-distances'); + expect(timeout).toHaveBeenCalledWith(5000); + timeout.mockRestore(); }); - // only the tiles holding a marked patch are encoded: identical result, - // a fraction of the time - it('asks the service for the marked tiles only', async () => { - const { service } = await init(); + it('falls back to its default timeout for a negative DINOV2_TIMEOUT_MS', async () => { + const { service } = await init({ DINOV2_URL: 'http://dinov2:8000', DINOV2_TIMEOUT_MS: '-1' }); + const timeout = jest.spyOn(AbortSignal, 'timeout'); fetchMock.mockResolvedValue(answer({}) as Response); await confirm(service); + expect(timeout).toHaveBeenCalledWith(60_000); + timeout.mockRestore(); + }); + + it('confirms noise when the content did not change', async () => { + const { service } = await init(); + fetchMock.mockResolvedValue(answer({ [at(3, 3)]: 0.05, [at(3, 4)]: 0.04 }) as Response); + + expect(await confirm(service)).toEqual({ noise: true, score: 0.04 }); const [url, request] = fetchMock.mock.calls[0]; expect(url).toBe('http://dinov2:8000/v1/patch-distances'); expect(JSON.parse(request?.body as string)).toEqual({ @@ -125,37 +114,50 @@ describe('Dinov2Service', () => { }); }); - it('takes the service address with or without a trailing slash, and its own timeout', async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [ - Dinov2Service, - { - provide: ConfigService, - useValue: { get: (key: string) => ({ DINOV2_URL: 'http://dinov2:8000/', DINOV2_TIMEOUT_MS: '5000' })[key] }, - }, - { provide: DiffWorkerPool, useValue: { run: jest.fn().mockResolvedValue({ gate }) } }, - ], - }).compile(); - const timeout = jest.spyOn(AbortSignal, 'timeout'); - fetchMock.mockResolvedValue(answer({}) as Response); + it('sees a change in content', async () => { + const { service } = await init(); + fetchMock.mockResolvedValue(answer({ [at(3, 3)]: 0.6, [at(3, 4)]: 0.5 }) as Response); - await confirm(module.get(Dinov2Service)); + expect(await confirm(service)).toEqual({ noise: false, score: 0.5 }); + }); - expect(fetchMock.mock.calls[0][0]).toBe('http://dinov2:8000/v1/patch-distances'); - expect(timeout).toHaveBeenCalledWith(5000); - timeout.mockRestore(); + it('confirms noise without a call when no patch is marked', async () => { + const { service } = await init(); + + expect(await confirm(service, { ...gate, cells: [] })).toEqual({ noise: true, score: 0 }); + expect(fetchMock).not.toHaveBeenCalled(); }); - // nothing but anti-aliasing changed: nothing for DINOv2 to look at - it('confirms noise without a call when pixelmatch marked no patch', async () => { - const { service } = await init({ poolOutput: { gate: { ...gate, cells: [] } } }); + it('does not confirm noise without a service configured', async () => { + const { service } = await init({ DINOV2_URL: '' }); - expect(await confirm(service)).toEqual({ noise: true, score: 0 }); + expect((await confirm(service)).noise).toBe(false); expect(fetchMock).not.toHaveBeenCalled(); }); + it('does not confirm noise when the answer does not fit the screenshots', async () => { + const { service } = await init(); + fetchMock.mockResolvedValue({ + ok: true, + json: async () => ({ patch: 14, width: 3, height: 3, distances: [0, 0, 0, 0, 0, 0, 0, 0, 0] }), + } as Response); + + expect((await confirm(service)).noise).toBe(false); + }); + + it('does not confirm noise when the service fails or is unreachable', async () => { + const { service } = await init(); + fetchMock.mockResolvedValueOnce({ ok: false, status: 500, json: async () => ({}) } as Response); + fetchMock.mockRejectedValueOnce(new TypeError('fetch failed')); + fetchMock.mockRejectedValueOnce('socket hang up'); + + expect((await confirm(service)).noise).toBe(false); + expect((await confirm(service)).noise).toBe(false); + expect((await confirm(service)).noise).toBe(false); + }); + describe('status', () => { - const status = async (url: string) => (await init({ url })).service.status(); + const status = async (url: string) => (await init({ DINOV2_URL: url })).service.status(); it('is not configured without a service address', async () => { expect(await status('')).toEqual({ configured: false, available: false }); @@ -175,66 +177,13 @@ describe('Dinov2Service', () => { expect(await status('http://dinov2:8000')).toEqual({ configured: true, available: false }); }); - it('is configured but not available when the service is unreachable', async () => { - fetchMock.mockRejectedValue(new TypeError('fetch failed')); + it.each([ + ['the service is unreachable', new TypeError('fetch failed')], + ['the check throws something that is not an Error', 'socket hang up'], + ])('is configured but not available when %s', async (_, failure) => { + fetchMock.mockRejectedValue(failure); expect(await status('http://dinov2:8000')).toEqual({ configured: true, available: false }); }); - - it('is configured but not available when the check throws something that is not an Error', async () => { - fetchMock.mockRejectedValue('socket hang up'); - - expect(await status('http://dinov2:8000')).toEqual({ configured: true, available: false }); - }); - }); - - // the check exists to stop the blur passing a real change; when it cannot - // check, the run goes to review rather than through - describe('when it cannot check', () => { - it('does not confirm noise without a service configured', async () => { - // an unset variable reads as empty - const { service } = await init({ url: '' }); - - expect((await confirm(service)).noise).toBe(false); - expect(fetchMock).not.toHaveBeenCalled(); - }); - - it('does not confirm noise when the service fails', async () => { - const { service } = await init(); - fetchMock.mockResolvedValue({ ok: false, status: 500, json: async () => ({}) } as Response); - - expect((await confirm(service)).noise).toBe(false); - }); - - it('does not confirm noise when the check throws something that is not an Error', async () => { - const { service } = await init(); - fetchMock.mockRejectedValue('socket hang up'); - - expect((await confirm(service)).noise).toBe(false); - }); - - it('does not confirm noise when the service is unreachable', async () => { - const { service } = await init(); - fetchMock.mockRejectedValue(new TypeError('fetch failed')); - - expect((await confirm(service)).noise).toBe(false); - }); - - it('does not confirm noise when the answer does not fit the screenshots', async () => { - const { service } = await init(); - fetchMock.mockResolvedValue({ - ok: true, - json: async () => ({ patch: 14, width: 3, height: 3, distances: [0, 0, 0, 0, 0, 0, 0, 0, 0] }), - } as Response); - - expect((await confirm(service)).noise).toBe(false); - }); - - it('does not confirm noise when the screenshots cannot be lined up', async () => { - const { service } = await init({ poolOutput: { gate: null } }); - - expect((await confirm(service)).noise).toBe(false); - expect(fetchMock).not.toHaveBeenCalled(); - }); }); }); diff --git a/src/compare/libs/dinov2/dinov2.service.ts b/src/compare/libs/dinov2/dinov2.service.ts index 9d395341..30d6ef53 100644 --- a/src/compare/libs/dinov2/dinov2.service.ts +++ b/src/compare/libs/dinov2/dinov2.service.ts @@ -1,36 +1,22 @@ import { Injectable, Logger } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; -import { IgnoreAreaDto } from '../../../test-runs/dto/ignore-area.dto'; -import { DiffWorkerPool } from '../../diff-worker-pool'; +import { PatchGate } from '../pixelmatch/patch-gate.core'; +import { positiveNumber } from '../../../utils/env'; -// the model's patch size, and the patches along one side of a 224 px tile -const PATCH = 14; const TILE_PATCHES = 16; const DEFAULT_TIMEOUT_MS = 60_000; -// the status is asked for while a form opens: a service this slow to say it -// is up counts as down const STATUS_TIMEOUT_MS = 3_000; export interface Dinov2Status { - // DINOV2_URL is set: a project may turn the check on configured: boolean; - // the service answered its health check just now available: boolean; } export interface Dinov2Verdict { - // true only when DINOv2 looked and saw no change of content noise: boolean; - // the strongest content change where pixelmatch saw one, when it looked score?: number; } -/** - * The strongest content distance among the patches pixelmatch marked, counting - * a patch only when a marked neighbour (up, down, left, right) changed too: - * a change worth a reviewer covers more than one 14 px patch, a stray patch is - * how anti-aliasing shows up. The score is the weaker of the pair. - */ export function blobScore(distances: number[], width: number, height: number, cells: number[]): number { const marked = new Set(cells); const at = (row: number, col: number) => @@ -47,7 +33,6 @@ export function blobScore(distances: number[], width: number, height: number, ce return score; } -/** The 224 px tiles, as [row, col], that hold at least one marked patch. */ export function tilesOf(cells: number[], width: number): [number, number][] { const tiles = new Map(); for (const cell of cells) { @@ -60,30 +45,12 @@ export function tilesOf(cells: number[], width: number): [number, number][] { return [...tiles.values()].sort((a, b) => a[0] - b[0] || a[1] - b[1]); } -/** - * A second look at a change the blur filter would pass as rendering noise: - * the DINOv2 service compares the two screenshots' content patch by patch, - * which catches what is faint in pixels but clear in meaning — a missing light - * icon, a removed input border. Calibrated in research/jepa-screenshot-diff. - * - * The check exists to stop the filter passing a real change, so whenever it - * cannot look — no service configured, the service down, screenshots that do - * not line up — it does not confirm noise and the run goes to review. - */ @Injectable() export class Dinov2Service { private readonly logger: Logger = new Logger(Dinov2Service.name); - constructor( - private readonly configService: ConfigService, - private readonly diffWorkerPool: DiffWorkerPool - ) {} + constructor(private readonly configService: ConfigService) {} - /** - * Whether this server can run the check, for the project form: a project - * that turns it on without a service sends every run the filter would pass - * to review. - */ async status(): Promise { const url = this.serviceUrl(); if (!url) { @@ -98,10 +65,10 @@ export class Dinov2Service { } } - async confirmsNoise( + async confirmsNoiseForGate( + gate: PatchGate, baseline: Buffer, image: Buffer, - ignoreAreas: IgnoreAreaDto[], threshold: number ): Promise { const url = this.serviceUrl(); @@ -110,17 +77,6 @@ export class Dinov2Service { return { noise: false }; } try { - const { gate } = await this.diffWorkerPool.run({ - kind: 'patch-gate', - baseline, - image, - ignoreAreas, - patch: PATCH, - }); - if (!gate) { - return { noise: false }; - } - // nothing but anti-aliasing changed: nothing for the model to look at if (gate.cells.length === 0) { return { noise: true, score: 0 }; } @@ -131,11 +87,9 @@ export class Dinov2Service { body: JSON.stringify({ baseline: baseline.toString('base64'), image: image.toString('base64'), - // only the tiles that hold a marked patch: tiles are encoded - // independently, so the others would cost time and change nothing tiles: tilesOf(gate.cells, gate.width), }), - signal: AbortSignal.timeout(Number(this.configService.get('DINOV2_TIMEOUT_MS')) || DEFAULT_TIMEOUT_MS), + signal: AbortSignal.timeout(positiveNumber(this.configService.get('DINOV2_TIMEOUT_MS'), DEFAULT_TIMEOUT_MS)), }); if (!response.ok) { throw new Error(`service answered ${response.status}`); @@ -150,16 +104,16 @@ export class Dinov2Service { const score = blobScore(answer.distances, gate.width, gate.height, gate.cells); return { noise: score <= threshold, score }; } catch (error) { - this.logger.warn( - // read as an Error, not checked with instanceof: fetch errors come - // from another realm under Jest - `DINOv2 check failed, the run goes to review: ${(error as Error)?.message ?? error}` - ); - return { noise: false }; + return this.failed(error); } } - // an unset variable reads as empty; a trailing slash is the operator's choice + private failed(error: unknown): Dinov2Verdict { + // a cast, not instanceof: fetch errors come from another realm under Jest + this.logger.warn(`DINOv2 check failed, the run goes to review: ${(error as Error)?.message ?? error}`); + return { noise: false }; + } + private serviceUrl(): string | undefined { return this.configService.get('DINOV2_URL')?.replace(/\/$/, '') || undefined; } diff --git a/src/compare/libs/image-comparator.interface.ts b/src/compare/libs/image-comparator.interface.ts index bb28b043..8ff42a0a 100644 --- a/src/compare/libs/image-comparator.interface.ts +++ b/src/compare/libs/image-comparator.interface.ts @@ -8,12 +8,6 @@ import { VlmConfig } from './vlm/vlm.types'; export type ImageCompareConfig = PixelmatchConfig | LooksSameConfig | OdiffConfig | VlmConfig; export interface ImageComparator { - /** - * True when getDiff applies `data.blurAmplitude` itself — cheaper inside a - * pass that has the images decoded already, and before anything expensive - * (a diff written to storage, a VLM call). Otherwise CompareService measures - * the pair afterwards and may overturn a failing verdict. - */ readonly appliesBlurAmplitudeFilter?: boolean; getDiff(data: ImageCompareInput, config: ImageCompareConfig): Promise; parseConfig(configJson: string): ImageCompareConfig; diff --git a/src/compare/libs/looks-same/looks-same.core.spec.ts b/src/compare/libs/looks-same/looks-same.core.spec.ts new file mode 100644 index 00000000..08fdc754 --- /dev/null +++ b/src/compare/libs/looks-same/looks-same.core.spec.ts @@ -0,0 +1,177 @@ +import { readFileSync } from 'fs'; +import { join } from 'path'; +import { PNG } from 'pngjs'; +import { pngBytes, solidPng } from '../../test-utils'; +import { computeLooksSame, LOOKS_SAME_DEFAULT_CONFIG, LooksSameJobInput } from './looks-same.core'; + +const fixture = (name: string) => readFileSync(join(__dirname, '../../../../test', name)); + +const filled = (width: number, height: number, value: number): Buffer => + pngBytes(solidPng(width, height, [value, value, value])); + +const job = (overrides: Partial = {}): LooksSameJobInput => ({ + kind: 'looks-same', + baseline: fixture('image.png'), + image: fixture('image_edited.png'), + ignoreAreas: [], + config: LOOKS_SAME_DEFAULT_CONFIG, + saveDiff: false, + ...overrides, +}); + +describe('computeLooksSame', () => { + it('reports an unreadable baseline', async () => { + await expect(computeLooksSame(job({ baseline: Buffer.from('nope') }))).resolves.toStrictEqual({ + outcome: 'baseline-unreadable', + }); + }); + + it('reports an unreadable image', async () => { + await expect(computeLooksSame(job({ image: Buffer.from('nope') }))).resolves.toStrictEqual({ + outcome: 'image-unreadable', + }); + }); + + it('reports a baseline that is unreadable before an unreadable image', async () => { + const output = await computeLooksSame(job({ baseline: Buffer.from('a'), image: Buffer.from('b') })); + + expect(output).toStrictEqual({ outcome: 'baseline-unreadable' }); + }); + + it('does not call a 60x30 and a 30x60 image of the same pixel bytes equal', async () => { + const output = await computeLooksSame(job({ baseline: filled(60, 30, 128), image: filled(30, 60, 128) })); + + expect(output).toStrictEqual({ outcome: 'diff-dimension' }); + }); + + it('compares them when other sizes are allowed', async () => { + const output = await computeLooksSame( + job({ + baseline: filled(60, 30, 128), + image: filled(30, 60, 128), + config: { ...LOOKS_SAME_DEFAULT_CONFIG, allowDiffDimensions: true }, + }) + ); + + expect(output).toMatchObject({ outcome: 'compared', isSameDimension: false }); + }); + + it('answers equal for the same pixels', async () => { + const output = await computeLooksSame(job({ baseline: filled(10, 10, 7), image: filled(10, 10, 7) })); + + expect(output).toStrictEqual({ outcome: 'equal' }); + }); + + it('counts the different pixels of the fixtures', async () => { + const output = await computeLooksSame(job()); + + expect(output).toStrictEqual({ + outcome: 'compared', + isSameDimension: true, + result: { equal: false, differentPixels: 31068, totalPixels: 129600 }, + errors: [], + }); + }); + + it('returns the diff image when asked to', async () => { + const output = await computeLooksSame(job({ saveDiff: true })); + + expect(output.outcome).toBe('compared'); + const diffPng = (output as { diffPng?: Uint8Array }).diffPng!; + expect(diffPng).toBeInstanceOf(Uint8Array); + const diff = PNG.sync.read(Buffer.from(diffPng)); + expect([diff.width, diff.height]).toEqual([360, 360]); + }); + + it('returns no diff image for an equal result even when asked to', async () => { + const baseline = filled(10, 10, 255); + const changed = PNG.sync.read(baseline); + changed.data[0] = 0; + + const output = await computeLooksSame( + job({ + baseline, + image: PNG.sync.write(changed), + ignoreAreas: [{ x: 0, y: 0, width: 1, height: 1 }], + saveDiff: true, + }) + ); + + expect(output).toStrictEqual({ + outcome: 'compared', + isSameDimension: true, + result: { equal: true, differentPixels: undefined, totalPixels: undefined }, + errors: [], + }); + }); + + it('falls back to the defaults when looks-same refuses the project config', async () => { + const output = await computeLooksSame( + job({ config: { ...LOOKS_SAME_DEFAULT_CONFIG, strict: true, tolerance: 5 } }) + ); + + expect(output).toMatchObject({ + outcome: 'compared', + result: { equal: false, differentPixels: 31068, totalPixels: 129600 }, + errors: [ + 'Unable to use "strict" and "tolerance" options together', + 'Looks-Same cannot compare with the project config, fallback to default one', + ], + }); + }); + + it('masks the ignore areas of both images before comparing', async () => { + const baseline = filled(10, 10, 255); + const changed = PNG.sync.read(baseline); + changed.data[0] = 0; + changed.data[4 * 55] = 0; + + const output = await computeLooksSame( + job({ + baseline, + image: PNG.sync.write(changed), + ignoreAreas: [{ x: 0, y: 0, width: 1, height: 1 }], + config: { strict: true, ignoreAntialiasing: false, ignoreCaret: false }, + }) + ); + + expect(output).toMatchObject({ + outcome: 'compared', + result: { equal: false, differentPixels: 1, totalPixels: 100 }, + }); + }); + + it('produces an output that survives structured cloning', async () => { + const output = await computeLooksSame(job({ saveDiff: true })); + + const clone = structuredClone(output) as { diffPng: Uint8Array }; + + expect(clone).toStrictEqual(output); + expect(clone.diffPng.byteLength).toBeGreaterThan(0); + }); +}); + +describe('computeLooksSame when looks-same fails', () => { + afterEach(() => { + jest.resetModules(); + jest.dontMock('looks-same'); + }); + + it('gives no result and all three messages when the defaults fail too', async () => { + jest.resetModules(); + jest.doMock('looks-same', () => ({ + __esModule: true, + default: jest.fn().mockRejectedValueOnce(new Error('first')).mockRejectedValueOnce(new Error('second')), + })); + const core: typeof import('./looks-same.core') = require('./looks-same.core'); + + const output = await core.computeLooksSame(job()); + + expect(output).toStrictEqual({ + outcome: 'compared', + isSameDimension: true, + result: null, + errors: ['first', 'Looks-Same cannot compare with the project config, fallback to default one', 'second'], + }); + }); +}); diff --git a/src/compare/libs/looks-same/looks-same.core.ts b/src/compare/libs/looks-same/looks-same.core.ts new file mode 100644 index 00000000..09c4ce81 --- /dev/null +++ b/src/compare/libs/looks-same/looks-same.core.ts @@ -0,0 +1,92 @@ +import looksSame from 'looks-same'; +import { PNG } from 'pngjs'; +import { IgnoreAreaDto } from '../../../test-runs/dto/ignore-area.dto'; +import { applyIgnoreAreas, readPng } from '../../utils'; +import { LooksSameConfig } from './looks-same.types'; + +export const LOOKS_SAME_DEFAULT_CONFIG: LooksSameConfig = { + strict: false, + ignoreAntialiasing: true, + ignoreCaret: true, + allowDiffDimensions: false, +}; + +export interface LooksSameJobInput { + kind: 'looks-same'; + baseline: Buffer | Uint8Array; + image: Buffer | Uint8Array; + ignoreAreas: IgnoreAreaDto[]; + config: LooksSameConfig; + saveDiff: boolean; +} + +export type LooksSameJobOutput = + | { outcome: 'baseline-unreadable' } + | { outcome: 'image-unreadable' } + | { outcome: 'equal' } + | { outcome: 'diff-dimension' } + | { + outcome: 'compared'; + isSameDimension: boolean; + result: { equal: boolean; differentPixels?: number; totalPixels?: number } | null; + diffPng?: Uint8Array; + errors: string[]; + }; + +const FAST_PNG = { deflateLevel: 1, filterType: 0 } as const; + +export async function computeLooksSame(input: LooksSameJobInput): Promise { + let baseline: PNG; + try { + baseline = readPng(input.baseline); + } catch { + return { outcome: 'baseline-unreadable' }; + } + let image: PNG; + try { + image = readPng(input.image); + } catch { + return { outcome: 'image-unreadable' }; + } + + const isSameDimension = baseline.width === image.width && baseline.height === image.height; + if (!isSameDimension && !input.config.allowDiffDimensions) { + return { outcome: 'diff-dimension' }; + } + if (isSameDimension && baseline.data.equals(image.data)) { + return { outcome: 'equal' }; + } + + applyIgnoreAreas(baseline, input.ignoreAreas); + applyIgnoreAreas(image, input.ignoreAreas); + const baselinePng = PNG.sync.write(baseline, FAST_PNG); + const imagePng = PNG.sync.write(image, FAST_PNG); + + const errors: string[] = []; + const run = (options: LooksSameConfig) => looksSame(baselinePng, imagePng, { ...options, createDiffImage: true }); + let result: looksSame.LooksSameResult | undefined; + try { + result = await run(input.config); + } catch (error) { + errors.push((error as Error).message, 'Looks-Same cannot compare with the project config, fallback to default one'); + result = await run(LOOKS_SAME_DEFAULT_CONFIG).catch((fallbackError: Error) => { + errors.push(fallbackError.message); + return undefined; + }); + } + if (!result) { + return { outcome: 'compared', isSameDimension, result: null, errors }; + } + + const diffPng = + input.saveDiff && !result.equal && result.diffImage + ? new Uint8Array(await result.diffImage.createBuffer('png')) + : undefined; + return { + outcome: 'compared', + isSameDimension, + result: { equal: result.equal, differentPixels: result.differentPixels, totalPixels: result.totalPixels }, + ...(diffPng ? { diffPng } : {}), + errors, + }; +} diff --git a/src/compare/libs/looks-same/looks-same.service.boundaries.spec.ts b/src/compare/libs/looks-same/looks-same.service.boundaries.spec.ts deleted file mode 100644 index e7889ac1..00000000 --- a/src/compare/libs/looks-same/looks-same.service.boundaries.spec.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { Test } from '@nestjs/testing'; -import { TestStatus } from '@prisma/client'; -import { PNG } from 'pngjs'; -import { StaticService } from '../../../static/static.service'; -import { DIFF_DIMENSION_RESULT } from '../consts'; -import { DEFAULT_CONFIG, LookSameService } from './looks-same.service'; - -const filled = (width: number, height: number, value: number): PNG => { - const image = new PNG({ width, height }); - image.data.fill(value); - for (let i = 3; i < image.data.length; i += 4) { - image.data[i] = 255; - } - return image; -}; - -const initService = async (baseline: PNG, image: PNG) => { - const saveImage = jest.fn().mockResolvedValue('diff.png'); - const module = await Test.createTestingModule({ - providers: [ - LookSameService, - { - provide: StaticService, - useValue: { getImage: jest.fn().mockResolvedValueOnce(baseline).mockResolvedValueOnce(image), saveImage }, - }, - ], - }).compile(); - return { service: module.get(LookSameService), saveImage }; -}; - -const input = { baseline: 'baseline', image: 'image', diffTollerancePercent: 0, ignoreAreas: [], saveDiffAsFile: true }; - -describe('LookSameService.getDiff dimensions', () => { - it('reports a different height as a dimension change', async () => { - const { service } = await initService(filled(10, 10, 0), filled(10, 5, 0)); - - await expect(service.getDiff(input, DEFAULT_CONFIG)).resolves.toBe(DIFF_DIMENSION_RESULT); - }); - - it('reports a different width as a dimension change', async () => { - const { service } = await initService(filled(10, 10, 0), filled(5, 10, 0)); - - await expect(service.getDiff(input, DEFAULT_CONFIG)).resolves.toBe(DIFF_DIMENSION_RESULT); - }); -}); - -describe('LookSameService.getDiff saving the diff', () => { - it('stores the diff as a "diff" image', async () => { - const { service, saveImage } = await initService(filled(10, 10, 0), filled(10, 10, 255)); - - const result = await service.getDiff(input, DEFAULT_CONFIG); - - expect(result.status).toBe(TestStatus.unresolved); - expect(result.diffName).toBe('diff.png'); - expect(saveImage).toHaveBeenCalledWith('diff', expect.any(Buffer)); - expect(PNG.sync.read(saveImage.mock.calls[0][1]).width).toBe(10); - }); -}); diff --git a/src/compare/libs/looks-same/looks-same.service.real-images.spec.ts b/src/compare/libs/looks-same/looks-same.service.real-images.spec.ts new file mode 100644 index 00000000..980c92ad --- /dev/null +++ b/src/compare/libs/looks-same/looks-same.service.real-images.spec.ts @@ -0,0 +1,169 @@ +import { readFileSync } from 'fs'; +import { join } from 'path'; +import { Test } from '@nestjs/testing'; +import { TestStatus } from '@prisma/client'; +import { PNG } from 'pngjs'; +import { StaticService } from '../../../static/static.service'; +import { DiffWorkerPool } from '../../diff-worker-pool'; +import { DIFF_DIMENSION_RESULT, EQUAL_RESULT } from '../consts'; +import { solidPng } from '../../test-utils'; +import { DEFAULT_CONFIG, LookSameService } from './looks-same.service'; + +const filled = (width: number, height: number, value: number): PNG => solidPng(width, height, [value, value, value]); + +const encode = (png: PNG) => PNG.sync.write(png, { filterType: 0, deflateLevel: 1 }); + +const initService = async (baseline: PNG, image: PNG) => { + const saveImage = jest.fn().mockResolvedValue('diff.png'); + const getImageBuffer = jest + .fn() + .mockImplementationOnce(async () => encode(baseline)) + .mockImplementationOnce(async () => encode(image)); + const module = await Test.createTestingModule({ + providers: [ + LookSameService, + DiffWorkerPool, + { + provide: StaticService, + useValue: { + getImageBuffer, + saveImage, + }, + }, + ], + }).compile(); + return { service: module.get(LookSameService), saveImage }; +}; + +const input = { baseline: 'baseline', image: 'image', diffTollerancePercent: 0, ignoreAreas: [], saveDiffAsFile: true }; + +describe('LookSameService.getDiff dimensions', () => { + it.each([ + ['a different height', filled(10, 10, 0), filled(10, 5, 0)], + ['a different width', filled(10, 10, 0), filled(5, 10, 0)], + ['a 60x30 and a 30x60 image of the same pixel bytes', filled(60, 30, 128), filled(30, 60, 128)], + ])('reports %s as a dimension change', async (_, baseline, image) => { + const { service } = await initService(baseline, image); + + await expect(service.getDiff(input, DEFAULT_CONFIG)).resolves.toBe(DIFF_DIMENSION_RESULT); + }); + + it('compares them when other sizes are allowed', async () => { + const { service } = await initService(filled(60, 30, 128), filled(30, 60, 128)); + + const result = await service.getDiff(input, { ...DEFAULT_CONFIG, allowDiffDimensions: true }); + + expect(result).not.toBe(EQUAL_RESULT); + expect(result.status).not.toBe(TestStatus.ok); + expect(result.isSameDimension).toBe(false); + }); +}); + +describe('LookSameService.getDiff saving the diff', () => { + it('stores the diff as a "diff" image', async () => { + const { service, saveImage } = await initService(filled(10, 10, 0), filled(10, 10, 255)); + + const result = await service.getDiff(input, DEFAULT_CONFIG); + + expect(result.status).toBe(TestStatus.unresolved); + expect(result.diffName).toBe('diff.png'); + expect(saveImage).toHaveBeenCalledWith('diff', expect.any(Buffer)); + expect(PNG.sync.read(saveImage.mock.calls[0][1]).width).toBe(10); + }); +}); + +describe('characterization on the real looks-same', () => { + const decoded = new Map(); + const fixture = (name: string) => { + if (!decoded.has(name)) { + decoded.set(name, PNG.sync.read(readFileSync(join(__dirname, '../../../../test', name)))); + } + return decoded.get(name) as PNG; + }; + const fixtures = () => initService(fixture('image.png'), fixture('image_edited.png')); + const ignoreFirstPixel = { ...input, ignoreAreas: [{ x: 0, y: 0, width: 1, height: 1 }] }; + + it('passes a pair that is equal only after masking, without counting pixels', async () => { + const changed = filled(10, 10, 255); + changed.data[0] = 0; + const { service } = await initService(filled(10, 10, 255), changed); + + const result = await service.getDiff(ignoreFirstPixel, DEFAULT_CONFIG); + + expect(result).toStrictEqual({ + status: TestStatus.ok, + diffName: null, + pixelMisMatchCount: undefined, + diffPercent: 0, + isSameDimension: true, + }); + }); + + it('counts zero pixels when only the gamma differs after masking', async () => { + const baseline = filled(10, 10, 255); + baseline.gamma = 0.45455; + const changed = filled(10, 10, 255); + changed.data[0] = 0; + const { service } = await initService(baseline, changed); + + const result = await service.getDiff(ignoreFirstPixel, DEFAULT_CONFIG); + + expect(result).toStrictEqual({ + status: TestStatus.ok, + diffName: null, + pixelMisMatchCount: 0, + diffPercent: 0, + isSameDimension: true, + }); + }); + + it('reports the pixels looks-same counts on the fixtures', async () => { + const { service } = await fixtures(); + + const result = await service.getDiff({ ...input, saveDiffAsFile: false }, DEFAULT_CONFIG); + + expect(result.status).toBe(TestStatus.unresolved); + expect(result.pixelMisMatchCount).toBe(31068); + expect(result.diffPercent).toBeCloseTo((31068 * 100) / 129600, 10); + expect(result.isSameDimension).toBe(true); + }); + + it('counts more pixels in the strict mode without antialiasing and caret handling', async () => { + const { service } = await fixtures(); + + const result = await service.getDiff( + { ...input, saveDiffAsFile: false }, + { strict: true, ignoreAntialiasing: false, ignoreCaret: false } + ); + + expect(result.pixelMisMatchCount).toBe(33915); + }); + + it('saves the looks-same diff of the fixtures', async () => { + const { service, saveImage } = await fixtures(); + + const result = await service.getDiff(input, DEFAULT_CONFIG); + + expect(result.diffName).toBe('diff.png'); + expect(saveImage).toHaveBeenCalledWith('diff', expect.any(Buffer)); + const diff = PNG.sync.read(saveImage.mock.calls[0][1]); + expect([diff.width, diff.height]).toEqual([360, 360]); + }); + + it('falls back to the defaults when looks-same refuses the project config', async () => { + const { service } = await fixtures(); + + const result = await service.getDiff( + { ...input, saveDiffAsFile: false }, + { ...DEFAULT_CONFIG, strict: true, tolerance: 5 } + ); + + expect(result).toStrictEqual({ + status: TestStatus.unresolved, + diffName: null, + pixelMisMatchCount: 31068, + diffPercent: (31068 * 100) / 129600, + isSameDimension: true, + }); + }); +}); diff --git a/src/compare/libs/looks-same/looks-same.service.spec.ts b/src/compare/libs/looks-same/looks-same.service.spec.ts index 01e4d5af..530cd0db 100644 --- a/src/compare/libs/looks-same/looks-same.service.spec.ts +++ b/src/compare/libs/looks-same/looks-same.service.spec.ts @@ -1,31 +1,86 @@ -import { TestingModule, Test } from '@nestjs/testing'; +import { Test } from '@nestjs/testing'; +import { Logger } from '@nestjs/common'; import { TestStatus } from '@prisma/client'; import { PNG } from 'pngjs'; import { StaticService } from '../../../static/static.service'; +import { DiffWorkerPool } from '../../diff-worker-pool'; import { DIFF_DIMENSION_RESULT, EQUAL_RESULT, NO_BASELINE_RESULT } from '../consts'; import { DEFAULT_CONFIG, LookSameService } from './looks-same.service'; import { LooksSameConfig } from './looks-same.types'; import { ImageCompareInput } from '../ImageCompareInput'; -const initService = async ({ getImageMock = jest.fn(), saveImageMock = jest.fn(), deleteImageMock = jest.fn() }) => { - const module: TestingModule = await Test.createTestingModule({ +const baselineBuffer = PNG.sync.write(new PNG({ width: 20, height: 20 })); +const another = new PNG({ width: 20, height: 20 }); +another.data[0] = 1; +const imageBuffer = PNG.sync.write(another); + +interface Deps { + getImageBuffer?: jest.Mock; + saveImage?: jest.Mock; + run?: jest.Mock; +} + +const setup = async ({ + getImageBuffer = jest.fn().mockResolvedValueOnce(baselineBuffer).mockResolvedValueOnce(imageBuffer), + saveImage = jest.fn(), + run = jest.fn(), +}: Deps = {}) => { + const module = await Test.createTestingModule({ providers: [ LookSameService, - { - provide: StaticService, - useValue: { - getImage: getImageMock, - saveImage: saveImageMock, - deleteImage: deleteImageMock, - }, - }, + { provide: DiffWorkerPool, useValue: { run } }, + { provide: StaticService, useValue: { getImageBuffer, saveImage } }, ], }).compile(); + return { service: module.get(LookSameService), getImageBuffer, saveImage, run }; +}; - return module.get(LookSameService); +const input: ImageCompareInput = { + baseline: 'baseline', + image: 'image', + diffTollerancePercent: 0.1, + ignoreAreas: [], + saveDiffAsFile: true, }; -let service: LookSameService; +const diff = ( + service: LookSameService, + extra: Partial = {}, + config: LooksSameConfig = DEFAULT_CONFIG +) => service.getDiff({ ...input, ...extra }, config); + +const outcome = (name: string) => jest.fn().mockResolvedValue({ outcome: name }); + +const compared = ( + result: { equal: boolean; differentPixels?: number; totalPixels?: number } | null, + extra: { diffPng?: Uint8Array; errors?: string[] } = {} +) => jest.fn().mockResolvedValue({ outcome: 'compared', isSameDimension: true, result, errors: [], ...extra }); + +const verdict = ( + status: TestStatus, + diffName: string | null, + diffPercent: number | undefined, + pixelMisMatchCount?: number +) => ({ + status, + diffName, + diffPercent, + pixelMisMatchCount, + isSameDimension: true, +}); + +const twentyOfFourHundred = { equal: false, differentPixels: 20, totalPixels: 400 }; +const diffPng = new Uint8Array(Buffer.from('png')); + +let loggerError: jest.SpyInstance; + +beforeEach(() => { + loggerError = jest.spyOn(Logger.prototype, 'error').mockImplementation(); +}); + +afterEach(() => { + jest.restoreAllMocks(); +}); describe('parseConfig', () => { it.each<[string, LooksSameConfig]>([ @@ -43,264 +98,202 @@ describe('parseConfig', () => { ['', DEFAULT_CONFIG], ['invalid', DEFAULT_CONFIG], ])('should parse config', async (json, expected) => { - service = await initService({}); + const { service } = await setup(); - const config = service.parseConfig(json); - - expect(config).toStrictEqual(expected); + expect(service.parseConfig(json)).toStrictEqual(expected); }); }); describe('getDiff', () => { - const image = new PNG({ - width: 20, - height: 20, - }); - const anotherImage = new PNG({ - width: 20, - height: 20, + it('no baseline', async () => { + const { service, run } = await setup({ + getImageBuffer: jest.fn().mockResolvedValueOnce(null).mockResolvedValueOnce(imageBuffer), + }); + + expect(await diff(service, { baseline: null })).toStrictEqual(NO_BASELINE_RESULT); + expect(run).not.toHaveBeenCalled(); }); - anotherImage.data[0] = 1; // alterate pixel to have it different from 0 - it('no baseline', async () => { - const getImageMock = jest.fn().mockReturnValueOnce(undefined).mockReturnValueOnce(image); - service = await initService({ getImageMock }); + it('compares the image it is given without reading it again', async () => { + const { service, getImageBuffer, run } = await setup({ + getImageBuffer: jest.fn().mockResolvedValue(baselineBuffer), + run: outcome('equal'), + }); - const result = await service.getDiff( - { - baseline: null, - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: true, - }, - DEFAULT_CONFIG - ); + await diff(service, { imageBuffer }); - expect(result).toStrictEqual(NO_BASELINE_RESULT); + expect(getImageBuffer).toHaveBeenCalledTimes(1); + expect(getImageBuffer).toHaveBeenCalledWith('baseline'); + expect(run).toHaveBeenCalledWith(expect.objectContaining({ baseline: baselineBuffer, image: imageBuffer })); }); - it('equal images', async () => { - const getImageMock = jest.fn().mockReturnValueOnce(image).mockReturnValueOnce(image); - service = await initService({ getImageMock }); + it('reports no baseline when the baseline cannot be read', async () => { + const { service, run } = await setup({ + getImageBuffer: jest.fn().mockRejectedValueOnce(new Error('disk')).mockResolvedValueOnce(imageBuffer), + }); - const result = await service.getDiff( - { - baseline: 'image', - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: true, - }, - DEFAULT_CONFIG - ); + expect(await diff(service)).toStrictEqual(NO_BASELINE_RESULT); + expect(run).not.toHaveBeenCalled(); + expect(loggerError).toHaveBeenCalledWith('disk'); + }); + + it('reports no baseline when the job finds it unreadable', async () => { + const { service } = await setup({ run: outcome('baseline-unreadable') }); - expect(result).toStrictEqual(EQUAL_RESULT); + await expect(diff(service)).resolves.toStrictEqual(NO_BASELINE_RESULT); }); - it('diff image dimensions mismatch', async () => { - const baseline = new PNG({ - width: 10, - height: 10, + it('answers EQUAL for byte-equal well-formed PNGs without a job', async () => { + const { service, run } = await setup({ getImageBuffer: jest.fn().mockResolvedValue(baselineBuffer) }); + + expect(await diff(service)).toStrictEqual(EQUAL_RESULT); + expect(run).not.toHaveBeenCalled(); + }); + + it('hands byte-equal files that are not well-formed PNGs to the job', async () => { + const { service, run } = await setup({ + getImageBuffer: jest.fn().mockResolvedValue(Buffer.from('not a png')), + run: outcome('baseline-unreadable'), }); - const getImageMock = jest.fn().mockReturnValueOnce(image).mockReturnValueOnce(baseline); - service = await initService({ getImageMock }); - const result = await service.getDiff( - { - baseline: 'image', - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: true, - }, - DEFAULT_CONFIG - ); + await diff(service); - expect(result).toStrictEqual(DIFF_DIMENSION_RESULT); + expect(run).toHaveBeenCalledTimes(1); }); - const input: Omit = { - baseline: 'image', - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - }; + it.each([ + ['equal images', 'equal', EQUAL_RESULT], + ['diff image dimensions mismatch', 'diff-dimension', DIFF_DIMENSION_RESULT], + ])('%s', async (_, name, expected) => { + const { service } = await setup({ run: outcome(name) }); - it('diff not found', async () => { - const getImageMock = jest.fn().mockReturnValueOnce(image).mockReturnValueOnce(anotherImage); - service = await initService({ getImageMock }); - service.compare = jest.fn().mockReturnValueOnce({ equal: true, differentPixels: 0, totalPixels: 400 }); - - const result = await service.getDiff({ ...input, saveDiffAsFile: true }, DEFAULT_CONFIG); + expect(await diff(service)).toStrictEqual(expected); + }); - expect(result).toStrictEqual({ - status: TestStatus.ok, - diffName: null, - diffPercent: 0, - pixelMisMatchCount: 0, - isSameDimension: true, + it('throws a clear error when the image cannot be read', async () => { + const { service, run } = await setup({ + getImageBuffer: jest.fn().mockResolvedValueOnce(baselineBuffer).mockResolvedValueOnce(null), }); + + await expect(diff(service, { saveDiffAsFile: false })).rejects.toThrow('Cannot get image: image'); + expect(run).not.toHaveBeenCalled(); + }); + + it('throws a clear error when the job finds the image unreadable', async () => { + const { service } = await setup({ run: outcome('image-unreadable') }); + + await expect(diff(service, { saveDiffAsFile: false })).rejects.toThrow('Cannot get image: image'); + }); + + it('diff not found', async () => { + const { service } = await setup({ run: compared({ equal: true, differentPixels: 0, totalPixels: 400 }) }); + + expect(await diff(service)).toStrictEqual(verdict(TestStatus.ok, null, 0, 0)); }); it('diff found', async () => { - const getImageMock = jest.fn().mockReturnValueOnce(image).mockReturnValueOnce(anotherImage); - service = await initService({ getImageMock }); - service.compare = jest.fn().mockReturnValueOnce({ equal: false, differentPixels: 20, totalPixels: 400 }); - service.saveDiff = jest.fn(); + const { service, saveImage } = await setup({ run: compared(twentyOfFourHundred) }); - const result = await service.getDiff({ ...input, saveDiffAsFile: false }, DEFAULT_CONFIG); + const result = await diff(service, { saveDiffAsFile: false }); - expect(service.compare).toHaveBeenCalledWith(image, anotherImage, DEFAULT_CONFIG); - expect(service.saveDiff).not.toHaveBeenCalled(); - expect(result).toStrictEqual({ - status: TestStatus.unresolved, - diffName: null, - diffPercent: 5, - pixelMisMatchCount: 20, - isSameDimension: true, - }); + expect(saveImage).not.toHaveBeenCalled(); + expect(result).toStrictEqual(verdict(TestStatus.unresolved, null, 5, 20)); }); it('diff found and save diff', async () => { - const diffImage = { createBuffer: jest.fn() }; - const getImageMock = jest.fn().mockReturnValueOnce(image).mockReturnValueOnce(anotherImage); - service = await initService({ getImageMock }); - service.compare = jest.fn().mockReturnValueOnce({ equal: false, differentPixels: 20, totalPixels: 400, diffImage }); - service.saveDiff = jest.fn().mockReturnValueOnce('diff name'); + const { service, saveImage } = await setup({ + run: compared(twentyOfFourHundred, { diffPng }), + saveImage: jest.fn().mockResolvedValueOnce('diff name'), + }); - const result = await service.getDiff({ ...input, saveDiffAsFile: true }, DEFAULT_CONFIG); + const result = await diff(service); - expect(service.compare).toHaveBeenCalledWith(image, anotherImage, DEFAULT_CONFIG); - expect(service.saveDiff).toHaveBeenCalledWith(diffImage); - expect(result).toStrictEqual({ - status: TestStatus.unresolved, - diffName: 'diff name', - diffPercent: 5, - pixelMisMatchCount: 20, - isSameDimension: true, - }); + expect(saveImage).toHaveBeenCalledWith('diff', Buffer.from('png')); + expect(saveImage.mock.calls[0][1]).toBeInstanceOf(Buffer); + expect(result).toStrictEqual(verdict(TestStatus.unresolved, 'diff name', 5, 20)); }); it('ok, without a diff, when the change is within the tolerance', async () => { - const getImageMock = jest.fn().mockReturnValueOnce(image).mockReturnValueOnce(anotherImage); - service = await initService({ getImageMock }); - service.compare = jest.fn().mockReturnValueOnce({ equal: false, differentPixels: 20, totalPixels: 400 }); - service.saveDiff = jest.fn(); + const { service, saveImage } = await setup({ run: compared(twentyOfFourHundred) }); - const result = await service.getDiff({ ...input, diffTollerancePercent: 5, saveDiffAsFile: true }, DEFAULT_CONFIG); + const result = await diff(service, { diffTollerancePercent: 5 }); - expect(service.saveDiff).not.toHaveBeenCalled(); - expect(result).toStrictEqual({ - status: TestStatus.ok, - diffName: null, - diffPercent: 5, - pixelMisMatchCount: 20, - isSameDimension: true, - }); + expect(saveImage).not.toHaveBeenCalled(); + expect(result).toStrictEqual(verdict(TestStatus.ok, null, 5, 20)); }); it('comparison failed', async () => { - const getImageMock = jest.fn().mockReturnValueOnce(image).mockReturnValueOnce(anotherImage); - service = await initService({ getImageMock }); - service.compare = jest.fn().mockReturnValueOnce(undefined); - - const result = await service.getDiff({ ...input, saveDiffAsFile: true }, DEFAULT_CONFIG); + const { service } = await setup({ run: compared(null) }); - expect(result).toStrictEqual({ - status: TestStatus.unresolved, - diffName: null, - diffPercent: undefined, - pixelMisMatchCount: undefined, - isSameDimension: true, - }); + expect(await diff(service)).toStrictEqual(verdict(TestStatus.unresolved, null, undefined)); }); it('reports a 0% diff when Looks-Same counts no pixels at all', async () => { - const getImageMock = jest.fn().mockReturnValueOnce(image).mockReturnValueOnce(anotherImage); - service = await initService({ getImageMock }); - service.compare = jest.fn().mockReturnValueOnce({ equal: false, differentPixels: 0, totalPixels: 0 }); + const { service } = await setup({ run: compared({ equal: false, differentPixels: 0, totalPixels: 0 }) }); - const result = await service.getDiff({ ...input, saveDiffAsFile: false }, DEFAULT_CONFIG); + expect(await diff(service, { saveDiffAsFile: false })).toStrictEqual(verdict(TestStatus.unresolved, null, 0, 0)); + }); - expect(result).toStrictEqual({ - status: TestStatus.unresolved, - diffName: null, - diffPercent: 0, - pixelMisMatchCount: 0, - isSameDimension: true, - }); + it('logs what looks-same refused', async () => { + const { service } = await setup({ run: compared(null, { errors: ['a', 'b'] }) }); + + await diff(service, { saveDiffAsFile: false }); + + expect(loggerError.mock.calls).toEqual([['a'], ['b']]); }); - it('throws when the baseline exists but the image cannot be read', async () => { - const getImageMock = jest.fn().mockReturnValueOnce(image).mockReturnValueOnce(undefined); - service = await initService({ getImageMock }); + it('passes the project config, the ignore areas and saveDiffAsFile to the job', async () => { + const { service, run } = await setup({ run: outcome('equal') }); + const config = { ...DEFAULT_CONFIG, tolerance: 3, strict: undefined }; + const ignoreAreas = [{ x: 1, y: 2, width: 3, height: 4 }]; - // NOTE: current behavior — unlike pixelmatch there is no explicit check, so a - // missing image surfaces as a TypeError rather than a descriptive error. - await expect(service.getDiff({ ...input, saveDiffAsFile: false }, DEFAULT_CONFIG)).rejects.toThrow(TypeError); + await diff(service, { ignoreAreas }, config); + + expect(run).toHaveBeenCalledWith({ + kind: 'looks-same', + baseline: baselineBuffer, + image: imageBuffer, + ignoreAreas, + config, + saveDiff: true, + }); }); - it('saves the Looks-Same diff with its metrics', async () => { - const baseline = new PNG({ width: 2, height: 1 }); - const changed = new PNG({ width: 2, height: 1 }); - baseline.data.fill(255); - changed.data.fill(255); - changed.data[0] = 0; - const getImageMock = jest.fn().mockReturnValueOnce(baseline).mockReturnValueOnce(changed); - const saveImageMock = jest.fn().mockReturnValueOnce('diff name'); - service = await initService({ getImageMock, saveImageMock }); - - const result = await service.getDiff( - { ...input, saveDiffAsFile: true }, - { ...DEFAULT_CONFIG, strict: true, ignoreAntialiasing: false } - ); - - expect(result).toStrictEqual({ + it('keeps the compared outcome when the diff cannot be saved', async () => { + const { service } = await setup({ + run: compared(twentyOfFourHundred, { diffPng }), + saveImage: jest.fn().mockRejectedValueOnce(new Error('disk full')), + }); + + expect(await diff(service)).toMatchObject({ status: TestStatus.unresolved, - diffName: 'diff name', - diffPercent: 50, - pixelMisMatchCount: 1, - isSameDimension: true, + diffName: null, + pixelMisMatchCount: 20, }); - expect(saveImageMock).toHaveBeenCalledWith('diff', expect.any(Buffer)); }); }); -describe('compare', () => { - it('returns undefined when Looks-Same fails', async () => { - service = await initService({}); - const broken = new PNG({ width: 1, height: 1 }); - jest.spyOn(PNG.sync, 'write').mockImplementationOnce(() => Buffer.from('not a png')); - - const result = await service.compare(broken, broken, DEFAULT_CONFIG); +describe('saveDiff', () => { + it('returns null when there is no diff image', async () => { + const { service, saveImage } = await setup(); - expect(result).toBeUndefined(); + await expect(service.saveDiff(undefined)).resolves.toBeNull(); + expect(saveImage).not.toHaveBeenCalled(); }); - it('compares with the defaults when Looks-Same refuses the project config', async () => { - service = await initService({}); - const baseline = new PNG({ width: 2, height: 1 }); - const changed = new PNG({ width: 2, height: 1 }); - baseline.data.fill(255); - changed.data.fill(255); - changed.data[0] = 0; - - // looks-same refuses strict together with tolerance - const result = await service.compare(baseline, changed, { ...DEFAULT_CONFIG, strict: true, tolerance: 5 }); + it('returns null when the diff cannot be saved', async () => { + const { service } = await setup({ saveImage: jest.fn().mockRejectedValueOnce(new Error('disk full')) }); - expect(result).toMatchObject({ equal: false, differentPixels: 1, totalPixels: 2 }); + expect(await service.saveDiff(diffPng)).toBeNull(); }); -}); -describe('saveDiff', () => { - it('returns null when the diff cannot be saved', async () => { - const saveImageMock = jest.fn().mockRejectedValueOnce(new Error('disk full')); - service = await initService({ saveImageMock }); - const diffImage = { createBuffer: jest.fn().mockResolvedValueOnce(Buffer.from('png')) }; + it('saves only the bytes of a view into a larger buffer', async () => { + const { service, saveImage } = await setup({ saveImage: jest.fn().mockResolvedValueOnce('diff name') }); + const whole = Buffer.from('xxpngxx'); - const result = await service.saveDiff(diffImage as never); + const result = await service.saveDiff(new Uint8Array(whole.buffer, whole.byteOffset + 2, 3)); - expect(result).toBeNull(); + expect(result).toBe('diff name'); + expect(saveImage.mock.calls[0][1].toString()).toBe('png'); }); }); diff --git a/src/compare/libs/looks-same/looks-same.service.ts b/src/compare/libs/looks-same/looks-same.service.ts index 015a8d88..fc356996 100644 --- a/src/compare/libs/looks-same/looks-same.service.ts +++ b/src/compare/libs/looks-same/looks-same.service.ts @@ -1,113 +1,104 @@ import { Injectable, Logger } from '@nestjs/common'; import { TestStatus } from '@prisma/client'; -import { PNG } from 'pngjs'; import { StaticService } from '../../../static/static.service'; +import { asBuffer, isByteEqualWellFormedPng } from '../../../static/utils'; import { DiffResult } from '../../../test-runs/diffResult'; -import { applyIgnoreAreas, parseConfig } from '../../utils'; +import { parseConfig } from '../../utils'; +import { DiffWorkerPool } from '../../diff-worker-pool'; import { ImageComparator } from '../image-comparator.interface'; import { ImageCompareInput } from '../ImageCompareInput'; import { LooksSameConfig } from './looks-same.types'; -import looksSame from 'looks-same'; +import { LOOKS_SAME_DEFAULT_CONFIG } from './looks-same.core'; import { DIFF_DIMENSION_RESULT, EQUAL_RESULT, NO_BASELINE_RESULT } from '../consts'; -export const DEFAULT_CONFIG: LooksSameConfig = { - strict: false, - ignoreAntialiasing: true, - ignoreCaret: true, - allowDiffDimensions: false, -}; +export const DEFAULT_CONFIG = LOOKS_SAME_DEFAULT_CONFIG; @Injectable() export class LookSameService implements ImageComparator { private readonly logger: Logger = new Logger(LookSameService.name); - constructor(private readonly staticService: StaticService) {} + constructor( + private readonly staticService: StaticService, + private readonly diffWorkerPool: DiffWorkerPool + ) {} parseConfig(configJson: string): LooksSameConfig { return parseConfig(configJson, DEFAULT_CONFIG, this.logger); } async getDiff(data: ImageCompareInput, config: LooksSameConfig): Promise { - const result: DiffResult = { - ...NO_BASELINE_RESULT, - }; - - const baseline = await this.staticService.getImage(data.baseline); - // a missing checkpoint image is not expected here, and fails on first use - const image = (await this.staticService.getImage(data.image))!; - - if (!baseline) { + const [baselineBuffer, imageBuffer] = await Promise.all([ + this.readBaseline(data.baseline), + data.imageBuffer ?? this.staticService.getImageBuffer(data.image), + ]); + if (!baselineBuffer) { return NO_BASELINE_RESULT; } + if (!imageBuffer) { + throw new Error(`Cannot get image: ${data.image}`); + } - if (baseline.data.equals(new Uint8Array(image.data.buffer, image.data.byteOffset, image.data.byteLength))) { + if (isByteEqualWellFormedPng(baselineBuffer, imageBuffer)) { return EQUAL_RESULT; } - result.isSameDimension = baseline.width === image.width && baseline.height === image.height; - if (!result.isSameDimension && !config.allowDiffDimensions) { - return DIFF_DIMENSION_RESULT; + const output = await this.diffWorkerPool.run({ + kind: 'looks-same', + baseline: baselineBuffer, + image: imageBuffer, + ignoreAreas: data.ignoreAreas, + config, + saveDiff: data.saveDiffAsFile, + }); + switch (output.outcome) { + case 'baseline-unreadable': + return NO_BASELINE_RESULT; + case 'image-unreadable': + throw new Error(`Cannot get image: ${data.image}`); + case 'equal': + return EQUAL_RESULT; + case 'diff-dimension': + return DIFF_DIMENSION_RESULT; } - // apply ignore areas - const baselineIgnored = applyIgnoreAreas(baseline, data.ignoreAreas); - const imageIgnored = applyIgnoreAreas(image, data.ignoreAreas); - - // compare - const compareResult = await this.compare(baselineIgnored, imageIgnored, config); - if (!compareResult) { - // looks-same failed: keep the run for review instead of failing it + output.errors.forEach((message) => this.logger.error(message)); + const result: DiffResult = { ...NO_BASELINE_RESULT, isSameDimension: output.isSameDimension }; + if (!output.result) { result.status = TestStatus.unresolved; return result; } - result.pixelMisMatchCount = compareResult.differentPixels; - result.diffPercent = - compareResult.totalPixels > 0 ? (compareResult.differentPixels * 100) / compareResult.totalPixels : 0; + const { equal, differentPixels, totalPixels } = output.result; + result.pixelMisMatchCount = differentPixels; + result.diffPercent = totalPixels && totalPixels > 0 ? ((differentPixels as number) * 100) / totalPixels : 0; - // The run's tolerance forgives a change as small as it, as pixelmatch and - // odiff do. Only a counted change: one looks-same reports but cannot - // measure (no pixels counted) is left for review whatever the tolerance. const withinTolerance = result.diffPercent > 0 && result.diffPercent <= data.diffTollerancePercent; - if (compareResult.equal || withinTolerance) { + if (equal || withinTolerance) { result.status = TestStatus.ok; } else { result.status = TestStatus.unresolved; - if (data.saveDiffAsFile) { - result.diffName = await this.saveDiff(compareResult.diffImage); - } + result.diffName = await this.saveDiff(output.diffPng); } return result; } - // one pass gives the verdict, the pixel counts and the diff image together - async compare( - baseline: PNG, - image: PNG, - config: LooksSameConfig - ): Promise | undefined> { - const [baselinePng, imagePng] = [PNG.sync.write(baseline), PNG.sync.write(image)]; - const run = (options: LooksSameConfig) => looksSame(baselinePng, imagePng, { ...options, createDiffImage: true }); + private async readBaseline(name: string | null): Promise { try { - return await run(config); + return await this.staticService.getImageBuffer(name); } catch (error) { this.logger.error((error as Error).message); + return null; } - // Most likely a project config looks-same refuses ("strict" together with - // "tolerance", say). That is the project's problem, not this screenshot's: - // compare with the defaults, as for a config that is not JSON, rather than - // leave every changed screenshot without numbers. - this.logger.error('Looks-Same cannot compare with the project config, fallback to default one'); - return run(DEFAULT_CONFIG).catch((error): undefined => { - this.logger.error(error.message); - return undefined; - }); } - async saveDiff(diffImage: looksSame.DiffImage): Promise { + async saveDiff(diffPng?: Uint8Array): Promise { + if (!diffPng) { + return null; + } try { - return await this.staticService.saveImage('diff', await diffImage.createBuffer('png')); + // saveImage checks the PNG signature with Buffer#equals, which a Uint8Array from a worker lacks + return await this.staticService.saveImage('diff', asBuffer(diffPng)); } catch (error) { this.logger.error((error as Error).message); return null; diff --git a/src/compare/libs/looks-same/looks-same.types.ts b/src/compare/libs/looks-same/looks-same.types.ts index 41084b1b..5e0d8818 100644 --- a/src/compare/libs/looks-same/looks-same.types.ts +++ b/src/compare/libs/looks-same/looks-same.types.ts @@ -1,47 +1,10 @@ import { BlurAmplitudeConfig } from '../blur-amplitude.config'; export interface LooksSameConfig extends BlurAmplitudeConfig { - /** - * strict comparsion - */ strict?: boolean; - /** - * ΔE value that will be treated as error in non-strict mode - */ tolerance?: number; - /** - * makes the search algorithm of the antialiasing less strict - */ antialiasingTolerance?: number; - /** - * Ability to ignore antialiasing - */ ignoreAntialiasing?: boolean; - /** - * Ability to ignore text caret - */ ignoreCaret?: boolean; allowDiffDimensions?: boolean; } - -/** - * coordinate bounds - */ -export interface CoordBounds { - /** - * X-coordinate of upper left corner - */ - left: number; - /** - * Y-coordinate of upper left corner - */ - top: number; - /** - * X-coordinate of bottom right corner - */ - right: number; - /** - * Y-coordinate of bottom right corner - */ - bottom: number; -} diff --git a/src/compare/libs/odiff/odiff.service.spec.ts b/src/compare/libs/odiff/odiff.service.spec.ts index e7d9c89a..394092ca 100644 --- a/src/compare/libs/odiff/odiff.service.spec.ts +++ b/src/compare/libs/odiff/odiff.service.spec.ts @@ -1,9 +1,10 @@ -import { TestingModule, Test } from '@nestjs/testing'; +import { Test } from '@nestjs/testing'; import { TestStatus } from '@prisma/client'; import { IgnoreAreaDto } from '../../../test-runs/dto/ignore-area.dto'; import { StaticService } from '../../../static/static.service'; import { HddService } from '../../../static/hdd/hdd.service'; import { DIFF_DIMENSION_RESULT, EQUAL_RESULT, NO_BASELINE_RESULT } from '../consts'; +import { ImageCompareInput } from '../ImageCompareInput'; import { OdiffService, DEFAULT_CONFIG, ignoreAreaToRegionMapper } from './odiff.service'; import { OdiffConfig, OdiffIgnoreRegions } from './odiff.types'; import { compare } from 'odiff-bin'; @@ -12,225 +13,112 @@ jest.mock('odiff-bin', () => ({ compare: jest.fn(), })); +type CompareResult = Awaited>; + const compareMock = compare as jest.MockedFunction; -const initService = async ({ - generateNewImageMock = jest.fn(), - getImagePathMock = jest.fn(), - deleteImageMock = jest.fn(), -}) => { - const module: TestingModule = await Test.createTestingModule({ +const setup = async (compareResult?: CompareResult) => { + if (compareResult) { + compareMock.mockResolvedValueOnce(compareResult); + } + const deleteImage = jest.fn(); + const module = await Test.createTestingModule({ providers: [ OdiffService, - { - provide: StaticService, - useValue: { - deleteImage: deleteImageMock, - }, - }, + { provide: StaticService, useValue: { deleteImage } }, { provide: HddService, useValue: { - generateNewImage: generateNewImageMock, - getImagePath: getImagePathMock, + generateNewImage: jest.fn().mockReturnValueOnce({ imagePath: './diff.png', imageName: 'diff.png' }), + getImagePath: jest.fn().mockReturnValueOnce('./baseline.png').mockReturnValueOnce('./image.png'), }, }, ], }).compile(); + return { service: module.get(OdiffService), deleteImage }; +}; - return module.get(OdiffService); +const input: ImageCompareInput = { + baseline: 'baseline', + image: 'image', + diffTollerancePercent: 0.1, + ignoreAreas: [], + saveDiffAsFile: true, }; -let service: OdiffService; +const pixelDiff = (diffPercentage: number, diffCount = 33.33) => + ({ match: false, reason: 'pixel-diff', diffPercentage, diffCount }) as CompareResult; + +const verdict = (status: TestStatus, diffName: string | null, diffPercent: number | undefined, count = 33.33) => ({ + diffName, + diffPercent, + isSameDimension: true, + pixelMisMatchCount: diffPercent === undefined ? undefined : count, + status, +}); describe('parseConfig', () => { it.each<[string, OdiffConfig]>([ [ '{"threshold":0,"antialiasing":true,"failOnLayoutDiff":true,"outputDiffMask":true}', - { - threshold: 0, - antialiasing: true, - failOnLayoutDiff: true, - outputDiffMask: true, - }, + { threshold: 0, antialiasing: true, failOnLayoutDiff: true, outputDiffMask: true }, ], ['', DEFAULT_CONFIG], ['invalid', DEFAULT_CONFIG], ])('should parse config', async (json, expected) => { - service = await initService({}); - - const config = service.parseConfig(json); + const { service } = await setup(); - expect(config).toStrictEqual(expected); + expect(service.parseConfig(json)).toStrictEqual(expected); }); }); describe('getDiff', () => { it('no baseline', async () => { - service = await initService({}); - - const result = await service.getDiff( - { - baseline: null, - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: true, - }, - DEFAULT_CONFIG - ); + const { service } = await setup(); - expect(result).toStrictEqual(NO_BASELINE_RESULT); + expect(await service.getDiff({ ...input, baseline: null }, DEFAULT_CONFIG)).toStrictEqual(NO_BASELINE_RESULT); }); it('diff layout', async () => { - const generateNewImageMock = jest.fn().mockReturnValueOnce({ imagePath: './diff.png', imageName: 'diff.png' }); - const getImagePathMock = jest.fn().mockReturnValueOnce('./baseline.png').mockReturnValueOnce('./image.png'); - const deleteImageMock = jest.fn(); - compareMock.mockResolvedValueOnce({ match: false, reason: 'layout-diff' }); - service = await initService({ - getImagePathMock, - generateNewImageMock, - deleteImageMock, - }); + const { service, deleteImage } = await setup({ match: false, reason: 'layout-diff' }); - const result = await service.getDiff( - { - baseline: 'baseline', - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: true, - }, - { - ...DEFAULT_CONFIG, - failOnLayoutDiff: true, - } - ); + const result = await service.getDiff(input, { ...DEFAULT_CONFIG, failOnLayoutDiff: true }); expect(result).toStrictEqual(DIFF_DIMENSION_RESULT); - expect(deleteImageMock).not.toHaveBeenCalled(); + expect(deleteImage).not.toHaveBeenCalled(); }); it('diff found', async () => { - const generateNewImageMock = jest.fn().mockReturnValueOnce({ imagePath: './diff.png', imageName: 'diff.png' }); - const getImagePathMock = jest.fn().mockReturnValueOnce('./baseline.png').mockReturnValueOnce('./image.png'); - const deleteImageMock = jest.fn(); - compareMock.mockResolvedValueOnce({ match: false, reason: 'pixel-diff', diffPercentage: 12.34, diffCount: 33.33 }); - service = await initService({ - getImagePathMock, - generateNewImageMock, - deleteImageMock, - }); + const { service, deleteImage } = await setup(pixelDiff(12.34)); - const result = await service.getDiff( - { - baseline: 'baseline', - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: true, - }, - { - ...DEFAULT_CONFIG, - failOnLayoutDiff: true, - } - ); - - expect(deleteImageMock).not.toHaveBeenCalled(); - expect(result).toStrictEqual({ - diffName: 'diff.png', - diffPercent: 12.34, - isSameDimension: true, - pixelMisMatchCount: 33.33, - status: TestStatus.unresolved, - }); + const result = await service.getDiff(input, { ...DEFAULT_CONFIG, failOnLayoutDiff: true }); + + expect(deleteImage).not.toHaveBeenCalled(); + expect(result).toStrictEqual(verdict(TestStatus.unresolved, 'diff.png', 12.34)); }); it('diff found < tollerance', async () => { - const generateNewImageMock = jest.fn().mockReturnValueOnce({ imagePath: './diff.png', imageName: 'diff.png' }); - const getImagePathMock = jest.fn().mockReturnValueOnce('./baseline.png').mockReturnValueOnce('./image.png'); - const deleteImageMock = jest.fn(); - compareMock.mockResolvedValueOnce({ match: false, reason: 'pixel-diff', diffPercentage: 0.05, diffCount: 33.33 }); - service = await initService({ - getImagePathMock, - generateNewImageMock, - deleteImageMock, - }); + const { service, deleteImage } = await setup(pixelDiff(0.05)); - const result = await service.getDiff( - { - baseline: 'baseline', - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: true, - }, - DEFAULT_CONFIG - ); - - // odiff writes the diff image for every pixel diff, so one below the tolerance is not kept - expect(deleteImageMock).toHaveBeenCalledWith('diff.png'); - expect(result).toStrictEqual({ - diffName: null, - diffPercent: 0.05, - isSameDimension: true, - pixelMisMatchCount: 33.33, - status: TestStatus.ok, - }); + const result = await service.getDiff(input, DEFAULT_CONFIG); + + expect(deleteImage).toHaveBeenCalledWith('diff.png'); + expect(result).toStrictEqual(verdict(TestStatus.ok, null, 0.05)); }); it('diff found no save', async () => { - const generateNewImageMock = jest.fn().mockReturnValueOnce({ imagePath: './diff.png', imageName: 'diff.png' }); - const getImagePathMock = jest.fn().mockReturnValueOnce('./baseline.png').mockReturnValueOnce('./image.png'); - const deleteImageMock = jest.fn(); - compareMock.mockResolvedValueOnce({ match: false, reason: 'pixel-diff', diffPercentage: 3, diffCount: 33.33 }); - service = await initService({ - getImagePathMock, - generateNewImageMock, - deleteImageMock, - }); + const { service, deleteImage } = await setup(pixelDiff(3)); - const result = await service.getDiff( - { - baseline: 'baseline', - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: false, - }, - DEFAULT_CONFIG - ); - - expect(deleteImageMock).toHaveBeenCalledWith('diff.png'); - expect(result).toStrictEqual({ - diffName: null, - diffPercent: 3, - isSameDimension: true, - pixelMisMatchCount: 33.33, - status: TestStatus.unresolved, - }); + const result = await service.getDiff({ ...input, saveDiffAsFile: false }, DEFAULT_CONFIG); + + expect(deleteImage).toHaveBeenCalledWith('diff.png'); + expect(result).toStrictEqual(verdict(TestStatus.unresolved, null, 3)); }); it('passes ignore areas to odiff as regions', async () => { - const generateNewImageMock = jest.fn().mockReturnValueOnce({ imagePath: './diff.png', imageName: 'diff.png' }); - const getImagePathMock = jest.fn().mockReturnValueOnce('./baseline.png').mockReturnValueOnce('./image.png'); - compareMock.mockResolvedValueOnce({ match: true }); - service = await initService({ - getImagePathMock, - generateNewImageMock, - }); + const { service } = await setup({ match: true }); - await service.getDiff( - { - baseline: 'baseline', - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [{ x: 1, y: 2, width: 3, height: 4 }], - saveDiffAsFile: true, - }, - DEFAULT_CONFIG - ); + await service.getDiff({ ...input, ignoreAreas: [{ x: 1, y: 2, width: 3, height: 4 }] }, DEFAULT_CONFIG); expect(compareMock).toHaveBeenLastCalledWith('./baseline.png', './image.png', './diff.png', { ...DEFAULT_CONFIG, @@ -240,179 +128,61 @@ describe('getDiff', () => { }); }); - it('does not pass empty ignore regions to odiff', async () => { - // odiff 4 rejects an empty --ignore= argument with "Invalid ignore regions format" - const generateNewImageMock = jest.fn().mockReturnValueOnce({ imagePath: './diff.png', imageName: 'diff.png' }); - const getImagePathMock = jest.fn().mockReturnValueOnce('./baseline.png').mockReturnValueOnce('./image.png'); - compareMock.mockResolvedValueOnce({ match: true }); - service = await initService({ - getImagePathMock, - generateNewImageMock, - }); + it.each([ + ['there are none', []], + ['every area lies outside the image', [{ x: -20, y: 0, width: 10, height: 5 }]], + ])('does not pass ignore regions when %s', async (_, ignoreAreas) => { + const { service } = await setup({ match: true }); - await service.getDiff( - { - baseline: 'baseline', - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: true, - }, - DEFAULT_CONFIG - ); + await service.getDiff({ ...input, ignoreAreas }, DEFAULT_CONFIG); expect(compareMock.mock.lastCall![3]).not.toHaveProperty('ignoreRegions'); }); it('ok', async () => { - const generateNewImageMock = jest.fn().mockReturnValueOnce({ imagePath: './diff.png', imageName: 'diff.png' }); - const getImagePathMock = jest.fn().mockReturnValueOnce('./baseline.png').mockReturnValueOnce('./image.png'); - const deleteImageMock = jest.fn(); - compareMock.mockResolvedValueOnce({ match: true }); - service = await initService({ - getImagePathMock, - generateNewImageMock, - deleteImageMock, - }); + const { service, deleteImage } = await setup({ match: true }); - const result = await service.getDiff( - { - baseline: 'baseline', - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: false, - }, - DEFAULT_CONFIG - ); + const result = await service.getDiff({ ...input, saveDiffAsFile: false }, DEFAULT_CONFIG); - // odiff writes no diff image when the images match - expect(deleteImageMock).not.toHaveBeenCalled(); + expect(deleteImage).not.toHaveBeenCalled(); expect(result).toStrictEqual(EQUAL_RESULT); }); it('diff found without parsable numbers', async () => { - // odiff-bin resolves a pixel diff without numbers when it cannot parse the binary output - const generateNewImageMock = jest.fn().mockReturnValueOnce({ imagePath: './diff.png', imageName: 'diff.png' }); - const getImagePathMock = jest.fn().mockReturnValueOnce('./baseline.png').mockReturnValueOnce('./image.png'); - compareMock.mockResolvedValueOnce({ match: false, reason: 'pixel-diff' } as Awaited>); - service = await initService({ - getImagePathMock, - generateNewImageMock, - }); + const { service } = await setup({ match: false, reason: 'pixel-diff' } as CompareResult); - const result = await service.getDiff( - { - baseline: 'baseline', - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: true, - }, - DEFAULT_CONFIG - ); - - expect(result).toStrictEqual({ - diffName: 'diff.png', - diffPercent: undefined, - isSameDimension: true, - pixelMisMatchCount: undefined, - status: TestStatus.unresolved, - }); + const result = await service.getDiff(input, DEFAULT_CONFIG); + + expect(result).toStrictEqual(verdict(TestStatus.unresolved, 'diff.png', undefined)); }); it('baseline file missing', async () => { - const generateNewImageMock = jest.fn().mockReturnValueOnce({ imagePath: './diff.png', imageName: 'diff.png' }); - const getImagePathMock = jest.fn().mockReturnValueOnce('./baseline.png').mockReturnValueOnce('./image.png'); - compareMock.mockResolvedValueOnce({ match: false, reason: 'file-not-exists', file: './baseline.png' }); - service = await initService({ - getImagePathMock, - generateNewImageMock, - }); + const { service } = await setup({ match: false, reason: 'file-not-exists', file: './baseline.png' }); - const result = await service.getDiff( - { - baseline: 'baseline', - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: true, - }, - DEFAULT_CONFIG - ); - - expect(result).toStrictEqual(NO_BASELINE_RESULT); + expect(await service.getDiff(input, DEFAULT_CONFIG)).toStrictEqual(NO_BASELINE_RESULT); }); it('image file missing', async () => { - const generateNewImageMock = jest.fn().mockReturnValueOnce({ imagePath: './diff.png', imageName: 'diff.png' }); - const getImagePathMock = jest.fn().mockReturnValueOnce('./baseline.png').mockReturnValueOnce('./image.png'); - compareMock.mockResolvedValueOnce({ match: false, reason: 'file-not-exists', file: './image.png' }); - service = await initService({ - getImagePathMock, - generateNewImageMock, - }); + const { service } = await setup({ match: false, reason: 'file-not-exists', file: './image.png' }); - await expect( - service.getDiff( - { - baseline: 'baseline', - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: true, - }, - DEFAULT_CONFIG - ) - ).rejects.toThrow('Cannot get image: image'); + await expect(service.getDiff(input, DEFAULT_CONFIG)).rejects.toThrow('Cannot get image: image'); }); it('odiff error', async () => { - const generateNewImageMock = jest.fn().mockReturnValueOnce({ imagePath: './diff.png', imageName: 'diff.png' }); - const getImagePathMock = jest.fn().mockReturnValueOnce('./baseline.png').mockReturnValueOnce('./image.png'); + const { service } = await setup(); compareMock.mockRejectedValueOnce(new TypeError('Error: Could not load comparison image: ./image.png')); - service = await initService({ - getImagePathMock, - generateNewImageMock, - }); - await expect( - service.getDiff( - { - baseline: 'baseline', - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: true, - }, - DEFAULT_CONFIG - ) - ).rejects.toThrow('Could not load comparison image'); + await expect(service.getDiff(input, DEFAULT_CONFIG)).rejects.toThrow('Could not load comparison image'); }); }); it.each<[IgnoreAreaDto[], OdiffIgnoreRegions]>([ [[], []], - [ - [ - { - x: 1, - y: 2, - width: 3, - height: 4, - }, - ], - [ - { - x1: 1, - y1: 2, - x2: 4, - y2: 6, - }, - ], - ], + [[{ x: 1, y: 2, width: 3, height: 4 }], [{ x1: 1, y1: 2, x2: 4, y2: 6 }]], + [[{ x: 1.5, y: 2.2, width: 3, height: 4 }], [{ x1: 1, y1: 2, x2: 5, y2: 7 }]], + [[{ x: -5, y: -1, width: 10, height: 3 }], [{ x1: 0, y1: 0, x2: 5, y2: 2 }]], + [[{ x: -20, y: 0, width: 10, height: 5 }], []], + [[{ x: 10, y: 10, width: 0, height: 5 }], [{ x1: 10, y1: 10, x2: 10, y2: 15 }]], ])('ignoreAreaToRegionMapper', async (ignoreAreas, ignoreRegions) => { - const result = ignoreAreaToRegionMapper(ignoreAreas); - - expect(result).toStrictEqual(ignoreRegions); + expect(ignoreAreaToRegionMapper(ignoreAreas)).toStrictEqual(ignoreRegions); }); diff --git a/src/compare/libs/odiff/odiff.service.ts b/src/compare/libs/odiff/odiff.service.ts index ee11ee5a..c2ebdad2 100644 --- a/src/compare/libs/odiff/odiff.service.ts +++ b/src/compare/libs/odiff/odiff.service.ts @@ -32,31 +32,25 @@ export class OdiffService implements ImageComparator { } async getDiff(data: ImageCompareInput, config: OdiffConfig): Promise { - const result: DiffResult = { - ...NO_BASELINE_RESULT, - }; - if (!data.baseline) { return NO_BASELINE_RESULT; } - // compare const baselinePath = this.hddService.getImagePath(data.baseline); const diff = this.hddService.generateNewImage('diff'); + const ignoreRegions = ignoreAreaToRegionMapper(data.ignoreAreas); const compareResult = await compare(baselinePath, this.hddService.getImagePath(data.image), diff.imagePath, { ...config, diffColor: '#cd2cc9', - // a missing file comes back as a result instead of an error naming the file noFailOnFsErrors: true, // odiff rejects an empty --ignore= argument, so only pass regions when there are any - ...(data.ignoreAreas.length > 0 && { ignoreRegions: ignoreAreaToRegionMapper(data.ignoreAreas) }), + ...(ignoreRegions.length > 0 && { ignoreRegions }), }); if (compareResult.match === true) { return EQUAL_RESULT; } if (compareResult.reason === 'file-not-exists') { - // same as the other comparators: a missing baseline file means there is nothing to compare with if (compareResult.file === baselinePath) { return NO_BASELINE_RESULT; } @@ -66,34 +60,31 @@ export class OdiffService implements ImageComparator { return DIFF_DIMENSION_RESULT; } - // pixel diff: odiff has written the diff image - result.isSameDimension = true; - result.diffPercent = compareResult.diffPercentage; - result.pixelMisMatchCount = compareResult.diffCount; - - // process result - // odiff-bin leaves the numbers out when it cannot parse the binary output: keep the run for review - if (!(result.diffPercent <= data.diffTollerancePercent)) { - // save diff - if (data.saveDiffAsFile) { - result.diffName = diff.imageName; - } else { - await this.staticService.deleteImage(diff.imageName); - } - result.status = TestStatus.unresolved; + const withinTolerance = compareResult.diffPercentage <= data.diffTollerancePercent; + const result: DiffResult = { + ...NO_BASELINE_RESULT, + isSameDimension: true, + diffPercent: compareResult.diffPercentage, + pixelMisMatchCount: compareResult.diffCount, + status: withinTolerance ? TestStatus.ok : TestStatus.unresolved, + }; + if (!withinTolerance && data.saveDiffAsFile) { + result.diffName = diff.imageName; } else { await this.staticService.deleteImage(diff.imageName); - result.status = TestStatus.ok; } return result; } } -export const ignoreAreaToRegionMapper = (ignoreAreas: IgnoreAreaDto[]): OdiffIgnoreRegions => { - return ignoreAreas.map((item) => ({ - x1: item.x, - y1: item.y, - x2: item.x + item.width, - y2: item.y + item.height, - })); -}; +export const ignoreAreaToRegionMapper = (ignoreAreas: IgnoreAreaDto[]): OdiffIgnoreRegions => + ignoreAreas.flatMap((item) => { + const region = { + x1: Math.max(0, Math.floor(item.x)), + y1: Math.max(0, Math.floor(item.y)), + x2: Math.ceil(item.x + item.width), + y2: Math.ceil(item.y + item.height), + }; + const usable = Object.values(region).every(Number.isFinite) && region.x2 >= region.x1 && region.y2 >= region.y1; + return usable ? [region] : []; + }); diff --git a/src/compare/libs/odiff/odiff.types.ts b/src/compare/libs/odiff/odiff.types.ts index 5d19d181..97b0ab96 100644 --- a/src/compare/libs/odiff/odiff.types.ts +++ b/src/compare/libs/odiff/odiff.types.ts @@ -1,13 +1,9 @@ import { BlurAmplitudeConfig } from '../blur-amplitude.config'; export type OdiffConfig = BlurAmplitudeConfig & { - /** Output full diff image. */ outputDiffMask: boolean; - /** Do not compare images and produce output if images layout is different. */ failOnLayoutDiff: boolean; - /** Color difference threshold (from 0 to 1). Less more precise. */ threshold: number; - /** If this is true, antialiased pixels are not counted to the diff of an image */ antialiasing: boolean; }; diff --git a/src/compare/libs/pixelmatch/blur-amplitude.core.spec.ts b/src/compare/libs/pixelmatch/blur-amplitude.core.spec.ts index c7b1964d..467ebd96 100644 --- a/src/compare/libs/pixelmatch/blur-amplitude.core.spec.ts +++ b/src/compare/libs/pixelmatch/blur-amplitude.core.spec.ts @@ -10,6 +10,7 @@ import { MAX_SOLID_SHARE, measureBlurredDifference, } from './blur-amplitude.core'; +import { DINOV2_PATCH, gateOfDecoded } from './patch-gate.core'; import { runWorkerJob } from './worker-job'; const SIZE = 20; @@ -28,104 +29,70 @@ const image = (paint: (x: number, y: number) => Rgba | undefined = () => undefin }; const blank = image(); +const pixel = (px: number, py: number, rgba: Rgba = BLACK) => + image((x, y) => (x === px && y === py ? rgba : undefined)); +const column = (col: number) => image((x) => (x === col ? BLACK : undefined)); +const box = (rgba: Rgba = BLACK) => image((x, y) => (x >= 4 && x < 16 && y >= 4 && y < 16 ? rgba : undefined)); -// the measure of a pair that is known to be measurable -const measured = (a: RawImage, b: RawImage, sigma: number): BlurredDifference => { - const measure = measureBlurredDifference(a, b, sigma); +const measured = (a: RawImage, b: RawImage): BlurredDifference => { + const measure = measureBlurredDifference(a, b); if (!measure) { throw new Error('expected a measure'); } return measure; }; -const maxBlurredDifference = (a: RawImage, b: RawImage, sigma: number): number => measured(a, b, sigma).amplitude; +const maxBlurredDifference = (a: RawImage, b: RawImage): number => measured(a, b).amplitude; -// Expected values are scipy.ndimage.gaussian_filter (sigma=1, truncate=4, -// mode='reflect') on the same black-and-white images, where every channel and -// the luma agree. +// expected: scipy.ndimage.gaussian_filter(sigma=1, truncate=4, mode='reflect') describe('measureBlurredDifference', () => { it('is zero for identical images', () => { - expect(maxBlurredDifference(blank, image(), 1)).toBe(0); + expect(maxBlurredDifference(blank, image())).toBe(0); }); - // what anti-aliasing mostly is: single pixels flipping along edges it('keeps an isolated pixel well under the default threshold', () => { - const amplitude = maxBlurredDifference( - blank, - image((x, y) => (x === 10 && y === 10 ? BLACK : undefined)), - 1 - ); + const amplitude = maxBlurredDifference(blank, pixel(10, 10)); expect(amplitude).toBeCloseTo(40.585, 2); expect(amplitude).toBeLessThan(DEFAULT_BLUR_AMPLITUDE_THRESHOLD); }); - // the other thing it is: an edge landing one pixel further over it('keeps a hairline moved by one pixel under the default threshold', () => { - const before = image((x) => (x === 10 ? BLACK : undefined)); - const after = image((x) => (x === 11 ? BLACK : undefined)); - - const amplitude = maxBlurredDifference(before, after, 1); + const amplitude = maxBlurredDifference(column(10), column(11)); expect(amplitude).toBeCloseTo(47.935, 2); expect(amplitude).toBeLessThan(DEFAULT_BLUR_AMPLITUDE_THRESHOLD); }); it('reports the full contrast of a solid element that appeared', () => { - const block = image((x, y) => (x >= 4 && x < 16 && y >= 4 && y < 16 ? BLACK : undefined)); - - expect(maxBlurredDifference(blank, block, 1)).toBeCloseTo(255, 2); + expect(maxBlurredDifference(blank, box())).toBeCloseTo(255, 2); }); - // the image is mirrored at its border, not padded with nothing, so a change - // in a corner is not underrated it('mirrors the image at its border', () => { - const corner = image((x, y) => (x === 0 && y === 0 ? BLACK : undefined)); - - expect(maxBlurredDifference(blank, corner, 1)).toBeCloseTo(104.747, 2); + expect(maxBlurredDifference(blank, pixel(0, 0))).toBeCloseTo(104.747, 2); }); - it('spreads a change thinner under a wider blur', () => { - const dot = image((x, y) => (x === 10 && y === 10 ? BLACK : undefined)); - - expect(maxBlurredDifference(blank, dot, 2)).toBeCloseTo(10.146, 2); - }); - - // ignore areas are blanked to transparent black, and pixelmatch reads - // transparent pixels as white; reading them the same way keeps a masked area - // from looking like a change against a white background it('reads transparent pixels as white', () => { - const transparent = image((x, y) => (x === 10 && y === 10 ? [0, 0, 0, 0] : undefined)); - - expect(maxBlurredDifference(blank, transparent, 1)).toBe(0); + expect(maxBlurredDifference(blank, pixel(10, 10, [0, 0, 0, 0]))).toBe(0); }); - // luma alone would call this nothing: the grey has the red's brightness it('sees a change of hue at equal brightness', () => { - const red = image((x, y) => (x >= 4 && x < 16 && y >= 4 && y < 16 ? [255, 0, 0, 255] : undefined)); - const grey = image((x, y) => (x >= 4 && x < 16 && y >= 4 && y < 16 ? [76, 76, 76, 255] : undefined)); - - expect(maxBlurredDifference(red, grey, 1)).toBeGreaterThan(DEFAULT_BLUR_AMPLITUDE_THRESHOLD); + expect(maxBlurredDifference(box([255, 0, 0, 255]), box([76, 76, 76, 255]))).toBeGreaterThan( + DEFAULT_BLUR_AMPLITUDE_THRESHOLD + ); }); it('refuses images of different sizes', () => { const smaller: RawImage = { data: Buffer.alloc(4 * 4 * 4), width: 4, height: 4 }; - expect(() => maxBlurredDifference(blank, smaller, 1)).toThrow(); + expect(() => maxBlurredDifference(blank, smaller)).toThrow(); }); - // anti-aliasing lives on edges, which the blur flattens; a filled shape's - // inside it leaves as it was it('counts the pixels of a solid change, not those of an edge', () => { - const block = image((x, y) => (x >= 4 && x < 16 && y >= 4 && y < 16 ? [200, 200, 200, 255] : undefined)); - const hairline = image((x) => (x === 10 ? BLACK : undefined)); - - expect(measured(blank, block, 1).solidPixels).toBeGreaterThan(50); - expect(measured(blank, hairline, 1).solidPixels).toBe(0); + expect(measured(blank, box([200, 200, 200, 255])).solidPixels).toBeGreaterThan(50); + expect(measured(blank, column(10)).solidPixels).toBe(0); }); - // measuring a change spread over a huge screenshot would cost seconds and - // hundreds of megabytes in a worker it('gives no measure for a changed area beyond the size limit', () => { const width = 4000; const height = Math.ceil(MAX_MEASURED_PIXELS / width) + 1; @@ -134,7 +101,7 @@ describe('measureBlurredDifference', () => { b.data[0] = 0; b.data[b.data.length - 4] = 0; - expect(measureBlurredDifference(a, b, 1)).toBeNull(); + expect(measureBlurredDifference(a, b)).toBeNull(); }); }); @@ -147,7 +114,6 @@ describe('isRenderingNoise', () => { expect(isRenderingNoise({ amplitude: 81, solidPixels: 0, solidShare: 0 }, 80)).toBe(false); }); - // a panel turned light grey: faint, so under any threshold, but solid it('is not noise when too much of the change is solid, however faint', () => { expect(isRenderingNoise({ amplitude: 31, solidPixels: 1, solidShare: MAX_SOLID_SHARE + 0.001 }, 80)).toBe(false); }); @@ -156,7 +122,6 @@ describe('isRenderingNoise', () => { expect(isRenderingNoise(null, 80)).toBe(false); }); - // measured on a real-size picture rather than on numbers it('sees a faint panel shade as a change', () => { const size = 200; const flat = (shade: number): RawImage => { @@ -169,7 +134,7 @@ describe('isRenderingNoise', () => { return { data, width: size, height: size }; }; - const measure = measured(flat(255), flat(224), 1); + const measure = measured(flat(255), flat(224)); expect(measure.amplitude).toBeLessThan(DEFAULT_BLUR_AMPLITUDE_THRESHOLD); expect(isRenderingNoise(measure, DEFAULT_BLUR_AMPLITUDE_THRESHOLD)).toBe(false); @@ -177,19 +142,17 @@ describe('isRenderingNoise', () => { }); describe('computeBlurAmplitude', () => { - // the worker-pool job: PNG bytes in, so the decoding happens off the event loop const encode = (raw: RawImage): Buffer => { const png = new PNG({ width: raw.width, height: raw.height }); raw.data.copy(png.data); return PNG.sync.write(png); }; - const block = image((x, y) => (x >= 4 && x < 16 && y >= 4 && y < 16 ? BLACK : undefined)); + const block = box(); const job = (overrides: Partial = {}): BlurAmplitudeJobInput => ({ kind: 'blur-amplitude', baseline: encode(blank), image: encode(block), ignoreAreas: [], - sigma: 1, ...overrides, }); @@ -197,7 +160,6 @@ describe('computeBlurAmplitude', () => { expect(computeBlurAmplitude(job()).measure?.amplitude).toBeCloseTo(255, 2); }); - // what the project has masked out must not decide its verdict it('leaves out what the ignore areas cover', () => { expect(computeBlurAmplitude(job({ ignoreAreas: [{ x: 0, y: 0, width: SIZE, height: SIZE }] })).measure).toEqual({ amplitude: 0, @@ -206,25 +168,67 @@ describe('computeBlurAmplitude', () => { }); }); - // a comparator allowed to compare different sizes has nothing to line up it('has no answer for screenshots of different sizes', () => { const smaller = encode({ data: Buffer.alloc(4 * 4 * 4, 255), width: 4, height: 4 }); expect(computeBlurAmplitude(job({ image: smaller })).measure).toBeNull(); }); - // the caller keeps the comparator's verdict rather than failing the upload it('has no answer for bytes that are not a PNG', () => { expect(computeBlurAmplitude(job({ image: Buffer.from('not a png') })).measure).toBeNull(); }); - // what a Buffer arrives as on the far side of postMessage it('reads the Uint8Array views a worker message turns Buffers into', () => { const viaMessage = job({ baseline: new Uint8Array(encode(blank)), image: new Uint8Array(encode(block)) }); expect(computeBlurAmplitude(viaMessage).measure?.amplitude).toBeCloseTo(255, 2); }); + describe('with the gate asked for', () => { + const dot = pixel(10, 10); + const gated = (overrides: Partial = {}) => + job({ image: encode(dot), gateIfNoise: { threshold: DEFAULT_BLUR_AMPLITUDE_THRESHOLD }, ...overrides }); + + it('returns the DINOv2 patch gate of a pair it takes for rendering noise', () => { + const output = computeBlurAmplitude(gated()); + + expect(isRenderingNoise(output.measure, DEFAULT_BLUR_AMPLITUDE_THRESHOLD)).toBe(true); + expect(output.gate).toEqual(gateOfDecoded(blank, dot, DINOV2_PATCH)); + expect(output.gate?.cells).toEqual([0]); + }); + + it('leaves out what the ignore areas cover', () => { + const output = computeBlurAmplitude(gated({ ignoreAreas: [{ x: 8, y: 8, width: 4, height: 4 }] })); + + expect(output.gate?.cells).toEqual([]); + }); + + it('returns no gate for a change stronger than rendering noise', () => { + const output = computeBlurAmplitude(gated({ image: encode(block) })); + + expect(output.measure?.amplitude).toBeGreaterThan(DEFAULT_BLUR_AMPLITUDE_THRESHOLD); + expect(output.gate).toBeUndefined(); + }); + + it('judges the noise by the threshold it is given', () => { + expect(computeBlurAmplitude(gated({ gateIfNoise: { threshold: 30 } })).gate).toBeUndefined(); + }); + + it('returns no gate when it was not asked for', () => { + expect(computeBlurAmplitude(job({ image: encode(dot) })).gate).toBeUndefined(); + }); + + it('returns no gate for screenshots it cannot line up', () => { + expect(computeBlurAmplitude(gated({ image: Buffer.from('not a png') }))).toEqual({ measure: null }); + }); + + it('returns a gate that survives a structured clone', () => { + const output = computeBlurAmplitude(gated()); + + expect(structuredClone(output)).toEqual(output); + }); + }); + it('is what the worker runs for a blur-amplitude job', () => { expect(runWorkerJob(job())).toEqual(computeBlurAmplitude(job())); }); diff --git a/src/compare/libs/pixelmatch/blur-amplitude.core.ts b/src/compare/libs/pixelmatch/blur-amplitude.core.ts index 24c8a178..87cfbb5e 100644 --- a/src/compare/libs/pixelmatch/blur-amplitude.core.ts +++ b/src/compare/libs/pixelmatch/blur-amplitude.core.ts @@ -1,109 +1,56 @@ -import { PNG } from 'pngjs'; import { IgnoreAreaDto } from '../../../test-runs/dto/ignore-area.dto'; -import { applyIgnoreAreas, RawImage } from '../../utils'; +import { RawImage, readSameSizePair } from '../../utils'; +import { DINOV2_PATCH, gateOfDecoded, PatchGate } from './patch-gate.core'; -/** - * Filter for rendering noise, calibrated in research/jepa-screenshot-diff: - * blurring the difference between two screenshots by a pixel or so makes - * anti-aliasing and sub-pixel rasterisation differences — single pixels - * flipping, edges landing a fraction of a pixel over — average out to a weak - * change, while anything a person would notice (a letter, an icon, a moved - * block) keeps most of its contrast. - * - * The blur cannot tell a faint change that is *solid* — a panel turned light - * grey — from noise by its strength alone: blurring a uniform patch leaves it - * as it is. So the pixels the blur does not attenuate are counted too, and a - * change with many of them is never noise. - */ export interface BlurAmplitudeOptions { - // Gaussian sigma in pixels - sigma: number; - // strongest blurred colour-channel difference (0-255) still treated as - // rendering noise threshold: number; - // a second look at what the blur would pass (see dinov2/dinov2.service.ts): - // largest content distance between the two screenshots still treated as noise dinov2?: { threshold: number }; } export interface BlurredDifference { - // strongest blurred colour-channel difference, 0-255 amplitude: number; - // pixels whose change the blur leaves (nearly) intact: a solid change solidPixels: number; - // the same as a share of the screenshot, 0-1 solidShare: number; } -/** - * The worker-pool job CompareService runs for comparators that do not apply the - * filter in their own pass (looks-same, odiff). PNG bytes in, because decoding - * is the expensive part and belongs off the event loop; all input/output must - * stay structured-clone serializable. - */ export interface BlurAmplitudeJobInput { kind: 'blur-amplitude'; baseline: Buffer | Uint8Array; image: Buffer | Uint8Array; ignoreAreas: IgnoreAreaDto[]; - sigma: number; + gateIfNoise?: { threshold: number }; } export interface BlurAmplitudeJobOutput { - // null when there is nothing to measure: unreadable bytes, different sizes, - // or a change too large to measure within bounds measure: BlurredDifference | null; + gate?: PatchGate; } export const DEFAULT_BLUR_SIGMA = 1; -// Above the strongest of 57 anti-aliasing-only pairs of real VRT pages (75.1) -// in the research dataset; at this value 72% of the injected bugs still fail. +// strongest of 57 anti-aliasing-only real pairs: 75.1; 72% of injected bugs still fail export const DEFAULT_BLUR_AMPLITUDE_THRESHOLD = 80; -// A pixel counts as solid when its change is at least this many levels and the -// blur keeps at least this share of it. Sub-pixel shifts of large flat shapes -// make up to 0.7% of a page solid in the research dataset (6,300 of 1280x720); -// a panel turning #e0e0e0 makes 19%. Taken as a share of the screenshot, so it -// holds for a small component and a tall page alike. +// solid share: sub-pixel shifts up to 0.7%, #e0e0e0 panel 19% const SOLID_MIN_CHANGE = 6; const SOLID_KEEP = 0.9; export const MAX_SOLID_SHARE = 0.01; -// Beyond this many pixels in the changed area, measuring would cost seconds -// and hundreds of megabytes in a worker; the comparator's verdict stands. export const MAX_MEASURED_PIXELS = 16_000_000; -/** The filter's verdict: true only for a measured change no stronger than noise. */ export function isRenderingNoise(measure: BlurredDifference | null, threshold: number): boolean { return measure !== null && measure.amplitude <= threshold && measure.solidShare <= MAX_SOLID_SHARE; } -// postMessage turns Buffers into Uint8Array views over their own ArrayBuffer. -function toBuffer(data: Buffer | Uint8Array): Buffer { - return Buffer.isBuffer(data) ? data : Buffer.from(data.buffer, data.byteOffset, data.byteLength); -} - export function computeBlurAmplitude(input: BlurAmplitudeJobInput): BlurAmplitudeJobOutput { - // The caller keeps the comparator's verdict when this has no answer, so a - // corrupt screenshot costs the filter, not the upload. - let baseline: PNG; - let image: PNG; - try { - baseline = PNG.sync.read(toBuffer(input.baseline)); - image = PNG.sync.read(toBuffer(input.image)); - } catch { - return { measure: null }; - } - if (baseline.width !== image.width || baseline.height !== image.height) { - return { measure: null }; - } - applyIgnoreAreas(baseline, input.ignoreAreas); - applyIgnoreAreas(image, input.ignoreAreas); - return { measure: measureBlurredDifference(baseline, image, input.sigma) }; + const pair = readSameSizePair(input.baseline, input.image, input.ignoreAreas); + const measure = pair && measureBlurredDifference(pair.baseline, pair.image); + const gate = + pair && input.gateIfNoise && isRenderingNoise(measure, input.gateIfNoise.threshold) + ? gateOfDecoded(pair.baseline, pair.image, DINOV2_PATCH) + : undefined; + return { measure, ...(gate ? { gate } : {}) }; } -// Same truncation as scipy.ndimage.gaussian_filter, which the research -// prototype used: the kernel reaches out four sigmas. function gaussianKernel(sigma: number): Float32Array { const radius = Math.floor(4 * sigma + 0.5); const kernel = new Float64Array(2 * radius + 1); @@ -115,17 +62,12 @@ function gaussianKernel(sigma: number): Float32Array { return Float32Array.from(kernel, (weight) => weight / sum); } -// Mirrors an index at the image border (scipy's 'reflect': -1 -> 0, n -> n-1), -// so a change in a corner is not diluted by an imaginary blank margin. function reflect(index: number, size: number): number { const period = 2 * size; const wrapped = ((index % period) + period) % period; return wrapped < size ? wrapped : period - 1 - wrapped; } -// A colour channel of a pixel over white, the way pixelmatch reads -// transparency: the ignore areas are blanked to transparent black and must look -// like nothing. function channel(data: Buffer | Uint8Array, offset: number, rgb: number): number { return 255 + (data[offset + rgb] - 255) * (data[offset + 3] / 255); } @@ -137,7 +79,6 @@ interface Box { maxY: number; } -// the bounding box of every pixel that differs at all, or null when none does function changedBox({ data: a, width, height }: RawImage, { data: b }: RawImage): Box | null { let minX = width; let minY = height; @@ -146,7 +87,6 @@ function changedBox({ data: a, width, height }: RawImage, { data: b }: RawImage) for (let y = 0; y < height; y++) { for (let x = 0; x < width; x++) { const o = (y * width + x) * 4; - // equal bytes are the overwhelming case and need no blending to compare if (a[o] === b[o] && a[o + 1] === b[o + 1] && a[o + 2] === b[o + 2] && a[o + 3] === b[o + 3]) { continue; } @@ -166,9 +106,6 @@ function changedBox({ data: a, width, height }: RawImage, { data: b }: RawImage) return maxX < 0 ? null : { minX, minY, maxX, maxY }; } -// For each position from `from - radius` to `to + radius`, the index into the -// box it reads once mirrored at the image border, or -1 outside the box (where -// the difference is zero). function sourceIndex(from: number, to: number, radius: number, size: number, boxMin: number, boxMax: number) { const index = new Int32Array(to - from + 1 + 2 * radius); for (let i = 0; i < index.length; i++) { @@ -178,14 +115,7 @@ function sourceIndex(from: number, to: number, radius: number, size: number, box return index; } -/** - * The strongest Gaussian-blurred difference between two images of equal size, - * per colour channel — so a change of hue at equal brightness is not mistaken - * for noise — and how many pixels the blur leaves intact. Blurring is linear, - * so the difference is blurred rather than each image, and only around the box - * where it is non-zero. Null when that box is too large to measure. - */ -export function measureBlurredDifference(baseline: RawImage, image: RawImage, sigma: number): BlurredDifference | null { +export function measureBlurredDifference(baseline: RawImage, image: RawImage): BlurredDifference | null { const { width, height } = baseline; if (image.width !== width || image.height !== height) { throw new Error(`Cannot blur-compare ${width}x${height} with ${image.width}x${image.height}`); @@ -201,9 +131,8 @@ export function measureBlurredDifference(baseline: RawImage, image: RawImage, si return null; } - const kernel = gaussianKernel(sigma); + const kernel = gaussianKernel(DEFAULT_BLUR_SIGMA); const radius = (kernel.length - 1) / 2; - // where the blurred difference can be non-zero const left = Math.max(0, minX - radius); const right = Math.min(width - 1, maxX + radius); const top = Math.max(0, minY - radius); @@ -212,7 +141,6 @@ export function measureBlurredDifference(baseline: RawImage, image: RawImage, si const columns = sourceIndex(left, right, radius, width, minX, maxX); const rowsOf = sourceIndex(top, bottom, radius, height, minY, maxY); - // shared by the three channels: one set of buffers, not three const difference = new Float32Array(boxWidth * boxHeight); const rows = new Float32Array(regionWidth * boxHeight); const solid = new Uint8Array(boxWidth * boxHeight); @@ -226,7 +154,6 @@ export function measureBlurredDifference(baseline: RawImage, image: RawImage, si } } - // horizontal pass, over the box's rows only: the others blur to zero for (let by = 0; by < boxHeight; by++) { const rowStart = by * boxWidth; for (let rx = 0; rx < regionWidth; rx++) { @@ -239,7 +166,6 @@ export function measureBlurredDifference(baseline: RawImage, image: RawImage, si } } - // vertical pass, keeping the extreme and marking what the blur left intact for (let ry = 0; ry <= bottom - top; ry++) { const y = top + ry; const insideBox = y >= minY && y <= maxY; diff --git a/src/compare/libs/pixelmatch/patch-gate.core.spec.ts b/src/compare/libs/pixelmatch/patch-gate.core.spec.ts index 306a5735..00712fcc 100644 --- a/src/compare/libs/pixelmatch/patch-gate.core.spec.ts +++ b/src/compare/libs/pixelmatch/patch-gate.core.spec.ts @@ -1,67 +1,43 @@ import { PNG } from 'pngjs'; -import { computePatchGate, PatchGateJobInput } from './patch-gate.core'; -import { runWorkerJob } from './worker-job'; +import { DINOV2_PATCH, gateOfDecoded } from './patch-gate.core'; +import { applyIgnoreAreas } from '../../utils'; +import { IgnoreAreaDto } from '../../../test-runs/dto/ignore-area.dto'; +import { paintedPng, pngBytes, solidPng, square } from '../../test-utils'; const WIDTH = 60; const HEIGHT = 40; -const png = (paint: (x: number, y: number) => [number, number, number] | undefined = () => undefined): Buffer => { - const image = new PNG({ width: WIDTH, height: HEIGHT }); - for (let y = 0; y < HEIGHT; y++) { - for (let x = 0; x < WIDTH; x++) { - const [r, g, b] = paint(x, y) ?? [255, 255, 255]; - image.data.set([r, g, b, 255], (y * WIDTH + x) * 4); - } - } - return PNG.sync.write(image); -}; - -const blank = png(); -// a solid block covering patch (1, 2) only: rows 14-27, columns 28-41 -const block = png((x, y) => (x >= 30 && x < 40 && y >= 16 && y < 26 ? [200, 40, 40] : undefined)); +const blank = pngBytes(solidPng(WIDTH, HEIGHT)); +const block = pngBytes(paintedPng(WIDTH, HEIGHT, square(30, 16, 10, [200, 40, 40]))); -const job = (overrides: Partial = {}): PatchGateJobInput => ({ - kind: 'patch-gate', - baseline: blank, - image: block, - ignoreAreas: [], - patch: 14, - ...overrides, -}); +const gateOf = (baseline: Buffer, image: Buffer, ignoreAreas: IgnoreAreaDto[] = []) => { + const decodedBaseline = PNG.sync.read(baseline); + const decodedImage = PNG.sync.read(image); + applyIgnoreAreas(decodedBaseline, ignoreAreas); + applyIgnoreAreas(decodedImage, ignoreAreas); + return gateOfDecoded(decodedBaseline, decodedImage, DINOV2_PATCH); +}; -describe('computePatchGate', () => { +describe('gateOfDecoded', () => { it('marks the patches where pixelmatch sees a change', () => { - expect(computePatchGate(job()).gate).toEqual({ width: 5, height: 3, cells: [1 * 5 + 2] }); + expect(gateOf(blank, block)).toEqual({ width: 5, height: 3, cells: [1 * 5 + 2] }); }); it('marks nothing for identical screenshots', () => { - expect(computePatchGate(job({ image: blank })).gate).toEqual({ width: 5, height: 3, cells: [] }); + expect(gateOf(blank, blank)).toEqual({ width: 5, height: 3, cells: [] }); }); - // what the project has masked out must not decide the verdict it('leaves out what the ignore areas cover', () => { - expect(computePatchGate(job({ ignoreAreas: [{ x: 28, y: 14, width: 14, height: 14 }] })).gate?.cells).toEqual([]); + expect(gateOf(blank, block, [{ x: 28, y: 14, width: 14, height: 14 }]).cells).toEqual([]); }); - it('has no answer for screenshots of different sizes', () => { - const taller = new PNG({ width: WIDTH, height: HEIGHT + 4 }); - taller.data.fill(255); - - expect(computePatchGate(job({ image: PNG.sync.write(taller) })).gate).toBeNull(); - }); - - it('has no answer for bytes that are not a PNG', () => { - expect(computePatchGate(job({ image: Buffer.from('not a png') })).gate).toBeNull(); - }); + it('gives a gate that survives a structured clone', () => { + const gate = gateOf(blank, block); - // what a Buffer arrives as on the far side of postMessage - it('reads the Uint8Array views a worker message turns Buffers into', () => { - expect( - computePatchGate(job({ baseline: new Uint8Array(blank), image: new Uint8Array(block) })).gate?.cells - ).toEqual([7]); + expect(structuredClone(gate)).toEqual(gate); }); - it('is what the worker runs for a patch-gate job', () => { - expect(runWorkerJob(job())).toEqual(computePatchGate(job())); + it('uses the 14 px patch of DINOv2', () => { + expect(DINOV2_PATCH).toBe(14); }); }); diff --git a/src/compare/libs/pixelmatch/patch-gate.core.ts b/src/compare/libs/pixelmatch/patch-gate.core.ts index 0f8e7160..9b7451f4 100644 --- a/src/compare/libs/pixelmatch/patch-gate.core.ts +++ b/src/compare/libs/pixelmatch/patch-gate.core.ts @@ -1,76 +1,32 @@ import { PNG } from 'pngjs'; import Pixelmatch from 'pixelmatch'; -import { IgnoreAreaDto } from '../../../test-runs/dto/ignore-area.dto'; -import { applyIgnoreAreas } from '../../utils'; +import { RawImage } from '../../utils'; -/** - * Where a DINOv2 look at two screenshots should count (see - * dinov2/dinov2.service.ts): the patches in which pixelmatch, with its - * anti-aliasing detector on, sees a change. Elsewhere a content distance is - * the model's context bleeding across a tile, not a change — measured in - * research/jepa-screenshot-diff, where this gate took DINOv2 from 14% to 73% - * of bugs caught. Runs on the diff worker pool; all input/output must stay - * structured-clone serializable. - */ -export interface PatchGateJobInput { - kind: 'patch-gate'; - baseline: Buffer | Uint8Array; - image: Buffer | Uint8Array; - ignoreAreas: IgnoreAreaDto[]; - // the model's patch size in pixels - patch: number; -} +export const DINOV2_PATCH = 14; export interface PatchGate { - // patches across and down, covering the screenshot width: number; height: number; - // indices (row * width + col) of the patches holding a change cells: number[]; } -export interface PatchGateJobOutput { - // null when there is nothing to line up: unreadable bytes or different sizes - gate: PatchGate | null; -} - -// postMessage turns Buffers into Uint8Array views over their own ArrayBuffer. -function toBuffer(data: Buffer | Uint8Array): Buffer { - return Buffer.isBuffer(data) ? data : Buffer.from(data.buffer, data.byteOffset, data.byteLength); -} - -export function computePatchGate(input: PatchGateJobInput): PatchGateJobOutput { - let baseline: PNG; - let image: PNG; - try { - baseline = PNG.sync.read(toBuffer(input.baseline)); - image = PNG.sync.read(toBuffer(input.image)); - } catch { - return { gate: null }; - } +export function gateOfDecoded(baseline: RawImage, image: RawImage, patch: number): PatchGate { const { width, height } = baseline; - if (image.width !== width || image.height !== height) { - return { gate: null }; - } - applyIgnoreAreas(baseline, input.ignoreAreas); - applyIgnoreAreas(image, input.ignoreAreas); - // the research's settings, whatever the project's: this asks where the - // content may have changed, not whether the run fails const mask = new PNG({ width, height }); Pixelmatch(baseline.data, image.data, mask.data, width, height, { threshold: 0.1, includeAA: false, diffMask: true }); - const columns = Math.ceil(width / input.patch); - const rows = Math.ceil(height / input.patch); + const columns = Math.ceil(width / patch); + const rows = Math.ceil(height / patch); const marked = new Uint8Array(columns * rows); for (let y = 0; y < height; y++) { for (let x = 0; x < width; x++) { if (mask.data[(y * width + x) * 4 + 3] > 0) { - marked[Math.floor(y / input.patch) * columns + Math.floor(x / input.patch)] = 1; + marked[Math.floor(y / patch) * columns + Math.floor(x / patch)] = 1; } } } const cells: number[] = []; marked.forEach((value, index) => value && cells.push(index)); - return { gate: { width: columns, height: rows, cells } }; + return { width: columns, height: rows, cells }; } diff --git a/src/compare/libs/pixelmatch/pixelmatch.core.boundaries.spec.ts b/src/compare/libs/pixelmatch/pixelmatch.core.real-images.spec.ts similarity index 63% rename from src/compare/libs/pixelmatch/pixelmatch.core.boundaries.spec.ts rename to src/compare/libs/pixelmatch/pixelmatch.core.real-images.spec.ts index 2abae522..b0ea534c 100644 --- a/src/compare/libs/pixelmatch/pixelmatch.core.boundaries.spec.ts +++ b/src/compare/libs/pixelmatch/pixelmatch.core.real-images.spec.ts @@ -1,17 +1,9 @@ import { PNG } from 'pngjs'; +import { paintedPng, pngBytes } from '../../test-utils'; import { computePixelmatchDiff, PixelmatchJobInput } from './pixelmatch.core'; -// white image with the first `redPixels` pixels (row by row) painted red -const png = (width: number, height: number, redPixels = 0): Buffer => { - const image = new PNG({ width, height }); - for (let i = 0; i < width * height; i++) { - image.data[i * 4] = 255; - image.data[i * 4 + 1] = i < redPixels ? 0 : 255; - image.data[i * 4 + 2] = i < redPixels ? 0 : 255; - image.data[i * 4 + 3] = 255; - } - return PNG.sync.write(image); -}; +const png = (width: number, height: number, redPixels = 0): Buffer => + pngBytes(paintedPng(width, height, (x, y) => (y * width + x < redPixels ? [255, 0, 0] : undefined))); const job = (baseline: Buffer, image: Buffer, overrides: Partial = {}) => computePixelmatchDiff({ @@ -20,7 +12,6 @@ const job = (baseline: Buffer, image: Buffer, overrides: Partial { - // 36 of 3600 pixels differ: exactly 1% const baseline = png(60, 60); const image = png(60, 60, 36); - it('reports the mismatch count and percent', () => { + it('reports the mismatch count and percent, and does not keep the diff when the percent equals the tolerance', () => { const result = job(baseline, image, { diffTolerancePercent: 1 }); expect(result.equal).toBe(false); expect(result.isSameDimension).toBe(true); expect(result.pixelMisMatchCount).toBe(36); expect(result.diffPercent).toBe(1); - }); - - it('does not keep the diff when the percent equals the tolerance', () => { - const result = job(baseline, image, { diffTolerancePercent: 1, withThumbnails: true }); - expect(result.diffBuffer).toBeUndefined(); expect(result.imageThumbnail).toBeUndefined(); expect(result.diffThumbnail).toBeUndefined(); }); it('keeps the diff and thumbnails when the percent is just over the tolerance', () => { - const result = job(baseline, image, { diffTolerancePercent: 0.99, withThumbnails: true }); + const result = job(baseline, image, { diffTolerancePercent: 0.99 }); expect(PNG.sync.read(Buffer.from(result.diffBuffer!)).width).toBe(60); expect(result.imageThumbnail).toBeDefined(); @@ -59,7 +44,7 @@ describe('computePixelmatchDiff tolerance boundary', () => { }); it('does not keep the diff over the tolerance when saving is off', () => { - const result = job(baseline, image, { diffTolerancePercent: 0, saveDiff: false, withThumbnails: true }); + const result = job(baseline, image, { diffTolerancePercent: 0, saveDiff: false }); expect(result.diffBuffer).toBeUndefined(); expect(result.imageThumbnail).toBeUndefined(); @@ -67,16 +52,13 @@ describe('computePixelmatchDiff tolerance boundary', () => { }); describe('computePixelmatchDiff dimensions', () => { - it('stops at a different height when different dimensions are not allowed', () => { - expect(job(png(60, 60), png(60, 30, 1))).toStrictEqual({ equal: false, isSameDimension: false }); - }); - - it('stops at a different width when different dimensions are not allowed', () => { - expect(job(png(60, 60), png(30, 60, 1))).toStrictEqual({ equal: false, isSameDimension: false }); + it.each([ + ['height', png(60, 30, 1)], + ['width', png(30, 60, 1)], + ])('stops at a different %s when different dimensions are not allowed', (_, image) => { + expect(job(png(60, 60), image)).toStrictEqual({ equal: false, isSameDimension: false }); }); - // 60x30 and 30x60 with the first 60 pixels red hold the very same bytes: - // one red row against two half-width red rows const wide = png(60, 30, 60); const tall = png(30, 60, 60); @@ -93,16 +75,13 @@ describe('computePixelmatchDiff dimensions', () => { }); it('compares on the larger canvas when different dimensions are allowed', () => { - // the checkpoint is padded with transparent pixels, which pixelmatch blends to white: - // only the baseline's red first row differs - const result = job(png(10, 10, 10), png(10, 5), { allowDiffDimensions: true, withSignature: true }); + const result = job(png(10, 10, 10), png(10, 5), { allowDiffDimensions: true }); expect(result.equal).toBe(false); expect(result.isSameDimension).toBe(false); expect(result.pixelMisMatchCount).toBe(10); expect(result.diffPercent).toBe(10); expect(PNG.sync.read(Buffer.from(result.diffBuffer!)).height).toBe(10); - // no signature for screenshots of different sizes expect(result.signature).toBeUndefined(); }); }); diff --git a/src/compare/libs/pixelmatch/pixelmatch.core.spec.ts b/src/compare/libs/pixelmatch/pixelmatch.core.spec.ts index c237a668..3e2b94c8 100644 --- a/src/compare/libs/pixelmatch/pixelmatch.core.spec.ts +++ b/src/compare/libs/pixelmatch/pixelmatch.core.spec.ts @@ -1,77 +1,55 @@ +import Pixelmatch from 'pixelmatch'; import { PNG } from 'pngjs'; -import { DEFAULT_BLUR_AMPLITUDE_THRESHOLD, DEFAULT_BLUR_SIGMA } from './blur-amplitude.core'; +import { DEFAULT_BLUR_AMPLITUDE_THRESHOLD } from './blur-amplitude.core'; +import { readSameSizePair } from '../../utils'; +import { DINOV2_PATCH, gateOfDecoded } from './patch-gate.core'; import { computePixelmatchDiff, PixelmatchJobInput } from './pixelmatch.core'; import { computeChangeSignature } from './signature.core'; import { THUMBNAIL_MAX_DIMENSION } from './thumbnail.core'; +import { paintedPng, pngBytes, Rgb, solidPng, square } from '../../test-utils'; const WIDTH = 60; const HEIGHT = 60; -const png = (paint: (set: (x: number, y: number, rgb: [number, number, number]) => void) => void): Buffer => { - const image = new PNG({ width: WIDTH, height: HEIGHT }); - for (let i = 0; i < WIDTH * HEIGHT; i++) { - image.data[i * 4] = 255; - image.data[i * 4 + 1] = 255; - image.data[i * 4 + 2] = 255; - image.data[i * 4 + 3] = 255; - } - paint((x, y, [r, g, b]) => { - const index = (y * WIDTH + x) * 4; - image.data[index] = r; - image.data[index + 1] = g; - image.data[index + 2] = b; - }); - return PNG.sync.write(image); -}; - -const block = (left: number, top: number, rgb: [number, number, number]): Buffer => - png((set) => { - for (let y = top; y < top + 12; y++) { - for (let x = left; x < left + 12; x++) { - set(x, y, rgb); - } - } - }); +const png = (paint?: (x: number, y: number) => Rgb | undefined): Buffer => pngBytes(paintedPng(WIDTH, HEIGHT, paint)); -const blank = png(() => undefined); -const RED: [number, number, number] = [255, 0, 0]; +const block = (left: number, top: number, rgb: Rgb): Buffer => png(square(left, top, 12, rgb)); + +const blank = png(); +const RED: Rgb = [255, 0, 0]; const diffJob = ( baseline: Buffer, image: Buffer, - withSignature: boolean, - withThumbnails = false, { saveDiff = true, diffTolerancePercent = 0, includeAA = false, + allowDiffDimensions = false, blurAmplitude = undefined as PixelmatchJobInput['blurAmplitude'], + ignoreAreas = [] as PixelmatchJobInput['ignoreAreas'], } = {} ) => computePixelmatchDiff({ kind: 'diff', baseline, image, - ignoreAreas: [], + ignoreAreas, threshold: 0.1, includeAA, - allowDiffDimensions: false, + allowDiffDimensions, diffTolerancePercent, saveDiff, - withSignature, - withThumbnails, blurAmplitude, }); describe('computePixelmatchDiff with a signature', () => { - // The whole point of computing it here is to reuse the decode the diff - // already paid for. If the fused version answered differently from the - // standalone one, stored signatures would stop matching computed ones and - // variations would silently stop grouping. + afterEach(() => jest.restoreAllMocks()); + it('gives the same signature the standalone job would', () => { const image = block(10, 10, RED); - const fused = diffJob(blank, image, true).signature; + const fused = diffJob(blank, image).signature; const standalone = computeChangeSignature({ kind: 'signature', baseline: blank, @@ -85,27 +63,63 @@ describe('computePixelmatchDiff with a signature', () => { expect(fused).not.toBeNull(); }); - it('leaves the signature out when it was not asked for', () => { - expect(diffJob(blank, block(10, 10, RED), false).signature).toBeUndefined(); + it('leaves the signature out when the diff is not kept', () => { + expect(diffJob(blank, block(10, 10, RED), { saveDiff: false }).signature).toBeUndefined(); }); it('has no signature when the screenshots are identical', () => { - expect(diffJob(blank, blank, true).signature).toBeUndefined(); + expect(diffJob(blank, blank).signature).toBeUndefined(); }); it('still reports the diff it was asked for', () => { - const result = diffJob(blank, block(10, 10, RED), true); + const result = diffJob(blank, block(10, 10, RED)); expect(result.equal).toBe(false); expect(result.pixelMisMatchCount).toBe(144); }); + + it('encodes the diff with the Sub filter without changing a pixel of it', () => { + const image = block(10, 10, RED); + const write = jest.spyOn(PNG.sync, 'write'); + const expected = new PNG({ width: WIDTH, height: HEIGHT }); + Pixelmatch(PNG.sync.read(blank).data, PNG.sync.read(image).data, expected.data, WIDTH, HEIGHT, { + includeAA: false, + threshold: 0.1, + }); + + const result = diffJob(blank, image); + const decoded = PNG.sync.read(Buffer.from(result.diffBuffer!)); + + expect(write).toHaveBeenCalledWith(expect.any(PNG), expect.objectContaining({ filterType: 1 })); + expect([decoded.width, decoded.height]).toStrictEqual([WIDTH, HEIGHT]); + expect(decoded.data.equals(expected.data)).toBe(true); + }); +}); + +describe('computePixelmatchDiff with byte-equal input', () => { + afterEach(() => jest.restoreAllMocks()); + + it('answers equal for byte-equal input without decoding it', () => { + const read = jest.spyOn(PNG.sync, 'read'); + const other = new Uint8Array(blank.length + 3); + other.set(blank, 3); + + const result = diffJob(blank, other.subarray(3) as unknown as Buffer); + + expect(result).toStrictEqual({ equal: true }); + expect(read).not.toHaveBeenCalled(); + }); + + it('decodes and rejects a byte-equal but truncated pair as before', () => { + const truncated = blank.subarray(0, blank.length - 20); + + expect(() => diffJob(truncated, Buffer.from(truncated))).toThrow(); + }); }); describe('computePixelmatchDiff with thumbnails', () => { - // Made here from the pixels the diff has already decoded and already drawn, - // so a thumbnail costs a resize rather than a second read of the screenshot. it('returns a small picture of both the checkpoint and the diff', () => { - const result = diffJob(blank, block(10, 10, RED), false, true); + const result = diffJob(blank, block(10, 10, RED)); const image = PNG.sync.read(Buffer.from(result.imageThumbnail!)); const diff = PNG.sync.read(Buffer.from(result.diffThumbnail!)); @@ -113,64 +127,36 @@ describe('computePixelmatchDiff with thumbnails', () => { expect(Math.max(diff.width, diff.height)).toBeLessThanOrEqual(THUMBNAIL_MAX_DIMENSION); }); - it('makes none when they were not asked for', () => { - const result = diffJob(blank, block(10, 10, RED), false, false); - - expect(result.imageThumbnail).toBeUndefined(); - expect(result.diffThumbnail).toBeUndefined(); - }); - - // A build is mostly runs that pass. Resizing two images for each of those, - // when the diff they belong to is thrown away seconds later, is work nobody - // ever sees the result of. it('makes none when the change is under tolerance', () => { - const result = diffJob(blank, block(10, 10, RED), false, true, { diffTolerancePercent: 90 }); + const result = diffJob(blank, block(10, 10, RED), { diffTolerancePercent: 90 }); expect(result.imageThumbnail).toBeUndefined(); expect(result.diffThumbnail).toBeUndefined(); }); it('makes none when the caller is not keeping the diff', () => { - const result = diffJob(blank, block(10, 10, RED), false, true, { saveDiff: false }); + const result = diffJob(blank, block(10, 10, RED), { saveDiff: false }); expect(result.imageThumbnail).toBeUndefined(); }); - // nothing to show and nothing to shrink it('makes none when the screenshots are identical', () => { - expect(diffJob(blank, blank, false, true).imageThumbnail).toBeUndefined(); + expect(diffJob(blank, blank).imageThumbnail).toBeUndefined(); }); }); describe('computePixelmatchDiff with the blur-amplitude filter', () => { - const BLACK: [number, number, number] = [0, 0, 0]; - const hairline = (column: number): Buffer => - png((set) => { - for (let y = 0; y < HEIGHT; y++) { - set(column, y, BLACK); - } - }); - const filter = { sigma: DEFAULT_BLUR_SIGMA, threshold: DEFAULT_BLUR_AMPLITUDE_THRESHOLD }; - // AA counted, as the project default `ignoreAntialiasing: true` actually - // configures pixelmatch: the moved hairline is then a real mismatch for it - const withFilter = (baseline: Buffer, image: Buffer, options: Parameters[4] = {}) => - diffJob(baseline, image, false, true, { includeAA: true, blurAmplitude: filter, ...options }); - - // an edge rasterised one pixel further over: pixelmatch sees two columns of - // changed pixels, a person sees the same line - it('marks a change no stronger than rendering noise', () => { + const hairline = (column: number): Buffer => png((x) => (x === column ? [0, 0, 0] : undefined)); + const filter = { threshold: DEFAULT_BLUR_AMPLITUDE_THRESHOLD }; + const withFilter = (baseline: Buffer, image: Buffer, options: Parameters[2] = {}) => + diffJob(baseline, image, { includeAA: true, blurAmplitude: filter, ...options }); + + it('marks a change no stronger than rendering noise, and produces no diff for it', () => { const result = withFilter(hairline(10), hairline(11)); expect(result.pixelMisMatchCount).toBeGreaterThan(0); expect(result.blurAmplitude).toBeLessThan(DEFAULT_BLUR_AMPLITUDE_THRESHOLD); expect(result.renderingNoiseOnly).toBe(true); - }); - - // the run is about to pass, so a diff and thumbnails for it would be thrown - // away unseen - it('produces no diff for a change it passes', () => { - const result = withFilter(hairline(10), hairline(11)); - expect(result.diffBuffer).toBeUndefined(); expect(result.imageThumbnail).toBeUndefined(); expect(result.diffThumbnail).toBeUndefined(); @@ -185,50 +171,23 @@ describe('computePixelmatchDiff with the blur-amplitude filter', () => { expect(result.imageThumbnail).toBeDefined(); }); - // faint, so under the threshold, but a whole panel of it: not noise it('lets a faint but solid change through', () => { - const panel = png((set) => { - for (let y = 0; y < HEIGHT; y++) { - for (let x = 0; x < WIDTH; x++) { - set(x, y, [224, 224, 224]); - } - } - }); - - const result = withFilter(blank, panel); + const result = withFilter(blank, pngBytes(solidPng(WIDTH, HEIGHT, [224, 224, 224]))); expect(result.blurAmplitude).toBeLessThan(DEFAULT_BLUR_AMPLITUDE_THRESHOLD); expect(result.renderingNoiseOnly).toBe(false); }); - // as CompareService does for the other comparators: a size change is a - // finding of its own, whatever the padded pixels look like it('does not judge screenshots of different sizes', () => { - const taller = (() => { - const image = new PNG({ width: WIDTH, height: HEIGHT + 5 }); - image.data.fill(255); - return PNG.sync.write(image); - })(); - - const result = computePixelmatchDiff({ - kind: 'diff', - baseline: blank, - image: taller, - ignoreAreas: [], - threshold: 0.1, - includeAA: true, - allowDiffDimensions: true, - diffTolerancePercent: 0, - saveDiff: true, - blurAmplitude: filter, - }); + const taller = pngBytes(solidPng(WIDTH, HEIGHT + 5)); + + const result = withFilter(blank, taller, { allowDiffDimensions: true }); expect(result.isSameDimension).toBe(false); expect(result.renderingNoiseOnly).toBeUndefined(); expect(result.blurAmplitude).toBeUndefined(); }); - // a second check may yet send it to review, and then it needs its diff it('keeps the diff of a noise-like change when a DINOv2 check will follow', () => { const result = withFilter(hairline(10), hairline(11), { blurAmplitude: { ...filter, dinov2: { threshold: 0.15 } }, @@ -239,15 +198,57 @@ describe('computePixelmatchDiff with the blur-amplitude filter', () => { expect(result.imageThumbnail).toBeDefined(); }); - it('measures nothing when the filter is off', () => { - const result = diffJob(hairline(10), hairline(11), false, false, { includeAA: true }); + describe('with the DINOv2 check', () => { + const checked = { ...filter, dinov2: { threshold: 0.15 } }; + const gated = (baseline: Buffer, image: Buffer, options: Parameters[2] = {}) => + withFilter(baseline, image, { blurAmplitude: checked, ...options }); + const gateOf = (baseline: Buffer, image: Buffer, ignoreAreas: PixelmatchJobInput['ignoreAreas']) => { + const pair = readSameSizePair(baseline, image, ignoreAreas)!; + return gateOfDecoded(pair.baseline, pair.image, DINOV2_PATCH); + }; + + it('computes the DINOv2 patch gate of a noise-like change', () => { + const ignoreAreas = [{ x: 40, y: 40, width: 5, height: 5 }]; + const result = gated(hairline(10), hairline(11), { ignoreAreas }); + + expect(result.renderingNoiseOnly).toBe(true); + expect(result.gate).toEqual(gateOf(hairline(10), hairline(11), ignoreAreas)); + expect(result.gate?.cells.length).toBeGreaterThan(0); + }); + + it('leaves out what the ignore areas cover', () => { + const ignoreAreas = [{ x: 0, y: 0, width: 30, height: 28 }]; + const result = gated(hairline(10), hairline(11), { ignoreAreas }); + + expect(result.gate?.cells.length).toBeGreaterThan(0); + expect(result.gate?.cells.every((cell) => Math.floor(cell / 5) >= 2)).toBe(true); + expect(result.gate).toEqual(gateOf(hairline(10), hairline(11), ignoreAreas)); + }); + + it('computes no gate for a change that is not noise', () => { + const result = gated(blank, block(10, 10, RED)); + + expect(result.renderingNoiseOnly).toBe(false); + expect(result.gate).toBeUndefined(); + }); + + it('computes no gate when the check is off', () => { + const result = withFilter(hairline(10), hairline(11), { blurAmplitude: filter }); + + expect(result.renderingNoiseOnly).toBe(true); + expect(result.gate).toBeUndefined(); + }); + }); + + it('measures nothing and computes no gate when the filter is off', () => { + const result = diffJob(hairline(10), hairline(11), { includeAA: true }); + expect(result.gate).toBeUndefined(); expect(result.blurAmplitude).toBeUndefined(); expect(result.renderingNoiseOnly).toBeUndefined(); expect(result.diffBuffer).toBeDefined(); }); - // a run under tolerance passes anyway; blurring it would be work for nothing it('measures nothing when the change is under tolerance', () => { const result = withFilter(hairline(10), hairline(11), { diffTolerancePercent: 90 }); diff --git a/src/compare/libs/pixelmatch/pixelmatch.core.ts b/src/compare/libs/pixelmatch/pixelmatch.core.ts index 5cf732e2..05322f27 100644 --- a/src/compare/libs/pixelmatch/pixelmatch.core.ts +++ b/src/compare/libs/pixelmatch/pixelmatch.core.ts @@ -1,23 +1,18 @@ import { PNG } from 'pngjs'; import Pixelmatch from 'pixelmatch'; +import { isByteEqualWellFormedPng } from '../../../static/utils'; import { IgnoreAreaDto } from '../../../test-runs/dto/ignore-area.dto'; -import { applyIgnoreAreas, scaleImageToSize } from '../../utils'; +import { applyIgnoreAreas, readPng, scaleImageToSize } from '../../utils'; import { BlurAmplitudeOptions, BlurredDifference, isRenderingNoise, measureBlurredDifference, } from './blur-amplitude.core'; +import { DINOV2_PATCH, gateOfDecoded, PatchGate } from './patch-gate.core'; import { signatureOfDecoded } from './signature.core'; import { encodeThumbnail } from './thumbnail.core'; -/** - * CPU-bound part of the pixelmatch comparison, extracted so it can run inside - * a worker thread (see pixelmatch.worker.ts / DiffWorkerPool): PNG decode, - * scaling, pixelmatch and diff encode of full-size screenshots block the event - * loop for ~a second each, which makes the whole API unresponsive during - * build ingestion. All input/output must stay structured-clone serializable. - */ export interface PixelmatchJobInput { kind: 'diff'; baseline: Buffer | Uint8Array; @@ -28,23 +23,6 @@ export interface PixelmatchJobInput { allowDiffDimensions?: boolean; diffTolerancePercent: number; saveDiff: boolean; - /** - * Also produce the change signature the variations dialog matches on. Free - * here next to the diff — the images are decoded already — and computed once - * at ingest so reviewing never has to fetch and decode a build's screenshots - * again just to group them. - */ - withSignature?: boolean; - /** - * Also return small PNGs of the checkpoint and of the diff. The grids draw - * these a hundred-odd pixels wide; sending the full-size files and letting - * CSS shrink them is what made opening the variations dialog cost megabytes. - */ - withThumbnails?: boolean; - /** - * Prototype: pass a change that is over tolerance but, once blurred, no - * stronger than rendering noise (see blur-amplitude.core.ts). Off when absent. - */ blurAmplitude?: BlurAmplitudeOptions; } @@ -54,59 +32,41 @@ export interface PixelmatchJobOutput { pixelMisMatchCount?: number; diffPercent?: number; diffBuffer?: Buffer | Uint8Array; - // absent when not asked for, when the screenshots are identical, or when - // their dimensions differ and there is nothing meaningful to sign signature?: number[]; - // absent when not asked for, or when the screenshots matched and there is - // nothing to show imageThumbnail?: Buffer | Uint8Array; diffThumbnail?: Buffer | Uint8Array; - // absent unless the blur-amplitude filter was asked for and the change was - // over tolerance between screenshots of one size — the only case where its - // verdict could matter; the two numbers are absent when it could not measure blurAmplitude?: number; solidPixels?: number; renderingNoiseOnly?: boolean; -} - -// postMessage turns Buffers into Uint8Array views over their own ArrayBuffer. -function toBuffer(data: Buffer | Uint8Array): Buffer { - return Buffer.isBuffer(data) ? data : Buffer.from(data.buffer, data.byteOffset, data.byteLength); + gate?: PatchGate; } export function computePixelmatchDiff(input: PixelmatchJobInput): PixelmatchJobOutput { - const baseline = PNG.sync.read(toBuffer(input.baseline)); - const image = PNG.sync.read(toBuffer(input.image)); + if (isByteEqualWellFormedPng(input.baseline, input.image)) { + return { equal: true }; + } + + const baseline = readPng(input.baseline); + const image = readPng(input.image); const isSameDimension = baseline.width === image.width && baseline.height === image.height; if (!isSameDimension && !input.allowDiffDimensions) { return { equal: false, isSameDimension }; } - // Identical bytes only mean identical screenshots at the same size: a 60x30 - // and a 30x60 image of the same pixels decode to the very same buffer. - if ( - isSameDimension && - baseline.data.equals(new Uint8Array(image.data.buffer, image.data.byteOffset, image.data.byteLength)) - ) { + if (isSameDimension && baseline.data.equals(image.data)) { return { equal: true }; } - // scale image to max size const maxWidth = Math.max(baseline.width, image.width); const maxHeight = Math.max(baseline.height, image.height); const scaledBaseline = scaleImageToSize(baseline, maxWidth, maxHeight); const scaledImage = scaleImageToSize(image, maxWidth, maxHeight); - // apply ignore areas const baselineIgnored = applyIgnoreAreas(scaledBaseline, input.ignoreAreas); const imageIgnored = applyIgnoreAreas(scaledImage, input.ignoreAreas); - // compare - const diff = new PNG({ - width: maxWidth, - height: maxHeight, - }); + const diff = new PNG({ width: maxWidth, height: maxHeight }); const pixelMisMatchCount = Pixelmatch(baselineIgnored.data, imageIgnored.data, diff.data, maxWidth, maxHeight, { includeAA: input.includeAA, threshold: input.threshold, @@ -114,42 +74,33 @@ export function computePixelmatchDiff(input: PixelmatchJobInput): PixelmatchJobO const diffPercent = (pixelMisMatchCount * 100) / (scaledImage.width * scaledImage.height); const overTolerance = diffPercent > input.diffTolerancePercent; - // A size change is a finding of its own, as CompareService treats it for the - // other comparators: the filter only judges screenshots of one size. const blurAmplitude = overTolerance && isSameDimension ? input.blurAmplitude : undefined; let measure: BlurredDifference | null = null; - let renderingNoiseOnly: boolean | undefined; + let renderingNoiseOnly = false; if (blurAmplitude) { - measure = measureBlurredDifference(baselineIgnored, imageIgnored, blurAmplitude.sigma); + measure = measureBlurredDifference(baselineIgnored, imageIgnored); renderingNoiseOnly = isRenderingNoise(measure, blurAmplitude.threshold); } - // A build is mostly runs that pass, so anything produced for a diff that is - // about to be thrown away is work nobody ever sees the result of. - // ...unless a DINOv2 check is still to come, which may send the run to - // review after all — and then it needs its diff. - const keepingTheDiff = overTolerance && input.saveDiff && (!renderingNoiseOnly || !!input.blurAmplitude?.dinov2); + const gate = + renderingNoiseOnly && blurAmplitude?.dinov2 + ? gateOfDecoded(baselineIgnored, imageIgnored, DINOV2_PATCH) + : undefined; - let diffBuffer: Buffer | undefined; - if (keepingTheDiff) { - diffBuffer = PNG.sync.write(diff); - } + const keepingTheDiff = overTolerance && input.saveDiff && (!renderingNoiseOnly || !!blurAmplitude?.dinov2); + const diffBuffer = keepingTheDiff ? PNG.sync.write(diff, { filterType: 1 }) : undefined; - // Same source images, same ignore areas, same threshold as the standalone - // signature job — sharing signatureOfDecoded is what keeps the two answers - // identical, so a stored signature still matches one computed on the fly. const signature = - input.withSignature && isSameDimension + input.saveDiff && isSameDimension ? signatureOfDecoded(baselineIgnored, imageIgnored, { threshold: input.threshold, includeAA: input.includeAA, }) : null; - const thumbnails = - input.withThumbnails && keepingTheDiff - ? { imageThumbnail: encodeThumbnail(imageIgnored), diffThumbnail: encodeThumbnail(diff) } - : {}; + const thumbnails = keepingTheDiff + ? { imageThumbnail: encodeThumbnail(imageIgnored), diffThumbnail: encodeThumbnail(diff) } + : {}; return { equal: false, @@ -159,6 +110,7 @@ export function computePixelmatchDiff(input: PixelmatchJobInput): PixelmatchJobO diffBuffer, ...(signature ? { signature } : {}), ...thumbnails, + ...(gate ? { gate } : {}), ...(blurAmplitude ? { renderingNoiseOnly, diff --git a/src/compare/libs/pixelmatch/pixelmatch.service.boundaries.spec.ts b/src/compare/libs/pixelmatch/pixelmatch.service.real-images.spec.ts similarity index 100% rename from src/compare/libs/pixelmatch/pixelmatch.service.boundaries.spec.ts rename to src/compare/libs/pixelmatch/pixelmatch.service.real-images.spec.ts diff --git a/src/compare/libs/pixelmatch/pixelmatch.service.spec.ts b/src/compare/libs/pixelmatch/pixelmatch.service.spec.ts index b820a721..72c0c682 100644 --- a/src/compare/libs/pixelmatch/pixelmatch.service.spec.ts +++ b/src/compare/libs/pixelmatch/pixelmatch.service.spec.ts @@ -6,39 +6,71 @@ import { mocked } from 'jest-mock'; import { StaticService } from '../../../static/static.service'; import { DiffWorkerPool } from '../../diff-worker-pool'; import { DIFF_DIMENSION_RESULT, EQUAL_RESULT, NO_BASELINE_RESULT } from '../consts'; +import { ImageCompareInput } from '../ImageCompareInput'; import { DEFAULT_CONFIG, PixelmatchService } from './pixelmatch.service'; import { PixelmatchConfig } from './pixelmatch.types'; +import { computePixelmatchDiff } from './pixelmatch.core'; import { Dinov2Service } from '../dinov2/dinov2.service'; jest.mock('pixelmatch'); -const toBuffer = (png: PNG | undefined): Buffer | undefined => (png ? PNG.sync.write(png) : undefined); - -const initService = async ({ - getImageBufferMock = jest.fn(), - saveImageMock = jest.fn(), - deleteImageMock = jest.fn(), -}) => { +const pngOf = (width: number, height: number) => PNG.sync.write(new PNG({ width, height })); +const toBuffer = (png: PNG): Buffer => PNG.sync.write(png); +const sequence = (...results: unknown[]) => results.reduce((m, r) => m.mockResolvedValueOnce(r), jest.fn()); +const pngReader = () => jest.fn().mockResolvedValue(Buffer.from('png')); +const poolOf = (output: Record) => ({ run: jest.fn().mockResolvedValue(output) }); + +interface Deps { + getImageBuffer?: jest.Mock; + saveImage?: jest.Mock; + deleteImage?: jest.Mock; + pool?: { run: unknown }; + confirmsNoiseForGate?: jest.Mock; +} + +const setup = async ({ + getImageBuffer = jest.fn(), + saveImage = jest.fn(), + deleteImage = jest.fn(), + pool, + confirmsNoiseForGate = jest.fn(), +}: Deps = {}) => { const module: TestingModule = await Test.createTestingModule({ providers: [ PixelmatchService, - DiffWorkerPool, - { provide: Dinov2Service, useValue: { confirmsNoise: jest.fn() } }, - { - provide: StaticService, - useValue: { - getImageBuffer: getImageBufferMock, - saveImage: saveImageMock, - deleteImage: deleteImageMock, - }, - }, + pool ? { provide: DiffWorkerPool, useValue: pool } : DiffWorkerPool, + { provide: Dinov2Service, useValue: { confirmsNoiseForGate } }, + { provide: StaticService, useValue: { getImageBuffer, saveImage, deleteImage } }, ], }).compile(); - - return module.get(PixelmatchService); + return { + service: module.get(PixelmatchService), + run: pool?.run as jest.Mock, + getImageBuffer, + saveImage, + deleteImage, + confirmsNoiseForGate, + }; }; -let service: PixelmatchService; +const compare = ( + service: PixelmatchService, + data: Partial = {}, + config: PixelmatchConfig = DEFAULT_CONFIG +) => + service.getDiff( + { + baseline: 'baseline.png', + image: 'image.png', + ignoreAreas: [], + diffTollerancePercent: 0, + saveDiffAsFile: false, + ...data, + }, + config + ); + +const mismatch = { equal: false, isSameDimension: true, pixelMisMatchCount: 10, diffPercent: 5 }; describe('parseConfig', () => { it.each<[string, PixelmatchConfig]>([ @@ -50,150 +82,65 @@ describe('parseConfig', () => { ['invalid', DEFAULT_CONFIG], ['null', DEFAULT_CONFIG], ])('should parse config', async (json, expected) => { - service = await initService({}); - - const config = service.parseConfig(json); + const { service } = await setup(); - expect(config).toStrictEqual(expected); + expect(service.parseConfig(json)).toStrictEqual(expected); }); }); describe('getDiff', () => { - const image = new PNG({ - width: 20, - height: 20, - }); + const image = new PNG({ width: 20, height: 20 }); + const flagged = (baseline: number | undefined = undefined) => { + const png = new PNG({ width: 100, height: 100 }); + if (baseline !== undefined) png.data[0] = baseline; + return toBuffer(png); + }; it('no baseline', async () => { - const getImageBufferMock = jest.fn().mockReturnValueOnce(undefined).mockReturnValueOnce(toBuffer(image)); - service = await initService({ getImageBufferMock }); - - const result = await service.getDiff( - { - baseline: null, - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: true, - }, - DEFAULT_CONFIG - ); + const { service } = await setup({ getImageBuffer: sequence(undefined, toBuffer(image)) }); - expect(result).toStrictEqual(NO_BASELINE_RESULT); + expect(await compare(service, { baseline: null, saveDiffAsFile: true })).toStrictEqual(NO_BASELINE_RESULT); }); it('throws when the image cannot be read', async () => { - const getImageBufferMock = jest.fn().mockResolvedValueOnce(toBuffer(image)).mockResolvedValueOnce(undefined); - service = await initService({ getImageBufferMock }); - - await expect( - service.getDiff( - { - baseline: 'baseline', - image: 'missing.png', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: true, - }, - DEFAULT_CONFIG - ) - ).rejects.toThrow('Cannot get image: missing.png'); - expect(getImageBufferMock).toHaveBeenNthCalledWith(2, 'missing.png'); + const { service, getImageBuffer } = await setup({ getImageBuffer: sequence(toBuffer(image), undefined) }); + + await expect(compare(service, { image: 'missing.png', saveDiffAsFile: true })).rejects.toThrow( + 'Cannot get image: missing.png' + ); + expect(getImageBuffer).toHaveBeenNthCalledWith(2, 'missing.png'); }); it('diff not found', async () => { - const getImageBufferMock = jest.fn().mockReturnValueOnce(toBuffer(image)).mockReturnValueOnce(toBuffer(image)); - service = await initService({ getImageBufferMock }); - - const result = await service.getDiff( - { - baseline: 'image', - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: true, - }, - DEFAULT_CONFIG - ); + const { service } = await setup({ getImageBuffer: sequence(toBuffer(image), toBuffer(image)) }); - expect(result).toStrictEqual(EQUAL_RESULT); + expect(await compare(service, { saveDiffAsFile: true })).toStrictEqual(EQUAL_RESULT); }); it('diff image dimensions mismatch', async () => { - const baseline = new PNG({ - width: 10, - height: 10, - }); - const getImageBufferMock = jest.fn().mockReturnValueOnce(toBuffer(baseline)).mockReturnValueOnce(toBuffer(image)); - service = await initService({ getImageBufferMock }); - - const result = await service.getDiff( - { - baseline: 'image', - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: true, - }, - DEFAULT_CONFIG - ); + const { service } = await setup({ getImageBuffer: sequence(pngOf(10, 10), toBuffer(image)) }); - expect(result).toStrictEqual(DIFF_DIMENSION_RESULT); + expect(await compare(service, { saveDiffAsFile: true })).toStrictEqual(DIFF_DIMENSION_RESULT); }); it('diff image dimensions mismatch ALLOWED', async () => { - const baseline = new PNG({ - width: 1, - height: 5, - }); - const image = new PNG({ - width: 2, - height: 4, - }); - const getImageBufferMock = jest.fn().mockReturnValueOnce(toBuffer(baseline)).mockReturnValueOnce(toBuffer(image)); const diffName = 'diff name'; - const saveImageMock = jest.fn().mockReturnValueOnce(diffName); mocked(Pixelmatch).mockReturnValueOnce(5); - service = await initService({ saveImageMock, getImageBufferMock }); - - const result = await service.getDiff( - { - baseline: 'image', - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: true, - }, - { - allowDiffDimensions: true, - ignoreAntialiasing: true, - threshold: 0.1, - } - ); + const { service, saveImage } = await setup({ + getImageBuffer: sequence(pngOf(1, 5), pngOf(2, 4)), + saveImage: sequence(diffName), + }); - expect(mocked(Pixelmatch)).toHaveBeenCalledWith( - new PNG({ - width: 2, - height: 5, - }).data, - new PNG({ - width: 2, - height: 5, - }).data, - new PNG({ - width: 2, - height: 5, - }).data, - 2, - 5, - { - includeAA: true, - threshold: 0.1, - } + const result = await compare( + service, + { diffTollerancePercent: 0.1, saveDiffAsFile: true }, + { allowDiffDimensions: true, ignoreAntialiasing: true, threshold: 0.1 } ); - // the full-size diff first, then the two thumbnails made beside it - expect(saveImageMock.mock.calls[0][0]).toBe('diff'); - expect(saveImageMock).toHaveBeenCalledTimes(3); + + const padded = new PNG({ width: 2, height: 5 }).data; + expect(mocked(Pixelmatch)).toHaveBeenCalledWith(padded, padded, padded, 2, 5, { includeAA: true, threshold: 0.1 }); + expect(saveImage.mock.calls[0][0]).toBe('diff'); + expect(saveImage).toHaveBeenCalledTimes(3); expect(result).toStrictEqual({ status: TestStatus.unresolved, diffName, @@ -203,141 +150,150 @@ describe('getDiff', () => { }); }); - it('diff found < tollerance', async () => { - const baseline = new PNG({ - width: 100, - height: 100, - }); - baseline.data[0] = 1; - const image = new PNG({ - width: 100, - height: 100, - }); - const getImageBufferMock = jest.fn().mockReturnValueOnce(toBuffer(baseline)).mockReturnValueOnce(toBuffer(image)); - const saveImageMock = jest.fn(); - service = await initService({ saveImageMock, getImageBufferMock }); - const pixelMisMatchCount = 150; - mocked(Pixelmatch).mockReturnValueOnce(pixelMisMatchCount); - - const result = await service.getDiff( - { - baseline: 'image', - image: 'image', - diffTollerancePercent: 2, - ignoreAreas: [], - saveDiffAsFile: false, - }, - DEFAULT_CONFIG - ); + it.each([ + ['below', 2, false, 150, 1.5, TestStatus.ok, null, 0], + ['above', 0.5, true, 200, 2, TestStatus.unresolved, 'diff name', 3], + ])( + 'diff found %s tollerance', + async (_, diffTollerancePercent, saveDiffAsFile, pixelMisMatchCount, diffPercent, status, diffName, saves) => { + mocked(Pixelmatch).mockReturnValueOnce(pixelMisMatchCount); + const { service, saveImage } = await setup({ + getImageBuffer: sequence(flagged(1), flagged()), + saveImage: sequence('diff name'), + }); + + const result = await compare(service, { diffTollerancePercent, saveDiffAsFile }); + + if (saves) expect(saveImage.mock.calls[0][0]).toBe('diff'); + expect(saveImage).toHaveBeenCalledTimes(saves); + expect(result).toStrictEqual({ status, diffName, pixelMisMatchCount, diffPercent, isSameDimension: true }); + } + ); +}); - expect(saveImageMock).toHaveBeenCalledTimes(0); - expect(result).toStrictEqual({ - status: TestStatus.ok, - diffName: null, - pixelMisMatchCount, - diffPercent: 1.5, - isSameDimension: true, - }); +describe('byte-equal fast path', () => { + const valid = pngOf(4, 4); + + it('answers EQUAL for byte-equal well-formed PNGs without queueing a job', async () => { + const pool = { run: jest.fn() }; + const { service } = await setup({ pool, getImageBuffer: sequence(Buffer.from(valid), Buffer.from(valid)) }); + + await expect(compare(service, { saveDiffAsFile: true })).resolves.toStrictEqual(EQUAL_RESULT); + expect(pool.run).not.toHaveBeenCalled(); }); - it('diff found > tollerance', async () => { - const baseline = new PNG({ - width: 100, - height: 100, - }); - baseline.data[0] = 1; - const image = new PNG({ - width: 100, - height: 100, - }); - const getImageBufferMock = jest.fn().mockReturnValueOnce(toBuffer(baseline)).mockReturnValueOnce(toBuffer(image)); - const pixelMisMatchCount = 200; - mocked(Pixelmatch).mockReturnValueOnce(pixelMisMatchCount); - const diffName = 'diff name'; - const saveImageMock = jest.fn().mockReturnValueOnce(diffName); - service = await initService({ - saveImageMock, - getImageBufferMock, + it('still sends a byte-equal pair to the worker when the PNG is not well formed', async () => { + const truncated = valid.subarray(0, valid.length - 20); + const pool = new DiffWorkerPool(); + const run = jest.spyOn(pool, 'run'); + const { service } = await setup({ pool, getImageBuffer: sequence(Buffer.from(truncated), Buffer.from(truncated)) }); + + await expect(compare(service, { saveDiffAsFile: true })).rejects.toThrow(); + expect(run).toHaveBeenCalledTimes(1); + }); + + it('sends different buffers to the worker', async () => { + const other = new PNG({ width: 4, height: 4 }); + other.data.fill(255); + const pool = poolOf({ equal: true }); + const { service } = await setup({ pool, getImageBuffer: sequence(Buffer.from(valid), toBuffer(other)) }); + + await compare(service, { saveDiffAsFile: true }); + + expect(pool.run).toHaveBeenCalledTimes(1); + }); +}); + +describe('image already in memory', () => { + it('uses the image it is given and reads only the baseline', async () => { + const pool = poolOf({ equal: true }); + const { service, getImageBuffer } = await setup({ + pool, + getImageBuffer: jest.fn().mockResolvedValue(Buffer.from('baseline bytes')), }); + const imageBuffer = Buffer.from('image bytes'); - const result = await service.getDiff( - { - baseline: 'image', - image: 'image', - diffTollerancePercent: 0.5, - ignoreAreas: [], - saveDiffAsFile: true, - }, - DEFAULT_CONFIG - ); + await compare(service, { imageBuffer }); - // the full-size diff first, then the two thumbnails made beside it - expect(saveImageMock.mock.calls[0][0]).toBe('diff'); - expect(saveImageMock).toHaveBeenCalledTimes(3); - expect(result).toStrictEqual({ - status: TestStatus.unresolved, - diffName, - pixelMisMatchCount, - diffPercent: 2, - isSameDimension: true, + expect(getImageBuffer).toHaveBeenCalledTimes(1); + expect(getImageBuffer).toHaveBeenCalledWith('baseline.png'); + expect(pool.run.mock.calls[0][0].image).toBe(imageBuffer); + }); + + it('reads the image itself when none is given and fails as before when it is missing', async () => { + const { service, getImageBuffer } = await setup({ + pool: { run: jest.fn() }, + getImageBuffer: sequence(Buffer.from('baseline bytes'), undefined), }); + + await expect(compare(service)).rejects.toThrow('Cannot get image: image.png'); + expect(getImageBuffer).toHaveBeenNthCalledWith(2, 'image.png'); }); }); -describe('change signature', () => { - // Correctness of the signature itself is pinned in pixelmatch.core.spec - // against the real pixelmatch; Pixelmatch is mocked in this file, so what is - // worth testing here is only the plumbing — that the service asks for one and - // carries what comes back into the result. - const initWithPool = async (output: Record) => { - const run = jest.fn().mockResolvedValue(output); - const module: TestingModule = await Test.createTestingModule({ - providers: [ - PixelmatchService, - { provide: DiffWorkerPool, useValue: { run } }, - { provide: Dinov2Service, useValue: { confirmsNoise: jest.fn() } }, - { - provide: StaticService, - useValue: { - getImageBuffer: jest.fn().mockResolvedValue(Buffer.from('png')), - saveImage: jest.fn(), - deleteImage: jest.fn(), - }, - }, - ], - }).compile(); - return { service: module.get(PixelmatchService), run }; - }; +describe('screenshots of another size', () => { + const setupSizes = (baseline: Buffer, image: Buffer, run: jest.Mock) => + setup({ pool: { run }, getImageBuffer: sequence(baseline, image) }); - const compare = (service: PixelmatchService, saveDiffAsFile = false) => - service.getDiff( - { - baseline: 'baseline.png', - image: 'image.png', - ignoreAreas: [], - diffTollerancePercent: 0, - saveDiffAsFile, - }, - DEFAULT_CONFIG - ); + it('answers different sizes from the headers without the worker', async () => { + const run = jest.fn(); + const { service } = await setupSizes(pngOf(2, 3), pngOf(3, 2), run); + + expect(await compare(service)).toBe(DIFF_DIMENSION_RESULT); + expect(run).not.toHaveBeenCalled(); + }); + + it('still sends different sizes to the worker when they are allowed', async () => { + const run = jest + .fn() + .mockResolvedValue({ equal: false, isSameDimension: false, pixelMisMatchCount: 1, diffPercent: 1 }); + const { service } = await setupSizes(pngOf(2, 3), pngOf(3, 2), run); + + await compare(service, {}, { ...DEFAULT_CONFIG, allowDiffDimensions: true }); + + expect(run).toHaveBeenCalledTimes(1); + }); + + it('sends buffers without a readable header to the worker', async () => { + const run = jest.fn().mockResolvedValue({ equal: false, isSameDimension: false }); + const { service } = await setupSizes(Buffer.from('a'), Buffer.from('b'), run); + + expect(await compare(service)).toBe(DIFF_DIMENSION_RESULT); + expect(run).toHaveBeenCalledTimes(1); + }); - // Asked for on every comparison, not only when the project has bulk approve - // switched on: computing it lazily would leave every run ingested before the - // flag was turned on without one. + it('gives the result the worker gave for screenshots of different sizes', async () => { + const baseline = pngOf(2, 3); + const image = pngOf(3, 2); + const output = computePixelmatchDiff({ + kind: 'diff', + baseline, + image, + ignoreAreas: [], + threshold: DEFAULT_CONFIG.threshold, + includeAA: DEFAULT_CONFIG.ignoreAntialiasing, + diffTolerancePercent: 0, + saveDiff: false, + }); + expect(output.isSameDimension).toBe(false); + expect(output.equal).toBeFalsy(); + + const run = jest.fn().mockImplementation(async (input) => computePixelmatchDiff(input)); + const { service } = await setupSizes(baseline, image, run); + expect(await compare(service)).toStrictEqual(DIFF_DIMENSION_RESULT); + }); +}); + +describe('change signature', () => { it('asks for a signature and carries it into the result', async () => { - const { service, run } = await initWithPool({ - equal: false, - isSameDimension: true, - pixelMisMatchCount: 10, - diffPercent: 5, - signature: [0.25, 0.75], + const { service, run } = await setup({ + pool: poolOf({ ...mismatch, signature: [0.25, 0.75] }), + getImageBuffer: pngReader(), }); - const result = await compare(service); + const result = await compare(service, { saveDiffAsFile: true }); - expect(run).toHaveBeenCalledWith(expect.objectContaining({ kind: 'diff', withSignature: true })); - // stamped with the settings it was computed under, so a stored one can be - // discarded when the project's config moves on + expect(run).toHaveBeenCalledWith(expect.objectContaining({ kind: 'diff', saveDiff: true })); expect(result.changeSignature).toEqual({ threshold: DEFAULT_CONFIG.threshold, includeAA: DEFAULT_CONFIG.ignoreAntialiasing, @@ -345,144 +301,113 @@ describe('change signature', () => { }); }); - const initSaving = async (output: Record) => { - const saveImage = jest - .fn() - .mockResolvedValueOnce('diff.png') - .mockResolvedValueOnce('image.thumb.png') - .mockResolvedValueOnce('diff.thumb.png'); - const module: TestingModule = await Test.createTestingModule({ - providers: [ - PixelmatchService, - { provide: DiffWorkerPool, useValue: { run: jest.fn().mockResolvedValue(output) } }, - { provide: Dinov2Service, useValue: { confirmsNoise: jest.fn() } }, - { - provide: StaticService, - useValue: { - getImageBuffer: jest.fn().mockResolvedValue(Buffer.from('png')), - saveImage, - deleteImage: jest.fn(), - }, - }, - ], - }).compile(); - return { service: module.get(PixelmatchService), saveImage }; - }; + it('does not ask for a signature for a comparison whose diff is thrown away', async () => { + const { service, run } = await setup({ pool: poolOf(mismatch), getImageBuffer: pngReader() }); - const overTolerance = { - equal: false, - isSameDimension: true, - pixelMisMatchCount: 10, - diffPercent: 5, - imageThumbnail: Buffer.from('small image'), - diffThumbnail: Buffer.from('small diff'), - }; + const result = await compare(service); - it('saves the thumbnails and reports the names they went under', async () => { - const { service } = await initSaving({ ...overTolerance, diffBuffer: Buffer.from('the diff') }); + expect(run).toHaveBeenCalledWith(expect.objectContaining({ saveDiff: false })); + expect(result.changeSignature).toBeUndefined(); + }); - const result = await compare(service, true); + describe('thumbnails', () => { + const overTolerance = { + ...mismatch, + imageThumbnail: Buffer.from('small image'), + diffThumbnail: Buffer.from('small diff'), + diffBuffer: Buffer.from('the diff'), + }; + const setupSaving = (saveImage: jest.Mock, output: Record = overTolerance) => + setup({ pool: poolOf(output), getImageBuffer: pngReader(), saveImage }); + const savedAs = (...names: (string | Error)[]) => + names.reduce( + (m, name) => (name instanceof Error ? m.mockRejectedValueOnce(name) : m.mockResolvedValueOnce(name)), + jest.fn() + ); - expect(result.imageThumbnailName).toBe('image.thumb.png'); - expect(result.diffThumbnailName).toBe('diff.thumb.png'); - }); + it('saves the thumbnails and reports the names they went under', async () => { + const { service } = await setupSaving(savedAs('diff.png', 'image.thumb.png', 'diff.thumb.png')); - // shouldAutoApprove compares against past baselines with saveDiffAsFile off - // and throws the result away. Storing thumbnails for those comparisons would - // leave two objects per attempt that nothing ever references or deletes. - it('stores nothing when the caller is not keeping the diff', async () => { - const { service, saveImage } = await initSaving(overTolerance); + const result = await compare(service, { saveDiffAsFile: true }); - const result = await compare(service, false); + expect(result.imageThumbnailName).toBe('image.thumb.png'); + expect(result.diffThumbnailName).toBe('diff.thumb.png'); + }); - expect(saveImage).not.toHaveBeenCalled(); - expect(result.imageThumbnailName).toBeUndefined(); - expect(result.diffThumbnailName).toBeUndefined(); - }); + it('keeps the diff and the verdict when a thumbnail cannot be saved, and deletes the one that was', async () => { + const { service, deleteImage } = await setupSaving(savedAs('diff.png', new Error('disk full'), 'diff.thumb.png')); + + const result = await compare(service, { saveDiffAsFile: true }); + + expect(result).toMatchObject({ status: TestStatus.unresolved, diffName: 'diff.png' }); + expect(result.imageThumbnailName).toBeUndefined(); + expect(result.diffThumbnailName).toBeUndefined(); + expect(deleteImage).toHaveBeenCalledTimes(1); + expect(deleteImage).toHaveBeenCalledWith('diff.thumb.png'); + }); + + it('drops a thumbnail pair when one of the names comes back empty', async () => { + const { service, deleteImage } = await setupSaving(savedAs('diff.png', '', 'diff.thumb.png')); + + const result = await compare(service, { saveDiffAsFile: true }); - it('leaves it out when the comparison produced none', async () => { - const { service } = await initWithPool({ - equal: false, - isSameDimension: true, - pixelMisMatchCount: 10, - diffPercent: 5, + expect(result.diffName).toBe('diff.png'); + expect(result.diffThumbnailName).toBeUndefined(); + expect(deleteImage).toHaveBeenCalledWith('diff.thumb.png'); }); - expect((await compare(service)).changeSignature).toBeUndefined(); + it('stores nothing when the caller is not keeping the diff', async () => { + const saveImage = savedAs('diff.png', 'image.thumb.png', 'diff.thumb.png'); + const { service } = await setupSaving(saveImage, { ...overTolerance, diffBuffer: undefined }); + + const result = await compare(service); + + expect(saveImage).not.toHaveBeenCalled(); + expect(result.imageThumbnailName).toBeUndefined(); + expect(result.diffThumbnailName).toBeUndefined(); + }); }); }); describe('blur-amplitude filter', () => { - // The filter itself is pinned in pixelmatch.core.spec and - // blur-amplitude.core.spec, and whether a project has it on is - // CompareService's business; here only the plumbing: what the service asks - // the worker for, and what it makes of the answer. - const init = async (output: Record, confirmsNoise = jest.fn()) => { - const run = jest.fn().mockResolvedValue(output); - const saveImage = jest.fn().mockResolvedValue('diff.png'); - const module: TestingModule = await Test.createTestingModule({ - providers: [ - PixelmatchService, - { provide: DiffWorkerPool, useValue: { run } }, - { provide: Dinov2Service, useValue: { confirmsNoise } }, - { - provide: StaticService, - useValue: { - getImageBuffer: jest.fn().mockResolvedValue(Buffer.from('png')), - saveImage, - deleteImage: jest.fn(), - }, - }, - ], - }).compile(); - return { service: module.get(PixelmatchService), run, saveImage, confirmsNoise }; - }; - - const filter: { sigma: number; threshold: number; dinov2?: { threshold: number } } = { sigma: 1, threshold: 80 }; - const compare = (service: PixelmatchService, blurAmplitude?: typeof filter) => - service.getDiff( - { - baseline: 'baseline.png', - image: 'image.png', - ignoreAreas: [], - diffTollerancePercent: 0, - saveDiffAsFile: true, - ...(blurAmplitude ? { blurAmplitude } : {}), - }, - DEFAULT_CONFIG - ); + const init = (output: Record, confirmsNoiseForGate = jest.fn()) => + setup({ + pool: poolOf(output), + getImageBuffer: pngReader(), + saveImage: jest.fn().mockResolvedValue('diff.png'), + confirmsNoiseForGate, + }); - const overTolerance = { equal: false, isSameDimension: true, pixelMisMatchCount: 10, diffPercent: 5 }; + const filter: { threshold: number; dinov2?: { threshold: number } } = { threshold: 80 }; + const compareWith = (service: PixelmatchService, blurAmplitude?: typeof filter) => + compare(service, { saveDiffAsFile: true, ...(blurAmplitude ? { blurAmplitude } : {}) }); - // so CompareService does not measure the same pair a second time it('applies the filter itself', async () => { - const { service } = await init(overTolerance); + const { service } = await init(mismatch); expect(service.appliesBlurAmplitudeFilter).toBe(true); }); it('asks for no measurement when the comparison does not carry the filter', async () => { - const { service, run } = await init(overTolerance); + const { service, run } = await init(mismatch); - await compare(service); + await compareWith(service); expect(run.mock.calls[0][0].blurAmplitude).toBeUndefined(); }); it('measures with the settings the comparison carries', async () => { - const { service, run } = await init(overTolerance); + const { service, run } = await init(mismatch); - await compare(service, { sigma: 2, threshold: 50 }); + await compareWith(service, { threshold: 50 }); - expect(run).toHaveBeenCalledWith(expect.objectContaining({ blurAmplitude: { sigma: 2, threshold: 50 } })); + expect(run).toHaveBeenCalledWith(expect.objectContaining({ blurAmplitude: { threshold: 50 } })); }); - // the measured mismatch stays on the run, so a reviewer can still see the - // pixels moved; only the verdict changes it('passes a run whose change is only rendering noise', async () => { - const { service, saveImage } = await init({ ...overTolerance, blurAmplitude: 40, renderingNoiseOnly: true }); + const { service, saveImage } = await init({ ...mismatch, blurAmplitude: 40, renderingNoiseOnly: true }); - const result = await compare(service, filter); + const result = await compareWith(service, filter); expect(result.status).toBe(TestStatus.ok); expect(result.diffPercent).toBe(5); @@ -492,13 +417,13 @@ describe('blur-amplitude filter', () => { it('fails a run whose change is stronger than rendering noise', async () => { const { service } = await init({ - ...overTolerance, + ...mismatch, blurAmplitude: 200, renderingNoiseOnly: false, diffBuffer: Buffer.from('the diff'), }); - const result = await compare(service, filter); + const result = await compareWith(service, filter); expect(result.status).toBe(TestStatus.unresolved); expect(result.diffName).toBe('diff.png'); @@ -506,71 +431,80 @@ describe('blur-amplitude filter', () => { describe('with the DINOv2 check', () => { const checked = { ...filter, dinov2: { threshold: 0.15 } }; + const gate = { width: 5, height: 3, cells: [7] }; const noise = { - ...overTolerance, + ...mismatch, blurAmplitude: 40, renderingNoiseOnly: true, diffBuffer: Buffer.from('the diff'), + gate, }; - // what the blur would pass, DINOv2 looks at once more - it('passes a run DINOv2 confirms is only noise', async () => { - const confirmsNoise = jest.fn().mockResolvedValue({ noise: true, score: 0.05 }); - const { service, saveImage } = await init(noise, confirmsNoise); + it('passes a run DINOv2 confirms is only noise, with the gate the diff job computed', async () => { + const confirmsNoiseForGate = jest.fn().mockResolvedValue({ noise: true, score: 0.05 }); + const { service, saveImage } = await init(noise, confirmsNoiseForGate); - const result = await compare(service, checked); + const result = await compareWith(service, checked); - expect(confirmsNoise).toHaveBeenCalledWith(Buffer.from('png'), Buffer.from('png'), [], 0.15); + expect(confirmsNoiseForGate).toHaveBeenCalledWith(gate, Buffer.from('png'), Buffer.from('png'), 0.15); expect(result.status).toBe(TestStatus.ok); expect(saveImage).not.toHaveBeenCalled(); }); it('fails a run in which DINOv2 sees a change of content, with its diff', async () => { - const confirmsNoise = jest.fn().mockResolvedValue({ noise: false, score: 0.56 }); - const { service, saveImage } = await init(noise, confirmsNoise); + const confirmsNoiseForGate = jest.fn().mockResolvedValue({ noise: false, score: 0.56 }); + const { service, saveImage } = await init(noise, confirmsNoiseForGate); - const result = await compare(service, checked); + const result = await compareWith(service, checked); expect(result.status).toBe(TestStatus.unresolved); expect(result.diffName).toBe('diff.png'); expect(saveImage).toHaveBeenCalledWith('diff', Buffer.from('the diff')); }); - // the worker must keep the diff of a noise-like change, or a run DINOv2 - // sends to review would arrive with nothing to show + it('fails a run whose diff job returned no gate, without asking DINOv2', async () => { + const confirmsNoiseForGate = jest.fn(); + const { service, saveImage } = await init({ ...noise, gate: undefined }, confirmsNoiseForGate); + + const result = await compareWith(service, checked); + + expect(confirmsNoiseForGate).not.toHaveBeenCalled(); + expect(result.status).toBe(TestStatus.unresolved); + expect(saveImage).toHaveBeenCalledWith('diff', Buffer.from('the diff')); + }); + it('asks the worker to keep the diff when a check will follow', async () => { const { service, run } = await init(noise, jest.fn().mockResolvedValue({ noise: true })); - await compare(service, checked); + await compareWith(service, checked); expect(run).toHaveBeenCalledWith(expect.objectContaining({ blurAmplitude: checked })); }); - // a check that could not look answers without a score: still to review it('fails a run DINOv2 could not confirm', async () => { const { service } = await init(noise, jest.fn().mockResolvedValue({ noise: false })); - expect((await compare(service, checked)).status).toBe(TestStatus.unresolved); + expect((await compareWith(service, checked)).status).toBe(TestStatus.unresolved); }); it('does not ask DINOv2 about a change the blur already fails', async () => { - const confirmsNoise = jest.fn(); + const confirmsNoiseForGate = jest.fn(); const { service } = await init( - { ...overTolerance, blurAmplitude: 200, renderingNoiseOnly: false }, - confirmsNoise + { ...mismatch, blurAmplitude: 200, renderingNoiseOnly: false }, + confirmsNoiseForGate ); - await compare(service, checked); + await compareWith(service, checked); - expect(confirmsNoise).not.toHaveBeenCalled(); + expect(confirmsNoiseForGate).not.toHaveBeenCalled(); }); it('does not ask DINOv2 when the project did not turn the check on', async () => { - const confirmsNoise = jest.fn(); - const { service } = await init(noise, confirmsNoise); + const confirmsNoiseForGate = jest.fn(); + const { service } = await init(noise, confirmsNoiseForGate); - expect((await compare(service, filter)).status).toBe(TestStatus.ok); - expect(confirmsNoise).not.toHaveBeenCalled(); + expect((await compareWith(service, filter)).status).toBe(TestStatus.ok); + expect(confirmsNoiseForGate).not.toHaveBeenCalled(); }); }); }); diff --git a/src/compare/libs/pixelmatch/pixelmatch.service.ts b/src/compare/libs/pixelmatch/pixelmatch.service.ts index f7136caf..128e53cc 100644 --- a/src/compare/libs/pixelmatch/pixelmatch.service.ts +++ b/src/compare/libs/pixelmatch/pixelmatch.service.ts @@ -1,17 +1,25 @@ import { Injectable, Logger } from '@nestjs/common'; import { TestStatus } from '@prisma/client'; import { StaticService } from '../../../static/static.service'; +import { isByteEqualWellFormedPng, readPngDimensions } from '../../../static/utils'; import { DiffResult } from '../../../test-runs/diffResult'; import { parseConfig } from '../../utils'; import { DIFF_DIMENSION_RESULT, EQUAL_RESULT, NO_BASELINE_RESULT } from '../consts'; import { ImageComparator } from '../image-comparator.interface'; import { ImageCompareInput } from '../ImageCompareInput'; import { PixelmatchConfig } from './pixelmatch.types'; +import { PixelmatchJobOutput } from './pixelmatch.core'; import { DiffWorkerPool } from '../../diff-worker-pool'; import { Dinov2Service } from '../dinov2/dinov2.service'; export const DEFAULT_CONFIG: PixelmatchConfig = { threshold: 0.1, ignoreAntialiasing: true }; +function sameDimensions(a: Buffer, b: Buffer): boolean { + const x = readPngDimensions(a); + const y = readPngDimensions(b); + return !x || !y || (x.width === y.width && x.height === y.height); +} + @Injectable() export class PixelmatchService implements ImageComparator { private readonly logger: Logger = new Logger(PixelmatchService.name); @@ -32,12 +40,19 @@ export class PixelmatchService implements ImageComparator { if (!baselineBuffer) { return NO_BASELINE_RESULT; } - const imageBuffer = await this.staticService.getImageBuffer(data.image); + const imageBuffer = data.imageBuffer ?? (await this.staticService.getImageBuffer(data.image)); if (!imageBuffer) { throw new Error(`Cannot get image: ${data.image}`); } - // decode + pixelmatch + diff encode run off the event loop + if (isByteEqualWellFormedPng(baselineBuffer, imageBuffer)) { + return EQUAL_RESULT; + } + + if (!config.allowDiffDimensions && !sameDimensions(baselineBuffer, imageBuffer)) { + return DIFF_DIMENSION_RESULT; + } + const output = await this.diffWorkerPool.run({ kind: 'diff', baseline: baselineBuffer, @@ -48,18 +63,6 @@ export class PixelmatchService implements ImageComparator { allowDiffDimensions: config.allowDiffDimensions, diffTolerancePercent: data.diffTollerancePercent, saveDiff: data.saveDiffAsFile, - // Asked for on every comparison, not only when the project has bulk - // approve of variations switched on: the pass costs little beside the - // full-size diff that has already been paid for, and computing it lazily - // would leave every run ingested before the flag was turned on without - // one — which is exactly the build someone then tries to review. - withSignature: true, - // Only when the caller is keeping the diff. shouldAutoApprove compares - // against past baselines with this off and throws the result away — - // making thumbnails for those would burn the CPU and, worse, store two - // objects per attempt that nothing ever references or deletes. - withThumbnails: data.saveDiffAsFile, - // measured here, next to the diff, while the images are decoded anyway ...(data.blurAmplitude ? { blurAmplitude: data.blurAmplitude } : {}), }); @@ -86,9 +89,7 @@ export class PixelmatchService implements ImageComparator { : {}), }; - // The mismatch stays on the run as measured, so a reviewer can still see - // the pixels moved; only the verdict is the filter's. - if (output.renderingNoiseOnly && (await this.confirmedByDinov2(data, baselineBuffer, imageBuffer))) { + if (output.renderingNoiseOnly && (await this.confirmedByDinov2(data, baselineBuffer, imageBuffer, output))) { this.logger.debug( `Diff of ${result.diffPercent}% passed as rendering noise (blur amplitude ${output.blurAmplitude?.toFixed(1)}, ${output.solidPixels} solid pixels)` ); @@ -100,23 +101,8 @@ export class PixelmatchService implements ImageComparator { if (output.diffBuffer) { result.diffName = await this.staticService.saveImage('diff', Buffer.from(output.diffBuffer)); } - // Kept to the same condition as the diff itself: a thumbnail of a diff - // that was never saved would point at a file that does not exist. Stored - // as ordinary images so they follow the same storage, the same deletion - // and the same URLs as everything else, with no naming convention for - // the UI to guess at. - // tied to the diff actually landing, not merely to being over tolerance - if (result.diffName && output.imageThumbnail && output.diffThumbnail) { - const [imageThumbnailName, diffThumbnailName] = await Promise.all([ - this.staticService.saveImage('screenshot', Buffer.from(output.imageThumbnail)), - this.staticService.saveImage('diff', Buffer.from(output.diffThumbnail)), - ]); - // only when both came back: a result carrying one half of a pair, or a - // key set to undefined, is worse than none at all - if (imageThumbnailName && diffThumbnailName) { - result.imageThumbnailName = imageThumbnailName; - result.diffThumbnailName = diffThumbnailName; - } + if (result.diffName) { + Object.assign(result, await this.saveThumbnails(output)); } result.status = TestStatus.unresolved; } else { @@ -125,17 +111,44 @@ export class PixelmatchService implements ImageComparator { return result; } - // The project's DINOv2 check, when it has one, gets the last word on what the - // blur would pass; without one the blur's verdict stands. - private async confirmedByDinov2(data: ImageCompareInput, baseline: Buffer, image: Buffer): Promise { + private async confirmedByDinov2( + data: ImageCompareInput, + baseline: Buffer, + image: Buffer, + output: PixelmatchJobOutput + ): Promise { const dinov2 = data.blurAmplitude?.dinov2; if (!dinov2) { return true; } - const verdict = await this.dinov2Service.confirmsNoise(baseline, image, data.ignoreAreas, dinov2.threshold); + if (!output.gate) { + return false; + } + const verdict = await this.dinov2Service.confirmsNoiseForGate(output.gate, baseline, image, dinov2.threshold); if (!verdict.noise) { this.logger.debug(`DINOv2 sees a change of content (score ${verdict.score?.toFixed(3) ?? 'n/a'}): to review`); } return verdict.noise; } + + private async saveThumbnails( + output: PixelmatchJobOutput + ): Promise<{ imageThumbnailName?: string; diffThumbnailName?: string }> { + if (!output.imageThumbnail || !output.diffThumbnail) { + return {}; + } + const [image, diff] = await Promise.allSettled([ + this.staticService.saveImage('screenshot', Buffer.from(output.imageThumbnail)), + this.staticService.saveImage('diff', Buffer.from(output.diffThumbnail)), + ]); + if (image.status === 'fulfilled' && diff.status === 'fulfilled' && image.value && diff.value) { + return { imageThumbnailName: image.value, diffThumbnailName: diff.value }; + } + for (const saved of [image, diff]) { + if (saved.status === 'fulfilled' && saved.value) void this.staticService.deleteImage(saved.value); + } + const failure = [image, diff].find((saved) => saved.status === 'rejected'); + this.logger.warn(`Thumbnails not saved: ${failure?.reason?.message ?? 'empty name'}`); + return {}; + } } diff --git a/src/compare/libs/pixelmatch/pixelmatch.worker.spec.ts b/src/compare/libs/pixelmatch/pixelmatch.worker.spec.ts index 9b40be0a..9e1254ca 100644 --- a/src/compare/libs/pixelmatch/pixelmatch.worker.spec.ts +++ b/src/compare/libs/pixelmatch/pixelmatch.worker.spec.ts @@ -27,10 +27,11 @@ const input: SignatureJobInput = { }; describe('pixelmatch.worker', () => { + let postedOnLoad: unknown[][]; + beforeAll(() => { - // a plain require: under `module: nodenext` an import() stays a real ESM import - // eslint-disable-next-line @typescript-eslint/no-require-imports require('./pixelmatch.worker'); + postedOnLoad = [...port.postMessage.mock.calls]; }); beforeEach(() => { @@ -38,31 +39,40 @@ describe('pixelmatch.worker', () => { runWorkerJobMock.mockReset(); }); + it('announces that it has loaded', () => { + expect(postedOnLoad).toEqual([[{ ready: true }]]); + }); + it('listens for jobs on the parent port', () => { expect(port.listenerCount('message')).toBe(1); }); - it('runs the job it receives and posts the result back', () => { + const flush = () => new Promise(setImmediate); + + it('runs the job it receives and posts the result back', async () => { runWorkerJobMock.mockReturnValue({ signature: [1] }); port.emit('message', input); + await flush(); expect(runWorkerJobMock).toHaveBeenCalledWith(input); expect(port.postMessage).toHaveBeenCalledWith({ signature: [1] }); }); - it('posts the error message back when the job throws an Error', () => { + it('posts the error message back when the job throws an Error', async () => { runWorkerJobMock.mockImplementation(() => { throw new Error('Invalid PNG'); }); port.emit('message', input); + await flush(); expect(port.postMessage).toHaveBeenCalledWith({ error: 'Invalid PNG' }); }); it('posts the stringified value back when the job throws something that is not an Error', () => { runWorkerJobMock.mockImplementation(() => { + // eslint-disable-next-line @typescript-eslint/only-throw-error -- worker must stringify non-Error throws throw 42; }); @@ -70,4 +80,24 @@ describe('pixelmatch.worker', () => { expect(port.postMessage).toHaveBeenCalledWith({ error: '42' }); }); + + it('awaits an async job before posting its result', async () => { + runWorkerJobMock.mockResolvedValue({ outcome: 'equal' }); + + port.emit('message', input); + await flush(); + + expect(port.postMessage).toHaveBeenCalledTimes(1); + expect(port.postMessage).toHaveBeenCalledWith({ outcome: 'equal' }); + }); + + it('posts the message of an async job that rejects', async () => { + runWorkerJobMock.mockRejectedValue(new Error('boom')); + + port.emit('message', input); + await flush(); + + expect(port.postMessage).toHaveBeenCalledTimes(1); + expect(port.postMessage).toHaveBeenCalledWith({ error: 'boom' }); + }); }); diff --git a/src/compare/libs/pixelmatch/pixelmatch.worker.ts b/src/compare/libs/pixelmatch/pixelmatch.worker.ts index 67520b56..40c6de79 100644 --- a/src/compare/libs/pixelmatch/pixelmatch.worker.ts +++ b/src/compare/libs/pixelmatch/pixelmatch.worker.ts @@ -1,13 +1,15 @@ import { parentPort } from 'worker_threads'; import { runWorkerJob, WorkerJobInput } from './worker-job'; -// this file is only ever loaded as a worker thread, where parentPort is set const port = parentPort!; -port.on('message', (input: WorkerJobInput) => { +// eslint-disable-next-line @typescript-eslint/no-misused-promises -- handler reports errors via postMessage +port.on('message', async (input: WorkerJobInput) => { try { - port.postMessage(runWorkerJob(input)); + port.postMessage(await runWorkerJob(input)); } catch (error) { port.postMessage({ error: error instanceof Error ? error.message : String(error) }); } }); + +port.postMessage({ ready: true }); diff --git a/src/compare/libs/pixelmatch/signature.core.spec.ts b/src/compare/libs/pixelmatch/signature.core.spec.ts index 3c48e99c..e8584fff 100644 --- a/src/compare/libs/pixelmatch/signature.core.spec.ts +++ b/src/compare/libs/pixelmatch/signature.core.spec.ts @@ -1,4 +1,4 @@ -import { PNG } from 'pngjs'; +import { paintedPng, pngBytes, Rgb, solidPng, square } from '../../test-utils'; import { computeChangeSignature, signaturesMatch, @@ -10,32 +10,13 @@ import { IgnoreAreaDto } from '../../../test-runs/dto/ignore-area.dto'; const WIDTH = 40; const HEIGHT = 40; -const png = (paint: (set: (x: number, y: number, rgb: [number, number, number]) => void) => void): Buffer => { - const image = new PNG({ width: WIDTH, height: HEIGHT }); - image.data.fill(255); - paint((x, y, [r, g, b]) => { - const index = (y * WIDTH + x) * 4; - image.data[index] = r; - image.data[index + 1] = g; - image.data[index + 2] = b; - image.data[index + 3] = 255; - }); - return PNG.sync.write(image); -}; - -const block = (left: number, top: number, rgb: [number, number, number]): Buffer => - png((set) => { - for (let y = top; y < top + 8; y++) { - for (let x = left; x < left + 8; x++) { - set(x, y, rgb); - } - } - }); +const block = (left: number, top: number, rgb: Rgb): Buffer => + pngBytes(paintedPng(WIDTH, HEIGHT, square(left, top, 8, rgb))); -const blank = png(() => undefined); +const blank = pngBytes(solidPng(WIDTH, HEIGHT)); -const RED: [number, number, number] = [255, 0, 0]; -const BLUE: [number, number, number] = [0, 0, 255]; +const RED: Rgb = [255, 0, 0]; +const BLUE: Rgb = [0, 0, 255]; const signatureOf = (baseline: Buffer, image: Buffer, ignoreAreas: IgnoreAreaDto[] = []): number[] | null => computeChangeSignature({ @@ -48,34 +29,18 @@ const signatureOf = (baseline: Buffer, image: Buffer, ignoreAreas: IgnoreAreaDto }).signature; describe('computeChangeSignature', () => { - it('has no signature when the images have different dimensions', () => { - const taller = new PNG({ width: WIDTH, height: HEIGHT * 2 }); - taller.data.fill(255); - - expect(signatureOf(blank, PNG.sync.write(taller))).toBeNull(); - }); - - it('has no signature for a checkpoint with the same pixel count but another shape', () => { - const wide = new PNG({ width: WIDTH * 2, height: HEIGHT / 2 }); - wide.data.fill(0); - - expect(signatureOf(blank, PNG.sync.write(wide))).toBeNull(); - }); - - it('has no signature for a checkpoint of another width only', () => { - const wider = new PNG({ width: WIDTH * 2, height: HEIGHT }); - wider.data.fill(0); - - expect(signatureOf(blank, PNG.sync.write(wider))).toBeNull(); + it.each([ + ['different dimensions', WIDTH, HEIGHT * 2], + ['the same pixel count but another shape', WIDTH * 2, HEIGHT / 2], + ['another width only', WIDTH * 2, HEIGHT], + ])('has no signature for a checkpoint with %s', (_, width, height) => { + expect(signatureOf(blank, pngBytes(solidPng(width, height)))).toBeNull(); }); it('has no signature when nothing changed', () => { expect(signatureOf(blank, blank)).toBeNull(); }); - // One unreadable screenshot must not reject the whole matching request: the - // siblings are signed in a single fan-out, so a throw here would take the - // variations dialog down with it rather than skipping that one candidate. it('has no signature when the checkpoint cannot be decoded', () => { expect(signatureOf(blank, Buffer.from('not a png'))).toBeNull(); }); @@ -88,7 +53,6 @@ describe('computeChangeSignature', () => { const signature = signatureOf(blank, block(0, 0, RED))!; const brightest = signature.indexOf(Math.max(...signature)); - // 4 buckets per channel: pure red is the last red bucket, first green/blue expect(brightest).toBe(3 * 16); expect(signature.reduce((sum, value) => sum + value, 0)).toBeCloseTo(1); }); @@ -109,7 +73,6 @@ describe('computeChangeSignature', () => { }); describe('computeChangeSignature with structured-clone input', () => { - // postMessage hands the worker Uint8Array views, possibly over a larger buffer it('reads Uint8Array views the same as Buffers', () => { const changed = block(0, 0, RED); const padded = new Uint8Array(changed.length + 16); @@ -147,9 +110,7 @@ describe('signaturesMatch', () => { it('matches at a cosine similarity of at least 0.9', () => { expect(SIGNATURE_SIMILARITY_THRESHOLD).toBe(0.9); - // cos = 0.95 / sqrt(0.95^2 + 0.3^2) ≈ 0.954 expect(signaturesMatch([1, 0], [0.95, 0.3])).toBe(true); - // cos = 0.8 / sqrt(0.8^2 + 0.6^2) = 0.8 expect(signaturesMatch([1, 0], [0.8, 0.6])).toBe(false); }); @@ -158,10 +119,8 @@ describe('signaturesMatch', () => { expect(signaturesMatch([1, 0], [0, 0])).toBe(false); }); - it('compares only as many buckets as the first signature has', () => { + it('compares only the buckets of the first signature (signatures are always SIGNATURE_LENGTH long)', () => { expect(SIGNATURE_LENGTH).toBe(64); - // NOTE: current behavior — a shorter second signature yields NaN (no match), - // while extra buckets in the second one are silently ignored (a match). expect(signaturesMatch([1, 0], [1])).toBe(false); expect(signaturesMatch([1], [1, 5])).toBe(true); }); diff --git a/src/compare/libs/pixelmatch/signature.core.ts b/src/compare/libs/pixelmatch/signature.core.ts index 020d37a7..4df8181f 100644 --- a/src/compare/libs/pixelmatch/signature.core.ts +++ b/src/compare/libs/pixelmatch/signature.core.ts @@ -1,21 +1,10 @@ import { PNG } from 'pngjs'; import Pixelmatch from 'pixelmatch'; import { IgnoreAreaDto } from '../../../test-runs/dto/ignore-area.dto'; -import { RawImage } from '../../utils'; -import { applyIgnoreAreas, downscale } from '../../utils'; +import { downscale, RawImage, readSameSizePair } from '../../utils'; -/** - * CPU-bound part of "do these two screens carry the same change?": PNG decode, - * downscale, pixelmatch and histogram. Extracted so it can run inside a worker - * thread (see pixelmatch.worker.ts / DiffWorkerPool) — the variations dialog - * asks for one of these per sibling, and decoding full-size screenshots on the - * main thread froze the whole API for as long as it took. All input/output must - * stay structured-clone serializable. - */ export interface SignatureJobInput { kind: 'signature'; - // PNG bytes, not decoded pixels: decoding is the expensive part and belongs - // in the worker. baseline: Buffer | Uint8Array; image: Buffer | Uint8Array; ignoreAreas: IgnoreAreaDto[]; @@ -27,70 +16,19 @@ export interface SignatureJobOutput { signature: number[] | null; } -// Colors are quantized to this many levels per RGB channel, giving -// COLOR_BUCKETS_PER_CHANNEL^3 histogram buckets. const COLOR_BUCKETS_PER_CHANNEL = 4; -// How many buckets a signature has. Exported so a stored one can be checked -// against the shape this build produces: comparing signatures of two different -// lengths yields a similarity score computed over undefined entries — a number, -// and a meaningless one. export const SIGNATURE_LENGTH = COLOR_BUCKETS_PER_CHANNEL ** 3; -// Longest side (px) images are downscaled to before computing the color -// signature — keeps the histogram representative while cutting pixelmatch cost. const SIGNATURE_MAX_DIMENSION = 500; -// Two changes are considered the same pattern when their color signatures' -// cosine similarity is at least this value. export const SIGNATURE_SIMILARITY_THRESHOLD = 0.9; -// postMessage turns Buffers into Uint8Array views over their own ArrayBuffer. -function toBuffer(data: Buffer | Uint8Array): Buffer { - return Buffer.isBuffer(data) ? data : Buffer.from(data.buffer, data.byteOffset, data.byteLength); -} - -/** - * Position-independent signature of a change: a normalized histogram of the - * colors that the changed pixels take in the new image. Because it ignores - * *where* the change is, it is robust to per-locale layout shifts (a title - * wrapping to a different number of lines, options moving down, etc.) while - * still capturing *what* changed (a selection highlight, a recolored button). - * Null when there is nothing to compare: unreadable bytes, differing - * dimensions, or no change outside the ignore areas. - */ export function computeChangeSignature(input: SignatureJobInput): SignatureJobOutput { - // A corrupt or truncated screenshot is one candidate the reviewer cannot be - // offered, not a failed request: the siblings are signed in a single - // fan-out, so throwing here would take the whole variations dialog down. - let baselineImage: PNG; - let checkpointImage: PNG; - try { - baselineImage = PNG.sync.read(toBuffer(input.baseline)); - checkpointImage = PNG.sync.read(toBuffer(input.image)); - } catch { - return { signature: null }; - } - - if (baselineImage.width !== checkpointImage.width || baselineImage.height !== checkpointImage.height) { - return { signature: null }; - } - - // Masked regions must not count toward the signature, so they are blanked at - // full resolution — the areas are given in full-resolution coordinates — - // before the downscale makes both images cheap to compare. - applyIgnoreAreas(baselineImage, input.ignoreAreas); - applyIgnoreAreas(checkpointImage, input.ignoreAreas); - - return { signature: signatureOfDecoded(baselineImage, checkpointImage, input) }; + const pair = readSameSizePair(input.baseline, input.image, input.ignoreAreas); + return { signature: pair && signatureOfDecoded(pair.baseline, pair.image, input) }; } -/** - * The signature of a pair that is already decoded, of equal size, and with its - * ignore areas blanked. Split out so the diff can produce a signature from the - * images it has just decoded rather than paying for a second decode — the two - * must not drift apart, so there is only ever one implementation. - */ export function signatureOfDecoded( baselineImage: RawImage, checkpointImage: RawImage, @@ -110,7 +48,7 @@ export function signatureOfDecoded( } const bucketSize = 256 / COLOR_BUCKETS_PER_CHANNEL; - const histogram = new Array(COLOR_BUCKETS_PER_CHANNEL ** 3).fill(0); + const histogram = new Array(SIGNATURE_LENGTH).fill(0); for (let i = 0; i < width * height; i++) { if (mask.data[i * 4 + 3] === 0) { continue; diff --git a/src/compare/libs/pixelmatch/thumbnail.core.spec.ts b/src/compare/libs/pixelmatch/thumbnail.core.spec.ts index 4dcfa7a8..3ec7665a 100644 --- a/src/compare/libs/pixelmatch/thumbnail.core.spec.ts +++ b/src/compare/libs/pixelmatch/thumbnail.core.spec.ts @@ -1,42 +1,28 @@ import { PNG } from 'pngjs'; +import { solidPng } from '../../test-utils'; import { encodeThumbnail, THUMBNAIL_MAX_DIMENSION } from './thumbnail.core'; -const solid = (width: number, height: number): PNG => { - const png = new PNG({ width, height }); - for (let i = 0; i < width * height; i++) { - png.data[i * 4] = 200; - png.data[i * 4 + 1] = 100; - png.data[i * 4 + 2] = 50; - png.data[i * 4 + 3] = 255; - } - return png; -}; +const solid = (width: number, height: number): PNG => solidPng(width, height, [200, 100, 50]); describe('encodeThumbnail', () => { - // The grids draw these a hundred-odd pixels wide. Sending the full - // screenshot and letting CSS shrink it cost ~7 MB and 68 requests every time - // the variations dialog opened. + const source = solid(1284, 2778); + it('brings a screenshot down to the thumbnail size', () => { - const thumbnail = PNG.sync.read(encodeThumbnail(solid(1284, 2778))); + const thumbnail = PNG.sync.read(encodeThumbnail(source)); expect(Math.max(thumbnail.width, thumbnail.height)).toBe(THUMBNAIL_MAX_DIMENSION); }); it('keeps the proportions of the screenshot it came from', () => { - const source = solid(1284, 2778); - const thumbnail = PNG.sync.read(encodeThumbnail(source)); expect(thumbnail.width / thumbnail.height).toBeCloseTo(source.width / source.height, 2); }); it('is a fraction of the bytes of the original', () => { - const source = solid(1284, 2778); - - expect(encodeThumbnail(source).length).toBeLessThan(PNG.sync.write(source).length / 10); + expect(encodeThumbnail(source).length).toBeLessThan(PNG.sync.write(source, { filterType: 1 }).length / 10); }); - // a small screenshot re-encoded larger would be worse than leaving it alone it('never enlarges an image that is already small', () => { const thumbnail = PNG.sync.read(encodeThumbnail(solid(100, 80))); diff --git a/src/compare/libs/pixelmatch/thumbnail.core.ts b/src/compare/libs/pixelmatch/thumbnail.core.ts index 1fef0cd6..901cef66 100644 --- a/src/compare/libs/pixelmatch/thumbnail.core.ts +++ b/src/compare/libs/pixelmatch/thumbnail.core.ts @@ -1,22 +1,8 @@ import { PNG } from 'pngjs'; import { downscale, RawImage } from '../../utils'; -/** - * Longest side, in pixels, of the picture the grids draw. They lay these out a - * hundred-odd pixels wide, so this leaves room for a retina screen and nothing - * more. Sending the full screenshot and letting CSS shrink it cost roughly 7 MB - * and 68 requests every time the variations dialog opened. - */ export const THUMBNAIL_MAX_DIMENSION = 400; -/** - * A small PNG of an already-decoded image, for the card grids. Produced beside - * the diff, from the pixels it has already decoded, so a thumbnail costs a - * resize rather than a second read of the screenshot. - * - * An image already within the bound is re-encoded as it is: enlarging it would - * make it bigger on the wire for no gain. - */ export function encodeThumbnail(source: RawImage): Buffer { const small = downscale(source, THUMBNAIL_MAX_DIMENSION); const png = new PNG({ width: small.width, height: small.height }); diff --git a/src/compare/libs/pixelmatch/worker-job.spec.ts b/src/compare/libs/pixelmatch/worker-job.spec.ts new file mode 100644 index 00000000..fdbc4872 --- /dev/null +++ b/src/compare/libs/pixelmatch/worker-job.spec.ts @@ -0,0 +1,41 @@ +import { computeLooksSame, LooksSameJobInput } from '../looks-same/looks-same.core'; +import { computeVlmInputs, VlmInputsJobInput } from '../vlm/vlm-inputs.core'; +import { runWorkerJob } from './worker-job'; + +jest.mock('../looks-same/looks-same.core', () => ({ + computeLooksSame: jest.fn(), +})); +jest.mock('../vlm/vlm-inputs.core', () => ({ + computeVlmInputs: jest.fn(), +})); + +describe('runWorkerJob', () => { + it('hands a looks-same job to computeLooksSame and returns its promise', async () => { + const input: LooksSameJobInput = { + kind: 'looks-same', + baseline: Buffer.from('baseline'), + image: Buffer.from('image'), + ignoreAreas: [], + config: {}, + saveDiff: false, + }; + (computeLooksSame as jest.Mock).mockResolvedValue({ outcome: 'equal' }); + + await expect(runWorkerJob(input)).resolves.toEqual({ outcome: 'equal' }); + expect(computeLooksSame).toHaveBeenCalledWith(input); + }); + + it('hands a vlm-inputs job to computeVlmInputs', () => { + const input: VlmInputsJobInput = { + kind: 'vlm-inputs', + baseline: Buffer.from('baseline'), + image: Buffer.from('image'), + ignoreAreas: [], + }; + const output = { baseline: null, image: null }; + (computeVlmInputs as jest.Mock).mockReturnValue(output); + + expect(runWorkerJob(input)).toBe(output); + expect(computeVlmInputs).toHaveBeenCalledWith(input); + }); +}); diff --git a/src/compare/libs/pixelmatch/worker-job.ts b/src/compare/libs/pixelmatch/worker-job.ts index 729f4727..9f574dc8 100644 --- a/src/compare/libs/pixelmatch/worker-job.ts +++ b/src/compare/libs/pixelmatch/worker-job.ts @@ -1,25 +1,26 @@ -import { BlurAmplitudeJobInput, BlurAmplitudeJobOutput, computeBlurAmplitude } from './blur-amplitude.core'; -import { computePatchGate, PatchGateJobInput, PatchGateJobOutput } from './patch-gate.core'; -import { computePixelmatchDiff, PixelmatchJobInput, PixelmatchJobOutput } from './pixelmatch.core'; -import { computeChangeSignature, SignatureJobInput, SignatureJobOutput } from './signature.core'; +import { computeBlurAmplitude } from './blur-amplitude.core'; +import { computeLooksSame } from '../looks-same/looks-same.core'; +import { computePixelmatchDiff } from './pixelmatch.core'; +import { computeVlmInputs } from '../vlm/vlm-inputs.core'; +import { computeChangeSignature } from './signature.core'; -/** - * The jobs {@link DiffWorkerPool} runs off the event loop, discriminated by - * `kind`. Both the worker thread and the pool's inline fallback go through - * {@link runWorkerJob}, so the two paths cannot answer a job differently. - */ -export type WorkerJobInput = PixelmatchJobInput | SignatureJobInput | BlurAmplitudeJobInput | PatchGateJobInput; -export type WorkerJobOutput = PixelmatchJobOutput | SignatureJobOutput | BlurAmplitudeJobOutput | PatchGateJobOutput; +const JOBS = { + diff: computePixelmatchDiff, + signature: computeChangeSignature, + 'blur-amplitude': computeBlurAmplitude, + 'looks-same': computeLooksSame, + 'vlm-inputs': computeVlmInputs, +} as const; -export function runWorkerJob(input: WorkerJobInput): WorkerJobOutput { - switch (input.kind) { - case 'signature': - return computeChangeSignature(input); - case 'blur-amplitude': - return computeBlurAmplitude(input); - case 'patch-gate': - return computePatchGate(input); - default: - return computePixelmatchDiff(input); - } +type Jobs = typeof JOBS; + +export type WorkerJobKind = keyof Jobs; +export type WorkerJobInput = { [K in WorkerJobKind]: Parameters[0] }[WorkerJobKind]; +export type WorkerJobOutput = { [K in WorkerJobKind]: Awaited> }[WorkerJobKind]; + +export type WorkerJobResult = Awaited>; +export type WorkerJobInputOf = Parameters[0] & { kind: K }; + +export function runWorkerJob(input: WorkerJobInput): WorkerJobOutput | Promise { + return (JOBS[input.kind] as (input: WorkerJobInput) => WorkerJobOutput | Promise)(input); } diff --git a/src/compare/libs/vlm/concurrency-limiter.spec.ts b/src/compare/libs/vlm/concurrency-limiter.spec.ts new file mode 100644 index 00000000..39fb966e --- /dev/null +++ b/src/compare/libs/vlm/concurrency-limiter.spec.ts @@ -0,0 +1,115 @@ +import { ConcurrencyLimiter } from './concurrency-limiter'; + +interface Deferred { + promise: Promise; + resolve: (value: T) => void; + reject: (error: Error) => void; +} + +const deferred = (): Deferred => { + let resolve!: (value: T) => void; + let reject!: (error: Error) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +}; + +const flush = async () => { + for (let i = 0; i < 5; i++) await Promise.resolve(); +}; + +describe('ConcurrencyLimiter', () => { + it('runs at most limit tasks at once', async () => { + const limiter = new ConcurrencyLimiter(2); + const gates = Array.from({ length: 5 }, () => deferred()); + let active = 0; + let peak = 0; + const started: number[] = []; + const results = gates.map((gate, index) => + limiter.run(async () => { + started.push(index); + active += 1; + peak = Math.max(peak, active); + try { + return await gate.promise; + } finally { + active -= 1; + } + }) + ); + + await flush(); + expect(started).toEqual([0, 1]); + + gates[0].resolve(10); + await flush(); + expect(started).toEqual([0, 1, 2]); + + gates[2].resolve(12); + gates[1].resolve(11); + await flush(); + expect(started).toEqual([0, 1, 2, 3, 4]); + + gates[3].resolve(13); + gates[4].resolve(14); + await expect(Promise.all(results)).resolves.toEqual([10, 11, 12, 13, 14]); + expect(peak).toBe(2); + }); + + it('frees the slot when a task rejects', async () => { + const limiter = new ConcurrencyLimiter(1); + const first = deferred(); + const failure = new Error('model is down'); + + const failing = limiter.run(() => first.promise); + const next = limiter.run(async () => 'second'); + first.reject(failure); + + await expect(failing).rejects.toBe(failure); + await expect(next).resolves.toBe('second'); + }); + + it('never exceeds the limit when a new task arrives while a waiting one is being woken', async () => { + const limiter = new ConcurrencyLimiter(1); + const gates = { a: deferred(), b: deferred(), c: deferred() }; + let active = 0; + let peak = 0; + const started: string[] = []; + const track = (name: keyof typeof gates) => + limiter.run(async () => { + started.push(name); + active += 1; + peak = Math.max(peak, active); + await gates[name].promise; + active -= 1; + }); + + const a = track('a'); + const b = track('b'); + await flush(); + expect(started).toEqual(['a']); + + gates.a.resolve(); + const c = track('c'); + await flush(); + expect(started).toEqual(['a', 'b']); + + gates.b.resolve(); + await flush(); + expect(started).toEqual(['a', 'b', 'c']); + gates.c.resolve(); + + await Promise.all([a, b, c]); + expect(peak).toBe(1); + }); + + it('lets a task start at once when a slot is free again', async () => { + const limiter = new ConcurrencyLimiter(1); + + await limiter.run(async () => 1); + + await expect(limiter.run(async () => 2)).resolves.toBe(2); + }); +}); diff --git a/src/compare/libs/vlm/concurrency-limiter.ts b/src/compare/libs/vlm/concurrency-limiter.ts new file mode 100644 index 00000000..bfad3caa --- /dev/null +++ b/src/compare/libs/vlm/concurrency-limiter.ts @@ -0,0 +1,28 @@ +export class ConcurrencyLimiter { + private active = 0; + private readonly waiting: (() => void)[] = []; + + constructor(private readonly limit: number) {} + + async run(task: () => Promise): Promise { + if (this.active < this.limit) { + this.active += 1; + } else { + await new Promise((resolve) => this.waiting.push(resolve)); + } + try { + return await task(); + } finally { + this.release(); + } + } + + private release(): void { + const next = this.waiting.shift(); + if (next) { + next(); + } else { + this.active -= 1; + } + } +} diff --git a/src/compare/libs/vlm/providers/gemini/gemini.service.spec.ts b/src/compare/libs/vlm/providers/gemini/gemini.service.spec.ts index fc1bdb2f..66641335 100644 --- a/src/compare/libs/vlm/providers/gemini/gemini.service.spec.ts +++ b/src/compare/libs/vlm/providers/gemini/gemini.service.spec.ts @@ -1,12 +1,8 @@ import { Test, TestingModule } from '@nestjs/testing'; +import { GoogleGenAI } from '@google/genai'; import { GeminiService } from './gemini.service'; import { GeminiVlmConfig } from '../../vlm.types'; -jest.mock('zod/v3', () => { - const actualZod = jest.requireActual('zod'); - return actualZod; -}); - const mockGenerateContent = jest.fn(); jest.mock('@google/genai', () => { @@ -104,8 +100,45 @@ describe('GeminiService', () => { const callArgs = mockGenerateContent.mock.calls[0][0]; expect(callArgs.config.responseMimeType).toBe('application/json'); const schema = callArgs.config.responseJsonSchema; - expect(schema).toBeDefined(); - expect(schema).toBeTruthy(); + expect(schema).toEqual({ + $schema: 'http://json-schema.org/draft-07/schema#', + type: 'object', + properties: { identical: { type: 'boolean' }, description: { type: 'string' } }, + required: ['identical', 'description'], + additionalProperties: false, + }); + }); + + describe('request timeout', () => { + afterEach(() => { + delete process.env.GEMINI_TIMEOUT_MS; + }); + + it('defaults to five minutes', async () => { + mockGenerateContent.mockResolvedValue(createMockResponse('{}')); + + await service.generate(createConfig(), []); + + expect(GoogleGenAI).toHaveBeenCalledWith({ apiKey: 'test-api-key', httpOptions: { timeout: 300000 } }); + }); + + it('falls back to five minutes for a negative GEMINI_TIMEOUT_MS', async () => { + process.env.GEMINI_TIMEOUT_MS = '-1'; + mockGenerateContent.mockResolvedValue(createMockResponse('{}')); + + await service.generate(createConfig(), []); + + expect(GoogleGenAI).toHaveBeenCalledWith({ apiKey: 'test-api-key', httpOptions: { timeout: 300000 } }); + }); + + it('is set by GEMINI_TIMEOUT_MS', async () => { + process.env.GEMINI_TIMEOUT_MS = '5000'; + mockGenerateContent.mockResolvedValue(createMockResponse('{}')); + + await service.generate(createConfig(), []); + + expect(GoogleGenAI).toHaveBeenCalledWith({ apiKey: 'test-api-key', httpOptions: { timeout: 5000 } }); + }); }); it.each([ diff --git a/src/compare/libs/vlm/providers/gemini/gemini.service.ts b/src/compare/libs/vlm/providers/gemini/gemini.service.ts index e3c0ff55..ffb751a6 100644 --- a/src/compare/libs/vlm/providers/gemini/gemini.service.ts +++ b/src/compare/libs/vlm/providers/gemini/gemini.service.ts @@ -2,8 +2,11 @@ import { Injectable, Logger } from '@nestjs/common'; import { GoogleGenAI } from '@google/genai'; import { zodToJsonSchema } from 'zod-to-json-schema'; import { VlmProvider, VlmProviderResponse } from '../../vlm-provider.interface'; -import { VlmConfig, GeminiVlmConfig } from '../../vlm.types'; +import { VlmConfig } from '../../vlm.types'; import { VlmComparisonResultSchema } from '../../vlm-result.schema'; +import { positiveNumber } from '../../../../../utils/env'; + +const DEFAULT_TIMEOUT_MS = 300_000; @Injectable() export class GeminiService implements VlmProvider { @@ -13,47 +16,32 @@ export class GeminiService implements VlmProvider { if (!apiKey) { throw new Error('Gemini API key is required'); } - return new GoogleGenAI({ apiKey }); - } - - private imageToBase64(imageBytes: Uint8Array): string { - const base64 = Buffer.from(imageBytes).toString('base64'); - return base64; + return new GoogleGenAI({ + apiKey, + httpOptions: { timeout: positiveNumber(process.env.GEMINI_TIMEOUT_MS, DEFAULT_TIMEOUT_MS) }, + }); } async generate(config: VlmConfig, images: Uint8Array[]): Promise { - // Type guard: ensure this is Gemini config if (config.provider !== 'gemini') { throw new Error(`GeminiService requires GeminiVlmConfig, got ${config.provider}`); } - - const geminiConfig: GeminiVlmConfig = config; - - const genAI = this.getGenAI(geminiConfig.apiKey); + const genAI = this.getGenAI(config.apiKey); try { const imageParts = images.map((img) => ({ - inlineData: { - data: this.imageToBase64(img), - mimeType: 'image/png', - }, + inlineData: { data: Buffer.from(img).toString('base64'), mimeType: 'image/png' }, })); - - const parts = [{ text: geminiConfig.prompt }, ...imageParts]; - const result = await genAI.models.generateContent({ - model: geminiConfig.model, - contents: parts, + model: config.model, + contents: [{ text: config.prompt }, ...imageParts], config: { - temperature: geminiConfig.temperature, - responseMimeType: 'application/json' as const, + temperature: config.temperature, + responseMimeType: 'application/json', responseJsonSchema: zodToJsonSchema(VlmComparisonResultSchema), }, }); - - return { - content: result.text, - }; + return { content: result.text }; } catch (error) { this.logger.error(`Gemini generate request failed: ${(error as Error).message}`, (error as Error).stack); throw error; diff --git a/src/compare/libs/vlm/providers/ollama/ollama.controller.spec.ts b/src/compare/libs/vlm/providers/ollama/ollama.controller.spec.ts index 0ce2ac1c..15f0e3ff 100644 --- a/src/compare/libs/vlm/providers/ollama/ollama.controller.spec.ts +++ b/src/compare/libs/vlm/providers/ollama/ollama.controller.spec.ts @@ -87,8 +87,6 @@ describe('OllamaController', () => { }); it('passes a missing temperature on as NaN', async () => { - // NOTE: current behavior — the query value is not validated, so leaving it - // out reaches the provider as NaN rather than its default temperature. ollamaService.generate.mockResolvedValue({}); await controller.compareImages([image('a'), image('b')], 'gemma3:12b', 'prompt', undefined as unknown as string); diff --git a/src/compare/libs/vlm/providers/ollama/ollama.service.spec.ts b/src/compare/libs/vlm/providers/ollama/ollama.service.spec.ts index c7b8df86..75528c84 100644 --- a/src/compare/libs/vlm/providers/ollama/ollama.service.spec.ts +++ b/src/compare/libs/vlm/providers/ollama/ollama.service.spec.ts @@ -1,13 +1,9 @@ import { Test, TestingModule } from '@nestjs/testing'; import { ConfigService } from '@nestjs/config'; +import { Ollama } from 'ollama'; import { OllamaService } from './ollama.service'; import { OllamaVlmConfig } from '../../vlm.types'; -jest.mock('zod/v3', () => { - const actualZod = jest.requireActual('zod'); - return actualZod; -}); - const mockChat = jest.fn(); const mockList = jest.fn(); @@ -25,11 +21,13 @@ describe('OllamaService', () => { let service: OllamaService; const mockConfigService = { getOrThrow: jest.fn().mockReturnValue('http://localhost:11434'), + get: jest.fn(), }; beforeEach(async () => { jest.clearAllMocks(); mockConfigService.getOrThrow.mockReturnValue('http://localhost:11434'); + mockConfigService.get.mockReset(); const module: TestingModule = await Test.createTestingModule({ providers: [ @@ -85,7 +83,13 @@ describe('OllamaService', () => { }, ], stream: false, - format: expect.any(Object), + format: { + $schema: 'http://json-schema.org/draft-07/schema#', + type: 'object', + properties: { identical: { type: 'boolean' }, description: { type: 'string' } }, + required: ['identical', 'description'], + additionalProperties: false, + }, options: { temperature: config.temperature, }, @@ -150,53 +154,117 @@ describe('OllamaService', () => { }); }); + describe('request timeout', () => { + type Fetch = (url: string, init?: RequestInit) => Promise; + const realFetch = global.fetch; + + afterEach(() => { + global.fetch = realFetch; + }); + + const sdkFetch = async (timeoutMs?: string): Promise => { + mockConfigService.get.mockReturnValue(timeoutMs); + mockChat.mockResolvedValue(createMockResponse('{}')); + const fresh = new OllamaService(mockConfigService as any); + await fresh.generate(createConfig(), []); + const [options] = (Ollama as unknown as jest.Mock).mock.calls.at(-1)!; + expect(options.host).toBe('http://localhost:11434'); + return options.fetch; + }; + + const stalledFetch = () => + jest.fn().mockImplementation( + (_url: string, init: RequestInit) => + new Promise((_, reject) => { + init.signal!.addEventListener('abort', () => reject(init.signal!.reason)); + }) + ); + + it('gives the SDK its own fetch and signals every request', async () => { + const fetchMock = jest.fn().mockResolvedValue({ ok: true }); + global.fetch = fetchMock; + const fetchForSdk = await sdkFetch(); + + await fetchForSdk('http://localhost:11434/api/chat', { method: 'POST' }); + + const [url, init] = fetchMock.mock.calls[0]; + expect(url).toBe('http://localhost:11434/api/chat'); + expect(init.method).toBe('POST'); + expect(init.signal).toBeInstanceOf(AbortSignal); + expect(init.signal.aborted).toBe(false); + }); + + it('aborts a request that outlasts OLLAMA_TIMEOUT_MS', async () => { + global.fetch = stalledFetch(); + const fetchForSdk = await sdkFetch('1'); + + await expect(fetchForSdk('http://localhost:11434/api/chat')).rejects.toMatchObject({ name: 'TimeoutError' }); + }); + + it('defaults to five minutes', async () => { + const timeout = jest.spyOn(AbortSignal, 'timeout'); + global.fetch = jest.fn().mockResolvedValue({}); + const fetchForSdk = await sdkFetch(); + + await fetchForSdk('http://localhost:11434/api/chat'); + + expect(timeout).toHaveBeenCalledWith(300_000); + timeout.mockRestore(); + }); + + it('falls back to five minutes for a negative OLLAMA_TIMEOUT_MS', async () => { + const timeout = jest.spyOn(AbortSignal, 'timeout'); + global.fetch = jest.fn().mockResolvedValue({}); + const fetchForSdk = await sdkFetch('-1'); + + await fetchForSdk('http://localhost:11434/api/chat'); + + expect(timeout).toHaveBeenCalledWith(300_000); + timeout.mockRestore(); + }); + + it('still honours the signal of the caller', async () => { + global.fetch = stalledFetch(); + const fetchForSdk = await sdkFetch('60000'); + const controller = new AbortController(); + + const request = fetchForSdk('http://localhost:11434/api/chat', { signal: controller.signal }); + controller.abort(); + + await expect(request).rejects.toMatchObject({ name: 'AbortError' }); + }); + }); + describe('listModels', () => { it('should return list of models', async () => { - const mockDate = new Date('2024-01-01'); - const mockResponse = { - models: [ - { - name: 'llava:7b', - model: 'llava:7b', - size: 1000, - digest: 'abc123', - modified_at: mockDate, - expires_at: mockDate, - size_vram: 500, - details: { - parent_model: '', - format: 'gguf', - family: 'llama', - families: ['llama'], - parameter_size: '7B', - quantization_level: 'Q4_0', - }, - }, - { - name: 'moondream', - model: 'moondream', - size: 2000, - digest: 'def456', - modified_at: mockDate, - expires_at: mockDate, - size_vram: 1000, - details: { - parent_model: '', - format: 'gguf', - family: 'moondream', - families: ['moondream'], - parameter_size: '1.6B', - quantization_level: 'Q4_0', - }, - }, - ], - }; - mockList.mockResolvedValue(mockResponse); + const date = new Date('2024-01-01'); + const modelEntry = (name: string, size: number, digest: string, family: string, parameterSize: string) => ({ + name, + model: name, + size, + digest, + modified_at: date, + expires_at: date, + size_vram: size / 2, + details: { + parent_model: '', + format: 'gguf', + family, + families: [family], + parameter_size: parameterSize, + quantization_level: 'Q4_0', + }, + }); + const models = [ + modelEntry('llava:7b', 1000, 'abc123', 'llama', '7B'), + modelEntry('moondream', 2000, 'def456', 'moondream', '1.6B'), + ]; + mockList.mockResolvedValue({ models }); const result = await service.listModels(); expect(mockList).toHaveBeenCalled(); - expect(result).toEqual(mockResponse.models); + expect(result).toEqual(models); }); it('should throw error when API fails', async () => { diff --git a/src/compare/libs/vlm/providers/ollama/ollama.service.ts b/src/compare/libs/vlm/providers/ollama/ollama.service.ts index e4d4ae78..2b2d2cd3 100644 --- a/src/compare/libs/vlm/providers/ollama/ollama.service.ts +++ b/src/compare/libs/vlm/providers/ollama/ollama.service.ts @@ -1,10 +1,13 @@ import { Injectable, Logger } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; -import { Ollama, ChatRequest, ChatResponse, ListResponse, ModelResponse } from 'ollama'; +import { Ollama, ModelResponse } from 'ollama'; import { zodToJsonSchema } from 'zod-to-json-schema'; import { VlmProvider, VlmProviderResponse } from '../../vlm-provider.interface'; -import { VlmConfig, OllamaVlmConfig } from '../../vlm.types'; +import { VlmConfig } from '../../vlm.types'; import { VlmComparisonResultSchema } from '../../vlm-result.schema'; +import { positiveNumber } from '../../../../../utils/env'; + +const DEFAULT_TIMEOUT_MS = 300_000; @Injectable() export class OllamaService implements VlmProvider { @@ -16,44 +19,36 @@ export class OllamaService implements VlmProvider { private getOllamaClient(): Ollama { if (!this.ollamaClient) { const baseUrl = this.configService.getOrThrow('OLLAMA_BASE_URL'); - this.ollamaClient = new Ollama({ host: baseUrl }); + const timeoutMs = positiveNumber(this.configService.get('OLLAMA_TIMEOUT_MS'), DEFAULT_TIMEOUT_MS); + this.ollamaClient = new Ollama({ + host: baseUrl, + // non-streaming chat sends headers only when done: without this a stalled model holds the upload for good + fetch: (url, init) => + fetch(url, { + ...init, + signal: init?.signal + ? AbortSignal.any([init.signal, AbortSignal.timeout(timeoutMs)]) + : AbortSignal.timeout(timeoutMs), + }), + }); } return this.ollamaClient; } async generate(config: VlmConfig, images: Uint8Array[]): Promise { - // Type guard: ensure this is Ollama config if (config.provider && config.provider !== 'ollama') { throw new Error(`OllamaService requires OllamaVlmConfig, got ${config.provider}`); } - - const ollamaConfig = config as OllamaVlmConfig; const client = this.getOllamaClient(); try { - const format = zodToJsonSchema(VlmComparisonResultSchema); - - const chatRequest: ChatRequest = { - model: ollamaConfig.model, - messages: [ - { - role: 'user', - content: ollamaConfig.prompt, - images, - }, - ], - stream: false, - format, - options: { - temperature: ollamaConfig.temperature, - }, - }; - - const response: ChatResponse = await client.chat({ - ...chatRequest, + const response = await client.chat({ + model: config.model, + messages: [{ role: 'user', content: config.prompt, images }], stream: false, + format: zodToJsonSchema(VlmComparisonResultSchema), + options: { temperature: config.temperature }, }); - return { content: response.message.content, thinking: response.message.thinking, @@ -67,8 +62,7 @@ export class OllamaService implements VlmProvider { async listModels(): Promise { const client = this.getOllamaClient(); try { - const response: ListResponse = await client.list(); - return response.models; + return (await client.list()).models; } catch (error) { this.logger.error(`Failed to list models: ${(error as Error).message}`); throw error; diff --git a/src/compare/libs/vlm/vlm-inputs.core.spec.ts b/src/compare/libs/vlm/vlm-inputs.core.spec.ts new file mode 100644 index 00000000..ce3d0242 --- /dev/null +++ b/src/compare/libs/vlm/vlm-inputs.core.spec.ts @@ -0,0 +1,94 @@ +import { PNG } from 'pngjs'; +import { pngBytes, solidPng } from '../../test-utils'; +import { computeVlmInputs } from './vlm-inputs.core'; + +const redPng = (width = 4, height = 4, gamma?: number): Buffer => { + const png = solidPng(width, height, [255, 0, 0]); + if (gamma !== undefined) { + png.gamma = gamma; + } + return pngBytes(png); +}; + +const pixelAt = (png: PNG, x: number, y: number) => + Array.from(png.data.subarray(4 * (png.width * y + x), 4 * (png.width * y + x) + 4)); + +describe('computeVlmInputs', () => { + afterEach(() => jest.restoreAllMocks()); + + it('masks the ignore areas of both pictures', () => { + const output = computeVlmInputs({ + kind: 'vlm-inputs', + baseline: redPng(), + image: redPng(), + ignoreAreas: [{ x: 1, y: 1, width: 2, height: 2 }], + }); + + for (const bytes of [output.baseline, output.image]) { + const png = PNG.sync.read(Buffer.from(bytes!)); + expect([png.width, png.height]).toEqual([4, 4]); + for (let y = 0; y < 4; y++) { + for (let x = 0; x < 4; x++) { + const ignored = x >= 1 && x < 3 && y >= 1 && y < 3; + expect(pixelAt(png, x, y)).toEqual(ignored ? [0, 0, 0, 0] : [255, 0, 0, 255]); + } + } + } + }); + + it('keeps the gamma of the source', () => { + const output = computeVlmInputs({ + kind: 'vlm-inputs', + baseline: redPng(4, 4, 0.45455), + image: redPng(), + ignoreAreas: [], + }); + + expect(PNG.sync.read(Buffer.from(output.baseline!)).gamma).toBe(0.45455); + }); + + it('encodes with the Sub filter', () => { + const [baseline, image] = [redPng(), redPng()]; + const write = jest.spyOn(PNG.sync, 'write'); + + computeVlmInputs({ kind: 'vlm-inputs', baseline, image, ignoreAreas: [] }); + + expect(write).toHaveBeenCalledTimes(2); + for (const [, options] of write.mock.calls) { + expect(options).toEqual(expect.objectContaining({ filterType: 1 })); + } + }); + + it('returns null for a picture that does not decode', () => { + const output = computeVlmInputs({ + kind: 'vlm-inputs', + baseline: Buffer.from('not a png'), + image: redPng(), + ignoreAreas: [], + }); + + expect(output.baseline).toBeNull(); + expect(output.image).toBeInstanceOf(Uint8Array); + }); + + it('accepts the Uint8Array views a worker receives', () => { + const source = redPng(); + const view = new Uint8Array(source.buffer.slice(source.byteOffset, source.byteOffset + source.byteLength)); + + const output = computeVlmInputs({ kind: 'vlm-inputs', baseline: view, image: view, ignoreAreas: [] }); + + expect(PNG.sync.read(Buffer.from(output.image!)).width).toBe(4); + }); + + it('returns a structured-clone serializable result', () => { + const output = computeVlmInputs({ + kind: 'vlm-inputs', + baseline: redPng(), + image: Buffer.from('not a png'), + ignoreAreas: [], + }); + + expect(() => structuredClone(output)).not.toThrow(); + expect(structuredClone(output).image).toBeNull(); + }); +}); diff --git a/src/compare/libs/vlm/vlm-inputs.core.ts b/src/compare/libs/vlm/vlm-inputs.core.ts new file mode 100644 index 00000000..d77d0753 --- /dev/null +++ b/src/compare/libs/vlm/vlm-inputs.core.ts @@ -0,0 +1,29 @@ +import { PNG } from 'pngjs'; +import { IgnoreAreaDto } from '../../../test-runs/dto/ignore-area.dto'; +import { applyIgnoreAreas, readPng } from '../../utils'; + +export interface VlmInputsJobInput { + kind: 'vlm-inputs'; + baseline: Buffer | Uint8Array; + image: Buffer | Uint8Array; + ignoreAreas: IgnoreAreaDto[]; +} + +export interface VlmInputsJobOutput { + baseline: Uint8Array | null; + image: Uint8Array | null; +} + +const masked = (bytes: Buffer | Uint8Array, ignoreAreas: IgnoreAreaDto[]): Uint8Array | null => { + let png: PNG; + try { + png = readPng(bytes); + } catch { + return null; + } + return new Uint8Array(PNG.sync.write(applyIgnoreAreas(png, ignoreAreas), { filterType: 1 })); +}; + +export function computeVlmInputs(input: VlmInputsJobInput): VlmInputsJobOutput { + return { baseline: masked(input.baseline, input.ignoreAreas), image: masked(input.image, input.ignoreAreas) }; +} diff --git a/src/compare/libs/vlm/vlm-provider.interface.ts b/src/compare/libs/vlm/vlm-provider.interface.ts index 7c6f2ed9..a086814c 100644 --- a/src/compare/libs/vlm/vlm-provider.interface.ts +++ b/src/compare/libs/vlm/vlm-provider.interface.ts @@ -1,29 +1,10 @@ import { VlmConfig } from './vlm.types'; -/** - * Interface for VLM (Vision Language Model) providers. - * This abstraction allows easy addition of future providers (OpenAI, Anthropic, etc.) - * without modifying VlmService logic. - * Each provider is responsible for converting its own config format to internal API calls. - */ export interface VlmProvider { - /** - * Generate a response from the VLM provider. - * @param config - Provider-specific configuration - * @param images - Array of images as Uint8Array (provider implementations handle format conversion) - * @returns Response containing content and optional thinking field - */ generate(config: VlmConfig, images: Uint8Array[]): Promise; } export interface VlmProviderResponse { - /** - * Main response content from the model - */ content?: string; - - /** - * Optional thinking/reasoning field (some models use this instead of content) - */ thinking?: string; } diff --git a/src/compare/libs/vlm/vlm-result.schema.ts b/src/compare/libs/vlm/vlm-result.schema.ts index a6ec8555..5b9e7fa1 100644 --- a/src/compare/libs/vlm/vlm-result.schema.ts +++ b/src/compare/libs/vlm/vlm-result.schema.ts @@ -1,8 +1,5 @@ import z from 'zod/v3'; -// The answer a VLM provider must give. Its own module because the providers -// validate with it and VlmService uses the providers: kept in vlm.service it -// made an import cycle, which only resolves when vlm.service loads first. export const VlmComparisonResultSchema = z.object({ identical: z.boolean(), description: z.string(), diff --git a/src/compare/libs/vlm/vlm.service.spec.ts b/src/compare/libs/vlm/vlm.service.spec.ts index 5629797c..bcffda3f 100644 --- a/src/compare/libs/vlm/vlm.service.spec.ts +++ b/src/compare/libs/vlm/vlm.service.spec.ts @@ -1,4 +1,4 @@ -import { Test, TestingModule } from '@nestjs/testing'; +import { Test } from '@nestjs/testing'; import { TestStatus } from '@prisma/client'; import { PNG } from 'pngjs'; import { StaticService } from '../../../static/static.service'; @@ -9,44 +9,17 @@ import { GeminiService } from './providers/gemini/gemini.service'; import { PixelmatchService } from '../pixelmatch/pixelmatch.service'; import { DiffResult } from '../../../test-runs/diffResult'; import { ImageCompareInput } from '../ImageCompareInput'; +import { VlmConfig } from './vlm.types'; +import { DiffWorkerPool } from '../../diff-worker-pool'; +import { pngBytes, solidPng } from '../../test-utils'; -jest.mock('zod/v3', () => { - const actualZod = jest.requireActual('zod'); - return actualZod; -}); +const pngBuffer = (width: number, height: number): Buffer => PNG.sync.write(new PNG({ width, height })); +const sequence = (...results: unknown[]) => results.reduce((m, r) => m.mockResolvedValueOnce(r), jest.fn()); +const reply = (identical: boolean, description: string) => ({ content: JSON.stringify({ identical, description }) }); +const modelSaying = (identical: boolean, description = 'ok') => + jest.fn().mockResolvedValue(reply(identical, description)); -const initService = async ({ - getImageMock = jest.fn(), - ollamaGenerateMock = jest.fn(), - geminiGenerateMock = jest.fn(), - pixelmatchGetDiffMock = jest.fn(), -}) => { - const module: TestingModule = await Test.createTestingModule({ - providers: [ - VlmService, - { - provide: StaticService, - useValue: { getImage: getImageMock }, - }, - { - provide: OllamaService, - useValue: { generate: ollamaGenerateMock }, - }, - { - provide: GeminiService, - useValue: { generate: geminiGenerateMock }, - }, - { - provide: PixelmatchService, - useValue: { getDiff: pixelmatchGetDiffMock }, - }, - ], - }).compile(); - - return module.get(VlmService); -}; - -const createPixelmatchResult = (overrides: Partial): DiffResult => ({ +const createPixelmatchResult = (overrides: Partial = {}): DiffResult => ({ status: TestStatus.unresolved, diffName: 'diff.png', pixelMisMatchCount: 100, @@ -55,197 +28,154 @@ const createPixelmatchResult = (overrides: Partial): DiffResult => ( ...overrides, }); -const createImageMocks = () => { - const image = new PNG({ width: 20, height: 20 }); - const diffImage = new PNG({ width: 20, height: 20 }); - return { - image, - diffImage, - getImageMock: jest.fn().mockReturnValueOnce(image).mockReturnValueOnce(image).mockReturnValueOnce(diffImage), - }; +interface Deps { + pixelmatch?: DiffResult; + buffers?: unknown[]; + getImageBuffer?: jest.Mock; + ollama?: jest.Mock; + gemini?: jest.Mock; +} + +const setup = async ({ + pixelmatch = createPixelmatchResult(), + buffers = [pngBuffer(20, 20), pngBuffer(20, 20), pngBuffer(20, 20)], + getImageBuffer = sequence(...buffers), + ollama = jest.fn(), + gemini = jest.fn(), +}: Deps = {}) => { + const pixelmatchGetDiff = jest.fn().mockResolvedValue(pixelmatch); + const module = await Test.createTestingModule({ + providers: [ + VlmService, + { provide: StaticService, useValue: { getImageBuffer } }, + { provide: OllamaService, useValue: { generate: ollama } }, + { provide: GeminiService, useValue: { generate: gemini } }, + { provide: PixelmatchService, useValue: { getDiff: pixelmatchGetDiff } }, + DiffWorkerPool, + ], + }).compile(); + return { service: module.get(VlmService), pixelmatchGetDiff, getImageBuffer, ollama, gemini }; }; -describe('VlmService', () => { - it('should return NO_BASELINE_RESULT when pixelmatch returns no baseline', async () => { - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(NO_BASELINE_RESULT); - const service = await initService({ pixelmatchGetDiffMock }); +const input: ImageCompareInput = { + baseline: 'baseline', + image: 'image', + diffTollerancePercent: 0.1, + ignoreAreas: [], + saveDiffAsFile: true, +}; - const result = await service.getDiff( - { baseline: null, image: 'image', diffTollerancePercent: 0.1, ignoreAreas: [], saveDiffAsFile: true }, - DEFAULT_CONFIG - ); +const geminiConfig = { + provider: 'gemini' as const, + model: 'gemini-1.5-pro', + prompt: DEFAULT_CONFIG.prompt, + temperature: 0.1, + apiKey: 'test-api-key', +}; - expect(result).toStrictEqual(NO_BASELINE_RESULT); - expect(pixelmatchGetDiffMock).toHaveBeenCalled(); - }); +const threeImages = expect.arrayContaining([expect.any(Uint8Array), expect.any(Uint8Array), expect.any(Uint8Array)]); +describe('VlmService', () => { it('should return OK immediately when pixelmatch finds no differences (VLM not called)', async () => { - const pixelmatchResult: DiffResult = { - ...EQUAL_RESULT, - status: TestStatus.ok, - pixelMisMatchCount: 0, - diffPercent: 0, - }; - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(pixelmatchResult); - const ollamaGenerateMock = jest.fn(); - const service = await initService({ pixelmatchGetDiffMock, ollamaGenerateMock }); - - const result = await service.getDiff( - { baseline: 'baseline', image: 'image', diffTollerancePercent: 0.1, ignoreAreas: [], saveDiffAsFile: false }, - DEFAULT_CONFIG - ); + const { service, ollama } = await setup({ + pixelmatch: { ...EQUAL_RESULT, status: TestStatus.ok, pixelMisMatchCount: 0, diffPercent: 0 }, + }); + + const result = await service.getDiff(input, DEFAULT_CONFIG); expect(result.status).toBe(TestStatus.ok); expect(result.pixelMisMatchCount).toBe(0); expect(result.diffPercent).toBe(0); - expect(ollamaGenerateMock).not.toHaveBeenCalled(); // VLM should not be called + expect(ollama).not.toHaveBeenCalled(); }); - // CompareService leaves the rendering-noise filter to VLM, which hands it to - // the pixelmatch pass it runs first: noise then passes before any model sees it it('applies the rendering-noise filter through pixelmatch, before the model', async () => { - const blurAmplitude = { sigma: 1, threshold: 80 }; - const pixelmatchGetDiffMock = jest - .fn() - .mockResolvedValue(createPixelmatchResult({ status: TestStatus.ok, diffName: null })); - const ollamaGenerateMock = jest.fn(); - const service = await initService({ pixelmatchGetDiffMock, ollamaGenerateMock }); - - const result = await service.getDiff( - { - baseline: 'baseline', - image: 'image', - diffTollerancePercent: 0, - ignoreAreas: [], - saveDiffAsFile: true, - blurAmplitude, - }, - DEFAULT_CONFIG - ); + const blurAmplitude = { threshold: 80 }; + const { service, pixelmatchGetDiff, ollama } = await setup({ + pixelmatch: createPixelmatchResult({ status: TestStatus.ok, diffName: null }), + }); + + const result = await service.getDiff({ ...input, diffTollerancePercent: 0, blurAmplitude }, DEFAULT_CONFIG); expect(service.appliesBlurAmplitudeFilter).toBe(true); - expect(pixelmatchGetDiffMock).toHaveBeenCalledWith(expect.objectContaining({ blurAmplitude }), expect.anything()); + expect(pixelmatchGetDiff).toHaveBeenCalledWith(expect.objectContaining({ blurAmplitude }), expect.anything()); expect(result.status).toBe(TestStatus.ok); - expect(ollamaGenerateMock).not.toHaveBeenCalled(); + expect(ollama).not.toHaveBeenCalled(); }); it.each([ [ 'override to OK when VLM says not noticeable', - { identical: true, description: 'Differences are minor rendering artifacts, not noticeable to humans.' }, + reply(true, 'Differences are minor rendering artifacts, not noticeable to humans.'), TestStatus.ok, { pixelMisMatchCount: 100, diffPercent: 2.5 }, ], [ 'keep unresolved when VLM confirms noticeable', - { - identical: false, - description: 'Button text changed from Submit to Send, and user count changed from 12 to 15.', - }, + reply(false, 'Button text changed from Submit to Send, and user count changed from 12 to 15.'), TestStatus.unresolved, { pixelMisMatchCount: 500, diffPercent: 12.5 }, ], ])('should %s', async (_, vlmResponse, expectedStatus, pixelmatchOverrides) => { - const pixelmatchResult = createPixelmatchResult(pixelmatchOverrides); - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(pixelmatchResult); - const { getImageMock } = createImageMocks(); - const ollamaGenerateMock = jest.fn().mockResolvedValue({ - content: JSON.stringify(vlmResponse), + const { service, ollama } = await setup({ + pixelmatch: createPixelmatchResult(pixelmatchOverrides), + ollama: jest.fn().mockResolvedValue(vlmResponse), }); - const service = await initService({ getImageMock, ollamaGenerateMock, pixelmatchGetDiffMock }); - const result = await service.getDiff( - { baseline: 'baseline', image: 'image', diffTollerancePercent: 0.1, ignoreAreas: [], saveDiffAsFile: false }, - DEFAULT_CONFIG - ); + const result = await service.getDiff(input, DEFAULT_CONFIG); expect(result.status).toBe(expectedStatus); - expect(result.vlmDescription).toBe(vlmResponse.description); + expect(result.vlmDescription).toBe(JSON.parse(vlmResponse.content).description); expect(result.pixelMisMatchCount).toBe(pixelmatchOverrides.pixelMisMatchCount); expect(result.diffPercent).toBe(pixelmatchOverrides.diffPercent); - expect(ollamaGenerateMock).toHaveBeenCalledWith( - DEFAULT_CONFIG, - expect.arrayContaining([expect.any(Uint8Array), expect.any(Uint8Array), expect.any(Uint8Array)]) - ); + expect(ollama).toHaveBeenCalledWith(DEFAULT_CONFIG, threeImages); + }); + + it('sends the image it is given to the model without reading it again', async () => { + const image = pngBuffer(20, 20); + const { service, getImageBuffer, ollama } = await setup({ + buffers: [image, pngBuffer(20, 20)], + ollama: modelSaying(true), + }); + + await service.getDiff({ ...input, imageBuffer: image }, DEFAULT_CONFIG); + + expect(getImageBuffer.mock.calls).toEqual([['baseline'], ['diff.png']]); + expect(ollama).toHaveBeenCalled(); }); it.each([ - [ - 'invalid JSON response', - { content: 'Invalid JSON response from model' }, - { pixelMisMatchCount: 200, diffPercent: 5 }, - ], - ['API error', null, { pixelMisMatchCount: 300, diffPercent: 7.5 }], - ])('should handle %s gracefully and return pixelmatch result', async (_, vlmResponse, pixelmatchOverrides) => { - const pixelmatchResult = createPixelmatchResult(pixelmatchOverrides); - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(pixelmatchResult); - const { getImageMock } = createImageMocks(); - const ollamaGenerateMock = vlmResponse - ? jest.fn().mockResolvedValue(vlmResponse) - : jest.fn().mockRejectedValue(new Error('Connection refused')); - const service = await initService({ getImageMock, ollamaGenerateMock, pixelmatchGetDiffMock }); - - const result = await service.getDiff( - { baseline: 'baseline', image: 'image', diffTollerancePercent: 0.1, ignoreAreas: [], saveDiffAsFile: false }, - DEFAULT_CONFIG - ); + ['invalid JSON response', jest.fn().mockResolvedValue({ content: 'Invalid JSON response from model' }), 200, 5], + ['API error', jest.fn().mockRejectedValue(new Error('Connection refused')), 300, 7.5], + ])('should handle %s gracefully and return pixelmatch result', async (_, ollama, pixelMisMatchCount, diffPercent) => { + const { service } = await setup({ + pixelmatch: createPixelmatchResult({ pixelMisMatchCount, diffPercent }), + ollama, + }); + + const result = await service.getDiff(input, DEFAULT_CONFIG); expect(result.status).toBe(TestStatus.unresolved); expect(result.vlmDescription).toContain('VLM analysis failed'); - expect(result.pixelMisMatchCount).toBe(pixelmatchOverrides.pixelMisMatchCount); - expect(result.diffPercent).toBe(pixelmatchOverrides.diffPercent); + expect(result.pixelMisMatchCount).toBe(pixelMisMatchCount); + expect(result.diffPercent).toBe(diffPercent); }); it.each([ ['ollama', DEFAULT_CONFIG], - [ - 'gemini', - { - provider: 'gemini' as const, - model: 'gemini-1.5-pro', - prompt: DEFAULT_CONFIG.prompt, - temperature: 0.1, - apiKey: 'test-api-key', - }, - ], + ['gemini', geminiConfig], ])('should send baseline and image with ignore areas masked to %s', async (_, config) => { - const createFilledImage = (r: number, g: number, b: number) => { - const png = new PNG({ width: 20, height: 20 }); - for (let k = 0; k < png.data.length; k += 4) { - png.data[k] = r; - png.data[k + 1] = g; - png.data[k + 2] = b; - png.data[k + 3] = 255; - } - return png; - }; - const baseline = createFilledImage(255, 255, 255); - const image = createFilledImage(255, 0, 0); - const diffImage = new PNG({ width: 20, height: 20 }); - const getImageMock = jest - .fn() - .mockReturnValueOnce(baseline) - .mockReturnValueOnce(image) - .mockReturnValueOnce(diffImage); - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(createPixelmatchResult({})); - const generateMock = jest.fn().mockResolvedValue({ - content: '{"identical": false, "description": "Changed."}', + const generate = modelSaying(false, 'Changed.'); + const { service } = await setup({ + buffers: [pngBytes(solidPng(20, 20)), pngBytes(solidPng(20, 20, [255, 0, 0])), pngBuffer(20, 20)], + ollama: generate, + gemini: generate, }); - const service = await initService({ - getImageMock, - pixelmatchGetDiffMock, - ollamaGenerateMock: generateMock, - geminiGenerateMock: generateMock, - }); - const ignoreAreas = [{ x: 2, y: 3, width: 5, height: 4 }]; - await service.getDiff( - { baseline: 'baseline', image: 'image', diffTollerancePercent: 0.1, ignoreAreas, saveDiffAsFile: false }, - config - ); + await service.getDiff({ ...input, ignoreAreas: [{ x: 2, y: 3, width: 5, height: 4 }] }, config); - expect(generateMock).toHaveBeenCalledTimes(1); - const [sentBaseline, sentImage] = (generateMock.mock.calls[0][1] as Uint8Array[]).map((bytes) => + expect(generate).toHaveBeenCalledTimes(1); + const [sentBaseline, sentImage] = (generate.mock.calls[0][1] as Uint8Array[]).map((bytes) => PNG.sync.read(Buffer.from(bytes)) ); const pixelAt = (png: PNG, x: number, y: number) => @@ -260,60 +190,35 @@ describe('VlmService', () => { }); it('should use custom model and temperature from config', async () => { - const pixelmatchResult = createPixelmatchResult({ pixelMisMatchCount: 150, diffPercent: 3.75 }); - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(pixelmatchResult); - const { getImageMock } = createImageMocks(); - const ollamaGenerateMock = jest.fn().mockResolvedValue({ - content: '{"identical": true, "description": "No differences."}', - }); - const service = await initService({ getImageMock, ollamaGenerateMock, pixelmatchGetDiffMock }); - + const { service, ollama } = await setup({ ollama: modelSaying(true, 'No differences.') }); const customConfig = { ...DEFAULT_CONFIG, model: 'llava:13b', prompt: 'Custom context', temperature: 0.2 }; - await service.getDiff( - { baseline: 'baseline', image: 'image', diffTollerancePercent: 0.1, ignoreAreas: [], saveDiffAsFile: false }, - customConfig - ); - expect(ollamaGenerateMock).toHaveBeenCalledWith( - customConfig, - expect.arrayContaining([expect.any(Uint8Array), expect.any(Uint8Array), expect.any(Uint8Array)]) - ); + await service.getDiff(input, customConfig); + + expect(ollama).toHaveBeenCalledWith(customConfig, threeImages); }); it('should use thinking field when useThinking is true', async () => { - const pixelmatchResult = createPixelmatchResult({ pixelMisMatchCount: 80, diffPercent: 2 }); - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(pixelmatchResult); - const { getImageMock } = createImageMocks(); - const ollamaGenerateMock = jest.fn().mockResolvedValue({ - content: '{"identical": false, "description": "Content field"}', - thinking: '{"identical": true, "description": "Thinking field"}', + const { service } = await setup({ + ollama: jest.fn().mockResolvedValue({ + content: JSON.stringify({ identical: false, description: 'Content field' }), + thinking: JSON.stringify({ identical: true, description: 'Thinking field' }), + }), }); - const service = await initService({ getImageMock, ollamaGenerateMock, pixelmatchGetDiffMock }); - const result = await service.getDiff( - { baseline: 'baseline', image: 'image', diffTollerancePercent: 0.1, ignoreAreas: [], saveDiffAsFile: false }, - { ...DEFAULT_CONFIG, useThinking: true } - ); + const result = await service.getDiff(input, { ...DEFAULT_CONFIG, useThinking: true }); expect(result.status).toBe(TestStatus.ok); expect(result.vlmDescription).toBe('Thinking field'); }); it('should handle missing diff image gracefully', async () => { - const pixelmatchResult = createPixelmatchResult({ diffName: null }); - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(pixelmatchResult); - const { image } = createImageMocks(); - const getImageMock = jest.fn().mockReturnValueOnce(image).mockReturnValueOnce(image).mockReturnValueOnce(null); - const ollamaGenerateMock = jest.fn(); - const service = await initService({ getImageMock, ollamaGenerateMock, pixelmatchGetDiffMock }); - - const result = await service.getDiff( - { baseline: 'baseline', image: 'image', diffTollerancePercent: 0.1, ignoreAreas: [], saveDiffAsFile: false }, - DEFAULT_CONFIG - ); + const pixelmatch = createPixelmatchResult({ diffName: null }); + const image = pngBuffer(20, 20); + const { service, ollama } = await setup({ pixelmatch, buffers: [image, image, null] }); - expect(result).toEqual(pixelmatchResult); - expect(ollamaGenerateMock).not.toHaveBeenCalled(); + expect(await service.getDiff(input, DEFAULT_CONFIG)).toEqual(pixelmatch); + expect(ollama).not.toHaveBeenCalled(); }); it.each([ @@ -321,7 +226,6 @@ describe('VlmService', () => { ['invalid JSON', 'invalid', DEFAULT_CONFIG], ['partial config, over the defaults', '{"model":"llava:7b"}', { ...DEFAULT_CONFIG, model: 'llava:7b' }], [ - // what a project created without a config holds: the column default is pixelmatch's 'the pixelmatch column default, over the defaults', '{ "threshold": 0.1, "ignoreAntialiasing": true, "allowDiffDimensions": false }', { ...DEFAULT_CONFIG, threshold: 0.1, ignoreAntialiasing: true, allowDiffDimensions: false }, @@ -329,105 +233,113 @@ describe('VlmService', () => { [ 'full config', '{"model":"llava:13b","prompt":"Custom prompt","temperature":0.2,"useThinking":true}', - { - provider: 'ollama', - model: 'llava:13b', - prompt: 'Custom prompt', - temperature: 0.2, - useThinking: true, - }, + { provider: 'ollama', model: 'llava:13b', prompt: 'Custom prompt', temperature: 0.2, useThinking: true }, ], ])('should parse config: %s', async (_, configJson, expected) => { - const service = await initService({}); + const { service } = await setup(); + expect(service.parseConfig(configJson)).toEqual(expected); }); describe('characterization', () => { - const input: ImageCompareInput = { - baseline: 'baseline', - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: false, - }; - - it('always asks pixelmatch for a saved diff with the pixelmatch defaults', async () => { - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(EQUAL_RESULT); - const service = await initService({ pixelmatchGetDiffMock }); + it('asks pixelmatch for a saved diff with the pixelmatch defaults when the diff is saved', async () => { + const { service, pixelmatchGetDiff } = await setup({ pixelmatch: EQUAL_RESULT }); await service.getDiff(input, { ...DEFAULT_CONFIG, temperature: 0.9 }); - expect(pixelmatchGetDiffMock).toHaveBeenCalledWith( - { ...input, saveDiffAsFile: true }, - { threshold: 0.1, ignoreAntialiasing: true } - ); + expect(pixelmatchGetDiff).toHaveBeenCalledWith(input, { threshold: 0.1, ignoreAntialiasing: true }); }); - it('returns a pixelmatch result with status new without asking the model', async () => { - const pixelmatchResult = createPixelmatchResult({ status: TestStatus.new }); - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(pixelmatchResult); - const getImageMock = jest.fn(); - const ollamaGenerateMock = jest.fn(); - const service = await initService({ pixelmatchGetDiffMock, getImageMock, ollamaGenerateMock }); + it.each([ + ['a pixelmatch result with status new', createPixelmatchResult({ status: TestStatus.new }), input], + ['the no-baseline result', NO_BASELINE_RESULT, { ...input, baseline: null }], + ])('returns %s without loading images or asking the model', async (_, pixelmatch, data) => { + const { service, getImageBuffer, ollama, pixelmatchGetDiff } = await setup({ + pixelmatch, + getImageBuffer: jest.fn(), + }); - const result = await service.getDiff(input, DEFAULT_CONFIG); + const result = await service.getDiff(data, DEFAULT_CONFIG); - expect(result).toBe(pixelmatchResult); - expect(getImageMock).not.toHaveBeenCalled(); - expect(ollamaGenerateMock).not.toHaveBeenCalled(); + expect(result).toBe(pixelmatch); + expect(pixelmatchGetDiff).toHaveBeenCalled(); + expect(getImageBuffer).not.toHaveBeenCalled(); + expect(ollama).not.toHaveBeenCalled(); }); - it('still loads images when pixelmatch reports no baseline', async () => { - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(NO_BASELINE_RESULT); - const getImageMock = jest.fn().mockResolvedValue(undefined); - const service = await initService({ pixelmatchGetDiffMock, getImageMock }); + describe('auto-approve probe (saveDiffAsFile: false)', () => { + const probe: ImageCompareInput = { ...input, saveDiffAsFile: false }; + + it('answers an auto-approve probe with the pixelmatch verdict, saving nothing and asking no model', async () => { + const pixelmatch = createPixelmatchResult({ status: TestStatus.unresolved }); + const { service, pixelmatchGetDiff, getImageBuffer, ollama, gemini } = await setup({ + pixelmatch, + getImageBuffer: jest.fn(), + }); + + const result = await service.getDiff(probe, DEFAULT_CONFIG); + + expect(result).toStrictEqual(pixelmatch); + expect(pixelmatchGetDiff).toHaveBeenCalledWith( + expect.objectContaining({ saveDiffAsFile: false }), + expect.anything() + ); + expect(ollama).not.toHaveBeenCalled(); + expect(gemini).not.toHaveBeenCalled(); + expect(getImageBuffer).not.toHaveBeenCalled(); + }); - const result = await service.getDiff({ ...input, baseline: null }, DEFAULT_CONFIG); + it('passes an ok probe through', async () => { + const pixelmatch = createPixelmatchResult({ status: TestStatus.ok, diffName: null }); + const { service, ollama } = await setup({ pixelmatch }); - // NOTE: current behavior — NO_BASELINE_RESULT has status undefined, not `new`, - // so the early return above never applies and the VLM path runs until it - // finds the images missing. - expect(getImageMock).toHaveBeenCalledWith(null); - expect(result).toBe(NO_BASELINE_RESULT); + expect(await service.getDiff(probe, DEFAULT_CONFIG)).toStrictEqual(pixelmatch); + expect(ollama).not.toHaveBeenCalled(); + }); }); it('reports an empty model response as a failed analysis', async () => { - const pixelmatchResult = createPixelmatchResult({}); - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(pixelmatchResult); - const { getImageMock } = createImageMocks(); - const ollamaGenerateMock = jest.fn().mockResolvedValue({ content: '', thinking: undefined }); - const service = await initService({ getImageMock, ollamaGenerateMock, pixelmatchGetDiffMock }); - - const result = await service.getDiff(input, DEFAULT_CONFIG); + const pixelmatch = createPixelmatchResult(); + const { service } = await setup({ + pixelmatch, + ollama: jest.fn().mockResolvedValue({ content: '', thinking: undefined }), + }); - expect(result).toStrictEqual({ - ...pixelmatchResult, + expect(await service.getDiff(input, DEFAULT_CONFIG)).toStrictEqual({ + ...pixelmatch, vlmDescription: 'VLM analysis failed: Empty response from model', }); }); - it('falls back to the thinking field when the content is empty', async () => { - const { getImageMock } = createImageMocks(); - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(createPixelmatchResult({})); - const ollamaGenerateMock = jest.fn().mockResolvedValue({ - content: '', - thinking: '{"identical": true, "description": "From thinking"}', - }); - const service = await initService({ getImageMock, ollamaGenerateMock, pixelmatchGetDiffMock }); - - const result = await service.getDiff(input, DEFAULT_CONFIG); - - expect(result.status).toBe(TestStatus.ok); - expect(result.vlmDescription).toBe('From thinking'); + it.each([ + [ + 'the thinking field when the content is empty', + { content: '', thinking: JSON.stringify({ identical: true, description: 'From thinking' }) }, + DEFAULT_CONFIG, + TestStatus.ok, + 'From thinking', + ], + [ + 'the content field when thinking is preferred but empty', + reply(false, 'From content'), + { ...DEFAULT_CONFIG, useThinking: true }, + TestStatus.unresolved, + 'From content', + ], + ])('falls back to %s', async (_, response, config, status, description) => { + const { service } = await setup({ ollama: jest.fn().mockResolvedValue(response) }); + + const result = await service.getDiff(input, config); + + expect(result.status).toBe(status); + expect(result.vlmDescription).toBe(description); }); it('marks the run unresolved when the model sees a difference, whatever status pixelmatch gave', async () => { - const { getImageMock } = createImageMocks(); - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(createPixelmatchResult({ status: TestStatus.failed })); - const ollamaGenerateMock = jest.fn().mockResolvedValue({ - content: '{"identical": false, "description": "Button moved"}', + const { service } = await setup({ + pixelmatch: createPixelmatchResult({ status: TestStatus.failed }), + ollama: modelSaying(false, 'Button moved'), }); - const service = await initService({ getImageMock, ollamaGenerateMock, pixelmatchGetDiffMock }); const result = await service.getDiff(input, DEFAULT_CONFIG); @@ -435,25 +347,8 @@ describe('VlmService', () => { expect(result.vlmDescription).toBe('Button moved'); }); - it('falls back to the content field when thinking is preferred but empty', async () => { - const { getImageMock } = createImageMocks(); - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(createPixelmatchResult({})); - const ollamaGenerateMock = jest.fn().mockResolvedValue({ - content: '{"identical": false, "description": "From content"}', - }); - const service = await initService({ getImageMock, ollamaGenerateMock, pixelmatchGetDiffMock }); - - const result = await service.getDiff(input, { ...DEFAULT_CONFIG, useThinking: true }); - - expect(result.status).toBe(TestStatus.unresolved); - expect(result.vlmDescription).toBe('From content'); - }); - it('fills in a description when the model returns an empty one', async () => { - const { getImageMock } = createImageMocks(); - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(createPixelmatchResult({})); - const ollamaGenerateMock = jest.fn().mockResolvedValue({ content: '{"identical": true, "description": ""}' }); - const service = await initService({ getImageMock, ollamaGenerateMock, pixelmatchGetDiffMock }); + const { service } = await setup({ ollama: modelSaying(true, '') }); const result = await service.getDiff(input, DEFAULT_CONFIG); @@ -461,11 +356,7 @@ describe('VlmService', () => { }); it('reports a response that does not match the schema as a failed analysis', async () => { - const { getImageMock } = createImageMocks(); - const pixelmatchResult = createPixelmatchResult({}); - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(pixelmatchResult); - const ollamaGenerateMock = jest.fn().mockResolvedValue({ content: '{"identical": "yes"}' }); - const service = await initService({ getImageMock, ollamaGenerateMock, pixelmatchGetDiffMock }); + const { service } = await setup({ ollama: jest.fn().mockResolvedValue({ content: '{"identical": "yes"}' }) }); const result = await service.getDiff(input, DEFAULT_CONFIG); @@ -477,98 +368,125 @@ describe('VlmService', () => { ['no provider', undefined], ['an unknown provider', 'openai'], ])('uses Ollama when the config names %s', async (_, provider) => { - const { getImageMock } = createImageMocks(); - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(createPixelmatchResult({})); - const ollamaGenerateMock = jest.fn().mockResolvedValue({ content: '{"identical": true, "description": "ok"}' }); - const geminiGenerateMock = jest.fn(); - const service = await initService({ - getImageMock, - ollamaGenerateMock, - geminiGenerateMock, - pixelmatchGetDiffMock, - }); - const config = { ...DEFAULT_CONFIG, provider } as any; + const { service, ollama, gemini } = await setup({ ollama: modelSaying(true) }); + const config = { ...DEFAULT_CONFIG, provider } as unknown as VlmConfig; await service.getDiff(input, config); - expect(ollamaGenerateMock).toHaveBeenCalledWith(config, expect.any(Array)); - expect(geminiGenerateMock).not.toHaveBeenCalled(); + expect(ollama).toHaveBeenCalledWith(config, expect.any(Array)); + expect(gemini).not.toHaveBeenCalled(); }); it('sends baseline, image and diff to the model as PNG bytes in that order', async () => { - const baseline = new PNG({ width: 1, height: 1 }); - const image = new PNG({ width: 2, height: 1 }); - const diff = new PNG({ width: 3, height: 1 }); - const getImageMock = jest - .fn() - .mockResolvedValueOnce(baseline) - .mockResolvedValueOnce(image) - .mockResolvedValueOnce(diff); - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(createPixelmatchResult({ diffName: 'diff.png' })); - const ollamaGenerateMock = jest.fn().mockResolvedValue({ content: '{"identical": true, "description": "ok"}' }); - const service = await initService({ getImageMock, ollamaGenerateMock, pixelmatchGetDiffMock }); + const diff = pngBuffer(3, 1); + const { service, getImageBuffer, ollama } = await setup({ + buffers: [pngBuffer(1, 1), pngBuffer(2, 1), diff], + ollama: modelSaying(true), + }); await service.getDiff(input, DEFAULT_CONFIG); - expect(getImageMock.mock.calls).toEqual([['baseline'], ['image'], ['diff.png']]); - const [[, images]] = ollamaGenerateMock.mock.calls; + expect(getImageBuffer.mock.calls).toEqual([['baseline'], ['image'], ['diff.png']]); + const [[, images]] = ollama.mock.calls; expect(images.map((bytes: Uint8Array) => PNG.sync.read(Buffer.from(bytes)).width)).toEqual([1, 2, 3]); + expect(Buffer.from(images[2]).equals(diff)).toBe(true); + }); + + it('masks ignore areas in a worker job', async () => { + const run = jest.spyOn(DiffWorkerPool.prototype, 'run'); + const opaque = new PNG({ width: 4, height: 4 }); + opaque.data.fill(255); + const picture = PNG.sync.write(opaque); + const { service, ollama } = await setup({ + buffers: [picture, picture, pngBuffer(4, 4)], + ollama: modelSaying(true), + }); + const ignoreAreas = [{ x: 1, y: 1, width: 2, height: 2 }]; + + await service.getDiff({ ...input, ignoreAreas }, DEFAULT_CONFIG); + + expect(run).toHaveBeenCalledWith(expect.objectContaining({ kind: 'vlm-inputs', ignoreAreas })); + const [[, [sentBaseline]]] = ollama.mock.calls; + const png = PNG.sync.read(Buffer.from(sentBaseline)); + const alpha = (x: number, y: number) => png.data[4 * (png.width * y + x) + 3]; + expect([alpha(0, 0), alpha(1, 1), alpha(2, 2), alpha(3, 3)]).toEqual([255, 0, 0, 255]); + run.mockRestore(); + }); + + it.each([ + ['a picture does not decode', [Buffer.from('not a png'), pngBuffer(2, 2), pngBuffer(2, 2)]], + ['the image does not decode', [pngBuffer(2, 2), Buffer.from('not a png'), pngBuffer(2, 2)]], + ['a stored picture is missing', [null, pngBuffer(2, 2), pngBuffer(2, 2)]], + ])('returns the pixelmatch result when %s', async (_, buffers) => { + const pixelmatch = createPixelmatchResult(); + const { service, ollama } = await setup({ pixelmatch, buffers }); + + expect(await service.getDiff(input, DEFAULT_CONFIG)).toStrictEqual(pixelmatch); + expect(ollama).not.toHaveBeenCalled(); + }); + }); + + describe('model concurrency', () => { + afterEach(() => { + delete process.env.VLM_CONCURRENCY; + }); + + it('asks the model at most VLM_CONCURRENCY times at once', async () => { + process.env.VLM_CONCURRENCY = '1'; + const gates: { resolve: (value: { content: string }) => void }[] = []; + const ollama = jest.fn().mockImplementation( + () => + new Promise((resolve) => { + gates.push({ resolve }); + }) + ); + const { service } = await setup({ + getImageBuffer: jest.fn().mockImplementation(async () => pngBuffer(2, 2)), + ollama, + }); + const answer = reply(true, 'ok'); + + const first = service.getDiff(input, DEFAULT_CONFIG); + const second = service.getDiff(input, DEFAULT_CONFIG); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(ollama).toHaveBeenCalledTimes(1); + + gates[0].resolve(answer); + await first; + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(ollama).toHaveBeenCalledTimes(2); + + gates[1].resolve(answer); + await second; + }); + + it('falls back to the default when VLM_CONCURRENCY is zero', async () => { + process.env.VLM_CONCURRENCY = '0'; + const { service } = await setup({ ollama: modelSaying(true) }); + + const result = await service.getDiff(input, DEFAULT_CONFIG); + + expect(result.status).toBe(TestStatus.ok); }); }); describe('Gemini provider', () => { it('should use GeminiService when provider is gemini', async () => { - const pixelmatchResult = createPixelmatchResult({}); - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(pixelmatchResult); - const { getImageMock } = createImageMocks(); - const geminiGenerateMock = jest.fn().mockResolvedValue({ - content: '{"identical": true, "description": "No noticeable differences."}', - }); - const ollamaGenerateMock = jest.fn(); - const service = await initService({ - getImageMock, - geminiGenerateMock, - ollamaGenerateMock, - pixelmatchGetDiffMock, - }); + const { service, gemini, ollama } = await setup({ gemini: modelSaying(true, 'No noticeable differences.') }); - const geminiConfig = { - provider: 'gemini' as const, - model: 'gemini-1.5-pro', - prompt: DEFAULT_CONFIG.prompt, - temperature: 0.1, - apiKey: 'test-api-key', - }; - - const result = await service.getDiff( - { baseline: 'baseline', image: 'image', diffTollerancePercent: 0.1, ignoreAreas: [], saveDiffAsFile: false }, - geminiConfig - ); + const result = await service.getDiff(input, geminiConfig); expect(result.status).toBe(TestStatus.ok); expect(result.vlmDescription).toBe('No noticeable differences.'); - expect(geminiGenerateMock).toHaveBeenCalledWith( - geminiConfig, - expect.arrayContaining([expect.any(Uint8Array), expect.any(Uint8Array), expect.any(Uint8Array)]) - ); - expect(ollamaGenerateMock).not.toHaveBeenCalled(); + expect(gemini).toHaveBeenCalledWith(geminiConfig, threeImages); + expect(ollama).not.toHaveBeenCalled(); }); it('should handle error when Gemini API key is missing', async () => { - const pixelmatchResult = createPixelmatchResult({}); - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(pixelmatchResult); - const { getImageMock } = createImageMocks(); - const service = await initService({ getImageMock, pixelmatchGetDiffMock }); - - const result = await service.getDiff( - { baseline: 'baseline', image: 'image', diffTollerancePercent: 0.1, ignoreAreas: [], saveDiffAsFile: false }, - { - provider: 'gemini' as const, - model: 'gemini-1.5-pro', - prompt: DEFAULT_CONFIG.prompt, - temperature: 0.1, - } as any - ); + const { service } = await setup(); + const { apiKey: _apiKey, ...withoutKey } = geminiConfig; + + const result = await service.getDiff(input, withoutKey as unknown as VlmConfig); expect(result.vlmDescription).toContain('VLM analysis failed'); expect(result.status).toBe(TestStatus.unresolved); diff --git a/src/compare/libs/vlm/vlm.service.ts b/src/compare/libs/vlm/vlm.service.ts index 7ecadd50..13983fc2 100644 --- a/src/compare/libs/vlm/vlm.service.ts +++ b/src/compare/libs/vlm/vlm.service.ts @@ -2,16 +2,17 @@ import { Injectable, Logger } from '@nestjs/common'; import { TestStatus } from '@prisma/client'; import { StaticService } from '../../../static/static.service'; import { DiffResult } from '../../../test-runs/diffResult'; -import { applyIgnoreAreas, parseConfig } from '../../utils'; +import { parseConfig } from '../../utils'; +import { ConcurrencyLimiter } from './concurrency-limiter'; import { ImageComparator } from '../image-comparator.interface'; import { ImageCompareInput } from '../ImageCompareInput'; import { VlmConfig, OllamaVlmConfig } from './vlm.types'; -import { VlmProvider } from './vlm-provider.interface'; +import { VlmProvider, VlmProviderResponse } from './vlm-provider.interface'; import { VlmComparisonResultSchema } from './vlm-result.schema'; import { OllamaService } from './providers/ollama/ollama.service'; import { GeminiService } from './providers/gemini/gemini.service'; +import { DiffWorkerPool } from '../../diff-worker-pool'; import { PixelmatchService, DEFAULT_CONFIG as PIXELMATCH_DEFAULT_CONFIG } from '../pixelmatch/pixelmatch.service'; -import { PNG } from 'pngjs'; export const DEFAULT_PROMPT = `You are provided with three images: 1. First image: baseline screenshot @@ -22,8 +23,6 @@ Spot any difference in text, color, shape and position of elements - treat as di Ignore minor rendering artifacts that are imperceptible to users like antialiasing. Describe the difference in about 100 words.`; -export { VlmComparisonResultSchema } from './vlm-result.schema'; - export const DEFAULT_CONFIG: OllamaVlmConfig = { provider: 'ollama', model: 'gemma3:12b', @@ -35,15 +34,15 @@ export const DEFAULT_CONFIG: OllamaVlmConfig = { @Injectable() export class VlmService implements ImageComparator { private readonly logger: Logger = new Logger(VlmService.name); - // through the pixelmatch pass below, which passes rendering noise before the - // model is ever asked about it readonly appliesBlurAmplitudeFilter = true; + private readonly limiter = new ConcurrencyLimiter(Math.max(1, Number(process.env.VLM_CONCURRENCY) || 4)); constructor( private readonly staticService: StaticService, private readonly ollamaService: OllamaService, private readonly geminiService: GeminiService, - private readonly pixelmatchService: PixelmatchService + private readonly pixelmatchService: PixelmatchService, + private readonly diffWorkerPool: DiffWorkerPool ) {} parseConfig(configJson: string): VlmConfig { @@ -51,55 +50,54 @@ export class VlmService implements ImageComparator { } async getDiff(data: ImageCompareInput, config: VlmConfig): Promise { - const pixelmatchResult = await this.pixelmatchService.getDiff( - { - ...data, - saveDiffAsFile: true, - }, - PIXELMATCH_DEFAULT_CONFIG - ); - - if (pixelmatchResult.status === TestStatus.new) { - return pixelmatchResult; - } - - if (pixelmatchResult.status === TestStatus.ok) { + const pixelmatchResult = await this.pixelmatchService.getDiff(data, PIXELMATCH_DEFAULT_CONFIG); + + if ( + !data.saveDiffAsFile || + !pixelmatchResult.status || + pixelmatchResult.status === TestStatus.new || + pixelmatchResult.status === TestStatus.ok + ) { return pixelmatchResult; } this.logger.debug('Pixel diff is being sent to VLM'); try { - const baseline = await this.staticService.getImage(data.baseline); - const image = await this.staticService.getImage(data.image); - const diffImage = pixelmatchResult.diffName ? await this.staticService.getImage(pixelmatchResult.diffName) : null; - - if (!baseline || !image || !diffImage) { + const [baselineBuffer, imageBuffer, diffBuffer] = await Promise.all([ + this.staticService.getImageBuffer(data.baseline), + data.imageBuffer ?? this.staticService.getImageBuffer(data.image), + pixelmatchResult.diffName ? this.staticService.getImageBuffer(pixelmatchResult.diffName) : null, + ]); + if (!baselineBuffer || !imageBuffer || !diffBuffer) { this.logger.warn('Missing images for VLM analysis, returning pixelmatch result'); return pixelmatchResult; } - // mask ignore areas the same way pixelmatch did, so the VLM does not flag changes inside them - const baselineBytes = new Uint8Array(PNG.sync.write(applyIgnoreAreas(baseline, data.ignoreAreas))); - const imageBytes = new Uint8Array(PNG.sync.write(applyIgnoreAreas(image, data.ignoreAreas))); - const diffBytes = new Uint8Array(PNG.sync.write(diffImage)); + const { baseline, image } = await this.diffWorkerPool.run({ + kind: 'vlm-inputs', + baseline: baselineBuffer, + image: imageBuffer, + ignoreAreas: data.ignoreAreas, + }); + if (!baseline || !image) { + this.logger.warn('Missing images for VLM analysis, returning pixelmatch result'); + return pixelmatchResult; + } - const { pass, description } = await this.compareImagesWithVLM(baselineBytes, imageBytes, diffBytes, config); + const { pass, description } = await this.compareImagesWithVLM( + baseline, + image, + new Uint8Array(diffBuffer), + config + ); - // Build result from pixelmatch, but override status based on VLM analysis - const result: DiffResult = { + return { ...pixelmatchResult, + status: pass ? TestStatus.ok : TestStatus.unresolved, vlmDescription: description, }; - - if (pass) { - result.status = TestStatus.ok; - } else { - result.status = TestStatus.unresolved; - } - - return result; } catch (error) { - // a cast, not instanceof: errors from Node's own fetch can come from another realm + // a cast, not instanceof: fetch errors come from another realm const { message, stack } = error as Error; this.logger.error(`VLM comparison failed: ${message}`, stack); return { @@ -110,18 +108,10 @@ export class VlmService implements ImageComparator { } private getProvider(config: VlmConfig): VlmProvider { - const provider = config.provider || 'ollama'; // Default to ollama for backward compatibility - - switch (provider) { - case 'gemini': - return this.geminiService; - case 'ollama': - default: - return this.ollamaService; - } + return config.provider === 'gemini' ? this.geminiService : this.ollamaService; } - private extractResponseContent(config: VlmConfig, response: { content?: string; thinking?: string }): string { + private extractResponseContent(config: VlmConfig, response: VlmProviderResponse): string { const preferred = config.useThinking ? response.thinking : response.content; const fallback = config.useThinking ? response.content : response.thinking; const content = preferred || fallback; @@ -135,9 +125,7 @@ export class VlmService implements ImageComparator { } private parseVlmResponse(responseContent: string): { pass: boolean; description: string } { - const parsed = JSON.parse(responseContent); - const validated = VlmComparisonResultSchema.parse(parsed); - + const validated = VlmComparisonResultSchema.parse(JSON.parse(responseContent)); return { pass: validated.identical, description: validated.description || 'No description provided', @@ -150,9 +138,9 @@ export class VlmService implements ImageComparator { diffBytes: Uint8Array, config: VlmConfig ): Promise<{ pass: boolean; description: string }> { - const provider = this.getProvider(config); - const images = [baselineBytes, imageBytes, diffBytes]; - const response = await provider.generate(config, images); + const response = await this.limiter.run(() => + this.getProvider(config).generate(config, [baselineBytes, imageBytes, diffBytes]) + ); const content = this.extractResponseContent(config, response); return this.parseVlmResponse(content); } diff --git a/src/compare/libs/vlm/vlm.types.ts b/src/compare/libs/vlm/vlm.types.ts index 1cf03060..36fa226a 100644 --- a/src/compare/libs/vlm/vlm.types.ts +++ b/src/compare/libs/vlm/vlm.types.ts @@ -1,63 +1,20 @@ import { BlurAmplitudeConfig } from '../blur-amplitude.config'; -/** - * Base configuration shared by all VLM providers - */ export interface BaseVlmConfig extends BlurAmplitudeConfig { - /** - * Custom prompt for image comparison. - */ prompt: string; - - /** - * Temperature parameter controlling response randomness (0.0-1.0). - * Lower values = more consistent results. - * @default 0.1 - */ temperature: number; - - /** - * Whether to prefer thinking field over content field for response. - * Some models return result in thinking field instead of response. - * @default false - */ useThinking?: boolean; } -/** - * Configuration for Ollama provider - */ export interface OllamaVlmConfig extends BaseVlmConfig { provider?: 'ollama'; - - /** - * Ollama vision model name. - * Examples: "gemma3:12b", "llava:13b" - * @default "gemma3:12b" - */ model: string; } -/** - * Configuration for Gemini provider - */ export interface GeminiVlmConfig extends BaseVlmConfig { provider: 'gemini'; - - /** - * Gemini model name. - * Examples: "gemini-1.5-pro", "gemini-1.5-flash" - */ model: string; - - /** - * Google Gemini API key (required). - * Stored in project settings for per-project configuration. - */ apiKey: string; } -/** - * Union type for all VLM provider configurations - */ export type VlmConfig = OllamaVlmConfig | GeminiVlmConfig; diff --git a/src/compare/test-utils.ts b/src/compare/test-utils.ts new file mode 100644 index 00000000..418140a3 --- /dev/null +++ b/src/compare/test-utils.ts @@ -0,0 +1,33 @@ +import { PNG } from 'pngjs'; + +export type Rgb = [number, number, number]; + +export const WHITE: Rgb = [255, 255, 255]; + +export const paintedPng = ( + width: number, + height: number, + paint: (x: number, y: number) => Rgb | undefined = () => undefined +): PNG => { + const png = new PNG({ width, height }); + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + const [r, g, b] = paint(x, y) ?? WHITE; + const index = (y * width + x) * 4; + png.data[index] = r; + png.data[index + 1] = g; + png.data[index + 2] = b; + png.data[index + 3] = 255; + } + } + return png; +}; + +export const solidPng = (width: number, height: number, rgb: Rgb = WHITE): PNG => paintedPng(width, height, () => rgb); + +export const pngBytes = (png: PNG): Buffer => PNG.sync.write(png); + +export const square = + (left: number, top: number, size: number, rgb: Rgb) => + (x: number, y: number): Rgb | undefined => + x >= left && x < left + size && y >= top && y < top + size ? rgb : undefined; diff --git a/src/compare/utils/index.spec.ts b/src/compare/utils/index.spec.ts index b514f537..ba397371 100644 --- a/src/compare/utils/index.spec.ts +++ b/src/compare/utils/index.spec.ts @@ -37,7 +37,6 @@ describe('parseConfig', () => { }); }); -// pixel i of a w x h image gets channels [i, 100 + i, 200 - i, 50 + i] so every channel is distinguishable const distinctPixels = (width: number, height: number): Buffer => { const data = Buffer.alloc(width * height * 4); for (let i = 0; i < width * height; i++) { @@ -59,9 +58,7 @@ describe('applyIgnoreAreas edges', () => { expect(pixelOf(image.data, 0)).toEqual([0, 0, 0, 0]); expect(pixelOf(image.data, 1)).toEqual([0, 0, 0, 0]); - // column right after the area expect(pixelOf(image.data, 2)).toEqual([255, 255, 255, 255]); - // row right after the area expect(pixelOf(image.data, 3)).toEqual([255, 255, 255, 255]); expect(pixelOf(image.data, 4)).toEqual([255, 255, 255, 255]); }); @@ -78,40 +75,122 @@ describe('applyIgnoreAreas edges', () => { }); }); -describe('scaleImageToSize', () => { - it('returns the same image when the requested size equals the image size', () => { - const image = generatePng(3, 2); +const masked = (image: PNG): number[] => { + const indexes: number[] = []; + for (let i = 0; i < image.width * image.height; i++) { + if (image.data[i * 4 + 3] === 0) { + indexes.push(i); + } + } + return indexes; +}; + +const legacyApply = (image: PNG, ignoreAreas: IgnoreAreaDto[]): PNG => { + ignoreAreas.forEach((area) => { + for (let y = area.y; y < Math.min(area.y + area.height, image.height); y++) { + for (let x = area.x; x < Math.min(area.x + area.width, image.width); x++) { + const k = 4 * (image.width * y + x); + image.data[k + 0] = 0; + image.data[k + 1] = 0; + image.data[k + 2] = 0; + image.data[k + 3] = 0; + } + } + }); + return image; +}; + +describe('applyIgnoreAreas rounding and clamping', () => { + it('masks every pixel a fractional area touches', () => { + const image = generatePng(4, 4); + + applyIgnoreAreas(image, [{ x: 0.5, y: 0.5, width: 1, height: 1 }]); + + expect(masked(image)).toEqual([0, 1, 4, 5]); + for (const index of masked(image)) { + expect(pixelOf(image.data, index)).toEqual([0, 0, 0, 0]); + } + }); + + it.each([ + ['does not wrap a negative x into the previous row', 4, { x: -2, y: 1, width: 3, height: 1 }, [4]], + ['clips a negative y to the first row', 4, { x: 1, y: -2, width: 1, height: 3 }, [1]], + [ + 'keeps the old ceil end for a fractional size at an integer start', + 4, + { x: 0, y: 0, width: 1.5, height: 1.2 }, + [0, 1, 4, 5], + ], + [ + 'returns at once for absurd coordinates', + 3, + { x: -1e12, y: -1e12, width: 2e12, height: 2e12 }, + [0, 1, 2, 3, 4, 5, 6, 7, 8], + ], + ])('%s', (_, size, area, expected) => { + const image = generatePng(size, size); + + applyIgnoreAreas(image, [area]); - expect(scaleImageToSize(image, 3, 2)).toBe(image); + expect(masked(image)).toEqual(expected); }); - it('returns the same image when only the height is requested smaller', () => { + it('skips null, empty and non-numeric areas', () => { const image = generatePng(3, 3); - expect(scaleImageToSize(image, 3, 2)).toBe(image); + applyIgnoreAreas(image, [ + null, + { x: 1, y: 1, width: 0, height: 2 }, + { x: NaN, y: 0, width: 2, height: 2 }, + ] as unknown as IgnoreAreaDto[]); + + expect(masked(image)).toEqual([]); }); - it('pads a wider canvas with transparent black, keeping the original pixels top-left', () => { - const image = new PNG({ width: 2, height: 1 }); - distinctPixels(2, 1).copy(image.data); + it('masks byte for byte what the old loop masked for integer areas', () => { + let seed = 12345; + const next = (min: number, max: number): number => { + seed = (seed * 1664525 + 1013904223) % 4294967296; + return min + Math.floor((seed / 4294967296) * (max - min)); + }; - const result = scaleImageToSize(image, 3, 1); + for (let i = 0; i < 300; i++) { + const area = { x: next(0, 12), y: next(-5, 12), width: next(-3, 12), height: next(-3, 12) }; + const legacy = new PNG({ width: 9, height: 7 }); + const current = new PNG({ width: 9, height: 7 }); + distinctPixels(9, 7).copy(legacy.data); + distinctPixels(9, 7).copy(current.data); - expect(result).not.toBe(image); - expect(result.width).toBe(3); - expect(result.height).toBe(1); - expect([...result.data]).toEqual([0, 100, 200, 50, 1, 101, 199, 51, 0, 0, 0, 0]); + legacyApply(legacy, [area]); + applyIgnoreAreas(current, [area]); + + expect(current.data.equals(legacy.data)).toBe(true); + } + }); +}); + +describe('scaleImageToSize', () => { + it.each([ + ['the requested size equals the image size', 3, 2, 3, 2], + ['only the height is requested smaller', 3, 3, 3, 2], + ])('returns the same image when %s', (_, imageWidth, imageHeight, width, height) => { + const image = generatePng(imageWidth, imageHeight); + + expect(scaleImageToSize(image, width, height)).toBe(image); }); - it('pads a taller canvas with transparent black, keeping the original pixels top-left', () => { - const image = new PNG({ width: 1, height: 1 }); - distinctPixels(1, 1).copy(image.data); + it.each([ + ['wider', [2, 1], [3, 1], [0, 100, 200, 50, 1, 101, 199, 51, 0, 0, 0, 0]], + ['taller', [1, 1], [1, 2], [0, 100, 200, 50, 0, 0, 0, 0]], + ])('pads a %s canvas with transparent black, keeping the original pixels top-left', (_, from, to, expected) => { + const image = new PNG({ width: from[0], height: from[1] }); + distinctPixels(from[0], from[1]).copy(image.data); - const result = scaleImageToSize(image, 1, 2); + const result = scaleImageToSize(image, to[0], to[1]); - expect(result.width).toBe(1); - expect(result.height).toBe(2); - expect([...result.data]).toEqual([0, 100, 200, 50, 0, 0, 0, 0]); + expect(result).not.toBe(image); + expect([result.width, result.height]).toEqual(to); + expect([...result.data]).toEqual(expected); }); }); @@ -130,7 +209,6 @@ describe('downscale', () => { expect(result.width).toBe(2); expect(result.height).toBe(2); expect(result.data.length).toBe(2 * 2 * 4); - // target (x, y) <- source (2x, 2y): source indexes 0, 2, 8, 10 expect([...result.data]).toEqual([ ...pixelOf(source.data, 0), ...pixelOf(source.data, 2), @@ -146,7 +224,6 @@ describe('downscale', () => { expect(result.width).toBe(2); expect(result.height).toBe(1); - // target x 0 <- source x 0, target x 1 <- source x 3 (scale 1/3) expect([...result.data]).toEqual([...pixelOf(source.data, 0), ...pixelOf(source.data, 3)]); }); }); diff --git a/src/compare/utils/index.ts b/src/compare/utils/index.ts index cd944b9e..4855b181 100644 --- a/src/compare/utils/index.ts +++ b/src/compare/utils/index.ts @@ -1,6 +1,7 @@ import { Logger } from '@nestjs/common'; import { PNG } from 'pngjs'; import { IgnoreAreaDto } from '../../test-runs/dto/ignore-area.dto'; +import { asBuffer } from '../../static/utils'; export function scaleImageToSize(image: PNG, width: number, height: number): PNG { if (width > image.width || height > image.height) { @@ -11,24 +12,50 @@ export function scaleImageToSize(image: PNG, width: number, height: number): PNG return image; } +export function readPng(data: Buffer | Uint8Array): PNG { + return PNG.sync.read(asBuffer(data)); +} + +export function readSameSizePair( + baselineBytes: Buffer | Uint8Array, + imageBytes: Buffer | Uint8Array, + ignoreAreas: IgnoreAreaDto[] +): { baseline: PNG; image: PNG } | null { + let baseline: PNG; + let image: PNG; + try { + baseline = readPng(baselineBytes); + image = readPng(imageBytes); + } catch { + return null; + } + if (baseline.width !== image.width || baseline.height !== image.height) { + return null; + } + applyIgnoreAreas(baseline, ignoreAreas); + applyIgnoreAreas(image, ignoreAreas); + return { baseline, image }; +} + export function applyIgnoreAreas(image: PNG, ignoreAreas: IgnoreAreaDto[]): PNG { - ignoreAreas.forEach((area) => { - for (let y = area.y; y < Math.min(area.y + area.height, image.height); y++) { - for (let x = area.x; x < Math.min(area.x + area.width, image.width); x++) { - const k = 4 * (image.width * y + x); - image.data[k + 0] = 0; - image.data[k + 1] = 0; - image.data[k + 2] = 0; - image.data[k + 3] = 0; - } + for (const area of ignoreAreas) { + if (!area) { + continue; + } + const x0 = Math.max(0, Math.floor(area.x)); + const y0 = Math.max(0, Math.floor(area.y)); + const x1 = Math.min(image.width, Math.ceil(area.x + area.width)); + const y1 = Math.min(image.height, Math.ceil(area.y + area.height)); + if (!(x1 > x0 && y1 > y0)) { + continue; } - }); + for (let y = y0; y < y1; y++) { + image.data.fill(0, 4 * (image.width * y + x0), 4 * (image.width * y + x1)); + } + } return image; } -// The stored config is laid over the comparator's defaults, not used in their -// place: a project that sets one option keeps the others, and a VLM project -// still holding the column's pixelmatch default still gets a model and prompt. export const parseConfig = (configJson: string, defaultConfig: T, logger: Logger): T => { try { const config = JSON.parse(configJson); @@ -50,8 +77,6 @@ export interface RawImage { height: number; } -// Nearest-neighbour downscale so the longest side is at most maxDimension. -// Returns the original when already small enough. export function downscale(source: RawImage, maxDimension: number): RawImage { const scale = maxDimension / Math.max(source.width, source.height); if (scale >= 1) { diff --git a/src/config/env.validation.spec.ts b/src/config/env.validation.spec.ts new file mode 100644 index 00000000..bd903121 --- /dev/null +++ b/src/config/env.validation.spec.ts @@ -0,0 +1,111 @@ +import { Logger } from '@nestjs/common'; +import { + DEFAULT_JWT_LIFE_TIME, + DEVELOPMENT_JWT_SECRET, + REFUSED_IN_PRODUCTION_JWT_SECRETS, + validateEnv, +} from './env.validation'; + +describe('validateEnv', () => { + const GOOD = { JWT_SECRET: 'a-random-secret', JWT_LIFE_TIME: '12h', APP_FRONTEND_URL: 'http://localhost:8080' }; + let logger: { warn: jest.Mock }; + + beforeEach(() => { + logger = { warn: jest.fn() }; + }); + + it('returns a valid config unchanged and silent', () => { + expect(validateEnv({ ...GOOD, OTHER: 'x' }, logger)).toEqual({ ...GOOD, OTHER: 'x' }); + expect(logger.warn).not.toHaveBeenCalled(); + }); + + it('does not mutate the config it was given', () => { + const config = { JWT_SECRET: 'a-random-secret' }; + + validateEnv(config, logger); + + expect(config).toEqual({ JWT_SECRET: 'a-random-secret' }); + }); + + describe('JWT_SECRET', () => { + it.each([undefined, '', ' '])('refuses to start without it (%p)', (value) => { + expect(() => validateEnv({ ...GOOD, JWT_SECRET: value }, logger)).toThrow('JWT_SECRET is not set'); + }); + + it('refuses the public sample value when NODE_ENV=production', () => { + expect(() => validateEnv({ ...GOOD, JWT_SECRET: 'jwtPrivateKey', NODE_ENV: 'production' }, logger)).toThrow( + 'Refusing to start' + ); + }); + + it.each([ + ['the public sample value when NODE_ENV=development', 'jwtPrivateKey', 'development'], + ['the public sample value when NODE_ENV is unset', 'jwtPrivateKey', undefined], + ['the committed development secret in production', DEVELOPMENT_JWT_SECRET, 'production'], + ])('only warns about %s', (_, secret, nodeEnv) => { + expect(() => validateEnv({ ...GOOD, JWT_SECRET: secret, NODE_ENV: nodeEnv }, logger)).not.toThrow(); + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('publicly known')); + }); + + it('lists the sample value among the refused ones', () => { + expect(REFUSED_IN_PRODUCTION_JWT_SECRETS).toContain('jwtPrivateKey'); + expect(REFUSED_IN_PRODUCTION_JWT_SECRETS).not.toContain(DEVELOPMENT_JWT_SECRET); + }); + }); + + describe('JWT_LIFE_TIME', () => { + it.each([undefined, '', ' ', 'abc', '1 fortnight', 'x'.repeat(101)])( + 'falls back to 1d and warns for %p', + (value) => { + const env = validateEnv({ ...GOOD, JWT_LIFE_TIME: value }, logger); + + expect(env.JWT_LIFE_TIME).toBe(DEFAULT_JWT_LIFE_TIME); + expect(DEFAULT_JWT_LIFE_TIME).toBe('1d'); + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('JWT_LIFE_TIME')); + } + ); + + it.each(['12h', '1d', '30 minutes', '2 days', '500ms', '1.5h'])('accepts %p without a warning', (value) => { + const env = validateEnv({ ...GOOD, JWT_LIFE_TIME: value }, logger); + + expect(env.JWT_LIFE_TIME).toBe(value); + expect(logger.warn).not.toHaveBeenCalled(); + }); + + it('keeps a number without a unit but warns that it counts milliseconds', () => { + const env = validateEnv({ ...GOOD, JWT_LIFE_TIME: '3600' }, logger); + + expect(env.JWT_LIFE_TIME).toBe('3600'); + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('milliseconds')); + }); + }); + + describe('other settings only warn', () => { + it('warns when APP_FRONTEND_URL is not set', () => { + validateEnv({ JWT_SECRET: 'a-random-secret', JWT_LIFE_TIME: '1d' }, logger); + + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('APP_FRONTEND_URL')); + }); + + it('warns when STATIC_SERVICE=s3 has no bucket', () => { + validateEnv({ ...GOOD, STATIC_SERVICE: 's3' }, logger); + + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('AWS_S3_BUCKET_NAME')); + }); + + it('is silent when STATIC_SERVICE=s3 has a bucket', () => { + validateEnv({ ...GOOD, STATIC_SERVICE: 's3', AWS_S3_BUCKET_NAME: 'vrt' }, logger); + + expect(logger.warn).not.toHaveBeenCalled(); + }); + }); + + it('logs through a Nest Logger by default', () => { + const warn = jest.spyOn(Logger.prototype, 'warn').mockImplementation(); + + validateEnv({ JWT_SECRET: 'a-random-secret', JWT_LIFE_TIME: 'abc' }); + + expect(warn).toHaveBeenCalledWith(expect.stringContaining('JWT_LIFE_TIME')); + warn.mockRestore(); + }); +}); diff --git a/src/config/env.validation.ts b/src/config/env.validation.ts new file mode 100644 index 00000000..53211a8d --- /dev/null +++ b/src/config/env.validation.ts @@ -0,0 +1,52 @@ +import { Logger } from '@nestjs/common'; + +export const DEFAULT_JWT_LIFE_TIME = '1d'; +export const REFUSED_IN_PRODUCTION_JWT_SECRETS: readonly string[] = ['jwtPrivateKey']; +export const DEVELOPMENT_JWT_SECRET = 'vrt-local-development-only-jwt-secret'; + +const TIMESPAN = + /^(-?(?:\d+)?\.?\d+) *(milliseconds?|msecs?|ms|seconds?|secs?|s|minutes?|mins?|m|hours?|hrs?|h|days?|d|weeks?|w|years?|yrs?|y)?$/i; + +const text = (value: unknown): string => (value === undefined || value === null ? '' : String(value).trim()); + +type Warn = Pick; + +export function validateEnv( + config: Record, + logger: Warn = new Logger('Config') +): Record { + const env = { ...config }; + + const secret = text(env.JWT_SECRET); + if (!secret) { + throw new Error('JWT_SECRET is not set. Generate one, e.g. `openssl rand -hex 32`.'); + } + if (REFUSED_IN_PRODUCTION_JWT_SECRETS.includes(secret) || secret === DEVELOPMENT_JWT_SECRET) { + const problem = + 'JWT_SECRET is a publicly known sample value: anyone can sign tokens for any user. Set a random one, e.g. `openssl rand -hex 32`.'; + if (text(env.NODE_ENV) === 'production' && REFUSED_IN_PRODUCTION_JWT_SECRETS.includes(secret)) { + throw new Error(`${problem} Refusing to start with NODE_ENV=production.`); + } + logger.warn(problem); + } + + const lifeTime = text(env.JWT_LIFE_TIME); + if (!lifeTime || lifeTime.length > 100 || !TIMESPAN.test(lifeTime)) { + logger.warn( + `JWT_LIFE_TIME ${lifeTime ? `"${lifeTime}" is not a timespan` : 'is not set'}; tokens expire after ${DEFAULT_JWT_LIFE_TIME}. Use e.g. "1d" or "12h".` + ); + env.JWT_LIFE_TIME = DEFAULT_JWT_LIFE_TIME; + } else if (/^\d+(\.\d+)?$/.test(lifeTime)) { + logger.warn( + `JWT_LIFE_TIME "${lifeTime}" has no unit and counts milliseconds; add "s", "h" or "d" if that is not meant.` + ); + } + + if (!text(env.APP_FRONTEND_URL)) { + logger.warn('APP_FRONTEND_URL is not set: links in test run results start with "undefined".'); + } + if (text(env.STATIC_SERVICE) === 's3' && !text(env.AWS_S3_BUCKET_NAME)) { + logger.warn('STATIC_SERVICE=s3 but AWS_S3_BUCKET_NAME is not set: storing images will fail.'); + } + return env; +} diff --git a/src/health/health.controller.spec.ts b/src/health/health.controller.spec.ts index af45bb2e..578072aa 100644 --- a/src/health/health.controller.spec.ts +++ b/src/health/health.controller.spec.ts @@ -1,34 +1,20 @@ import { ServiceUnavailableException } from '@nestjs/common'; -import { Test, TestingModule } from '@nestjs/testing'; -import { HealthCheckService, TerminusModule } from '@nestjs/terminus'; +import { Test } from '@nestjs/testing'; +import { HealthCheckService, PrismaHealthIndicator, TerminusModule } from '@nestjs/terminus'; import { PrismaService } from '../prisma/prisma.service'; import { HealthController } from './health.controller'; describe('Health Controller', () => { let controller: HealthController; - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - controllers: [HealthController], - providers: [ - { provide: HealthCheckService, useValue: {} }, - { provide: PrismaService, useValue: {} }, - ], - }).compile(); - - controller = module.get(HealthController); - }); - - it('should be defined', () => { - expect(controller).toBeDefined(); - }); - describe('check', () => { - it('runs a single indicator that queries SELECT 1', async () => { + it('runs a single indicator that pings the database with a timeout', async () => { const health = { check: jest.fn().mockResolvedValue('result') }; - const prisma = { $queryRaw: jest.fn().mockResolvedValue([{ '?column?': 1 }]) }; + const prismaHealth = { pingCheck: jest.fn().mockResolvedValue({ database: { status: 'up' } }) }; + const prisma = {}; const healthController = new HealthController( health as unknown as HealthCheckService, + prismaHealth as unknown as PrismaHealthIndicator, prisma as unknown as PrismaService ); @@ -36,17 +22,23 @@ describe('Health Controller', () => { const [indicators] = health.check.mock.calls[0]; expect(indicators).toHaveLength(1); - await indicators[0](); - const [strings] = prisma.$queryRaw.mock.calls[0]; - expect([...strings]).toEqual(['SELECT 1']); + await expect(indicators[0]()).resolves.toEqual({ database: { status: 'up' } }); + expect(prismaHealth.pingCheck).toHaveBeenCalledWith('database', prisma, { timeout: 3000 }); }); }); describe('check with the real terminus HealthCheckService', () => { - let prisma: { $queryRaw: jest.Mock }; + let prisma: { $runCommandRaw: jest.Mock; $queryRawUnsafe: jest.Mock }; beforeEach(async () => { - prisma = { $queryRaw: jest.fn() }; + prisma = { + $runCommandRaw: jest + .fn() + .mockRejectedValue( + new Error('The postgresql provider does not support $runCommandRaw. Use the mongodb provider.') + ), + $queryRawUnsafe: jest.fn(), + }; const module = await Test.createTestingModule({ imports: [TerminusModule.forRoot({ logger: false })], controllers: [HealthController], @@ -55,25 +47,40 @@ describe('Health Controller', () => { controller = module.get(HealthController); }); - // NOTE: current behavior — the raw query result is not a terminus - // HealthIndicatorResult, so the report names no "database" indicator at all. - it('reports ok with empty details when the database answers', async () => { - prisma.$queryRaw.mockResolvedValue([{ '?column?': 1 }]); + afterEach(() => { + jest.useRealTimers(); + }); - await expect(controller.check()).resolves.toEqual({ status: 'ok', info: {}, error: {}, details: {} }); + it('reports the database as up', async () => { + prisma.$queryRawUnsafe.mockResolvedValue([{ '?column?': 1 }]); + + await expect(controller.check()).resolves.toEqual({ + status: 'ok', + info: { database: { status: 'up', responseTime: expect.any(Number) } }, + error: {}, + details: { database: { status: 'up', responseTime: expect.any(Number) } }, + }); + expect(prisma.$queryRawUnsafe).toHaveBeenCalledWith('SELECT 1'); }); - // NOTE: current behavior — the indicator does not throw a terminus - // HealthCheckError, so terminus rethrows the raw error instead of - // answering 503; through the global filter /health then answers 500. - it('rethrows the raw database error instead of a 503', async () => { - const error = new Error('Can not reach database server'); - prisma.$queryRaw.mockRejectedValue(error); + it('answers 503 when the database fails', async () => { + prisma.$queryRawUnsafe.mockRejectedValue(new Error('Can not reach database server')); const result = controller.check(); - await expect(result).rejects.toBe(error); - await expect(result).rejects.not.toBeInstanceOf(ServiceUnavailableException); + await expect(result).rejects.toBeInstanceOf(ServiceUnavailableException); + const error = (await result.catch((e: ServiceUnavailableException) => e)) as ServiceUnavailableException; + expect((error.getResponse() as { error: Record }).error.database.status).toBe('down'); + }); + + it('answers 503 when the database does not answer within 3 s', async () => { + jest.useFakeTimers(); + prisma.$queryRawUnsafe.mockReturnValue(new Promise(() => undefined)); + + const result = controller.check().catch((e: unknown) => e); + await jest.advanceTimersByTimeAsync(3000); + + await expect(result).resolves.toBeInstanceOf(ServiceUnavailableException); }); }); }); diff --git a/src/health/health.controller.ts b/src/health/health.controller.ts index 22410d27..44965c0d 100644 --- a/src/health/health.controller.ts +++ b/src/health/health.controller.ts @@ -1,13 +1,16 @@ import { Controller, Get } from '@nestjs/common'; -import { HealthCheckService, HealthCheck, HealthIndicatorResult } from '@nestjs/terminus'; +import { HealthCheckService, HealthCheck, PrismaHealthIndicator } from '@nestjs/terminus'; import { PrismaService } from '../prisma/prisma.service'; import { ApiOkResponse, ApiTags } from '@nestjs/swagger'; +const DATABASE_TIMEOUT_MS = 3_000; + @ApiTags('health') @Controller('health') export class HealthController { constructor( private health: HealthCheckService, + private readonly prismaHealth: PrismaHealthIndicator, private readonly prismaService: PrismaService ) {} @@ -15,7 +18,8 @@ export class HealthController { @ApiOkResponse({ type: Object }) @HealthCheck() check() { - // the raw query result has always been passed through as the indicator result - return this.health.check([() => this.prismaService.$queryRaw`SELECT 1`]); + return this.health.check([ + () => this.prismaHealth.pingCheck('database', this.prismaService, { timeout: DATABASE_TIMEOUT_MS }), + ]); } } diff --git a/src/http-exception.filter.spec.ts b/src/http-exception.filter.spec.ts index feebef95..113a58d4 100644 --- a/src/http-exception.filter.spec.ts +++ b/src/http-exception.filter.spec.ts @@ -6,12 +6,13 @@ import { HttpStatus, Logger, NotFoundException, + ServiceUnavailableException, } from '@nestjs/common'; import { Prisma } from '@prisma/client'; import { HttpExceptionFilter } from './http-exception.filter'; const createHost = (url = '/some/path') => { - const response = { status: jest.fn(), json: jest.fn() }; + const response = { status: jest.fn(), json: jest.fn(), setHeader: jest.fn() }; response.status.mockReturnValue(response); const request = { url }; const host = { @@ -20,150 +21,195 @@ const createHost = (url = '/some/path') => { return { host, response }; }; -// the errors Prisma 7 raises through the pg adapter, as captured from PostgreSQL -const prisma7UniqueConstraintError = (method: string, constraint: string) => - new Prisma.PrismaClientKnownRequestError( - `\nInvalid \`this.prismaService.${method}\` invocation in\n/app/dist/x.js:1:1\n\n\nUnique constraint failed on the constraint: \`${constraint}\``, - { code: 'P2002', clientVersion: Prisma.prismaVersion.client } - ); +const prismaError = (code: string, message: string) => + new Prisma.PrismaClientKnownRequestError(message, { code, clientVersion: Prisma.prismaVersion.client }); const uniqueConstraintError = (method: string) => - new Prisma.PrismaClientKnownRequestError( - `\nInvalid \`prisma.${method}\` invocation:\n\n\nUnique constraint failed on the fields: (\`name\`)`, - { code: 'P2002', clientVersion: Prisma.prismaVersion.client } + prismaError( + 'P2002', + `\nInvalid \`prisma.${method}\` invocation:\n\n\nUnique constraint failed on the fields: (\`name\`)` + ); + +const prisma7UniqueConstraintError = (method: string, constraint: string) => + prismaError( + 'P2002', + `\nInvalid \`this.prismaService.${method}\` invocation in\n/app/dist/x.js:1:1\n\n\nUnique constraint failed on the constraint: \`${constraint}\`` ); describe('HttpExceptionFilter', () => { let filter: HttpExceptionFilter; - let loggerSpy: jest.SpyInstance; + let errorSpy: jest.SpyInstance; + let warnSpy: jest.SpyInstance; + + const catchWith = (exception: unknown, url?: string) => { + const { host, response } = createHost(url); + filter.catch(exception, host); + return response; + }; beforeEach(() => { filter = new HttpExceptionFilter(); - loggerSpy = jest.spyOn(Logger, 'error').mockImplementation(); + errorSpy = jest.spyOn(Logger.prototype, 'error').mockImplementation(); + warnSpy = jest.spyOn(Logger.prototype, 'warn').mockImplementation(); }); afterEach(() => { - loggerSpy.mockRestore(); + errorSpy.mockRestore(); + warnSpy.mockRestore(); }); describe('catch', () => { - it('answers an HttpException with its own status and the full JSON shape', () => { - const { host, response } = createHost('/projects/1'); - const exception = new NotFoundException('Project not found'); - - filter.catch(exception, host); + it('answers an HttpException with its own status and a body without the exception or its stack', () => { + const response = catchWith(new NotFoundException('Project not found'), '/projects/1'); expect(response.status).toHaveBeenCalledWith(404); expect(response.json).toHaveBeenCalledWith({ + statusCode: 404, path: '/projects/1', name: 'NotFoundException', message: 'Project not found', - exception, - stack: exception.stack, }); }); - it('logs the exception with its stack and its response body', () => { - const { host } = createHost(); - const exception = new BadRequestException('Bad input'); + it('logs a 5xx at error with the stack', () => { + const exception = new Error('Something broke'); - filter.catch(exception, host); + catchWith(exception, '/boom'); - expect(loggerSpy).toHaveBeenCalledWith(exception, exception.stack, { - message: 'Bad input', - error: 'Bad Request', - statusCode: 400, - }); + expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('500: Something broke'), exception.stack); + expect(warnSpy).not.toHaveBeenCalled(); }); - it('answers 500 for an error without getStatus and logs it without a response body', () => { - const { host, response } = createHost(); - const exception = new Error('Something broke'); + it('logs a 4xx at warn without the stack', () => { + catchWith(new BadRequestException('Bad input'), '/projects/1'); - filter.catch(exception as HttpException, host); + expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('400: Bad input')); + expect(warnSpy.mock.calls[0]).toHaveLength(1); + expect(errorSpy).not.toHaveBeenCalled(); + }); + + it('answers 500 for an error without getStatus', () => { + const response = catchWith(new Error('Something broke')); expect(response.status).toHaveBeenCalledWith(HttpStatus.INTERNAL_SERVER_ERROR); - expect(response.json).toHaveBeenCalledWith( - expect.objectContaining({ name: 'Error', message: 'Something broke', exception }) - ); - expect(loggerSpy).toHaveBeenCalledWith(exception, exception.stack, undefined); + expect(response.json).toHaveBeenCalledWith({ + statusCode: 500, + path: '/some/path', + name: 'Error', + message: 'Something broke', + }); }); - it('answers a ValidationPipe rejection with the validation messages', () => { - const { host, response } = createHost(); - const exception = new BadRequestException([ - 'name should not be empty', - 'imageComparisonConfig must be a json string', - ]); - - filter.catch(exception, host); + it('answers 500 for a thrown string', () => { + const response = catchWith('plain string'); - expect(response.status).toHaveBeenCalledWith(400); + expect(response.status).toHaveBeenCalledWith(500); expect(response.json).toHaveBeenCalledWith( - expect.objectContaining({ - message: 'name should not be empty, imageComparisonConfig must be a json string', - }) + expect.objectContaining({ statusCode: 500, name: 'Error', message: 'plain string' }) ); }); - it('answers 404 when Prisma finds no record to update or delete (P2025)', () => { - const { host, response } = createHost(); - const exception = new Prisma.PrismaClientKnownRequestError('No record was found for an update.', { - code: 'P2025', - clientVersion: Prisma.prismaVersion.client, - }); + it.each([ + ['an error with a status but no parser type', Object.assign(new Error('odd'), { status: 413 })], + [ + 'a typed error whose status is not a client error', + Object.assign(new Error('odd'), { statusCode: 502, type: 'x' }), + ], + [ + 'an exception whose getStatus throws', + Object.assign(new HttpException('Weird', 418), { + getStatus: () => { + throw new Error('no status'); + }, + }), + ], + ])('answers 500 for %s', (_, exception) => { + expect(catchWith(exception).status).toHaveBeenCalledWith(500); + }); - filter.catch(exception as unknown as HttpException, host); + it.each([ + ['a foreign key violation (P2003)', prismaError('P2003', 'Foreign key constraint violated.'), 409], + ['no record to update or delete (P2025)', prismaError('P2025', 'No record was found for an update.'), 404], + ])('answers %s with %i', (_, exception, status) => { + const response = catchWith(exception); - expect(response.status).toHaveBeenCalledWith(HttpStatus.NOT_FOUND); + expect(response.status).toHaveBeenCalledWith(status); expect(response.json).toHaveBeenCalledWith( - expect.objectContaining({ message: 'No record was found for an update.', exception }) + expect.objectContaining({ statusCode: status, message: exception.message }) ); }); - it('answers 500 when getStatus itself throws', () => { - const { host, response } = createHost(); - const exception = new HttpException('Weird', 418); - jest.spyOn(exception, 'getStatus').mockImplementation(() => { - throw new Error('no status'); - }); + it('sets Retry-After from retryAfterSeconds', () => { + const response = catchWith(Object.assign(new ServiceUnavailableException('busy'), { retryAfterSeconds: 5 })); - filter.catch(exception, host); + expect(response.setHeader).toHaveBeenCalledWith('Retry-After', '5'); + expect(response.status).toHaveBeenCalledWith(503); + }); - expect(response.status).toHaveBeenCalledWith(500); + it('sends no Retry-After without retryAfterSeconds', () => { + expect(catchWith(new ServiceUnavailableException('busy')).setHeader).not.toHaveBeenCalled(); }); - it.each([ - ['build.update()', 'There is already a build with this ci build id.'], - ['project.create()', 'Project exists with this name.'], - ['user.create()', 'This user already exists.'], - ])('answers 400 with a friendly message for a unique constraint failure in %s', (method, message) => { - const { host, response } = createHost(); - const exception = uniqueConstraintError(method); + it('answers 413 for a body over the parser limit', () => { + const response = catchWith( + Object.assign(new Error('request entity too large'), { + status: 413, + statusCode: 413, + type: 'entity.too.large', + }) + ); - filter.catch(exception as unknown as HttpException, host); + expect(response.status).toHaveBeenCalledWith(413); + expect(response.json).toHaveBeenCalledWith({ + statusCode: 413, + path: '/some/path', + name: 'Error', + message: 'request entity too large', + }); + }); - expect(response.status).toHaveBeenCalledWith(HttpStatus.BAD_REQUEST); + it('answers a ValidationPipe rejection with the validation messages', () => { + const response = catchWith( + new BadRequestException(['name should not be empty', 'imageComparisonConfig must be a json string']) + ); + + expect(response.status).toHaveBeenCalledWith(400); expect(response.json).toHaveBeenCalledWith( - expect.objectContaining({ name: 'PrismaClientKnownRequestError', message, exception }) + expect.objectContaining({ + message: 'name should not be empty, imageComparisonConfig must be a json string', + }) ); }); + describe.each([ + ['Prisma 6', (method: string, _constraint: string) => uniqueConstraintError(method)], + ['Prisma 7', prisma7UniqueConstraintError], + ])('a unique constraint failure from %s', (_, build) => { + it.each([ + ['build.update()', 'Build_projectId_ciBuildId_key', 'There is already a build with this ci build id.'], + ['project.create()', 'Project_name_key', 'Project exists with this name.'], + ['user.create()', 'User_email_key', 'This user already exists.'], + ])('in %s answers 400 with a friendly message', (method, constraint, message) => { + const response = catchWith(build(method, constraint)); + + expect(response.status).toHaveBeenCalledWith(HttpStatus.BAD_REQUEST); + expect(response.json).toHaveBeenCalledWith( + expect.objectContaining({ name: 'PrismaClientKnownRequestError', message }) + ); + }); + }); + it('answers 400 with the raw Prisma message for a unique constraint failure on another model', () => { - const { host, response } = createHost(); const exception = uniqueConstraintError('testVariation.create()'); - filter.catch(exception as unknown as HttpException, host); + const response = catchWith(exception); expect(response.status).toHaveBeenCalledWith(400); expect(response.json).toHaveBeenCalledWith(expect.objectContaining({ message: exception.message })); }); - // the exception Nest throws when a guard returns false it('keeps the 403 status of a guard rejection and rewrites its message', () => { - const { host, response } = createHost(); - - filter.catch(new ForbiddenException('Forbidden resource'), host); + const response = catchWith(new ForbiddenException('Forbidden resource')); expect(response.status).toHaveBeenCalledWith(403); expect(response.json).toHaveBeenCalledWith( @@ -174,92 +220,40 @@ describe('HttpExceptionFilter', () => { ); }); - // NOTE: current behavior — the "forbidden" rewrite is keyed on the message - // text only, so any error mentioning "Forbidden resource" is reworded while - // keeping its original status. - it('rewrites the message of a non-403 error that mentions "Forbidden resource"', () => { - const { host, response } = createHost(); - - filter.catch(new Error('Forbidden resource: storage bucket') as HttpException, host); + it('keeps the message of a non-403 error mentioning "Forbidden resource"', () => { + const response = catchWith(new Error('Forbidden resource: storage bucket')); expect(response.status).toHaveBeenCalledWith(500); expect(response.json).toHaveBeenCalledWith( - expect.objectContaining({ message: 'You do not have permission to perform this operation.' }) + expect.objectContaining({ message: 'Forbidden resource: storage bucket' }) ); }); - - // NOTE: current behavior — a thrown value without a string `message` (a - // thrown string, or an object literal) crashes the filter itself with a - // TypeError instead of producing a 500 response. - it('throws a TypeError for a thrown value without a message', () => { - const { host, response } = createHost(); - - expect(() => filter.catch('plain string' as unknown as HttpException, host)).toThrow(TypeError); - expect(response.status).not.toHaveBeenCalled(); - }); }); describe('isUniqueConstraintException', () => { - it('detects the Prisma unique constraint message', () => { - expect(filter.isUniqueConstraintException(uniqueConstraintError('user.create()') as never)).toBe(true); - }); - - it('ignores other messages', () => { - expect(filter.isUniqueConstraintException(new BadRequestException('nope'))).toBe(false); - }); - - it("detects Prisma 7's wording, which names the constraint instead of the fields", () => { - expect( - filter.isUniqueConstraintException( - prisma7UniqueConstraintError('project.create()', 'Project_name_key') as never - ) - ).toBe(true); - }); - - it('detects the P2002 code whatever the message says', () => { - const error = new Prisma.PrismaClientKnownRequestError('reworded in some later version', { - code: 'P2002', - clientVersion: Prisma.prismaVersion.client, - }); - - expect(filter.isUniqueConstraintException(error as never)).toBe(true); - }); - - it('does not take another Prisma error code for a duplicate', () => { - const error = new Prisma.PrismaClientKnownRequestError('Record to update not found.', { - code: 'P2025', - clientVersion: Prisma.prismaVersion.client, - }); - - expect(filter.isUniqueConstraintException(error as never)).toBe(false); - }); - }); - - describe.each([ - ['build.update()', 'Build_projectId_ciBuildId_key', 'There is already a build with this ci build id.'], - ['project.create()', 'Project_name_key', 'Project exists with this name.'], - ['user.create()', 'User_email_key', 'This user already exists.'], - ])('a Prisma 7 duplicate from %s', (method, constraint, message) => { - it(`answers 400 "${message}"`, () => { - const { host, response } = createHost(); - - filter.catch(prisma7UniqueConstraintError(method, constraint) as never, host); - - expect(response.status).toHaveBeenCalledWith(400); - expect(response.json).toHaveBeenCalledWith(expect.objectContaining({ message })); + it.each([ + ['the Prisma unique constraint message', uniqueConstraintError('user.create()'), true], + ['other messages', new BadRequestException('nope'), false], + [ + "Prisma 7's wording, which names the constraint instead of the fields", + prisma7UniqueConstraintError('project.create()', 'Project_name_key'), + true, + ], + ['the P2002 code whatever the message says', prismaError('P2002', 'reworded in some later version'), true], + ['no other Prisma error code as a duplicate', prismaError('P2025', 'Record to update not found.'), false], + ])('handles %s', (_, error, expected) => { + expect(filter.isUniqueConstraintException(error)).toBe(expected); }); }); describe('isForbiddenException', () => { - it('detects the message of a guard rejection', () => { - expect(filter.isForbiddenException(new ForbiddenException('Forbidden resource'))).toBe(true); - }); - - // NOTE: current behavior — detection is by message text only, so a - // ForbiddenException thrown by application code ("Forbidden" by default, - // or a custom message) is not recognised and keeps its message. - it.each([undefined, 'Admins only'])('does not detect a ForbiddenException with message %p', (message) => { - expect(filter.isForbiddenException(new ForbiddenException(message))).toBe(false); + it.each([ + ['the message of a guard rejection', new ForbiddenException('Forbidden resource'), true], + ['only a ForbiddenException with the guard message', new Error('Forbidden resource'), false], + ['no ForbiddenException without a message', new ForbiddenException(undefined), false], + ['no ForbiddenException with another message', new ForbiddenException('Admins only'), false], + ])('handles %s', (_, error, expected) => { + expect(filter.isForbiddenException(error)).toBe(expected); }); }); @@ -270,14 +264,9 @@ describe('HttpExceptionFilter', () => { ['... user.create() ...', 'This user already exists.'], ['Forbidden resource', 'You do not have permission to perform this operation.'], ['anything else', 'anything else'], + ['user.create() build.update() Forbidden resource', 'There is already a build with this ci build id.'], ])('maps %p to %p', (message, expected) => { - expect(filter.getCustomMessageForException(new Error(message) as HttpException)).toBe(expected); - }); - - it('prefers the build message when several markers are present', () => { - expect( - filter.getCustomMessageForException(new Error('user.create() build.update() Forbidden resource') as never) - ).toBe('There is already a build with this ci build id.'); + expect(filter.getCustomMessageForException(new Error(message))).toBe(expected); }); }); }); diff --git a/src/http-exception.filter.ts b/src/http-exception.filter.ts index 98a93eed..e7a15e6c 100644 --- a/src/http-exception.filter.ts +++ b/src/http-exception.filter.ts @@ -1,63 +1,68 @@ -import { ExceptionFilter, Catch, ArgumentsHost, HttpException, HttpStatus, Logger } from '@nestjs/common'; +import { ExceptionFilter, Catch, ArgumentsHost, ForbiddenException, HttpStatus, Logger } from '@nestjs/common'; import { Request, Response } from 'express'; @Catch() export class HttpExceptionFilter implements ExceptionFilter { - catch(exception: HttpException, host: ArgumentsHost): any { + private readonly logger = new Logger(HttpExceptionFilter.name); + + catch(exception: unknown, host: ArgumentsHost): void { const ctx = host.switchToHttp(); const response = ctx.getResponse(); const request = ctx.getRequest(); + const error = exception instanceof Error ? exception : new Error(String(exception)); - let status: number; - const isUniqueConstaintException = this.isUniqueConstraintException(exception); - const isForbiddenException = this.isForbiddenException(exception); - try { - status = isUniqueConstaintException - ? HttpStatus.BAD_REQUEST - : this.isRecordNotFoundException(exception) - ? HttpStatus.NOT_FOUND - : exception.getStatus(); - } catch { - status = HttpStatus.INTERNAL_SERVER_ERROR; - } + const status = this.getStatus(error); const message = - isUniqueConstaintException || isForbiddenException - ? this.getCustomMessageForException(exception) - : this.getMessage(exception); + this.isUniqueConstraintException(error) || this.isForbiddenException(error) + ? this.getCustomMessageForException(error) + : this.getMessage(error); + + if (status >= 500) { + this.logger.error(`${request.method} ${request.url} ${status}: ${message}`, error.stack); + } else { + this.logger.warn(`${request.method} ${request.url} ${status}: ${message}`); + } - Logger.error(exception, exception.stack, exception.getResponse?.()); + const retryAfter = (exception as { retryAfterSeconds?: unknown } | null)?.retryAfterSeconds; + if (typeof retryAfter === 'number') { + response.setHeader('Retry-After', String(retryAfter)); + } + + response.status(status).json({ statusCode: status, path: request.url, name: error.name, message }); + } - response.status(status).json({ - path: request.url, - name: exception.name, - message: message, - exception: exception, - stack: exception.stack, - }); + getStatus(exception: Error): number { + if (this.isUniqueConstraintException(exception)) return HttpStatus.BAD_REQUEST; + const code = (exception as { code?: unknown }).code; + if (code === 'P2025') return HttpStatus.NOT_FOUND; + if (code === 'P2003') return HttpStatus.CONFLICT; + try { + const status = (exception as { getStatus?: () => unknown }).getStatus?.(); + if (typeof status === 'number') return status; + } catch { + return HttpStatus.INTERNAL_SERVER_ERROR; + } + return this.getParserStatus(exception) ?? HttpStatus.INTERNAL_SERVER_ERROR; } - isForbiddenException(exception: HttpException): boolean { - return exception.message.includes('Forbidden resource'); + private getParserStatus(exception: Error): number | undefined { + const { status, statusCode, type } = exception as { status?: unknown; statusCode?: unknown; type?: unknown }; + const clientStatus = Number(status ?? statusCode); + return typeof type === 'string' && clientStatus >= 400 && clientStatus < 500 ? clientStatus : undefined; } - // P2025: an update or delete found no row to act on — the id the client sent - // names nothing, which is a 404 like any other lookup that comes back empty - isRecordNotFoundException(exception: HttpException): boolean { - return (exception as { code?: unknown }).code === 'P2025'; + isForbiddenException(exception: Error): boolean { + return exception instanceof ForbiddenException && exception.message === 'Forbidden resource'; } - // ValidationPipe puts the reasons in the response body and leaves the - // exception's own message at the generic "Bad Request Exception"; the - // reasons are what the client needs to see - getMessage(exception: HttpException): string { - const response = exception.getResponse?.(); - const details = typeof response === 'object' ? (response as { message?: unknown })?.message : undefined; + getMessage(exception: Error): string { + const response = (exception as { getResponse?: () => unknown }).getResponse?.(); + const details = typeof response === 'object' ? (response as { message?: unknown } | null)?.message : undefined; return Array.isArray(details) ? details.join(', ') : exception.message; } - isUniqueConstraintException(exception: HttpException): boolean { - // P2002 is Prisma's code for it in every version; the wording changed in - // Prisma 7 ("on the constraint" instead of "on the fields") + isUniqueConstraintException(exception: Error): boolean { + // P2002 message says "on the fields" before Prisma 7, "on the constraint" since return ( (exception as { code?: unknown }).code === 'P2002' || exception.message.includes('Unique constraint failed on the fields') || @@ -65,17 +70,12 @@ export class HttpExceptionFilter implements ExceptionFilter { ); } - getCustomMessageForException(exception: HttpException): string { - let message = exception.message; - message = message.includes('build.update()') - ? 'There is already a build with this ci build id.' - : message.includes('project.create()') - ? 'Project exists with this name.' - : message.includes('user.create()') - ? 'This user already exists.' - : message.includes('Forbidden resource') - ? 'You do not have permission to perform this operation.' - : message; + getCustomMessageForException(exception: Error): string { + const { message } = exception; + if (message.includes('build.update()')) return 'There is already a build with this ci build id.'; + if (message.includes('project.create()')) return 'Project exists with this name.'; + if (message.includes('user.create()')) return 'This user already exists.'; + if (message.includes('Forbidden resource')) return 'You do not have permission to perform this operation.'; return message; } } diff --git a/src/jest.config.ts b/src/jest.config.ts index a544aa21..914f623e 100644 --- a/src/jest.config.ts +++ b/src/jest.config.ts @@ -1,25 +1,21 @@ -/** @returns {Promise} */ module.exports = async () => { return { displayName: 'Unit Tests', roots: ['./'], moduleFileExtensions: ['js', 'json', 'ts'], transform: { - // SWC only transpiles; type errors are caught once by `npm run typecheck` '^.+\\.(t|j)s$': '@swc/jest', }, - // count every source file, not just the ones some test happens to import - // measured on the TypeScript source through source maps: istanbul would - // instrument SWC's output and count its decorator helpers as branches + // v8 on the TypeScript source via source maps: istanbul would count SWC's decorator helpers as branches coverageProvider: 'v8', collectCoverageFrom: [ '**/*.ts', '!**/*.spec.ts', '!jest.config.ts', - // startup only + '!jest.setup.ts', + '!**/test-utils.ts', '!main.ts', '!swagger.ts', - // types only: no code runs, so v8 would report them as never executed '!**/*.interface.ts', '!**/*.types.ts', '!**/*.model.ts', @@ -29,9 +25,10 @@ module.exports = async () => { '!test-variations/dto/test-variation-update.dto.ts', ], coverageThreshold: { - global: { statements: 98, branches: 85, functions: 98, lines: 98 }, + global: { statements: 98.5, branches: 98, functions: 99, lines: 98.5 }, }, coverageDirectory: '../coverage', testEnvironment: 'node', + setupFiles: ['/jest.setup.ts'], }; }; diff --git a/src/jest.setup.ts b/src/jest.setup.ts new file mode 100644 index 00000000..8cf16cbc --- /dev/null +++ b/src/jest.setup.ts @@ -0,0 +1,4 @@ +import { Logger } from '@nestjs/common'; + +Logger.overrideLogger(false); +Logger.overrideLogger = () => undefined; diff --git a/src/main.helpers.spec.ts b/src/main.helpers.spec.ts new file mode 100644 index 00000000..35815f2d --- /dev/null +++ b/src/main.helpers.spec.ts @@ -0,0 +1,101 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'fs'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { DEFAULT_HTTPS_CERT_PATH, DEFAULT_HTTPS_KEY_PATH, getHttpsOptions, parseLogLevels } from './main.helpers'; + +describe('getHttpsOptions', () => { + let dir: string; + let logger: { log: jest.Mock; warn: jest.Mock }; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'vrt-https-')); + logger = { log: jest.fn(), warn: jest.fn() }; + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + it('reads the key and certificate from HTTPS_KEY_PATH and HTTPS_CERT_PATH', () => { + writeFileSync(join(dir, 'a.key'), 'KEY'); + writeFileSync(join(dir, 'a.cert'), 'CERT'); + + const options = getHttpsOptions( + { HTTPS_KEY_PATH: join(dir, 'a.key'), HTTPS_CERT_PATH: join(dir, 'a.cert') }, + logger + ); + + expect(options).toEqual({ key: Buffer.from('KEY'), cert: Buffer.from('CERT') }); + expect(logger.warn).not.toHaveBeenCalled(); + }); + + it('keeps the documented default paths', () => { + expect(DEFAULT_HTTPS_KEY_PATH).toBe('./secrets/ssl.key'); + expect(DEFAULT_HTTPS_CERT_PATH).toBe('./secrets/ssl.cert'); + }); + + const inEmptyDir = (fn: () => T): T => { + const original = process.cwd(); + process.chdir(dir); + try { + return fn(); + } finally { + process.chdir(original); + } + }; + + it('falls back to HTTP with a log line when no files are there and nothing is configured', () => { + expect(inEmptyDir(() => getHttpsOptions({}, logger))).toBeUndefined(); + + expect(logger.log).toHaveBeenCalledWith('HTTPS config not found. Fall back to HTTP'); + expect(logger.warn).not.toHaveBeenCalled(); + }); + + it('falls back to HTTP with a warning when the configured files are missing', () => { + const env = { HTTPS_KEY_PATH: join(dir, 'missing.key'), HTTPS_CERT_PATH: join(dir, 'missing.cert') }; + + expect(getHttpsOptions(env, logger)).toBeUndefined(); + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('missing.key')); + expect(logger.log).not.toHaveBeenCalled(); + }); + + it('warns when only one of the two paths is configured and the other default is absent', () => { + writeFileSync(join(dir, 'a.key'), 'KEY'); + + expect(getHttpsOptions({ HTTPS_KEY_PATH: join(dir, 'a.key') }, logger)).toBeUndefined(); + expect(logger.warn).toHaveBeenCalled(); + }); + + it('treats blank paths as not configured', () => { + expect(inEmptyDir(() => getHttpsOptions({ HTTPS_KEY_PATH: ' ', HTTPS_CERT_PATH: '' }, logger))).toBeUndefined(); + expect(logger.warn).not.toHaveBeenCalled(); + expect(logger.log).toHaveBeenCalled(); + }); +}); + +describe('parseLogLevels', () => { + let logger: { warn: jest.Mock }; + + beforeEach(() => { + logger = { warn: jest.fn() }; + }); + + it.each([undefined, '', ' ,, '])('keeps Nest default (every level) for %p', (value) => { + expect(parseLogLevels(value, logger)).toBeUndefined(); + expect(logger.warn).not.toHaveBeenCalled(); + }); + + it('parses a comma separated list, trimming and lower-casing', () => { + expect(parseLogLevels('warn, ERROR', logger)).toEqual(['warn', 'error']); + }); + + it('ignores unknown levels with a warning', () => { + expect(parseLogLevels('warn,loud', logger)).toEqual(['warn']); + expect(logger.warn).toHaveBeenCalledWith('LOG_LEVEL: unknown levels ignored: loud'); + }); + + it('keeps Nest default when no level is known', () => { + expect(parseLogLevels('loud', logger)).toBeUndefined(); + expect(logger.warn).toHaveBeenCalled(); + }); +}); diff --git a/src/main.helpers.ts b/src/main.helpers.ts new file mode 100644 index 00000000..e7a3d74f --- /dev/null +++ b/src/main.helpers.ts @@ -0,0 +1,40 @@ +import { Logger, LogLevel } from '@nestjs/common'; +import { HttpsOptions } from '@nestjs/common/interfaces/external/https-options.interface'; +import { existsSync, readFileSync } from 'fs'; + +export const DEFAULT_HTTPS_KEY_PATH = './secrets/ssl.key'; +export const DEFAULT_HTTPS_CERT_PATH = './secrets/ssl.cert'; +const LOG_LEVELS: readonly LogLevel[] = ['log', 'error', 'warn', 'debug', 'verbose', 'fatal']; + +export function getHttpsOptions( + env: NodeJS.ProcessEnv = process.env, + logger: Pick = new Logger('Bootstrap') +): HttpsOptions | undefined { + const keyPath = env.HTTPS_KEY_PATH?.trim() || DEFAULT_HTTPS_KEY_PATH; + const certPath = env.HTTPS_CERT_PATH?.trim() || DEFAULT_HTTPS_CERT_PATH; + if (!existsSync(keyPath) || !existsSync(certPath)) { + if (env.HTTPS_KEY_PATH?.trim() || env.HTTPS_CERT_PATH?.trim()) { + logger.warn(`HTTPS key or certificate not found (${keyPath}, ${certPath}). Fall back to HTTP`); + } else { + logger.log('HTTPS config not found. Fall back to HTTP'); + } + return undefined; + } + return { key: readFileSync(keyPath), cert: readFileSync(certPath) }; +} + +export function parseLogLevels( + value: string | undefined, + logger: Pick = new Logger('Bootstrap') +): LogLevel[] | undefined { + const requested = (value ?? '') + .split(',') + .map((level) => level.trim().toLowerCase()) + .filter(Boolean); + const levels = requested.filter((level): level is LogLevel => (LOG_LEVELS as readonly string[]).includes(level)); + const unknown = requested.filter((level) => !levels.includes(level as LogLevel)); + if (unknown.length) { + logger.warn(`LOG_LEVEL: unknown levels ignored: ${unknown.join(', ')}`); + } + return levels.length ? levels : undefined; +} diff --git a/src/main.ts b/src/main.ts index e923b014..3b6a22eb 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,60 +1,40 @@ import { NestFactory } from '@nestjs/core'; import { AppModule } from './app.module'; import { setupSwagger } from './swagger'; -import { Logger, ValidationPipe } from '@nestjs/common'; +import { Logger } from '@nestjs/common'; import { join } from 'path'; -import * as bodyParser from 'body-parser'; import compression from 'compression'; -import { readFileSync, existsSync } from 'fs'; -import { HttpsOptions } from '@nestjs/common/interfaces/external/https-options.interface'; import { NestExpressApplication } from '@nestjs/platform-express'; import { HDD_IMAGE_PATH } from './static/hdd/constants'; import { RedisIoAdapter } from './redis-io.adapter'; - -function getHttpsOptions(): HttpsOptions | null { - const keyPath = './secrets/ssl.key'; - const certPath = './secrets/ssl.cert'; - if (!existsSync(keyPath) || !existsSync(certPath)) { - Logger.log('HTTPS config not found. Fall back to HTTP'); - return null; - } - return { - key: readFileSync(keyPath), - cert: readFileSync(certPath), - }; -} +import { getHttpsOptions, parseLogLevels } from './main.helpers'; +import { GlobalValidationPipe } from './shared/validation'; async function bootstrap() { + const logger = parseLogLevels(process.env.LOG_LEVEL); const app = await NestFactory.create(AppModule, { cors: true, - // null leaves HTTPS off, as undefined would - httpsOptions: getHttpsOptions() as HttpsOptions | undefined, + httpsOptions: getHttpsOptions(), + abortOnError: false, + ...(logger && { logger }), }); - app.useGlobalPipes(new ValidationPipe()); - // On SIGTERM, close the server and run the providers' teardown (diff - // workers, database pool) so a rolling deploy does not cut requests short + app.useGlobalPipes(new GlobalValidationPipe()); app.enableShutdownHooks(); - // Large builds return multi-megabyte test-run lists; gzip shrinks them ~10x. - // Images are already-compressed PNGs — recompressing them wastes CPU. - // Must be registered before any routes (incl. swagger) to cover them. app.use( compression({ filter: (req, res) => compression.filter(req, res) && res.getHeader('Content-Type') !== 'image/png' }) ); setupSwagger(app); - // Fan out socket.io events across API instances when running multiple - // replicas. Without REDIS_URL the default in-memory adapter is used. await RedisIoAdapter.use(app, process.env.REDIS_URL); if (process.env.BODY_PARSER_JSON_LIMIT) { - app.use(bodyParser.json({ limit: process.env.BODY_PARSER_JSON_LIMIT })); + app.useBodyParser('json', { limit: process.env.BODY_PARSER_JSON_LIMIT }); } app.useStaticAssets(join(process.cwd(), HDD_IMAGE_PATH), { maxAge: 31536000, - // allow cors setHeaders: (res) => { res.set('Access-Control-Allow-Origin', '*'); }, @@ -62,4 +42,10 @@ async function bootstrap() { await app.listen(process.env.APP_PORT || 3000); } -bootstrap(); +bootstrap().catch((error: unknown) => { + Logger.error( + `Failed to start: ${error instanceof Error ? error.message : String(error)}`, + error instanceof Error ? error.stack : undefined + ); + process.exit(1); +}); diff --git a/src/prisma/prisma-errors.spec.ts b/src/prisma/prisma-errors.spec.ts new file mode 100644 index 00000000..3331a5e4 --- /dev/null +++ b/src/prisma/prisma-errors.spec.ts @@ -0,0 +1,20 @@ +import { Prisma } from '@prisma/client'; +import { isPrismaError } from './prisma-errors'; + +describe('isPrismaError', () => { + const known = (code: string) => new Prisma.PrismaClientKnownRequestError(code, { code, clientVersion: 'test' }); + + it('matches a known request error with the same code', () => { + expect(isPrismaError(known('P2002'), 'P2002')).toBe(true); + }); + + it('rejects a known request error with another code', () => { + expect(isPrismaError(known('P2003'), 'P2002')).toBe(false); + }); + + it('rejects look-alikes and non-errors', () => { + expect(isPrismaError({ code: 'P2002' }, 'P2002')).toBe(false); + expect(isPrismaError(new Error('P2002'), 'P2002')).toBe(false); + expect(isPrismaError(undefined, 'P2002')).toBe(false); + }); +}); diff --git a/src/prisma/prisma-errors.ts b/src/prisma/prisma-errors.ts new file mode 100644 index 00000000..606b4d4d --- /dev/null +++ b/src/prisma/prisma-errors.ts @@ -0,0 +1,4 @@ +import { Prisma } from '@prisma/client'; + +export const isPrismaError = (e: unknown, code: string): e is Prisma.PrismaClientKnownRequestError => + e instanceof Prisma.PrismaClientKnownRequestError && e.code === code; diff --git a/src/prisma/prisma.module.spec.ts b/src/prisma/prisma.module.spec.ts index e97b36f0..687477f6 100644 --- a/src/prisma/prisma.module.spec.ts +++ b/src/prisma/prisma.module.spec.ts @@ -18,8 +18,6 @@ describe('PrismaModule', () => { await moduleRef.close(); }); - // Every PrismaService is a separate PrismaClient with its own connection pool, - // so the whole app must share a single instance (issue #443). it('creates a single PrismaService for the whole app', () => { const instances = new Set(); const declaredIn: string[] = []; diff --git a/src/prisma/prisma.module.ts b/src/prisma/prisma.module.ts index 3c11b27a..7207426f 100644 --- a/src/prisma/prisma.module.ts +++ b/src/prisma/prisma.module.ts @@ -1,9 +1,6 @@ import { Global, Module } from '@nestjs/common'; import { PrismaService } from './prisma.service'; -// PrismaService is a PrismaClient with its own connection pool. It must be -// provided only here: listing it in another module's providers creates an -// extra client (and pool) per module. @Global() @Module({ providers: [PrismaService], diff --git a/src/prisma/prisma.service.spec.ts b/src/prisma/prisma.service.spec.ts index 36b36705..80510c70 100644 --- a/src/prisma/prisma.service.spec.ts +++ b/src/prisma/prisma.service.spec.ts @@ -3,7 +3,7 @@ import { Test, TestingModule } from '@nestjs/testing'; import { mkdtempSync, writeFileSync } from 'fs'; import { availableParallelism, tmpdir } from 'os'; import { join, relative, resolve } from 'path'; -import { Client } from 'pg'; +import { Client, ClientConfig } from 'pg'; import { getPgOptions, negotiatePreferredSsl, physicalCpuCount, PrismaService } from './prisma.service'; describe('PrismaService', () => { @@ -63,7 +63,6 @@ describe('physicalCpuCount', () => { `processor\t: ${id}\nmodel name\t: x86\nphysical id\t: ${physicalId}\ncore id\t\t: ${coreId}\n`; it('counts each physical core once, however many hyper-threads it runs', () => { - // two sockets, two cores each, two threads per core const cpuinfo = [0, 1, 2, 3, 4, 5, 6, 7] .map((id) => processor(id, Math.floor(id / 4), Math.floor(id / 2) % 2)) .join('\n'); @@ -90,6 +89,11 @@ describe('getPgOptions', () => { const url = 'postgresql://user:secret@db:5432/vrt'; const defaultMax = physicalCpuCount() * 2 + 1; const noVerify = { rejectUnauthorized: false }; + const writeCa = (content: string) => { + const ca = join(mkdtempSync(join(tmpdir(), 'vrt-ca-')), 'server-ca.pem'); + writeFileSync(ca, content); + return ca; + }; it('keeps the Prisma 6 pool defaults when the URL sets none', () => { expect(getPgOptions(url)).toEqual({ @@ -129,24 +133,21 @@ describe('getPgOptions', () => { }); describe('TLS, as Prisma 6 read the URL', () => { - it('tries TLS without checking the certificate when the URL names no sslmode (prefer)', () => { - expect(getPgOptions(url)).toMatchObject({ pool: { ssl: noVerify }, preferSsl: true }); - }); - - it('treats an explicit sslmode=prefer the same, and takes it out of the URL for pg', () => { - const { pool, preferSsl } = getPgOptions(`${url}?sslmode=prefer&schema=vrt`); - - expect(preferSsl).toBe(true); - expect(pool.ssl).toEqual(noVerify); - expect(pool.connectionString).toBe(`${url}?schema=vrt`); - }); - - it('insists on TLS under sslmode=require, still without checking the certificate', () => { - const { pool, preferSsl } = getPgOptions(`${url}?sslmode=require`); - - expect(preferSsl).toBe(false); + it.each([ + ['takes sslmode=prefer out of the URL for pg', '?sslmode=prefer&schema=vrt', true, '?schema=vrt'], + ['insists on TLS under sslmode=require, unchecked', '?sslmode=require', false, ''], + [ + 'drops sslcert without sslaccept=strict (Prisma 6 accepted any certificate)', + '?sslmode=require&sslcert=a.pem', + false, + '', + ], + ])('%s', (_, query, preferSsl, remainingQuery) => { + const { pool, preferSsl: prefer } = getPgOptions(`${url}${query}`); + + expect(prefer).toBe(preferSsl); expect(pool.ssl).toEqual(noVerify); - expect(pool.connectionString).toBe(url); + expect(pool.connectionString).toBe(`${url}${remainingQuery}`); }); it('checks the certificate against the system CAs under sslaccept=strict', () => { @@ -154,19 +155,16 @@ describe('getPgOptions', () => { }); it('checks it against sslcert, the server certificate to Prisma, under sslaccept=strict', () => { - const ca = join(mkdtempSync(join(tmpdir(), 'vrt-ca-')), 'server-ca.pem'); - writeFileSync(ca, 'ca-pem'); + const ca = writeCa('ca-pem'); const { pool } = getPgOptions(`${url}?sslmode=require&sslaccept=strict&sslcert=${encodeURIComponent(ca)}`); expect(pool.ssl).toEqual({ ca: Buffer.from('ca-pem') }); - // pg would read sslcert as a client certificate expect(pool.connectionString).toBe(url); }); it('resolves a relative sslcert from the prisma/ folder, as Prisma did', () => { - const ca = join(mkdtempSync(join(tmpdir(), 'vrt-ca-')), 'server-ca.pem'); - writeFileSync(ca, 'relative-ca'); + const ca = writeCa('relative-ca'); const fromPrismaFolder = relative(resolve('prisma'), ca); const { pool } = getPgOptions(`${url}?sslaccept=strict&sslcert=${encodeURIComponent(fromPrismaFolder)}`); @@ -174,13 +172,6 @@ describe('getPgOptions', () => { expect(pool.ssl).toEqual({ ca: Buffer.from('relative-ca') }); }); - it('drops sslcert without sslaccept=strict, where Prisma 6 accepted any certificate', () => { - const { pool } = getPgOptions(`${url}?sslmode=require&sslcert=server-ca.pem`); - - expect(pool.ssl).toEqual(noVerify); - expect(pool.connectionString).toBe(url); - }); - it('connects without TLS under sslmode=disable', () => { expect(getPgOptions(`${url}?sslmode=disable`)).toMatchObject({ pool: { ssl: false }, preferSsl: false }); }); @@ -216,14 +207,16 @@ describe('negotiatePreferredSsl', () => { afterEach(() => jest.restoreAllMocks()); const stubClient = (connect: () => Promise) => { - jest.spyOn(Client.prototype, 'connect').mockImplementation(connect as never); + jest.spyOn(Client.prototype, 'connect').mockImplementation(connect); return jest.spyOn(Client.prototype, 'end').mockResolvedValue(undefined as never); }; + const preferringConfig = () => ({ connectionString: 'postgresql://db/vrt', ssl: { rejectUnauthorized: false } }); + it('falls back to plaintext when the server has no TLS', async () => { const end = stubClient(() => Promise.reject(new Error('The server does not support SSL connections'))); const log = jest.fn(); - const config = { connectionString: 'postgresql://db/vrt', ssl: { rejectUnauthorized: false } }; + const config: ClientConfig = preferringConfig(); await negotiatePreferredSsl(config, { log } as unknown as Logger); @@ -234,7 +227,7 @@ describe('negotiatePreferredSsl', () => { it('keeps TLS when the trial connection succeeds', async () => { const end = stubClient(() => Promise.resolve()); - const config = { connectionString: 'postgresql://db/vrt', ssl: { rejectUnauthorized: false } }; + const config = preferringConfig(); await negotiatePreferredSsl(config); @@ -244,7 +237,7 @@ describe('negotiatePreferredSsl', () => { it('keeps TLS when the connection fails for another reason, left for the real connection to report', async () => { stubClient(() => Promise.reject(new Error('password authentication failed'))); - const config = { connectionString: 'postgresql://db/vrt', ssl: { rejectUnauthorized: false } }; + const config = preferringConfig(); await expect(negotiatePreferredSsl(config)).resolves.toBeUndefined(); diff --git a/src/prisma/prisma.service.ts b/src/prisma/prisma.service.ts index 5e2f2f4a..98b452d9 100644 --- a/src/prisma/prisma.service.ts +++ b/src/prisma/prisma.service.ts @@ -6,16 +6,10 @@ import { availableParallelism } from 'os'; import { isAbsolute, resolve } from 'path'; import { Client, ClientConfig, PoolConfig } from 'pg'; -// Connection URL parameters Prisma 6 understood that neither the pg driver nor -// getPgOptions carries over. const UNSUPPORTED_PRISMA_PARAMS = ['pgbouncer', 'statement_cache_size', 'socket_timeout', 'sslidentity', 'sslpassword']; -// TLS parameters with Prisma 6's meaning: getPgOptions turns them into pg's -// `ssl` option and takes them out of the URL, where pg would read them its -// own way (sslcert as a client certificate, require as verify-full). +// Translated into pg's `ssl`: pg reads sslcert as a client certificate and require as verify-full const PRISMA_SSL_PARAMS = ['sslmode', 'sslaccept', 'sslcert']; -// Parameters only pg knows. A URL carrying one was written for pg, so its -// TLS settings are left to pg as they are. const PG_ONLY_SSL_PARAMS = ['sslrootcert', 'sslkey', 'uselibpqcompat']; const SSL_NOT_SUPPORTED = 'The server does not support SSL connections'; @@ -28,14 +22,7 @@ const readCpuinfo = (): string => { } }; -/** - * Physical cores, counted the way Prisma 6 sized its default pool - * (`num_physical_cpus * 2 + 1`, Rust's num_cpus::get_physical): distinct - * (physical id, core id) pairs in /proc/cpuinfo. Hyper-threads would double a - * logical count, and with it every replica's share of max_connections. Where - * cpuinfo has no such fields (ARM, macOS) it falls back to the logical count, - * as num_cpus does. - */ +// Prisma 6 sized its pool from physical cores; hyper-threads would double connections export function physicalCpuCount(cpuinfo: string = readCpuinfo()): number { const cores = new Set(); for (const block of cpuinfo.split(/\n\s*\n/)) { @@ -48,15 +35,6 @@ export function physicalCpuCount(cpuinfo: string = readCpuinfo()): number { return cores.size || availableParallelism(); } -/** - * Pool settings for the `pg` driver Prisma 7 talks to the database through. - * Prisma 6 read its pool and TLS settings from the connection URL and had its - * own defaults; `pg` ignores those parameters, defaults to a 10-connection pool - * that waits forever for a connection, and verifies the certificate under - * `sslmode=require`. Carry the URL parameters over and keep Prisma 6's - * behaviour, so a deployment tuned through DATABASE_URL works the same after - * the upgrade. - */ export function getPgOptions(databaseUrl: string | undefined): { pool: PoolConfig; schema?: string; @@ -65,14 +43,10 @@ export function getPgOptions(databaseUrl: string | undefined): { } { let url: URL | undefined; try { - // undefined throws here like any other unparsable URL url = new URL(databaseUrl as string); - } catch { - // an unparsable URL is left for pg to report when it connects - } + } catch {} const params = url?.searchParams ?? new URLSearchParams(); - // 0 means "no timeout" to Prisma; seconds otherwise const seconds = (name: string, fallback: number) => { const value = params.has(name) ? Number(params.get(name)) : NaN; return (Number.isFinite(value) && value >= 0 ? value : fallback) * 1000; @@ -84,12 +58,8 @@ export function getPgOptions(databaseUrl: string | undefined): { const pool: PoolConfig = { connectionString: databaseUrl, max: connectionLimit > 0 ? connectionLimit : physicalCpuCount() * 2 + 1, - // Prisma 6 waited up to 5s to open a connection and 10s for a free one; - // pg has a single timeout for both, so take the longer, and none at all - // when either was switched off connectionTimeoutMillis: connectTimeout && poolTimeout ? Math.max(connectTimeout, poolTimeout) : undefined, - // Prisma 6 kept idle connections for 300s; pg drops them after 10s, - // which reconnects over and over between ingestion bursts + // Prisma 6's value; pg's 10s default reconnects over and over between ingestion bursts idleTimeoutMillis: 300_000, }; @@ -98,19 +68,15 @@ export function getPgOptions(databaseUrl: string | undefined): { url !== undefined && ['prefer', 'require', 'disable'].includes(sslmode) && !PG_ONLY_SSL_PARAMS.some((name) => params.has(name)); - if (prismaTls) { - // Prisma 6: prefer (its default) tries TLS and falls back to plaintext, - // require insists on TLS, and neither checks the certificate unless - // sslaccept=strict, which checks it against sslcert (resolved, like - // Prisma did, from the prisma/ folder) or the system CAs + if (url && prismaTls) { + // Prisma 6: no certificate check unless sslaccept=strict const sslcert = params.get('sslcert'); const ssl = params.get('sslaccept') === 'strict' ? { ...(sslcert && { ca: readFileSync(isAbsolute(sslcert) ? sslcert : resolve('prisma', sslcert)) }) } : { rejectUnauthorized: false }; - // prismaTls is only true for a parsed URL - PRISMA_SSL_PARAMS.forEach((name) => url!.searchParams.delete(name)); - pool.connectionString = url!.toString(); + PRISMA_SSL_PARAMS.forEach((name) => url.searchParams.delete(name)); + pool.connectionString = url.toString(); pool.ssl = sslmode === 'disable' ? false : ssl; } @@ -122,11 +88,6 @@ export function getPgOptions(databaseUrl: string | undefined): { }; } -/** - * sslmode=prefer, which pg cannot do per connection: one trial connection - * over TLS, and plaintext from then on if the server has no TLS at all. - * Anything else that goes wrong is left for the real connection to report. - */ export async function negotiatePreferredSsl(config: ClientConfig, logger?: Logger): Promise { const client = new Client(config); try { @@ -144,7 +105,6 @@ export async function negotiatePreferredSsl(config: ClientConfig, logger?: Logge @Injectable() export class PrismaService extends PrismaClient implements OnModuleInit, OnApplicationShutdown { private readonly logger = new Logger(PrismaService.name); - // the very object the adapter builds its pool from when Prisma first connects private readonly pgConfig: PoolConfig; private readonly preferSsl: boolean; @@ -152,7 +112,6 @@ export class PrismaService extends PrismaClient implements OnModuleInit, OnAppli const { pool, schema, unsupportedParams, preferSsl } = getPgOptions(process.env.DATABASE_URL); super({ adapter: new PrismaPg(pool, schema ? { schema } : undefined), - // log: [{ emit: 'stdout', level: 'query' }], }); this.pgConfig = pool; this.preferSsl = preferSsl; @@ -163,6 +122,7 @@ export class PrismaService extends PrismaClient implements OnModuleInit, OnAppli ); } } + async onModuleInit() { if (this.preferSsl) { await negotiatePreferredSsl(this.pgConfig, this.logger); @@ -170,18 +130,11 @@ export class PrismaService extends PrismaClient implements OnModuleInit, OnAppli await this.$connect(); } - /** - * A connection of its own, with the same settings as the pool but outside - * it, for state bound to one connection that must not tie up a pooled one - * for long — such as the advisory lock the nightly cleanup holds while it runs. - * Not connected yet; the caller connects and ends it. - */ dedicatedClient(): Client { return new Client(this.pgConfig); } - // After the HTTP server has stopped and drained its requests (Nest runs - // onModuleDestroy before that), so none of them finds the pool closed + // Not onModuleDestroy: that runs before the HTTP server has drained its requests async onApplicationShutdown() { await this.$disconnect(); } diff --git a/src/projects/dto/project.dto.spec.ts b/src/projects/dto/project.dto.spec.ts index 49fe2776..028e4fd6 100644 --- a/src/projects/dto/project.dto.spec.ts +++ b/src/projects/dto/project.dto.spec.ts @@ -39,6 +39,8 @@ describe('ProjectDto', () => { ['imageComparisonConfig', { imageComparisonConfig: '{not json' }, 'isJson'], ['autoApproveFeature', { autoApproveFeature: 'yes' }, 'isBoolean'], ['bulkApproveGroupBy', { bulkApproveGroupBy: 1 }, 'isString'], + ['maxBuildAllowed', { maxBuildAllowed: 1.5 }, 'isInt'], + ['maxBranchLifetime', { maxBranchLifetime: 0.5 }, 'isInt'], ])('rejects an invalid %s', async (property, override, constraint) => { const errors = await validate(plainToInstance(ProjectDto, { ...plainProject, ...override })); @@ -46,4 +48,10 @@ describe('ProjectDto', () => { expect(errors[0].property).toBe(property); expect(Object.keys(errors[0].constraints!)).toEqual([constraint]); }); + + it.each([0, -1, 1, 100])('still accepts the integer %p for both counters', async (value) => { + const dto = plainToInstance(ProjectDto, { ...plainProject, maxBuildAllowed: value, maxBranchLifetime: value }); + + await expect(validate(dto)).resolves.toEqual([]); + }); }); diff --git a/src/projects/dto/project.dto.ts b/src/projects/dto/project.dto.ts index 1de2bf77..778838d9 100644 --- a/src/projects/dto/project.dto.ts +++ b/src/projects/dto/project.dto.ts @@ -1,5 +1,5 @@ import { ApiProperty } from '@nestjs/swagger'; -import { IsUUID, IsString, IsNumber, IsBoolean, IsEnum, IsJSON, IsDate } from 'class-validator'; +import { IsUUID, IsString, IsNumber, IsInt, IsBoolean, IsEnum, IsJSON, IsDate } from 'class-validator'; import { ImageComparison, Project } from '@prisma/client'; import { Type } from 'class-transformer'; @@ -39,11 +39,11 @@ export class ProjectDto implements Project { imageComparison!: ImageComparison; @ApiProperty() - @IsNumber() + @IsInt() maxBuildAllowed!: number; @ApiProperty() - @IsNumber() + @IsInt() maxBranchLifetime!: number; @ApiProperty() diff --git a/src/projects/projects.controller.spec.ts b/src/projects/projects.controller.spec.ts index 761beb89..0b2393f9 100644 --- a/src/projects/projects.controller.spec.ts +++ b/src/projects/projects.controller.spec.ts @@ -37,10 +37,6 @@ describe('Projects Controller', () => { controller = module.get(ProjectsController); }); - it('should be defined', () => { - expect(controller).toBeDefined(); - }); - it('lists all projects', async () => { projectsService.findAll.mockResolvedValue([TEST_PROJECT]); @@ -96,8 +92,6 @@ describe('Projects Controller', () => { let app: INestApplication; beforeEach(async () => { - // the guard is stubbed to sign in a guest: favourites are personal, so - // no role guard may stand in front of them const module: TestingModule = await Test.createTestingModule({ controllers: [ProjectsController], providers: [ diff --git a/src/projects/projects.controller.ts b/src/projects/projects.controller.ts index 816ca69c..a7869d81 100644 --- a/src/projects/projects.controller.ts +++ b/src/projects/projects.controller.ts @@ -42,9 +42,7 @@ export class ProjectsController { return this.projectsService.findAll(); } - // Favourites are personal, not project settings, so every role (guests - // included) may use them: no RoleGuard. Declared before any `:id` route so - // "favorites" is never read as a project id. + // No RoleGuard: favourites are personal. Declared before any `:id` route so "favorites" is not read as an id @Get('favorites') @ApiBearerAuth() @UseGuards(JwtAuthGuard) diff --git a/src/projects/projects.module.ts b/src/projects/projects.module.ts index 6344c6de..d40dd7cb 100644 --- a/src/projects/projects.module.ts +++ b/src/projects/projects.module.ts @@ -3,9 +3,10 @@ import { ProjectsService } from './projects.service'; import { ProjectsController } from './projects.controller'; import { BuildsModule } from '../builds/builds.module'; import { TestVariationsModule } from '../test-variations/test-variations.module'; +import { StaticModule } from '../static/static.module'; @Module({ - imports: [forwardRef(() => BuildsModule), forwardRef(() => TestVariationsModule)], + imports: [forwardRef(() => BuildsModule), forwardRef(() => TestVariationsModule), StaticModule], providers: [ProjectsService], controllers: [ProjectsController], exports: [ProjectsService], diff --git a/src/projects/projects.service.spec.ts b/src/projects/projects.service.spec.ts index 246a83c1..a10e97af 100644 --- a/src/projects/projects.service.spec.ts +++ b/src/projects/projects.service.spec.ts @@ -1,235 +1,342 @@ -import { Test, TestingModule } from '@nestjs/testing'; +import { Test } from '@nestjs/testing'; import { ProjectsService } from './projects.service'; import { PrismaService } from '../prisma/prisma.service'; -import { TestVariationsService } from '../test-variations/test-variations.service'; -import { BuildsService } from '../builds/builds.service'; -import { HttpException, HttpStatus } from '@nestjs/common'; -import { ImageComparison } from '@prisma/client'; +import { StaticService } from '../static/static.service'; +import { EventsGateway } from '../shared/events/events.gateway'; +import { BuildDto } from '../builds/dto/build.dto'; +import { BadRequestException, ConflictException, HttpException, HttpStatus } from '@nestjs/common'; +import { ImageComparison, Prisma } from '@prisma/client'; import { TEST_PROJECT } from '../_data_'; import { CreateProjectDto } from './dto/create-project.dto'; import { UpdateProjectDto } from './dto/update-project.dto'; -const initService = async ({ - projectFindUniqueMock = jest.fn(), - projectFindManyMock = jest.fn(), - projectCreateMock = jest.fn(), - projectUpdateMock = jest.fn(), - projectDeleteMock = jest.fn(), - userUpdateMock = jest.fn(), - buildRemoveMock = jest.fn(), - testVariationDeleteMock = jest.fn(), -}) => { - const module: TestingModule = await Test.createTestingModule({ +const projectId = 'a9385fc1-884d-4f9f-915e-40da0e7773d5'; +const notFound = new HttpException('Project not found', HttpStatus.NOT_FOUND); + +const setup = async () => { + const prisma = { + project: { + findUnique: jest.fn(), + findMany: jest.fn(), + create: jest.fn(), + update: jest.fn(), + delete: jest.fn(), + }, + user: { update: jest.fn() }, + $transaction: jest.fn(), + }; + const deleteImage = jest.fn(); + const buildDeleted = jest.fn(); + const module = await Test.createTestingModule({ providers: [ ProjectsService, + { provide: PrismaService, useValue: prisma }, { - provide: PrismaService, + provide: StaticService, useValue: { - project: { - findUnique: projectFindUniqueMock, - findMany: projectFindManyMock, - create: projectCreateMock, - update: projectUpdateMock, - delete: projectDeleteMock, - }, - user: { - update: userUpdateMock, + deleteImages: async (names: Iterable) => { + for (const name of names) if (name) await deleteImage(name); }, }, }, - { provide: TestVariationsService, useValue: { delete: testVariationDeleteMock } }, - { provide: BuildsService, useValue: { remove: buildRemoveMock } }, + { provide: EventsGateway, useValue: { buildDeleted } }, ], }).compile(); - return module.get(ProjectsService); + return { service: module.get(ProjectsService), prisma, deleteImage, buildDeleted }; }; -describe('ProjectsService', () => { - let service: ProjectsService; +describe('ProjectsService', () => { describe('findOne', () => { it.each([ ['name', 'someName', { id: undefined, name: 'someName' }], - ['id', 'a9385fc1-884d-4f9f-915e-40da0e7773d5', { id: 'a9385fc1-884d-4f9f-915e-40da0e7773d5', name: undefined }], + ['id', projectId, { id: projectId, name: undefined }], ])('should find by %s', async (_, query, expected) => { - const projectFindUniqueMock = jest.fn().mockResolvedValue({}); - service = await initService({ projectFindUniqueMock }); + const { service, prisma } = await setup(); + prisma.project.findUnique.mockResolvedValue({}); await service.findOne(query); - expect(projectFindUniqueMock).toHaveBeenCalledWith({ - where: expected, - }); + expect(prisma.project.findUnique).toHaveBeenCalledWith({ where: expected }); }); it('should throw exception if not found', async () => { - const projectFindUniqueMock = jest.fn().mockResolvedValueOnce(undefined); - service = await initService({ projectFindUniqueMock }); + const { service, prisma } = await setup(); + prisma.project.findUnique.mockResolvedValueOnce(undefined); - await expect(service.findOne('foo')).rejects.toThrow( - new HttpException('Project not found', HttpStatus.NOT_FOUND) - ); + await expect(service.findOne('foo')).rejects.toThrow(notFound); }); }); - describe('findAll', () => { - it('returns every project from the database', async () => { - const projectFindManyMock = jest.fn().mockResolvedValue([TEST_PROJECT]); - service = await initService({ projectFindManyMock }); - const result = await service.findAll(); + it('findAll returns every project from the database', async () => { + const { service, prisma } = await setup(); + prisma.project.findMany.mockResolvedValue([TEST_PROJECT]); - expect(projectFindManyMock).toHaveBeenCalledWith(); - expect(result).toEqual([TEST_PROJECT]); - }); + const result = await service.findAll(); + + expect(prisma.project.findMany).toHaveBeenCalledWith(); + expect(result).toEqual([TEST_PROJECT]); }); - describe('create', () => { - it('creates the project from the dto fields', async () => { - const projectCreateMock = jest.fn().mockResolvedValue(TEST_PROJECT); - service = await initService({ projectCreateMock }); - const dto: CreateProjectDto = { - name: 'New project', - mainBranchName: 'main', - autoApproveFeature: true, - imageComparison: ImageComparison.lookSame, - imageComparisonConfig: '{"strict":true}', - maxBuildAllowed: 50, - maxBranchLifetime: 10, - bulkApproveVariations: true, - bulkApproveGroupBy: 'name', - }; + it('create creates the project from the dto fields', async () => { + const { service, prisma } = await setup(); + prisma.project.create.mockResolvedValue(TEST_PROJECT); + const dto: CreateProjectDto = { + name: 'New project', + mainBranchName: 'main', + autoApproveFeature: true, + imageComparison: ImageComparison.lookSame, + imageComparisonConfig: '{"strict":true}', + maxBuildAllowed: 50, + maxBranchLifetime: 10, + bulkApproveVariations: true, + bulkApproveGroupBy: 'name', + }; + + const result = await service.create({ ...dto, extra: 'ignored' } as CreateProjectDto); + + expect(prisma.project.create).toHaveBeenCalledWith({ data: dto }); + expect(result).toBe(TEST_PROJECT); + }); - const result = await service.create({ ...dto, extra: 'ignored' } as CreateProjectDto); + it('update updates the project by id with the dto fields', async () => { + const { service, prisma } = await setup(); + prisma.project.update.mockResolvedValue(TEST_PROJECT); + const dto: UpdateProjectDto = { + id: projectId, + name: 'Renamed', + mainBranchName: 'develop', + autoApproveFeature: false, + imageComparison: ImageComparison.pixelmatch, + imageComparisonConfig: '{}', + maxBuildAllowed: 5, + maxBranchLifetime: 7, + bulkApproveVariations: false, + bulkApproveGroupBy: 'customTags', + }; + + const result = await service.update(dto); + + const { id, ...data } = dto; + expect(prisma.project.update).toHaveBeenCalledWith({ where: { id }, data }); + expect(result).toBe(TEST_PROJECT); + }); - expect(projectCreateMock).toHaveBeenCalledWith({ data: dto }); - expect(result).toBe(TEST_PROJECT); + describe('odiff on S3 storage', () => { + const originalStaticService = process.env.STATIC_SERVICE; + const odiffDto = { ...TEST_PROJECT, id: projectId, imageComparison: ImageComparison.odiff } as UpdateProjectDto; + + afterEach(() => { + if (originalStaticService === undefined) { + delete process.env.STATIC_SERVICE; + } else { + process.env.STATIC_SERVICE = originalStaticService; + } }); - }); - describe('update', () => { - it('updates the project by id with the dto fields', async () => { - const projectUpdateMock = jest.fn().mockResolvedValue(TEST_PROJECT); - service = await initService({ projectUpdateMock }); - const dto: UpdateProjectDto = { - id: 'a9385fc1-884d-4f9f-915e-40da0e7773d5', - name: 'Renamed', - mainBranchName: 'develop', - autoApproveFeature: false, - imageComparison: ImageComparison.pixelmatch, - imageComparisonConfig: '{}', - maxBuildAllowed: 5, - maxBranchLifetime: 7, - bulkApproveVariations: false, - bulkApproveGroupBy: 'customTags', - }; + it.each(['create', 'update'] as const)( + '%s rejects odiff on S3 storage with 400 and writes nothing', + async (method) => { + process.env.STATIC_SERVICE = 's3'; + const { service, prisma } = await setup(); - const result = await service.update(dto); + await expect(service[method](odiffDto)).rejects.toThrow(BadRequestException); - const { id, ...data } = dto; - expect(projectUpdateMock).toHaveBeenCalledWith({ where: { id }, data }); - expect(result).toBe(TEST_PROJECT); + expect(prisma.project.create).not.toHaveBeenCalled(); + expect(prisma.project.update).not.toHaveBeenCalled(); + } + ); + + it.each(['create', 'update'] as const)('%s accepts odiff on HDD storage', async (method) => { + delete process.env.STATIC_SERVICE; + const { service, prisma } = await setup(); + prisma.project.create.mockResolvedValue(TEST_PROJECT); + prisma.project.update.mockResolvedValue(TEST_PROJECT); + + await expect(service[method](odiffDto)).resolves.toBe(TEST_PROJECT); + }); + + it('still accepts other comparators on S3 storage', async () => { + process.env.STATIC_SERVICE = 's3'; + const { service, prisma } = await setup(); + prisma.project.create.mockResolvedValue(TEST_PROJECT); + + await expect(service.create({ ...odiffDto, imageComparison: ImageComparison.lookSame })).resolves.toBe( + TEST_PROJECT + ); }); }); describe('remove', () => { - it('removes every build and test variation one by one before deleting the project', async () => { + const knownRequestError = (code: string) => + new Prisma.PrismaClientKnownRequestError(code, { code, clientVersion: 'test' }); + + const initRemove = async (transaction?: (fn: (tx: unknown) => unknown) => unknown) => { + const ctx = await setup(); const calls: string[] = []; - const projectFindUniqueMock = jest.fn().mockResolvedValue({ - ...TEST_PROJECT, - builds: [{ id: 'build1' }, { id: 'build2' }], - testVariations: [{ id: 'variation1' }], - }); - const buildRemoveMock = jest.fn().mockImplementation(async (id) => calls.push(`build ${id}`)); - const testVariationDeleteMock = jest.fn().mockImplementation(async (id) => calls.push(`variation ${id}`)); - const projectDeleteMock = jest.fn().mockImplementation(async () => { - calls.push('project'); - return TEST_PROJECT; - }); - service = await initService({ - projectFindUniqueMock, - buildRemoveMock, - testVariationDeleteMock, - projectDeleteMock, - }); + const record = (name: string, result?: T) => + jest.fn().mockImplementation(async () => { + calls.push(name); + return result; + }); + const builds = ['build1', 'build2'].map((id) => ({ + id, + ciBuildId: null, + number: 1, + branchName: 'master', + status: null, + projectId, + updatedAt: new Date(), + createdAt: new Date(), + userId: null, + isRunning: false, + })); + const tx = { + build: { findMany: record('build.findMany', builds), deleteMany: record('build.deleteMany') }, + testRun: { + findMany: record('testRun.findMany', [ + { imageName: 'run1.png', diffName: 'run1.diff.png', imageThumbnailName: null, diffThumbnailName: null }, + { imageName: 'run2.png', diffName: null, imageThumbnailName: 'run2.thumb.png', diffThumbnailName: '' }, + ]), + deleteMany: record('testRun.deleteMany'), + }, + baseline: { + findMany: record('baseline.findMany', [{ baselineName: 'baseline1.png' }]), + deleteMany: record('baseline.deleteMany'), + }, + testVariation: { deleteMany: record('testVariation.deleteMany') }, + project: { delete: record('project.delete', TEST_PROJECT) }, + }; + ctx.prisma.project.findUnique.mockResolvedValue(TEST_PROJECT); + ctx.prisma.$transaction.mockImplementation(transaction ?? ((fn, _options) => fn(tx))); + ctx.deleteImage.mockImplementation(async (name) => calls.push(`deleteImage ${name}`)); + ctx.buildDeleted.mockImplementation((dto) => calls.push(`buildDeleted ${dto.id}`)); + return { ...ctx, calls, tx, builds, transaction: ctx.prisma.$transaction }; + }; + + it('removes rows in FK order inside one transaction, then files, then announces builds', async () => { + const { service, calls, tx, transaction } = await initRemove(); + + const result = await service.remove(projectId); + + expect(calls).toEqual([ + 'build.findMany', + 'testRun.findMany', + 'baseline.findMany', + 'testRun.deleteMany', + 'baseline.deleteMany', + 'testVariation.deleteMany', + 'build.deleteMany', + 'project.delete', + 'deleteImage run1.png', + 'deleteImage run1.diff.png', + 'deleteImage run2.png', + 'deleteImage run2.thumb.png', + 'deleteImage baseline1.png', + 'buildDeleted build1', + 'buildDeleted build2', + ]); + expect(tx.testRun.deleteMany).toHaveBeenCalledWith({ where: { build: { projectId } } }); + expect(tx.baseline.deleteMany).toHaveBeenCalledWith({ where: { testVariation: { projectId } } }); + expect(tx.project.delete).toHaveBeenCalledWith({ where: { id: projectId } }); + expect(transaction).toHaveBeenCalledTimes(1); + expect(transaction).toHaveBeenCalledWith(expect.any(Function), { maxWait: 10000, timeout: 120000 }); + expect(result).toBe(TEST_PROJECT); + }); + + it('announces every build with the build dto', async () => { + const { service, buildDeleted, builds } = await initRemove(); + + await service.remove(projectId); + + expect(buildDeleted).toHaveBeenNthCalledWith(1, new BuildDto(builds[0])); + expect(buildDeleted).toHaveBeenNthCalledWith(2, new BuildDto(builds[1])); + }); - const result = await service.remove('projectId'); + it('deletes no file when the transaction fails', async () => { + const { service, tx, deleteImage, buildDeleted } = await initRemove(); + tx.project.delete.mockRejectedValue(new Error('db down')); - expect(projectFindUniqueMock).toHaveBeenCalledWith({ - where: { id: 'projectId' }, - include: { builds: true, testVariations: true }, + await expect(service.remove(projectId)).rejects.toThrow('db down'); + + expect(deleteImage).not.toHaveBeenCalled(); + expect(buildDeleted).not.toHaveBeenCalled(); + }); + + it('retries when an upload added rows meanwhile (P2003)', async () => { + let attempt = 0; + const { service, transaction, tx } = await initRemove((fn) => { + if (++attempt === 1) { + throw knownRequestError('P2003'); + } + return fn(tx); }); - expect(calls).toEqual(['build build1', 'build build2', 'variation variation1', 'project']); - expect(projectDeleteMock).toHaveBeenCalledWith({ where: { id: 'projectId' } }); - expect(result).toBe(TEST_PROJECT); + + await expect(service.remove(projectId)).resolves.toBe(TEST_PROJECT); + + expect(transaction).toHaveBeenCalledTimes(2); }); - it('stops at the first failing build and leaves the project in place', async () => { - const projectFindUniqueMock = jest.fn().mockResolvedValue({ - ...TEST_PROJECT, - builds: [{ id: 'build1' }, { id: 'build2' }], - testVariations: [{ id: 'variation1' }], + it('answers 409 when uploads keep arriving', async () => { + const { service, transaction, deleteImage } = await initRemove(() => { + throw knownRequestError('P2003'); }); - const buildRemoveMock = jest.fn().mockRejectedValueOnce(new Error('cannot delete')); - const testVariationDeleteMock = jest.fn(); - const projectDeleteMock = jest.fn(); - service = await initService({ - projectFindUniqueMock, - buildRemoveMock, - testVariationDeleteMock, - projectDeleteMock, + + await expect(service.remove(projectId)).rejects.toThrow(ConflictException); + + expect(transaction).toHaveBeenCalledTimes(3); + expect(deleteImage).not.toHaveBeenCalled(); + }); + + it('answers 404 when the project disappears meanwhile (P2025)', async () => { + const { service, transaction } = await initRemove(() => { + throw knownRequestError('P2025'); }); - await expect(service.remove('projectId')).rejects.toThrow('cannot delete'); + await expect(service.remove(projectId)).rejects.toThrow(notFound); - expect(buildRemoveMock).toHaveBeenCalledTimes(1); - expect(testVariationDeleteMock).not.toHaveBeenCalled(); - expect(projectDeleteMock).not.toHaveBeenCalled(); + expect(transaction).toHaveBeenCalledTimes(1); }); - it('throws a TypeError when the project does not exist', async () => { - const projectFindUniqueMock = jest.fn().mockResolvedValue(null); - const projectDeleteMock = jest.fn(); - service = await initService({ projectFindUniqueMock, projectDeleteMock }); + it('answers 404 for an unknown project and deletes nothing', async () => { + const { service, prisma } = await setup(); + prisma.project.findUnique.mockResolvedValue(null); - // NOTE: current behavior — no not-found handling (unlike findOne), the null project is dereferenced - await expect(service.remove('missing')).rejects.toThrow(TypeError); - expect(projectDeleteMock).not.toHaveBeenCalled(); + await expect(service.remove('missing')).rejects.toThrow(notFound); + expect(prisma.project.delete).not.toHaveBeenCalled(); }); }); describe('favourites', () => { const userId = 'b1e2c3d4-0000-4000-8000-000000000001'; - const projectId = 'a9385fc1-884d-4f9f-915e-40da0e7773d5'; - it('lists the ids of the projects the user marked as favourite', async () => { - const projectFindManyMock = jest.fn().mockResolvedValue([{ id: 'p1' }, { id: 'p2' }]); - service = await initService({ projectFindManyMock }); + it.each([ + [ + [{ id: 'p1' }, { id: 'p2' }], + ['p1', 'p2'], + ], + [[], []], + ])('lists the ids of the projects the user marked as favourite (%j)', async (rows, ids) => { + const { service, prisma } = await setup(); + prisma.project.findMany.mockResolvedValue(rows); const result = await service.findFavoriteIds(userId); - expect(projectFindManyMock).toHaveBeenCalledWith({ + expect(prisma.project.findMany).toHaveBeenCalledWith({ where: { favoritedBy: { some: { id: userId } } }, select: { id: true }, }); - expect(result).toEqual(['p1', 'p2']); - }); - - it('lists nothing for a user without favourites', async () => { - const projectFindManyMock = jest.fn().mockResolvedValue([]); - service = await initService({ projectFindManyMock }); - - await expect(service.findFavoriteIds(userId)).resolves.toEqual([]); + expect(result).toEqual(ids); }); it('connects the project to the user favourites', async () => { - const projectFindUniqueMock = jest.fn().mockResolvedValue(TEST_PROJECT); - const userUpdateMock = jest.fn().mockResolvedValue({}); - service = await initService({ projectFindUniqueMock, userUpdateMock }); + const { service, prisma } = await setup(); + prisma.project.findUnique.mockResolvedValue(TEST_PROJECT); await service.addFavorite(userId, projectId); - expect(projectFindUniqueMock).toHaveBeenCalledWith({ where: { id: projectId, name: undefined } }); - expect(userUpdateMock).toHaveBeenCalledWith({ + expect(prisma.project.findUnique).toHaveBeenCalledWith({ where: { id: projectId, name: undefined } }); + expect(prisma.user.update).toHaveBeenCalledWith({ where: { id: userId }, data: { favoriteProjects: { connect: { id: projectId } } }, select: { id: true }, @@ -237,49 +344,34 @@ describe('ProjectsService', () => { }); it('adds a favourite twice without an error', async () => { - const projectFindUniqueMock = jest.fn().mockResolvedValue(TEST_PROJECT); - const userUpdateMock = jest.fn().mockResolvedValue({}); - service = await initService({ projectFindUniqueMock, userUpdateMock }); + const { service, prisma } = await setup(); + prisma.project.findUnique.mockResolvedValue(TEST_PROJECT); await service.addFavorite(userId, projectId); await expect(service.addFavorite(userId, projectId)).resolves.toBeUndefined(); - expect(userUpdateMock).toHaveBeenCalledTimes(2); + expect(prisma.user.update).toHaveBeenCalledTimes(2); }); it('answers 404 and changes nothing when the project does not exist', async () => { - const projectFindUniqueMock = jest.fn().mockResolvedValue(null); - const userUpdateMock = jest.fn(); - service = await initService({ projectFindUniqueMock, userUpdateMock }); + const { service, prisma } = await setup(); + prisma.project.findUnique.mockResolvedValue(null); - await expect(service.addFavorite(userId, projectId)).rejects.toThrow( - new HttpException('Project not found', HttpStatus.NOT_FOUND) - ); - expect(userUpdateMock).not.toHaveBeenCalled(); + await expect(service.addFavorite(userId, projectId)).rejects.toThrow(notFound); + expect(prisma.user.update).not.toHaveBeenCalled(); }); - it('disconnects the project from the user favourites', async () => { - const userUpdateMock = jest.fn().mockResolvedValue({}); - service = await initService({ userUpdateMock }); + it('disconnects the project from the user favourites without looking it up', async () => { + const { service, prisma } = await setup(); await expect(service.removeFavorite(userId, projectId)).resolves.toBeUndefined(); - expect(userUpdateMock).toHaveBeenCalledWith({ + expect(prisma.user.update).toHaveBeenCalledWith({ where: { id: userId }, data: { favoriteProjects: { disconnect: { id: projectId } } }, select: { id: true }, }); - }); - - it('removes a project that is not a favourite without an error', async () => { - // Prisma treats disconnecting an unconnected record as a no-op, so the - // service does no lookup of its own first - const projectFindUniqueMock = jest.fn(); - const userUpdateMock = jest.fn().mockResolvedValue({}); - service = await initService({ projectFindUniqueMock, userUpdateMock }); - - await expect(service.removeFavorite(userId, projectId)).resolves.toBeUndefined(); - expect(projectFindUniqueMock).not.toHaveBeenCalled(); + expect(prisma.project.findUnique).not.toHaveBeenCalled(); }); }); }); diff --git a/src/projects/projects.service.ts b/src/projects/projects.service.ts index 251b08d9..5831d082 100644 --- a/src/projects/projects.service.ts +++ b/src/projects/projects.service.ts @@ -1,12 +1,30 @@ -import { forwardRef, HttpException, HttpStatus, Inject, Injectable, Logger } from '@nestjs/common'; +import { BadRequestException, ConflictException, HttpException, HttpStatus, Injectable, Logger } from '@nestjs/common'; import { CreateProjectDto } from './dto/create-project.dto'; -import { BuildsService } from '../builds/builds.service'; -import { TestVariationsService } from '../test-variations/test-variations.service'; +import { BuildDto } from '../builds/dto/build.dto'; +import { EventsGateway } from '../shared/events/events.gateway'; +import { StaticService } from '../static/static.service'; import { PrismaService } from '../prisma/prisma.service'; -import { Project } from '@prisma/client'; +import { isPrismaError } from '../prisma/prisma-errors'; +import { ImageComparison, Prisma, Project } from '@prisma/client'; import { isUuidShaped } from '../utils/random-key'; +import { testRunImageNames } from '../utils'; import { UpdateProjectDto } from './dto/update-project.dto'; -import type { WrapperType } from '../shared/wrapper-type'; +import { isHddStaticServiceConfigured } from '../static/utils'; + +const MAX_REMOVE_ATTEMPTS = 3; +const REMOVE_TRANSACTION = { maxWait: 10_000, timeout: 120_000 }; + +const projectData = (dto: CreateProjectDto | UpdateProjectDto) => ({ + name: dto.name, + mainBranchName: dto.mainBranchName, + autoApproveFeature: dto.autoApproveFeature, + imageComparison: dto.imageComparison, + imageComparisonConfig: dto.imageComparisonConfig, + maxBuildAllowed: dto.maxBuildAllowed, + maxBranchLifetime: dto.maxBranchLifetime, + bulkApproveVariations: dto.bulkApproveVariations, + bulkApproveGroupBy: dto.bulkApproveGroupBy, +}); @Injectable() export class ProjectsService { @@ -14,15 +32,13 @@ export class ProjectsService { constructor( private prismaService: PrismaService, - @Inject(forwardRef(() => BuildsService)) - private buildsService: WrapperType, - @Inject(forwardRef(() => TestVariationsService)) - private testVariationsService: WrapperType + private staticService: StaticService, + private eventsGateway: EventsGateway ) {} - async findOne(idOrName: string): Promise { + async findOne(idOrName: string): Promise { const isUUID = isUuidShaped(idOrName); - const project: Project | null = await this.prismaService.project.findUnique({ + const project = await this.prismaService.project.findUnique({ where: { id: isUUID ? idOrName : undefined, name: !isUUID ? idOrName : undefined, @@ -48,10 +64,7 @@ export class ProjectsService { } async addFavorite(userId: string, projectId: string): Promise { - // connect alone would fail with a Prisma error for an unknown project; - // findOne turns that into the API's usual 404 await this.findOne(projectId); - // connecting an already connected project is a no-op, so adding twice is fine await this.prismaService.user.update({ where: { id: userId }, data: { favoriteProjects: { connect: { id: projectId } } }, @@ -60,8 +73,6 @@ export class ProjectsService { } async removeFavorite(userId: string, projectId: string): Promise { - // disconnecting a project that is not connected (or does not exist) is a - // no-op, which makes removal idempotent without a lookup await this.prismaService.user.update({ where: { id: userId }, data: { favoriteProjects: { disconnect: { id: projectId } } }, @@ -69,58 +80,64 @@ export class ProjectsService { }); } + private assertComparisonSupported(imageComparison: ImageComparison): void { + if (imageComparison === ImageComparison.odiff && !isHddStaticServiceConfigured()) { + throw new BadRequestException('Odiff can only be used with the HDD static service (STATIC_SERVICE=hdd)'); + } + } + async create(projectDto: CreateProjectDto): Promise { - return this.prismaService.project.create({ - data: { - name: projectDto.name, - mainBranchName: projectDto.mainBranchName, - autoApproveFeature: projectDto.autoApproveFeature, - imageComparison: projectDto.imageComparison, - imageComparisonConfig: projectDto.imageComparisonConfig, - maxBuildAllowed: projectDto.maxBuildAllowed, - maxBranchLifetime: projectDto.maxBranchLifetime, - bulkApproveVariations: projectDto.bulkApproveVariations, - bulkApproveGroupBy: projectDto.bulkApproveGroupBy, - }, - }); + this.assertComparisonSupported(projectDto.imageComparison); + return this.prismaService.project.create({ data: projectData(projectDto) }); } async update(projectDto: UpdateProjectDto): Promise { - return this.prismaService.project.update({ - where: { id: projectDto.id }, - data: { - name: projectDto.name, - mainBranchName: projectDto.mainBranchName, - autoApproveFeature: projectDto.autoApproveFeature, - imageComparison: projectDto.imageComparison, - maxBuildAllowed: projectDto.maxBuildAllowed, - maxBranchLifetime: projectDto.maxBranchLifetime, - imageComparisonConfig: projectDto.imageComparisonConfig, - bulkApproveVariations: projectDto.bulkApproveVariations, - bulkApproveGroupBy: projectDto.bulkApproveGroupBy, - }, - }); + this.assertComparisonSupported(projectDto.imageComparison); + return this.prismaService.project.update({ where: { id: projectDto.id }, data: projectData(projectDto) }); } async remove(id: string): Promise { this.logger.debug(`Going to remove Project ${id}`); - const project = (await this.prismaService.project.findUnique({ - where: { id }, - include: { - builds: true, - testVariations: true, - }, - }))!; - - for (const build of project.builds) { - await this.buildsService.remove(build.id); - } - for (const testVariation of project.testVariations) { - await this.testVariationsService.delete(testVariation.id); + await this.findOne(id); + for (let attempt = 1; ; attempt++) { + try { + const { project, builds, fileNames } = await this.prismaService.$transaction( + (tx) => this.removeRows(tx, id), + REMOVE_TRANSACTION + ); + // Files only after the commit: a failure leaks bytes instead of leaving rows that point at deleted images + await this.staticService.deleteImages(fileNames); + builds.forEach((build) => this.eventsGateway.buildDeleted(new BuildDto(build))); + return project; + } catch (e) { + if (isPrismaError(e, 'P2025')) { + throw new HttpException('Project not found', HttpStatus.NOT_FOUND); + } + if (!isPrismaError(e, 'P2003')) { + throw e; + } + if (attempt === MAX_REMOVE_ATTEMPTS) { + throw new ConflictException(`Project ${id} keeps receiving uploads, try again later`); + } + } } + } - return this.prismaService.project.delete({ - where: { id }, + private async removeRows(tx: Prisma.TransactionClient, projectId: string) { + const ofProjectBuilds = { build: { projectId } }; + const ofProjectVariations = { testVariation: { projectId } }; + const builds = await tx.build.findMany({ where: { projectId } }); + const testRuns = await tx.testRun.findMany({ + where: ofProjectBuilds, + select: { imageName: true, diffName: true, imageThumbnailName: true, diffThumbnailName: true }, }); + const baselines = await tx.baseline.findMany({ where: ofProjectVariations, select: { baselineName: true } }); + await tx.testRun.deleteMany({ where: ofProjectBuilds }); + await tx.baseline.deleteMany({ where: ofProjectVariations }); + await tx.testVariation.deleteMany({ where: { projectId } }); + await tx.build.deleteMany({ where: { projectId } }); + const project = await tx.project.delete({ where: { id: projectId } }); + const fileNames = [...testRuns.flatMap(testRunImageNames), ...baselines.map((b) => b.baselineName)]; + return { project, builds, fileNames }; } } diff --git a/src/redis-io.adapter.spec.ts b/src/redis-io.adapter.spec.ts index f7b87bf5..1f14d1cc 100644 --- a/src/redis-io.adapter.spec.ts +++ b/src/redis-io.adapter.spec.ts @@ -10,29 +10,37 @@ describe('RedisIoAdapter', () => { const createClientMock = createClient as jest.Mock; const createAdapterMock = createAdapter as jest.Mock; - let pubClient: { on: jest.Mock; connect: jest.Mock; duplicate: jest.Mock }; - let subClient: { on: jest.Mock; connect: jest.Mock }; + let pubClient: { on: jest.Mock; connect: jest.Mock; disconnect: jest.Mock; duplicate: jest.Mock }; + let subClient: { on: jest.Mock; connect: jest.Mock; disconnect: jest.Mock }; beforeEach(() => { jest.clearAllMocks(); - subClient = { on: jest.fn(), connect: jest.fn().mockResolvedValue(undefined) }; + subClient = { + on: jest.fn(), + connect: jest.fn().mockResolvedValue(undefined), + disconnect: jest.fn().mockResolvedValue(undefined), + }; pubClient = { on: jest.fn(), connect: jest.fn().mockResolvedValue(undefined), + disconnect: jest.fn().mockResolvedValue(undefined), duplicate: jest.fn(() => subClient), }; createClientMock.mockReturnValue(pubClient); createAdapterMock.mockReturnValue('ADAPTER_CONSTRUCTOR'); }); - const makeAdapter = () => new RedisIoAdapter(undefined as never); + const makeAdapter = () => new RedisIoAdapter(undefined); it('connects a pub/sub client pair and builds the adapter', async () => { const adapter = makeAdapter(); await adapter.connectToRedis('redis://localhost:6379'); - expect(createClientMock).toHaveBeenCalledWith({ url: 'redis://localhost:6379' }); + expect(createClientMock).toHaveBeenCalledWith({ + url: 'redis://localhost:6379', + socket: { reconnectStrategy: expect.any(Function) }, + }); expect(pubClient.duplicate).toHaveBeenCalledTimes(1); expect(pubClient.connect).toHaveBeenCalledTimes(1); expect(subClient.connect).toHaveBeenCalledTimes(1); @@ -58,11 +66,111 @@ describe('RedisIoAdapter', () => { expect(() => subErrorHandler(new Error('boom'))).not.toThrow(); }); - it('rejects when a client fails to connect', async () => { - pubClient.connect.mockRejectedValueOnce(new Error('redis down')); - const adapter = makeAdapter(); + describe('first connection deadline', () => { + const originalEnv = { ...process.env }; + const strategyOf = () => + createClientMock.mock.calls[0][0].socket.reconnectStrategy as (retries: number, cause: Error) => number | Error; + let now: jest.SpyInstance; + const connectPending = (url: string, timeoutMs?: number) => { + let release!: () => void; + pubClient.connect.mockReturnValue(new Promise((resolve) => (release = resolve))); + const pending = makeAdapter().connectToRedis(url, timeoutMs); + return { strategy: strategyOf(), finish: async () => (release(), pending) }; + }; + + beforeEach(() => { + now = jest.spyOn(Date, 'now').mockReturnValue(1_000_000); + }); + + afterEach(() => { + now.mockRestore(); + process.env = { ...originalEnv }; + }); + + it('gives up reconnecting once the connect timeout passes before the first connection', async () => { + const { strategy, finish } = connectPending('redis://:secret@redis-host:6379', 30_000); + + now.mockReturnValue(1_000_000 + 29_999); + expect(strategy(3, new Error('ECONNREFUSED'))).toBeLessThanOrEqual(5000); + now.mockReturnValue(1_000_000 + 30_000); + const result = strategy(4, new Error('ECONNREFUSED')); + + expect(result).toBeInstanceOf(Error); + expect((result as Error).message).toContain('redis-host:6379'); + expect((result as Error).message).toContain('ECONNREFUSED'); + expect((result as Error).message).not.toContain('secret'); + await finish(); + }); + + it('keeps reconnecting after it has connected once', async () => { + await makeAdapter().connectToRedis('redis://localhost:6379', 30_000); + const strategy = strategyOf(); + + now.mockReturnValue(1_000_000 + 10 * 60_000); + + expect(strategy(100, new Error('ECONNRESET'))).toBe(5000); + }); - await expect(adapter.connectToRedis('redis://localhost:6379')).rejects.toThrow('redis down'); + it('backs off by 200 ms per retry up to 5 s', async () => { + await makeAdapter().connectToRedis('redis://localhost:6379', 30_000); + const strategy = strategyOf(); + + expect(strategy(1, new Error('x'))).toBe(200); + expect(strategy(10, new Error('x'))).toBe(2000); + expect(strategy(1000, new Error('x'))).toBe(5000); + }); + + it('does not disable the offline queue', async () => { + await makeAdapter().connectToRedis('redis://localhost:6379'); + + expect(createClientMock.mock.calls[0][0]).not.toHaveProperty('disableOfflineQueue'); + expect(createClientMock.mock.calls[0][0].socket).not.toHaveProperty('disableOfflineQueue'); + }); + + it('disconnects both clients when the first connection fails', async () => { + pubClient.connect.mockRejectedValueOnce(new Error('redis down')); + + await expect(makeAdapter().connectToRedis('redis://localhost:6379')).rejects.toThrow('redis down'); + + expect(pubClient.disconnect).toHaveBeenCalledTimes(1); + expect(subClient.disconnect).toHaveBeenCalledTimes(1); + expect(createAdapterMock).not.toHaveBeenCalled(); + }); + + it('still reports the connect error when a disconnect fails too', async () => { + pubClient.connect.mockRejectedValueOnce(new Error('redis down')); + subClient.disconnect.mockRejectedValueOnce(new Error('already closed')); + + await expect(makeAdapter().connectToRedis('redis://localhost:6379')).rejects.toThrow('redis down'); + }); + + it.each([ + ['2000', 2_000], + ['abc', 30_000], + ['', 30_000], + ['0', 30_000], + ['-1', 30_000], + ])('gives up after the REDIS_CONNECT_TIMEOUT_MS=%p it reads (%p ms)', async (value, ms) => { + process.env.REDIS_CONNECT_TIMEOUT_MS = value; + const { strategy, finish } = connectPending('redis://localhost:6379'); + + now.mockReturnValue(1_000_000 + ms - 1); + expect(typeof strategy(1, new Error('x'))).toBe('number'); + now.mockReturnValue(1_000_000 + ms); + expect(strategy(2, new Error('x'))).toBeInstanceOf(Error); + await finish(); + }); + + it('does not put a malformed url into the error', async () => { + const { strategy, finish } = connectPending('not a url with secret', 1000); + + now.mockReturnValue(1_000_000 + 1000); + const result = strategy(1, new Error('x')) as Error; + + expect(result.message).toContain('REDIS_URL'); + expect(result.message).not.toContain('secret'); + await finish(); + }); }); it('attaches the redis adapter to the created server', async () => { @@ -102,7 +210,7 @@ describe('RedisIoAdapter', () => { const enabled = await RedisIoAdapter.use(app as never, 'redis://localhost:6379'); expect(enabled).toBe(true); - expect(createClientMock).toHaveBeenCalledWith({ url: 'redis://localhost:6379' }); + expect(createClientMock).toHaveBeenCalledWith(expect.objectContaining({ url: 'redis://localhost:6379' })); expect(app.useWebSocketAdapter).toHaveBeenCalledWith(expect.any(RedisIoAdapter)); }); }); diff --git a/src/redis-io.adapter.ts b/src/redis-io.adapter.ts index 888f3766..82d59367 100644 --- a/src/redis-io.adapter.ts +++ b/src/redis-io.adapter.ts @@ -1,45 +1,59 @@ -import { INestApplication, INestApplicationContext, Logger } from '@nestjs/common'; +import { INestApplication, Logger } from '@nestjs/common'; import { IoAdapter } from '@nestjs/platform-socket.io'; import { createAdapter } from '@socket.io/redis-adapter'; import { createClient } from 'redis'; import { ServerOptions } from 'socket.io'; +import { positiveNumber } from './utils/env'; + +const DEFAULT_CONNECT_TIMEOUT_MS = 30_000; +const MAX_RECONNECT_DELAY_MS = 5_000; + +const connectTimeoutFromEnv = (): number => + positiveNumber(process.env.REDIS_CONNECT_TIMEOUT_MS, DEFAULT_CONNECT_TIMEOUT_MS); + +const describeUrl = (url: string): string => { + try { + return new URL(url).host; + } catch { + return 'REDIS_URL'; + } +}; -/** - * Socket.io adapter backed by Redis pub/sub so that events emitted from any - * API instance are delivered to clients connected to every other instance. - * Without it a multi-replica deployment only broadcasts to clients that happen - * to be connected to the same instance that emitted the event. - */ export class RedisIoAdapter extends IoAdapter { private static readonly logger = new Logger(RedisIoAdapter.name); private adapterConstructor!: ReturnType; - constructor(app: INestApplicationContext) { - super(app); - } + async connectToRedis(url: string, connectTimeoutMs = connectTimeoutFromEnv()): Promise { + const startedAt = Date.now(); + let connected = false; + const reconnectStrategy = (retries: number, cause: Error): number | Error => + !connected && Date.now() - startedAt >= connectTimeoutMs + ? new Error(`Redis at ${describeUrl(url)} is not reachable after ${connectTimeoutMs} ms: ${cause.message}`) + : Math.min(retries * 200, MAX_RECONNECT_DELAY_MS); - async connectToRedis(url: string): Promise { - const pubClient = createClient({ url }); + // no disableOfflineQueue: @socket.io/redis-adapter publishes without catch, an offline rejection would crash the process + const pubClient = createClient({ url, socket: { reconnectStrategy } }); const subClient = pubClient.duplicate(); pubClient.on('error', (err) => RedisIoAdapter.logger.error(`Redis pub client error: ${err}`)); subClient.on('error', (err) => RedisIoAdapter.logger.error(`Redis sub client error: ${err}`)); - await Promise.all([pubClient.connect(), subClient.connect()]); + try { + await Promise.all([pubClient.connect(), subClient.connect()]); + } catch (error) { + await Promise.allSettled([pubClient.disconnect(), subClient.disconnect()]); + throw error; + } + connected = true; this.adapterConstructor = createAdapter(pubClient, subClient); } - createIOServer(port: number, options?: ServerOptions): any { + override createIOServer(port: number, options?: ServerOptions): any { const server = super.createIOServer(port, options); server.adapter(this.adapterConstructor); return server; } - /** - * Installs the Redis-backed socket.io adapter on the app when `redisUrl` is - * provided. Returns whether the Redis adapter was enabled (false = default - * in-memory adapter, i.e. single-instance mode). - */ static async use(app: INestApplication, redisUrl?: string): Promise { if (!redisUrl) { this.logger.log('REDIS_URL not set — socket.io running with the in-memory adapter (single instance only)'); diff --git a/src/shared/api-file.decorator.spec.ts b/src/shared/api-file.decorator.spec.ts index e47c76b4..770ad756 100644 --- a/src/shared/api-file.decorator.spec.ts +++ b/src/shared/api-file.decorator.spec.ts @@ -1,6 +1,5 @@ import { ApiFile } from './api-file.decorator'; -// @nestjs/swagger does not export its metadata keys const API_MODEL_PROPERTIES = 'swagger/apiModelProperties'; const API_MODEL_PROPERTIES_ARRAY = 'swagger/apiModelPropertiesArray'; @@ -39,8 +38,7 @@ describe('ApiFile', () => { }); }); - // NOTE: current behavior — every option other than isArray is dropped. - it('ignores options other than isArray', () => { + it('describes only isArray in the schema', () => { expect(propertyMetadata('explicitSingle')).toEqual({ type: 'string', format: 'binary', isArray: false }); }); }); diff --git a/src/shared/current-user.decorator.spec.ts b/src/shared/current-user.decorator.spec.ts index 4e24bf3e..aab26dae 100644 --- a/src/shared/current-user.decorator.spec.ts +++ b/src/shared/current-user.decorator.spec.ts @@ -2,7 +2,6 @@ import { ExecutionContext } from '@nestjs/common'; import { ROUTE_ARGS_METADATA } from '@nestjs/common/constants'; import { CurrentUser } from './current-user.decorator'; -// a param decorator only exposes its factory through the route metadata const getFactory = () => { class Controller { handler(@CurrentUser() _user: unknown) {} diff --git a/src/shared/dto/baseline.dto.ts b/src/shared/dto/baseline.dto.ts index 1598b90c..8c20252b 100644 --- a/src/shared/dto/baseline.dto.ts +++ b/src/shared/dto/baseline.dto.ts @@ -2,7 +2,6 @@ import { ApiProperty, OmitType } from '@nestjs/swagger'; import { TestRunDto } from '../../test-runs/dto/testRun.dto'; import { UserDto } from '../../users/dto/user.dto'; -// who approved a baseline, as anyone who can open a variation sees them export class BaselineUserDto extends OmitType(UserDto, ['apiKey'] as const) {} export class BaselineDto { diff --git a/src/shared/events/debounce.spec.ts b/src/shared/events/debounce.spec.ts new file mode 100644 index 00000000..fc820059 --- /dev/null +++ b/src/shared/events/debounce.spec.ts @@ -0,0 +1,94 @@ +import { debounce } from './debounce'; + +const setup = () => { + const fn = jest.fn(); + return { fn, debounced: debounce(fn, 1500, 3000) }; +}; + +describe('debounce', () => { + beforeEach(() => { + jest.useFakeTimers(); + }); + + afterEach(() => { + jest.clearAllTimers(); + jest.useRealTimers(); + }); + + it('runs the first call at once', () => { + const { fn, debounced } = setup(); + debounced(); + + expect(fn).toHaveBeenCalledTimes(1); + }); + + it('does not run again after the wait when it was called once', () => { + const { fn, debounced } = setup(); + debounced(); + jest.advanceTimersByTime(5000); + + expect(fn).toHaveBeenCalledTimes(1); + }); + + it('runs once more 1500ms after the last of the calls that followed the first', () => { + const { fn, debounced } = setup(); + + debounced(); + debounced(); + jest.advanceTimersByTime(1000); + debounced(); + jest.advanceTimersByTime(1499); + expect(fn).toHaveBeenCalledTimes(1); + + jest.advanceTimersByTime(1); + expect(fn).toHaveBeenCalledTimes(2); + }); + + it('runs a steady stream of calls at least every maxWait', () => { + const { fn, debounced } = setup(); + + debounced(); + for (let elapsed = 0; elapsed < 9000; elapsed += 1000) { + jest.advanceTimersByTime(1000); + debounced(); + } + + expect(fn).toHaveBeenCalledTimes(4); + }); + + it('runs at once again when called after a quiet period', () => { + const { fn, debounced } = setup(); + + debounced(); + jest.advanceTimersByTime(1500); + debounced(); + + expect(fn).toHaveBeenCalledTimes(2); + }); + + it('runs at once when maxWait has passed before its timer could fire', () => { + const { fn, debounced } = setup(); + + debounced(); + debounced(); + jest.setSystemTime(Date.now() + 3000); + debounced(); + expect(fn).toHaveBeenCalledTimes(2); + + jest.advanceTimersByTime(5000); + expect(fn).toHaveBeenCalledTimes(2); + }); + + it('drops the pending run when cancelled and starts afresh afterwards', () => { + const { fn, debounced } = setup(); + + debounced(); + debounced(); + debounced.cancel(); + jest.advanceTimersByTime(5000); + expect(fn).toHaveBeenCalledTimes(1); + + debounced(); + expect(fn).toHaveBeenCalledTimes(2); + }); +}); diff --git a/src/shared/events/debounce.ts b/src/shared/events/debounce.ts new file mode 100644 index 00000000..d0204ab9 --- /dev/null +++ b/src/shared/events/debounce.ts @@ -0,0 +1,57 @@ +export interface Debounced { + (): void; + cancel(): void; +} + +export function debounce(fn: () => void, wait: number, maxWait: number): Debounced { + let timer: NodeJS.Timeout | undefined; + let lastCallTime: number | undefined; + let lastInvokeTime = 0; + let pending = false; + + const invoke = (time: number) => { + lastInvokeTime = time; + pending = false; + fn(); + }; + + const shouldInvoke = (time: number) => { + if (lastCallTime === undefined) return true; + const sinceLastCall = time - lastCallTime; + return sinceLastCall >= wait || time - lastInvokeTime >= maxWait; + }; + + const timerExpired = () => { + const time = Date.now(); + if (shouldInvoke(time)) { + timer = undefined; + if (pending) invoke(time); + return; + } + timer = setTimeout(timerExpired, Math.min(wait - (time - lastCallTime!), maxWait - (time - lastInvokeTime))); + }; + + const debounced = () => { + const time = Date.now(); + const isInvoking = shouldInvoke(time); + pending = true; + lastCallTime = time; + if (!isInvoking) { + timer ??= setTimeout(timerExpired, wait); + return; + } + if (timer !== undefined) clearTimeout(timer); + timer = setTimeout(timerExpired, wait); + invoke(time); + }; + + debounced.cancel = () => { + if (timer !== undefined) clearTimeout(timer); + timer = undefined; + lastCallTime = undefined; + lastInvokeTime = 0; + pending = false; + }; + + return debounced; +} diff --git a/src/shared/events/events.gateway.spec.ts b/src/shared/events/events.gateway.spec.ts index 32ddae0b..8d32d50b 100644 --- a/src/shared/events/events.gateway.spec.ts +++ b/src/shared/events/events.gateway.spec.ts @@ -5,16 +5,14 @@ import { Server } from 'socket.io'; import { EventsGateway } from './events.gateway'; import { PrismaService } from '../../prisma/prisma.service'; import { BuildDto } from '../../builds/dto/build.dto'; +import { TestRunDto } from '../../test-runs/dto/testRun.dto'; import { generateTestRun, TEST_BUILD } from '../../_data_'; -// the debounced build_updated emit resolves Prisma before emitting; setImmediate -// is left real so the pending promise chain can be drained const flushPromises = () => new Promise((resolve) => setImmediate(resolve)); -const initGateway = async ({ - buildFindManyMock = jest.fn().mockResolvedValue([]), - testRunGroupByMock = jest.fn().mockResolvedValue([]), -}) => { +const initGateway = async () => { + const buildFindManyMock = jest.fn().mockResolvedValue([]); + const testRunGroupByMock = jest.fn().mockResolvedValue([]); const module: TestingModule = await Test.createTestingModule({ providers: [ EventsGateway, @@ -31,7 +29,7 @@ const initGateway = async ({ const gateway = module.get(EventsGateway); const emitMock = jest.fn(); gateway.server = { emit: emitMock } as unknown as Server; - return { gateway, emitMock }; + return { gateway, emitMock, buildFindManyMock, testRunGroupByMock }; }; const emitsOf = (emitMock: jest.Mock, event: string) => @@ -51,24 +49,17 @@ describe('EventsGateway', () => { jest.useRealTimers(); }); - it('emits build_created straight away with the build it is given', async () => { - const { gateway, emitMock } = await initGateway({}); + it.each([ + ['buildCreated', 'build_created'], + ['buildDeleted', 'build_deleted'], + ] as const)('%s emits %s straight away with the build it is given', async (method, event) => { + const { gateway, emitMock } = await initGateway(); const build = new BuildDto(TEST_BUILD); - gateway.buildCreated(build); + gateway[method](build); expect(emitMock).toHaveBeenCalledTimes(1); - expect(emitMock).toHaveBeenCalledWith('build_created', build); - }); - - it('emits build_deleted straight away with the build it is given', async () => { - const { gateway, emitMock } = await initGateway({}); - const build = new BuildDto(TEST_BUILD); - - gateway.buildDeleted(build); - - expect(emitMock).toHaveBeenCalledTimes(1); - expect(emitMock).toHaveBeenCalledWith('build_deleted', build); + expect(emitMock).toHaveBeenCalledWith(event, build); }); describe.each([ @@ -77,24 +68,24 @@ describe('EventsGateway', () => { ['testRunDeleted', 'testRun_deleted'], ] as const)('%s', (method, event) => { it('emits the first test run on the leading edge', async () => { - const { gateway, emitMock } = await initGateway({}); + const { gateway, emitMock } = await initGateway(); gateway[method](testRunA); - expect(emitsOf(emitMock, event)).toEqual([[testRunA]]); + expect(emitsOf(emitMock, event)).toEqual([[new TestRunDto(testRunA)]]); }); it('does not emit again after the wait when only one test run came in', async () => { - const { gateway, emitMock } = await initGateway({}); + const { gateway, emitMock } = await initGateway(); gateway[method](testRunA); jest.advanceTimersByTime(5000); - expect(emitsOf(emitMock, event)).toEqual([[testRunA]]); + expect(emitsOf(emitMock, event)).toEqual([[new TestRunDto(testRunA)]]); }); it('batches the test runs that follow the leading edge into one emit 1500ms after the last call', async () => { - const { gateway, emitMock } = await initGateway({}); + const { gateway, emitMock } = await initGateway(); gateway[method](testRunA); gateway[method](testRunB); @@ -102,15 +93,18 @@ describe('EventsGateway', () => { gateway[method](testRunC); jest.advanceTimersByTime(1499); - expect(emitsOf(emitMock, event)).toEqual([[testRunA]]); + expect(emitsOf(emitMock, event)).toEqual([[new TestRunDto(testRunA)]]); jest.advanceTimersByTime(1); - expect(emitsOf(emitMock, event)).toEqual([[testRunA], [testRunB, testRunC]]); + expect(emitsOf(emitMock, event)).toEqual([ + [new TestRunDto(testRunA)], + [new TestRunDto(testRunB), new TestRunDto(testRunC)], + ]); }); it('flushes a steady stream of test runs at the 3000ms max wait', async () => { - const { gateway, emitMock } = await initGateway({}); + const { gateway, emitMock } = await initGateway(); gateway[method](testRunA); jest.advanceTimersByTime(1000); @@ -119,16 +113,18 @@ describe('EventsGateway', () => { gateway[method](testRunC); jest.advanceTimersByTime(999); - expect(emitsOf(emitMock, event)).toEqual([[testRunA]]); + expect(emitsOf(emitMock, event)).toEqual([[new TestRunDto(testRunA)]]); jest.advanceTimersByTime(1); - expect(emitsOf(emitMock, event)).toEqual([[testRunA], [testRunB, testRunC]]); + expect(emitsOf(emitMock, event)).toEqual([ + [new TestRunDto(testRunA)], + [new TestRunDto(testRunB), new TestRunDto(testRunC)], + ]); }); it('queues a build update for the build of the test run', async () => { - const buildFindManyMock = jest.fn().mockResolvedValue([]); - const { gateway } = await initGateway({ buildFindManyMock }); + const { gateway, buildFindManyMock } = await initGateway(); gateway[method](testRunC); @@ -138,9 +134,8 @@ describe('EventsGateway', () => { describe('testRunDeleted', () => { it('drops the deleted test run from the created and updated queues', async () => { - const { gateway, emitMock } = await initGateway({}); + const { gateway, emitMock } = await initGateway(); - // leading edges go out at once; the second calls are queued gateway.testRunCreated(testRunA); gateway.testRunCreated(testRunB); gateway.testRunCreated(testRunC); @@ -150,12 +145,12 @@ describe('EventsGateway', () => { jest.advanceTimersByTime(1500); - expect(emitsOf(emitMock, 'testRun_created')).toEqual([[testRunA], [testRunC]]); - expect(emitsOf(emitMock, 'testRun_deleted')).toEqual([[testRunB]]); + expect(emitsOf(emitMock, 'testRun_created')).toEqual([[new TestRunDto(testRunA)], [new TestRunDto(testRunC)]]); + expect(emitsOf(emitMock, 'testRun_deleted')).toEqual([[new TestRunDto(testRunB)]]); }); it('keeps the other queued updates when one updated test run is deleted', async () => { - const { gateway, emitMock } = await initGateway({}); + const { gateway, emitMock } = await initGateway(); gateway.testRunUpdated(testRunA); gateway.testRunUpdated(testRunB); @@ -164,11 +159,11 @@ describe('EventsGateway', () => { jest.advanceTimersByTime(1500); - expect(emitsOf(emitMock, 'testRun_updated')).toEqual([[testRunA], [testRunC]]); + expect(emitsOf(emitMock, 'testRun_updated')).toEqual([[new TestRunDto(testRunA)], [new TestRunDto(testRunC)]]); }); - it('still emits an empty batch once every queued entry was deleted', async () => { - const { gateway, emitMock } = await initGateway({}); + it('emits nothing once every queued entry was deleted', async () => { + const { gateway, emitMock } = await initGateway(); gateway.testRunUpdated(testRunA); gateway.testRunUpdated(testRunB); @@ -176,18 +171,87 @@ describe('EventsGateway', () => { jest.advanceTimersByTime(1500); - // NOTE: current behavior — the trailing debounce call still fires with the - // now-empty queue, so clients receive `testRun_updated` with []. - expect(emitsOf(emitMock, 'testRun_updated')).toEqual([[testRunA], []]); + expect(emitsOf(emitMock, 'testRun_updated')).toEqual([[new TestRunDto(testRunA)]]); + }); + }); + + describe('payload', () => { + it.each([ + ['testRunCreated', 'testRun_created'], + ['testRunUpdated', 'testRun_updated'], + ['testRunDeleted', 'testRun_deleted'], + ] as const)('%s emits TestRunDto, not the database row', async (method, event) => { + const { gateway, emitMock } = await initGateway(); + const row = generateTestRun({ id: 'row', changeSignature: '[0.5]', pixelMisMatchCount: 7 }); + + gateway[method](row); + + const [[payload]] = emitsOf(emitMock, event); + expect(payload).toBeInstanceOf(TestRunDto); + expect(payload).toStrictEqual(new TestRunDto(row)); + expect(payload).not.toHaveProperty('changeSignature'); + expect(payload).not.toHaveProperty('createdAt'); + }); + }); + + describe('ordering and de-duplication', () => { + it('sends an update of a run still queued as created with the created event, not before it', async () => { + const { gateway, emitMock } = await initGateway(); + const diffed = { ...testRunB, status: TestStatus.unresolved, diffName: 'diff.png' }; + + gateway.testRunCreated(testRunA); + gateway.testRunCreated(testRunB); + gateway.testRunUpdated(diffed); + jest.advanceTimersByTime(1500); + + expect(emitsOf(emitMock, 'testRun_updated')).toEqual([]); + expect(emitsOf(emitMock, 'testRun_created')).toEqual([[new TestRunDto(testRunA)], [new TestRunDto(diffed)]]); + }); + + it('sends an update of a run already announced as created as an update', async () => { + const { gateway, emitMock } = await initGateway(); + const diffed = { ...testRunA, status: TestStatus.unresolved }; + + gateway.testRunCreated(testRunA); + gateway.testRunUpdated(diffed); + + expect(emitsOf(emitMock, 'testRun_created')).toEqual([[new TestRunDto(testRunA)]]); + expect(emitsOf(emitMock, 'testRun_updated')).toEqual([[new TestRunDto(diffed)]]); + }); + + it('keeps only the latest queued update of a run, in the order the runs first came in', async () => { + const { gateway, emitMock } = await initGateway(); + const first = { ...testRunB, status: TestStatus.unresolved }; + const latest = { ...testRunB, status: TestStatus.approved }; + + gateway.testRunUpdated(testRunA); + gateway.testRunUpdated(first); + gateway.testRunUpdated(testRunC); + gateway.testRunUpdated(latest); + jest.advanceTimersByTime(1500); + + expect(emitsOf(emitMock, 'testRun_updated')).toEqual([ + [new TestRunDto(testRunA)], + [new TestRunDto(latest), new TestRunDto(testRunC)], + ]); + }); + + it('announces a deleted run once even when it is deleted twice in a batch', async () => { + const { gateway, emitMock } = await initGateway(); + + gateway.testRunDeleted(testRunA); + gateway.testRunDeleted(testRunB); + gateway.testRunDeleted(testRunB); + jest.advanceTimersByTime(1500); + + expect(emitsOf(emitMock, 'testRun_deleted')).toEqual([[new TestRunDto(testRunA)], [new TestRunDto(testRunB)]]); }); }); describe('onApplicationShutdown', () => { it('drops the batches still waiting for their debounce', async () => { - const buildFindManyMock = jest.fn().mockResolvedValue([]); - const { gateway, emitMock } = await initGateway({ buildFindManyMock }); + const { gateway, emitMock, buildFindManyMock } = await initGateway(); - // the leading edge goes out at once; the second call waits for the trailing edge gateway.testRunCreated(testRunA); gateway.testRunCreated(testRunB); await flushPromises(); @@ -206,22 +270,19 @@ describe('EventsGateway', () => { describe('build_updated', () => { it('emits the build with stats aggregated by the database on the leading edge', async () => { const build: Build = { ...TEST_BUILD, id: 'buildX' }; - const buildFindManyMock = jest.fn().mockResolvedValue([build]); - const testRunGroupByMock = jest - .fn() - .mockResolvedValueOnce([ - { buildId: 'buildX', status: TestStatus.ok, _count: { _all: 3 } }, - { buildId: 'buildX', status: TestStatus.failed, _count: { _all: 1 } }, - ]) - .mockResolvedValueOnce([{ buildId: 'buildX' }]); - const { gateway, emitMock } = await initGateway({ buildFindManyMock, testRunGroupByMock }); + const { gateway, emitMock, buildFindManyMock, testRunGroupByMock } = await initGateway(); + buildFindManyMock.mockResolvedValue([build]); + testRunGroupByMock.mockResolvedValueOnce([ + { buildId: 'buildX', status: TestStatus.ok, merge: false, _count: { _all: 3 } }, + { buildId: 'buildX', status: TestStatus.failed, merge: true, _count: { _all: 1 } }, + ]); gateway.buildUpdated('buildX'); await flushPromises(); expect(buildFindManyMock).toHaveBeenCalledWith({ where: { id: { in: ['buildX'] } } }); expect(testRunGroupByMock).toHaveBeenCalledWith({ - by: ['buildId', 'status'], + by: ['buildId', 'status', 'merge'], where: { buildId: { in: ['buildX'] } }, _count: { _all: true }, }); @@ -234,8 +295,8 @@ describe('EventsGateway', () => { }); it('logs a failed read instead of leaving the rejection unhandled', async () => { - const buildFindManyMock = jest.fn().mockRejectedValue(new Error('Connection terminated')); - const { gateway, emitMock } = await initGateway({ buildFindManyMock }); + const { gateway, emitMock, buildFindManyMock } = await initGateway(); + buildFindManyMock.mockRejectedValue(new Error('Connection terminated')); const logError = jest.spyOn(Logger.prototype, 'error').mockImplementation(); const unhandled = jest.fn(); process.on('unhandledRejection', unhandled); @@ -252,9 +313,7 @@ describe('EventsGateway', () => { }); it('deduplicates the queued build ids before reading them', async () => { - const buildFindManyMock = jest.fn().mockResolvedValue([]); - const testRunGroupByMock = jest.fn().mockResolvedValue([]); - const { gateway } = await initGateway({ buildFindManyMock, testRunGroupByMock }); + const { gateway, buildFindManyMock, testRunGroupByMock } = await initGateway(); gateway.buildUpdated('first'); gateway.testRunCreated(testRunA); @@ -268,16 +327,17 @@ describe('EventsGateway', () => { expect(buildFindManyMock).toHaveBeenNthCalledWith(1, { where: { id: { in: ['first'] } } }); expect(buildFindManyMock).toHaveBeenNthCalledWith(2, { where: { id: { in: ['buildX', 'buildY'] } } }); expect(testRunGroupByMock).toHaveBeenCalledWith({ - by: ['buildId'], - where: { buildId: { in: ['buildX', 'buildY'] }, merge: true }, + by: ['buildId', 'status', 'merge'], + where: { buildId: { in: ['buildX', 'buildY'] } }, + _count: { _all: true }, }); }); it('emits the builds in the order Prisma returns them, without stats for builds it did not ask about', async () => { const known: Build = { ...TEST_BUILD, id: 'buildX' }; const unknown: Build = { ...TEST_BUILD, id: 'other' }; - const buildFindManyMock = jest.fn().mockResolvedValue([unknown, known]); - const { gateway, emitMock } = await initGateway({ buildFindManyMock }); + const { gateway, emitMock, buildFindManyMock } = await initGateway(); + buildFindManyMock.mockResolvedValue([unknown, known]); gateway.buildUpdated('buildX'); await flushPromises(); @@ -290,13 +350,13 @@ describe('EventsGateway', () => { ]); }); - it('emits an empty list when no queued build exists any more', async () => { - const { gateway, emitMock } = await initGateway({}); + it('emits nothing when no queued build exists any more', async () => { + const { gateway, emitMock } = await initGateway(); gateway.buildUpdated('gone'); await flushPromises(); - expect(emitsOf(emitMock, 'build_updated')).toEqual([[]]); + expect(emitsOf(emitMock, 'build_updated')).toEqual([]); }); }); }); diff --git a/src/shared/events/events.gateway.ts b/src/shared/events/events.gateway.ts index 87ae09e3..ae3dc893 100644 --- a/src/shared/events/events.gateway.ts +++ b/src/shared/events/events.gateway.ts @@ -4,8 +4,15 @@ import { Server } from 'socket.io'; import { TestRun } from '@prisma/client'; import { BuildDto } from '../../builds/dto/build.dto'; import { getBuildsStats } from '../../builds/build-stats'; -import { debounce } from 'lodash'; +import { TestRunDto } from '../../test-runs/dto/testRun.dto'; import { PrismaService } from '../../prisma/prisma.service'; +import { Debounced, debounce } from './debounce'; + +const DEBOUNCE_WAIT = 1500; +const MAX_WAIT = 3000; + +const TEST_RUN_EVENTS = ['testRun_created', 'testRun_updated', 'testRun_deleted'] as const; +type TestRunEvent = (typeof TEST_RUN_EVENTS)[number]; @WebSocketGateway({ cors: true }) export class EventsGateway implements OnApplicationShutdown { @@ -16,39 +23,40 @@ export class EventsGateway implements OnApplicationShutdown { constructor(private prismaService: PrismaService) {} - private debounceTimeout = 1500; - private maxWait = 3000; - private testRunsCreatedQueued: Array = []; - private testRunsDeletedQueued: Array = []; - private testRunsUpdatedQueued: Array = []; - private buildsUpdatedQueued: Array = []; + private readonly testRunQueues = Object.fromEntries( + TEST_RUN_EVENTS.map((event) => [event, new Map()]) + ) as Record>; + private readonly testRunEmitters = Object.fromEntries( + TEST_RUN_EVENTS.map((event) => [event, debounce(() => this.emitQueued(event), DEBOUNCE_WAIT, MAX_WAIT)]) + ) as Record; + private readonly buildsUpdatedQueued = new Set(); buildCreated(build: BuildDto): void { this.server.emit('build_created', build); } buildUpdated(id: string): void { - this.buildsUpdatedQueued.push(id); + this.buildsUpdatedQueued.add(id); this.buildUpdatedDebounced(); } testRunCreated(testRun: TestRun): void { - this.testRunsCreatedQueued.push(testRun); - this.testRunCreatedDebounced(); + this.enqueueTestRun('testRun_created', testRun); this.buildUpdated(testRun.buildId); } testRunUpdated(testRun: TestRun): void { - this.testRunsUpdatedQueued.push(testRun); - this.testRunUpdatedDebounced(); + this.enqueueTestRun( + this.testRunQueues.testRun_created.has(testRun.id) ? 'testRun_created' : 'testRun_updated', + testRun + ); this.buildUpdated(testRun.buildId); } testRunDeleted(testRun: TestRun): void { - this.testRunsCreatedQueued = this.testRunsCreatedQueued.filter((tr) => tr.id !== testRun.id); - this.testRunsUpdatedQueued = this.testRunsUpdatedQueued.filter((tr) => tr.id !== testRun.id); - this.testRunsDeletedQueued.push(testRun); - this.testRunDeletedDebounced(); + this.testRunQueues.testRun_created.delete(testRun.id); + this.testRunQueues.testRun_updated.delete(testRun.id); + this.enqueueTestRun('testRun_deleted', testRun); this.buildUpdated(testRun.buildId); } @@ -56,77 +64,43 @@ export class EventsGateway implements OnApplicationShutdown { this.server.emit('build_deleted', buildDto); } - // A batch still waiting for its debounce would otherwise fire after shutdown - // and query a database pool that is already closed. Runs once the HTTP - // server has drained, so no request queues another batch after it. + // Runs after the HTTP server drained: a pending batch would otherwise hit a closed database pool onApplicationShutdown(): void { - this.testRunCreatedDebounced.cancel(); - this.testRunUpdatedDebounced.cancel(); - this.testRunDeletedDebounced.cancel(); + TEST_RUN_EVENTS.forEach((event) => this.testRunEmitters[event].cancel()); this.buildUpdatedDebounced.cancel(); } - private testRunUpdatedDebounced = debounce( - () => { - this.server.emit('testRun_updated', this.testRunsUpdatedQueued); - this.testRunsUpdatedQueued = []; - }, - this.debounceTimeout, - { - leading: true, - maxWait: this.maxWait, - } - ); - - private testRunCreatedDebounced = debounce( - () => { - this.server.emit('testRun_created', this.testRunsCreatedQueued); - this.testRunsCreatedQueued = []; - }, - this.debounceTimeout, - { - leading: true, - maxWait: this.maxWait, - } - ); + private enqueueTestRun(event: TestRunEvent, testRun: TestRun): void { + this.testRunQueues[event].set(testRun.id, new TestRunDto(testRun)); + this.testRunEmitters[event](); + } - private testRunDeletedDebounced = debounce( - () => { - this.server.emit('testRun_deleted', this.testRunsDeletedQueued); - this.testRunsDeletedQueued = []; - }, - this.debounceTimeout, - { - leading: true, - maxWait: this.maxWait, - } - ); + private emitQueued(event: TestRunEvent): void { + const queue = this.testRunQueues[event]; + if (queue.size === 0) return; + const testRuns = [...queue.values()]; + queue.clear(); + this.server.emit(event, testRuns); + } private buildUpdatedDebounced = debounce( () => { - // Deduplicate: during ingestion the queue holds one entry per test run. - // Stats are aggregated in the database — a build can hold thousands of - // test runs and this fires on every debounced event burst. - const buildIds = [...new Set(this.buildsUpdatedQueued)]; - this.buildsUpdatedQueued = []; + const buildIds = [...this.buildsUpdatedQueued]; + this.buildsUpdatedQueued.clear(); Promise.all([ this.prismaService.build.findMany({ where: { id: { in: buildIds } } }), getBuildsStats(this.prismaService, buildIds), ]) .then(([builds, stats]) => { + if (builds.length === 0) return; this.server.emit( 'build_updated', builds.map((build) => new BuildDto(build, stats.get(build.id))) ); }) - // nothing awaits this chain: unhandled, a failed query would take the - // whole process down .catch((error) => this.logger.error(`Cannot emit build_updated: ${error}`)); }, - this.debounceTimeout, - { - leading: true, - maxWait: this.maxWait, - } + DEBOUNCE_WAIT, + MAX_WAIT ); } diff --git a/src/shared/fite-to-body.interceptor.spec.ts b/src/shared/fite-to-body.interceptor.spec.ts index ffcdbd43..665aaf4f 100644 --- a/src/shared/fite-to-body.interceptor.spec.ts +++ b/src/shared/fite-to-body.interceptor.spec.ts @@ -10,6 +10,16 @@ const contextWith = (request: object) => const handler = (): CallHandler & { handle: jest.Mock } => ({ handle: jest.fn().mockReturnValue(of('handled')) }); +const expectUntouched = (interceptor: { intercept: FilesToBodyInterceptor['intercept'] }, request: object) => { + const before = JSON.parse(JSON.stringify(request)); + const next = handler(); + + interceptor.intercept(contextWith(request), next); + + expect(JSON.parse(JSON.stringify(request))).toEqual(before); + expect(next.handle).toHaveBeenCalledTimes(1); +}; + describe('FilesToBodyInterceptor', () => { const interceptor = new FilesToBodyInterceptor(); @@ -35,8 +45,6 @@ describe('FilesToBodyInterceptor', () => { expect(request.body.a).toEqual([existing, added]); }); - // NOTE: current behavior — a non-array body value under the same field name - // has `.push` called on it, so a plain string value makes the interceptor throw. it('throws when the body already has a non-array value for the field', () => { const request = { body: { a: 'text' }, files: [file('a')] }; @@ -49,13 +57,7 @@ describe('FilesToBodyInterceptor', () => { ['an empty files array', { body: {}, files: [] }], ['files given as a field map', { body: {}, files: { a: [file('a')] } }], ])('leaves the request untouched with %s', (_label, request) => { - const before = JSON.parse(JSON.stringify(request)); - const next = handler(); - - interceptor.intercept(contextWith(request), next); - - expect(JSON.parse(JSON.stringify(request))).toEqual(before); - expect(next.handle).toHaveBeenCalledTimes(1); + expectUntouched(interceptor, request); }); }); @@ -86,12 +88,6 @@ describe('FileToBodyInterceptor', () => { ['no file', { body: {} }], ['a file without a field name', { body: {}, file: { originalname: 'x.png' } }], ])('leaves the request untouched with %s', (_label, request) => { - const before = JSON.parse(JSON.stringify(request)); - const next = handler(); - - interceptor.intercept(contextWith(request), next); - - expect(JSON.parse(JSON.stringify(request))).toEqual(before); - expect(next.handle).toHaveBeenCalledTimes(1); + expectUntouched(interceptor, request); }); }); diff --git a/src/shared/tasks/tasks.service.spec.ts b/src/shared/tasks/tasks.service.spec.ts index bcf475da..6d91f0e2 100644 --- a/src/shared/tasks/tasks.service.spec.ts +++ b/src/shared/tasks/tasks.service.spec.ts @@ -1,10 +1,9 @@ import { CLEANUP_LOCK_KEY, TasksService } from './tasks.service'; import { generateTestVariation, TEST_PROJECT } from '../../_data_'; -import { Test, TestingModule } from '@nestjs/testing'; +import { Test } from '@nestjs/testing'; import { PrismaService } from '../../prisma/prisma.service'; import { TestVariationsService } from '../../test-variations/test-variations.service'; -// the dedicated pg connection the cleanup takes its lock on const lockClient = (locked: boolean) => ({ on: jest.fn(), connect: jest.fn().mockResolvedValue(undefined), @@ -12,88 +11,72 @@ const lockClient = (locked: boolean) => ({ end: jest.fn().mockResolvedValue(undefined), }); -const initService = async ({ - client = lockClient(true), - projectFindManyMock = jest.fn().mockResolvedValue([]), - testVariationFindManyMock = jest.fn(), - testVariationsDeleteMock = jest.fn(), -}) => { - const module: TestingModule = await Test.createTestingModule({ +const secondProject = { ...TEST_PROJECT, id: '2', name: 'Second Project' }; +const variations = (...ids: string[]) => ids.map((id) => generateTestVariation({ id })); + +const setup = async (client = lockClient(true)) => { + const projectFindMany = jest.fn().mockResolvedValue([]); + const variationFindMany = jest.fn(); + const variationFindFirst = jest.fn(async ({ where }: { where: { id: string } }): Promise<{ id: string } | null> => ({ + id: where.id, + })); + const variationDelete = jest.fn(); + const module = await Test.createTestingModule({ providers: [ TasksService, { provide: PrismaService, useValue: { dedicatedClient: () => client, - testVariation: { - findMany: testVariationFindManyMock, - }, - project: { - findMany: projectFindManyMock, - }, - }, - }, - { - provide: TestVariationsService, - useValue: { - delete: testVariationsDeleteMock, + testVariation: { findMany: variationFindMany, findFirst: variationFindFirst }, + project: { findMany: projectFindMany }, }, }, + { provide: TestVariationsService, useValue: { delete: variationDelete } }, ], }).compile(); - return module.get(TasksService); + const service = module.get(TasksService); + const spyLogger = (method: 'log' | 'warn') => jest.spyOn((service as any).logger, method).mockImplementation(); + return { service, client, projectFindMany, variationFindMany, variationFindFirst, variationDelete, spyLogger }; }; describe('cleanOldTestVariations', () => { - let service: TasksService; - afterEach(() => { jest.useRealTimers(); }); it('findMany', async () => { - // .Arrange const project = TEST_PROJECT; const testVariation = generateTestVariation(); - const projectFindManyMock = jest.fn().mockResolvedValueOnce([project]); - const testVariationFindManyMock = jest.fn().mockResolvedValueOnce([testVariation]); - const testVariationsDeleteMock = jest.fn(); - service = await initService({ - projectFindManyMock: projectFindManyMock, - testVariationFindManyMock: testVariationFindManyMock, - testVariationsDeleteMock: testVariationsDeleteMock, - }); + const { service, projectFindMany, variationFindMany, variationDelete } = await setup(); + projectFindMany.mockResolvedValueOnce([project]); + variationFindMany.mockResolvedValueOnce([testVariation]); const dateNow = new Date('2022-10-23'); jest.useFakeTimers().setSystemTime(dateNow); const dateRemoveAfter: Date = new Date(dateNow); dateRemoveAfter.setDate(dateRemoveAfter.getDate() - project.maxBranchLifetime); - // .Act await service.cleanOldTestVariations(); - // .Assert - expect(testVariationFindManyMock).toHaveBeenCalledWith({ + expect(variationFindMany).toHaveBeenCalledWith({ where: { updatedAt: { lte: dateRemoveAfter }, - // nor run since: a run that matches the baseline does not move updatedAt testRuns: { none: { createdAt: { gt: dateRemoveAfter } } }, branchName: { not: project.mainBranchName }, projectId: project.id, }, }); - expect(testVariationsDeleteMock).toHaveBeenCalledWith(testVariation.id); + expect(variationDelete).toHaveBeenCalledWith(testVariation.id); }); it('removes only while holding the advisory lock, taken in a transaction on its own connection', async () => { - const client = lockClient(true); - const projectFindManyMock = jest.fn().mockResolvedValueOnce([TEST_PROJECT]); - const testVariationFindManyMock = jest.fn().mockResolvedValueOnce([generateTestVariation()]); - const testVariationsDeleteMock = jest.fn(async () => { - // the lock is still held: the connection carrying it has not been closed + const { service, client, projectFindMany, variationFindMany, variationDelete } = await setup(); + projectFindMany.mockResolvedValueOnce([TEST_PROJECT]); + variationFindMany.mockResolvedValueOnce([generateTestVariation()]); + variationDelete.mockImplementation(async () => { expect(client.end).not.toHaveBeenCalled(); }); - service = await initService({ client, projectFindManyMock, testVariationFindManyMock, testVariationsDeleteMock }); await service.cleanOldTestVariations(); @@ -102,30 +85,25 @@ describe('cleanOldTestVariations', () => { ['BEGIN'], ['SELECT pg_try_advisory_xact_lock($1) AS locked', [CLEANUP_LOCK_KEY]], ]); - expect(testVariationsDeleteMock).toHaveBeenCalledTimes(1); - // closing the connection ends the transaction and releases the lock + expect(variationDelete).toHaveBeenCalledTimes(1); expect(client.end).toHaveBeenCalledTimes(1); }); it('skips when another instance holds the lock', async () => { - const client = lockClient(false); - const projectFindManyMock = jest.fn(); - const testVariationsDeleteMock = jest.fn(); - service = await initService({ client, projectFindManyMock, testVariationsDeleteMock }); - const logSpy = jest.spyOn((service as any).logger, 'log').mockImplementation(); + const { service, client, projectFindMany, variationDelete, spyLogger } = await setup(lockClient(false)); + const logSpy = spyLogger('log'); await service.cleanOldTestVariations(); - expect(projectFindManyMock).not.toHaveBeenCalled(); - expect(testVariationsDeleteMock).not.toHaveBeenCalled(); + expect(projectFindMany).not.toHaveBeenCalled(); + expect(variationDelete).not.toHaveBeenCalled(); expect(logSpy).toHaveBeenCalledWith('Another instance is removing old test variations, skipping'); expect(client.end).toHaveBeenCalledTimes(1); }); it('releases the lock when the cleanup fails', async () => { - const client = lockClient(true); - const projectFindManyMock = jest.fn().mockRejectedValueOnce(new Error('connection reset')); - service = await initService({ client, projectFindManyMock }); + const { service, client, projectFindMany } = await setup(); + projectFindMany.mockRejectedValueOnce(new Error('connection reset')); await expect(service.cleanOldTestVariations()).rejects.toThrow('connection reset'); @@ -133,9 +111,8 @@ describe('cleanOldTestVariations', () => { }); it('logs a lost lock connection instead of letting the error event crash the process', async () => { - const client = lockClient(true); - service = await initService({ client }); - const warnSpy = jest.spyOn((service as any).logger, 'warn').mockImplementation(); + const { service, client, spyLogger } = await setup(); + const warnSpy = spyLogger('warn'); await service.cleanOldTestVariations(); const [event, listener] = client.on.mock.calls[0]; @@ -146,4 +123,63 @@ describe('cleanOldTestVariations', () => { 'Lost the cleanup lock connection: terminating connection due to administrator command' ); }); + + it('one failing variation does not stop the cleanup of the next variation and project', async () => { + const { service, projectFindMany, variationFindMany, variationDelete, spyLogger } = await setup(); + projectFindMany.mockResolvedValueOnce([TEST_PROJECT, secondProject]); + variationFindMany.mockResolvedValueOnce(variations('v1', 'v2')).mockResolvedValueOnce(variations('v3')); + variationDelete.mockRejectedValueOnce(new Error('FK')).mockResolvedValue(undefined); + spyLogger('warn'); + + await expect(service.cleanOldTestVariations()).resolves.toBeUndefined(); + + expect(variationDelete.mock.calls).toEqual([['v1'], ['v2'], ['v3']]); + }); + + it('a failing project does not stop the next one', async () => { + const { service, projectFindMany, variationFindMany, variationDelete, spyLogger } = await setup(); + projectFindMany.mockResolvedValueOnce([TEST_PROJECT, secondProject]); + variationFindMany.mockRejectedValueOnce(new Error('db down')).mockResolvedValueOnce(variations('v3')); + variationDelete.mockResolvedValue(undefined); + const warnSpy = spyLogger('warn'); + + await service.cleanOldTestVariations(); + + expect(variationDelete.mock.calls).toEqual([['v3']]); + expect(warnSpy).toHaveBeenCalledWith('Cannot clean up TestVariations of Test Project: Error: db down'); + }); + + it('logs a summary per project', async () => { + const { service, projectFindMany, variationFindMany, variationDelete, spyLogger } = await setup(); + projectFindMany.mockResolvedValueOnce([TEST_PROJECT]); + variationFindMany.mockResolvedValueOnce(variations('v1', 'v2')); + variationDelete.mockRejectedValueOnce(new Error('FK')).mockResolvedValue(undefined); + const logSpy = spyLogger('log'); + spyLogger('warn'); + + await service.cleanOldTestVariations(); + + expect(logSpy).toHaveBeenCalledWith('Removed 1 of 2 TestVariations of Test Project, 0 in use again, 1 failed'); + }); + + it('skips a variation used after the list was read', async () => { + const { service, projectFindMany, variationFindMany, variationFindFirst, variationDelete, spyLogger } = + await setup(); + projectFindMany.mockResolvedValueOnce([TEST_PROJECT]); + variationFindMany.mockResolvedValueOnce(variations('v1', 'v2')); + variationFindFirst.mockImplementation(async ({ where }) => (where.id === 'v1' ? null : { id: where.id })); + variationDelete.mockResolvedValue(undefined); + const logSpy = spyLogger('log'); + spyLogger('warn'); + + await service.cleanOldTestVariations(); + + const { where: listWhere } = variationFindMany.mock.calls[0][0]; + expect(variationFindFirst.mock.calls).toEqual([ + [{ where: { id: 'v1', ...listWhere }, select: { id: true } }], + [{ where: { id: 'v2', ...listWhere }, select: { id: true } }], + ]); + expect(variationDelete.mock.calls).toEqual([['v2']]); + expect(logSpy).toHaveBeenCalledWith('Removed 1 of 2 TestVariations of Test Project, 1 in use again, 0 failed'); + }); }); diff --git a/src/shared/tasks/tasks.service.ts b/src/shared/tasks/tasks.service.ts index a6b67176..84e44285 100644 --- a/src/shared/tasks/tasks.service.ts +++ b/src/shared/tasks/tasks.service.ts @@ -1,11 +1,10 @@ import { forwardRef, Inject, Injectable, Logger } from '@nestjs/common'; import { Cron, CronExpression } from '@nestjs/schedule'; +import { Prisma, Project } from '@prisma/client'; import { PrismaService } from '../../prisma/prisma.service'; import { TestVariationsService } from '../../test-variations/test-variations.service'; import type { WrapperType } from '../wrapper-type'; -// Advisory lock key of the cleanup, shared by every instance against the same -// database: "VRT" and 1, a value nothing else in the database locks on export const CLEANUP_LOCK_KEY = 0x56525401; @Injectable() @@ -18,27 +17,10 @@ export class TasksService { private testVariationService: WrapperType ) {} - /** - * Every API instance schedules this for the same midnight. One of them runs - * it and the others skip, or they would race to delete the same variations - * and baseline images. - * - * The lock is a transaction-level advisory lock, taken on a connection of - * its own and held in an otherwise idle transaction until the cleanup is - * done. Taking it only to claim the run and committing before the work would - * let an instance whose timer fires a moment later run alongside; a - * session-level lock behind a PgBouncer in transaction mode would stay on - * whichever server connection took it. This one goes with the transaction, - * which ends with the connection, also when the instance dies. The - * transaction holds no rows and, between statements, no snapshot, so it does - * not hold up vacuum; the deletions go through Prisma's pool as usual. An - * instance that comes after the run has finished finds nothing left to remove. - */ + // Transaction-level lock on a dedicated connection: a session-level one would stick to a PgBouncer connection @Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT) async cleanOldTestVariations() { const client = this.prismaService.dedicatedClient(); - // an idle pg client emits a lost connection as an event, which would - // otherwise crash the process; the lock goes with it, the cleanup goes on client.on('error', (error) => this.logger.warn(`Lost the cleanup lock connection: ${error.message}`)); await client.connect(); try { @@ -52,40 +34,64 @@ export class TasksService { } await this.removeOldTestVariations(); } finally { - // closing the connection rolls the transaction back and releases the lock await client.end(); } } - /** - * Feature-branch variations nobody has used within the project's - * maxBranchLifetime. Use is an approval or edit of the variation (its - * updatedAt) or a test run compared against it: a run that matches its - * baseline leaves the variation untouched, yet the branch is plainly alive, - * and dropping its baseline would turn its next run into "new". - */ private async removeOldTestVariations() { const projects = await this.prismaService.project.findMany(); for (const project of projects) { - const dateRemoveAfter: Date = new Date(); - dateRemoveAfter.setDate(dateRemoveAfter.getDate() - project.maxBranchLifetime); + try { + await this.removeOldTestVariationsOf(project); + } catch (e) { + this.logger.warn(`Cannot clean up TestVariations of ${project.name}: ${e}`); + } + } + } - const testVariations = await this.prismaService.testVariation.findMany({ - where: { - projectId: project.id, - updatedAt: { lte: dateRemoveAfter }, - testRuns: { none: { createdAt: { gt: dateRemoveAfter } } }, - branchName: { not: project.mainBranchName }, - }, - }); - this.logger.debug( - `Removing ${testVariations.length} TestVariations for ${project.name} unused since ${dateRemoveAfter}` - ); + private async removeOldTestVariationsOf(project: Project) { + const dateRemoveAfter: Date = new Date(); + dateRemoveAfter.setDate(dateRemoveAfter.getDate() - project.maxBranchLifetime); - for (const testVariation of testVariations) { + const testVariations = await this.prismaService.testVariation.findMany({ + where: this.staleVariationsWhere(project, dateRemoveAfter), + }); + this.logger.debug( + `Removing ${testVariations.length} TestVariations for ${project.name} unused since ${dateRemoveAfter}` + ); + + let removed = 0; + let skipped = 0; + let failed = 0; + for (const testVariation of testVariations) { + try { + const stillStale = await this.prismaService.testVariation.findFirst({ + where: { id: testVariation.id, ...this.staleVariationsWhere(project, dateRemoveAfter) }, + select: { id: true }, + }); + if (!stillStale) { + skipped++; + continue; + } await this.testVariationService.delete(testVariation.id); + removed++; + } catch (e) { + failed++; + this.logger.warn(`Cannot remove TestVariation ${testVariation.id}: ${e}`); } } + this.logger.log( + `Removed ${removed} of ${testVariations.length} TestVariations of ${project.name}, ${skipped} in use again, ${failed} failed` + ); + } + + private staleVariationsWhere(project: Project, removeAfter: Date): Prisma.TestVariationWhereInput { + return { + projectId: project.id, + updatedAt: { lte: removeAfter }, + testRuns: { none: { createdAt: { gt: removeAfter } } }, + branchName: { not: project.mainBranchName }, + }; } } diff --git a/src/shared/validation.audit.spec.ts b/src/shared/validation.audit.spec.ts new file mode 100644 index 00000000..cbfdb1cd --- /dev/null +++ b/src/shared/validation.audit.spec.ts @@ -0,0 +1,69 @@ +import { getMetadataStorage } from 'class-validator'; +import { ModifyBuildDto } from '../builds/dto/build-modify.dto'; +import { CreateBuildDto } from '../builds/dto/build-create.dto'; +import { CreateProjectDto } from '../projects/dto/create-project.dto'; +import { UpdateProjectDto } from '../projects/dto/update-project.dto'; +import { CreateTestRequestBase64Dto } from '../test-runs/dto/create-test-request-base64.dto'; +import { CreateTestRequestMultipartDto } from '../test-runs/dto/create-test-request-multipart.dto'; +import { CreateTestRequestDto } from '../test-runs/dto/create-test-request.dto'; +import { IgnoreAreaDto } from '../test-runs/dto/ignore-area.dto'; +import { UpdateIgnoreAreasDto } from '../test-runs/dto/update-ignore-area.dto'; +import { UpdateTestRunDto } from '../test-runs/dto/update-test.dto'; +import { MergeParams } from '../test-variations/types'; +import { AssignRoleDto } from '../users/dto/assign-role.dto'; +import { ChangePasswordDto } from '../users/dto/change-password.dto'; +import { CreateUserDto } from '../users/dto/user-create.dto'; +import { UserLoginRequestDto } from '../users/dto/user-login-request.dto'; +import { UpdateUserDto } from '../users/dto/user-update.dto'; +import { BaselineDataDto } from './dto/baseline-data.dto'; + +type DtoClass = abstract new (...args: any[]) => unknown; + +const declared = (dto: DtoClass): string[] => + ((Reflect.getMetadata('swagger/apiModelPropertiesArray', dto.prototype) ?? []) as string[]).map((name) => + name.replace(/^:/, '') + ); +const validated = (dto: DtoClass): Set => + new Set( + getMetadataStorage() + .getTargetValidationMetadatas(dto, '', true, false) + .map((m) => m.propertyName) + ); + +describe.each([ + IgnoreAreaDto, + UpdateIgnoreAreasDto, + UpdateTestRunDto, + BaselineDataDto, + CreateTestRequestDto, + CreateTestRequestBase64Dto, + CreateTestRequestMultipartDto, + CreateBuildDto, + ModifyBuildDto, + CreateProjectDto, + UpdateProjectDto, + CreateUserDto, + UserLoginRequestDto, + UpdateUserDto, + AssignRoleDto, + ChangePasswordDto, +])('%p', (dto) => { + it('validates every declared property', () => { + const properties = declared(dto); + expect(properties).not.toEqual([]); + expect(properties.filter((name) => !validated(dto).has(name))).toEqual([]); + }); +}); + +describe('MergeParams', () => { + it('validates every query parameter', () => { + expect([...validated(MergeParams)].sort()).toEqual(['fromBranch', 'projectId', 'toBranch']); + }); +}); + +describe('CreateTestRequestDto', () => { + it('validates sourceBranch, which the variation lookup reads', () => { + expect(validated(CreateTestRequestDto).has('sourceBranch')).toBe(true); + expect(validated(CreateTestRequestMultipartDto).has('sourceBranch')).toBe(true); + }); +}); diff --git a/src/shared/validation.spec.ts b/src/shared/validation.spec.ts new file mode 100644 index 00000000..8dab24f5 --- /dev/null +++ b/src/shared/validation.spec.ts @@ -0,0 +1,57 @@ +import { IsString } from 'class-validator'; +import { GlobalValidationPipe, ValidationOptions } from './validation'; + +class Plain { + @IsString() + a!: string; + + b?: string; +} + +@ValidationOptions({ forbidNonWhitelisted: true, transform: true }) +class Strict { + @IsString() + a!: string; +} + +class StrictChild extends Strict {} + +describe('GlobalValidationPipe', () => { + const pipe = new GlobalValidationPipe(); + + it('drops properties without a validator and keeps the declared ones', async () => { + expect(await pipe.transform({ a: 'x', b: 'y' }, { type: 'body', metatype: Plain })).toEqual({ a: 'x' }); + }); + + it('still rejects an invalid value', async () => { + await expect(pipe.transform({ a: 1 }, { type: 'body', metatype: Plain })).rejects.toThrow(); + }); + + describe('@ValidationOptions', () => { + it('rejects undeclared fields when the DTO says so', async () => { + await expect(pipe.transform({ a: 'x', b: 'y' }, { type: 'body', metatype: Strict })).rejects.toThrow(); + }); + + it('still whitelists beside the declared options and applies them', async () => { + const result = await pipe.transform({ a: 'x' }, { type: 'body', metatype: Strict }); + expect(result).toBeInstanceOf(Strict); + expect(result).toEqual({ a: 'x' }); + }); + + it('applies to a DTO that extends the decorated one', async () => { + await expect(pipe.transform({ a: 'x', b: 'y' }, { type: 'body', metatype: StrictChild })).rejects.toThrow(); + }); + }); + + it('passes a body without a metatype through the default whitelist pipe', async () => { + const body = { a: 1 }; + expect(await pipe.transform(body, { type: 'body' })).toBe(body); + }); + + it.each([ + ['an array', Array, [1, 'a']], + ['a primitive', String, 'text'], + ])('leaves %s alone', async (_name, metatype, value) => { + expect(await pipe.transform(value, { type: 'body', metatype })).toBe(value); + }); +}); diff --git a/src/shared/validation.ts b/src/shared/validation.ts new file mode 100644 index 00000000..7fff6495 --- /dev/null +++ b/src/shared/validation.ts @@ -0,0 +1,31 @@ +import { ArgumentMetadata, PipeTransform, ValidationPipe, ValidationPipeOptions } from '@nestjs/common'; + +const VALIDATION_OPTIONS = Symbol('validationOptions'); + +export const ValidationOptions = + (options: ValidationPipeOptions): ClassDecorator => + (target) => { + Reflect.defineMetadata(VALIDATION_OPTIONS, options, target); + }; + +export class GlobalValidationPipe implements PipeTransform { + private readonly defaultPipe = new ValidationPipe({ whitelist: true }); + private readonly pipes = new WeakMap(); + + transform(value: unknown, metadata: ArgumentMetadata): Promise { + return this.pipeFor(metadata.metatype).transform(value, metadata); + } + + private pipeFor(metatype: ArgumentMetadata['metatype']): ValidationPipe { + const options: ValidationPipeOptions | undefined = metatype && Reflect.getMetadata(VALIDATION_OPTIONS, metatype); + if (!metatype || !options) { + return this.defaultPipe; + } + let pipe = this.pipes.get(metatype); + if (!pipe) { + pipe = new ValidationPipe({ whitelist: true, ...options }); + this.pipes.set(metatype, pipe); + } + return pipe; + } +} diff --git a/src/shared/validation.whitelist.spec.ts b/src/shared/validation.whitelist.spec.ts new file mode 100644 index 00000000..4aecedb3 --- /dev/null +++ b/src/shared/validation.whitelist.spec.ts @@ -0,0 +1,158 @@ +import { BadRequestException, Type } from '@nestjs/common'; +import { ModifyBuildDto } from '../builds/dto/build-modify.dto'; +import { CreateBuildDto } from '../builds/dto/build-create.dto'; +import { UpdateProjectDto } from '../projects/dto/update-project.dto'; +import { CreateProjectDto } from '../projects/dto/create-project.dto'; +import { CreateTestRequestBase64Dto } from '../test-runs/dto/create-test-request-base64.dto'; +import { CreateTestRequestMultipartDto } from '../test-runs/dto/create-test-request-multipart.dto'; +import { UpdateIgnoreAreasDto } from '../test-runs/dto/update-ignore-area.dto'; +import { UpdateTestRunDto } from '../test-runs/dto/update-test.dto'; +import { MergeParams } from '../test-variations/types'; +import { AssignRoleDto } from '../users/dto/assign-role.dto'; +import { ChangePasswordDto } from '../users/dto/change-password.dto'; +import { CreateUserDto } from '../users/dto/user-create.dto'; +import { UserLoginRequestDto } from '../users/dto/user-login-request.dto'; +import { UpdateUserDto } from '../users/dto/user-update.dto'; +import { GlobalValidationPipe } from './validation'; + +const run = async ( + value: unknown, + metatype: Type, + type: 'body' | 'query' = 'body' +): Promise> => + (await new GlobalValidationPipe().transform(value, { type, metatype, data: undefined })) as Record; +const BUILD_ID = '146e7a8d-89f0-4565-aa2c-e61efabb0afd'; +const PROJECT_ID = '3bc4a5bc-006e-4d43-8e4e-eaa132627fcc'; +const area = { x: 1, y: 2, width: 3, height: 4 }; + +describe('global whitelist', () => { + it('UpdateTestRunDto keeps only the comment', async () => { + const body = await run({ comment: 'c', baselineName: 'x', ignoreAreas: 'bad' }, UpdateTestRunDto); + expect(body).toEqual({ comment: 'c' }); + }); + + it('UpdateIgnoreAreasDto keeps the id the web UI selects areas by', async () => { + const body = await run( + { ids: ['a'], ignoreAreas: [{ ...area, id: '1', junk: 'x' }], junk: 'x' }, + UpdateIgnoreAreasDto + ); + expect(body).toEqual({ ids: ['a'], ignoreAreas: [{ ...area, id: '1' }] }); + }); + + it('CreateTestRequestBase64Dto keeps sourceBranch and the area ids', async () => { + const body = await run( + { + buildId: BUILD_ID, + projectId: PROJECT_ID, + name: 'n', + branchName: 'b', + sourceBranch: 'release', + imageBase64: 'abc', + ignoreAreas: [{ ...area, id: '1', junk: 'x' }], + junk: 'x', + }, + CreateTestRequestBase64Dto + ); + expect(body).toMatchObject({ sourceBranch: 'release', imageBase64: 'abc', ignoreAreas: [{ ...area, id: '1' }] }); + expect(body).not.toHaveProperty('junk'); + expect(body.ignoreAreas[0]).not.toHaveProperty('junk'); + }); + + describe('CreateTestRequestMultipartDto', () => { + const form = { + buildId: BUILD_ID, + projectId: PROJECT_ID, + name: 'n', + branchName: 'b', + image: { fieldname: 'image', buffer: Buffer.from([1]) }, + }; + + it('is parsed once, with the image, the areas and sourceBranch kept', async () => { + const body = await run( + { + ...form, + merge: 'true', + diffTollerancePercent: '1.5', + ignoreAreas: JSON.stringify([{ ...area, id: '1' }]), + sourceBranch: 'release', + junk: 'x', + }, + CreateTestRequestMultipartDto + ); + expect(Buffer.isBuffer(body.image.buffer)).toBe(true); + expect(body.merge).toBe(true); + expect(body.diffTollerancePercent).toBe(1.5); + expect(body.ignoreAreas).toEqual([{ ...area, id: '1' }]); + expect(body.sourceBranch).toBe('release'); + expect(body).not.toHaveProperty('junk'); + }); + + it('turns empty form fields into the defaults', async () => { + const body = await run( + { ...form, merge: '', diffTollerancePercent: '', ignoreAreas: '' }, + CreateTestRequestMultipartDto + ); + expect(body).toMatchObject({ merge: false, diffTollerancePercent: 0, ignoreAreas: [] }); + }); + }); + + describe('ModifyBuildDto', () => { + it('still rejects fields it does not declare', async () => { + await expect(run({ branchName: 'x' }, ModifyBuildDto)).rejects.toBeInstanceOf(BadRequestException); + }); + + it('passes the declared fields', async () => { + expect(await run({ isRunning: false }, ModifyBuildDto)).toEqual({ isRunning: false }); + }); + + it('keeps an empty body empty', async () => { + expect(await run(undefined, ModifyBuildDto)).toBeUndefined(); + }); + }); + + it('MergeParams keeps its three query parameters', async () => { + const body = await run({ projectId: PROJECT_ID, fromBranch: 'a', toBranch: 'b', junk: 'x' }, MergeParams, 'query'); + expect(body).toEqual({ projectId: PROJECT_ID, fromBranch: 'a', toBranch: 'b' }); + }); + + describe('projects', () => { + const project = { + id: PROJECT_ID, + name: 'p', + mainBranchName: 'master', + autoApproveFeature: true, + imageComparison: 'pixelmatch', + maxBuildAllowed: 5, + maxBranchLifetime: 7, + imageComparisonConfig: '{}', + bulkApproveVariations: false, + bulkApproveGroupBy: 'name', + }; + + it('UpdateProjectDto drops the counters and dates, keeps the ten settings', async () => { + const body = await run({ ...project, buildsCounter: 3, createdAt: '2020-01-01', junk: 'x' }, UpdateProjectDto); + expect(body).toEqual(project); + }); + + it('CreateProjectDto has no id', async () => { + const { id: _id, ...fields } = project; + expect(await run({ ...project, junk: 'x' }, CreateProjectDto)).toEqual(fields); + }); + }); + + it.each([ + ['CreateBuildDto', CreateBuildDto, { project: 'p', branchName: 'b', ciBuildId: 'c' }], + ['CreateUserDto', CreateUserDto, { email: 'a@b.co', password: 'p', firstName: 'f', lastName: 'l' }], + ['UserLoginRequestDto', UserLoginRequestDto, { email: 'a@b.co', password: 'p' }], + ['UpdateUserDto', UpdateUserDto, { email: 'a@b.co', firstName: 'f', lastName: 'l' }], + ['AssignRoleDto', AssignRoleDto, { id: PROJECT_ID, role: 'editor' }], + ['ChangePasswordDto', ChangePasswordDto, { password: 'p' }], + ] as const)('%s keeps its declared fields and drops the rest', async (_name, dto, fields) => { + expect(await run({ ...fields, junk: 'x' }, dto as Type)).toEqual(fields); + }); + + it('leaves an array body of ids alone', async () => { + const ids = ['a', 'b']; + expect(await run(ids, Array as unknown as Type)).toBe(ids); + }); +}); diff --git a/src/shared/wrapper-type.ts b/src/shared/wrapper-type.ts index 50b5520a..8c799e3a 100644 --- a/src/shared/wrapper-type.ts +++ b/src/shared/wrapper-type.ts @@ -1,8 +1,2 @@ -/** - * The type of a dependency injected with forwardRef. A plain class type would - * be emitted into the constructor's design:paramtypes metadata, and with - * circular imports that class is not initialised yet when the decorator runs: - * tsc's output reads it as undefined, SWC's live bindings throw. Behind an - * alias the metadata says Object, and forwardRef hands Nest the real class. - */ +// Hides the type from design:paramtypes: a circular import is not initialised there yet and SWC throws export type WrapperType = T; diff --git a/src/static/aws/s3.service.spec.ts b/src/static/aws/s3.service.spec.ts index 73ed4d80..35d4f5cd 100644 --- a/src/static/aws/s3.service.spec.ts +++ b/src/static/aws/s3.service.spec.ts @@ -6,7 +6,6 @@ import { S3Client, } from '@aws-sdk/client-s3'; import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; -import { PNG } from 'pngjs'; import { Readable } from 'stream'; import { AWSS3Service, URL_WINDOW_SECONDS } from './s3.service'; import { generateNewImageName } from '../utils'; @@ -31,12 +30,28 @@ jest.mock('../utils', () => ({ generateNewImageName: jest.fn(), })); +const notFound = (props: object) => Object.assign(new Error('not found'), props); +const notFoundCases = [ + ['a NoSuchKey error', { name: 'NoSuchKey' }], + ['an HTTP 404', { $metadata: { httpStatusCode: 404 } }], +] as const; +const defaultTimeouts = { + requestHandler: { connectionTimeout: 5000, requestTimeout: 60000, throwOnRequestTimeout: true }, +}; + describe('AWSS3Service', () => { const originalAwsBucket = process.env.AWS_S3_BUCKET_NAME; + const originalEnv = { ...process.env }; beforeEach(() => { jest.clearAllMocks(); process.env.AWS_S3_BUCKET_NAME = 'vrt-bucket'; + mockSend.mockResolvedValue({}); + (getSignedUrl as jest.Mock).mockResolvedValue('https://signed-url'); + }); + + afterEach(() => { + process.env = { ...originalEnv }; }); afterAll(() => { @@ -46,7 +61,6 @@ describe('AWSS3Service', () => { describe('saveImage', () => { it('uploads the image buffer and returns the generated image name', async () => { (generateNewImageName as jest.Mock).mockReturnValue('generated.screenshot.png'); - mockSend.mockResolvedValue({}); const service = new AWSS3Service(); const imageBuffer = Buffer.from('png-data'); @@ -82,49 +96,7 @@ describe('AWSS3Service', () => { }); }); - describe('getImage', () => { - it('returns null when the file name is missing', async () => { - const service = new AWSS3Service(); - - await expect(service.getImage('')).resolves.toBeNull(); - expect(mockSend).not.toHaveBeenCalled(); - }); - - it('reads the image from S3 and parses it as PNG', async () => { - const service = new AWSS3Service(); - const png = new PNG({ width: 1, height: 1 }); - const pngBuffer = PNG.sync.write(png); - const stream = { - toArray: jest.fn().mockResolvedValue([pngBuffer.subarray(0, 8), pngBuffer.subarray(8)]), - } as unknown as Readable; - mockSend.mockResolvedValue({ Body: stream }); - - const result = await service.getImage('baseline.png'); - - expect(GetObjectCommand).toHaveBeenCalledWith({ - Bucket: 'vrt-bucket', - Key: 'baseline.png', - }); - expect(result).toMatchObject({ width: 1, height: 1 }); - }); - - it('logs failures and returns undefined when the image cannot be read', async () => { - const service = new AWSS3Service(); - const loggerSpy = jest.spyOn((service as any).logger, 'error').mockImplementation(); - mockSend.mockRejectedValue(new Error('download failed')); - - await expect(service.getImage('baseline.png')).resolves.toBeUndefined(); - - expect(loggerSpy).toHaveBeenCalledWith( - 'Error from read : Cannot get image: baseline.png. Error: download failed' - ); - }); - }); - describe('getImageUrl', () => { - // A browser caches by URL. Signing from the current instant produced a - // different URL on every request, so the bytes it had just downloaded - // could never be reused and every view of an image cost a fresh download. const signingDateOfLastCall = (): Date => (getSignedUrl as jest.Mock).mock.calls.at(-1)[2].signingDate; afterEach(() => { @@ -133,7 +105,6 @@ describe('AWSS3Service', () => { it('returns a signed URL for the requested object', async () => { const service = new AWSS3Service(); - (getSignedUrl as jest.Mock).mockResolvedValue('https://signed-url'); const result = await service.getImageUrl('image.png'); @@ -143,47 +114,57 @@ describe('AWSS3Service', () => { ); }); - it('hands out the same URL for an image asked for again soon after', async () => { + it('signs for twice the window, from the start of the current window, with a matching cache header', async () => { + jest.spyOn(Date, 'now').mockReturnValue(Date.parse('2026-09-01T10:42:31.500Z')); const service = new AWSS3Service(); - (getSignedUrl as jest.Mock).mockResolvedValue('https://signed-url'); - const nowSpy = jest.spyOn(Date, 'now'); - nowSpy.mockReturnValue(Date.parse('2026-09-01T10:00:05Z')); await service.getImageUrl('image.png'); - const first = signingDateOfLastCall(); - nowSpy.mockReturnValue(Date.parse('2026-09-01T10:42:31Z')); + expect(URL_WINDOW_SECONDS).toBe(3600); + expect(GetObjectCommand).toHaveBeenCalledWith({ + Bucket: 'vrt-bucket', + Key: 'image.png', + ResponseCacheControl: 'private, max-age=3600', + }); + expect(getSignedUrl).toHaveBeenCalledWith(expect.objectContaining({ send: mockSend }), expect.anything(), { + expiresIn: 7200, + signingDate: new Date('2026-09-01T10:00:00.000Z'), + }); + }); + + it('signs against "undefined" when no bucket is configured (startup warns about it)', async () => { + delete process.env.AWS_S3_BUCKET_NAME; + const service = new AWSS3Service(); + await service.getImageUrl('image.png'); - // asserted to be a real instant first: two undefineds are also equal - expect(first).toBeInstanceOf(Date); - expect(signingDateOfLastCall()).toEqual(first); + expect(GetObjectCommand).toHaveBeenCalledWith(expect.objectContaining({ Bucket: 'undefined' })); }); - it('moves on to a new URL once the window has passed', async () => { + it.each([ + ['hands out the same URL for an image asked for again soon after', '10:00:05', '10:42:31', true], + ['moves on to a new URL once the window has passed', '10:42:31', '11:00:01', false], + ])('%s', async (_, firstTime, secondTime, same) => { const service = new AWSS3Service(); - (getSignedUrl as jest.Mock).mockResolvedValue('https://signed-url'); const nowSpy = jest.spyOn(Date, 'now'); - nowSpy.mockReturnValue(Date.parse('2026-09-01T10:42:31Z')); + nowSpy.mockReturnValue(Date.parse(`2026-09-01T${firstTime}Z`)); await service.getImageUrl('image.png'); const first = signingDateOfLastCall(); - nowSpy.mockReturnValue(Date.parse('2026-09-01T11:00:01Z')); + nowSpy.mockReturnValue(Date.parse(`2026-09-01T${secondTime}Z`)); await service.getImageUrl('image.png'); - expect(signingDateOfLastCall()).not.toEqual(first); + expect(first).toBeInstanceOf(Date); + if (same) { + expect(signingDateOfLastCall()).toEqual(first); + } else { + expect(signingDateOfLastCall()).not.toEqual(first); + } }); - // a URL handed out at the very end of a window must still be usable for a - // whole window afterwards, or the browser's cached copy outlives its URL it('keeps a URL alive well past the window it was signed in', async () => { const service = new AWSS3Service(); - (getSignedUrl as jest.Mock).mockResolvedValue('https://signed-url'); - // The worst case, when the least life is left: minted at the last second - // of a window. Pinned rather than read live, or a run that straddles the - // hour boundary asks the clock on the far side of it and sees a shorter - // life than the URL actually has. const lastSecondOfWindow = Date.parse('2026-09-01T10:59:59Z'); jest.spyOn(Date, 'now').mockReturnValue(lastSecondOfWindow); @@ -194,11 +175,8 @@ describe('AWSS3Service', () => { expect(livesUntil - lastSecondOfWindow).toBeGreaterThanOrEqual(3600 * 1000); }); - // S3 answers with no cache headers of its own, so a stable URL alone still - // leaves the browser guessing whether it may keep the bytes it('asks S3 to answer with a cache header', async () => { const service = new AWSS3Service(); - (getSignedUrl as jest.Mock).mockResolvedValue('https://signed-url'); await service.getImageUrl('image.png'); @@ -209,17 +187,39 @@ describe('AWSS3Service', () => { }); describe('constructor', () => { - it('creates one S3 client with the default credential chain', () => { + it('creates one S3 client with the default credential chain and request timeouts', () => { + delete process.env.S3_CONNECTION_TIMEOUT_MS; + delete process.env.S3_REQUEST_TIMEOUT_MS; + new AWSS3Service(); expect(S3Client).toHaveBeenCalledTimes(1); - expect(S3Client).toHaveBeenCalledWith(); + expect(S3Client).toHaveBeenCalledWith(defaultTimeouts); + }); + + it('reads the timeouts from S3_CONNECTION_TIMEOUT_MS and S3_REQUEST_TIMEOUT_MS', () => { + process.env.S3_CONNECTION_TIMEOUT_MS = '2000'; + process.env.S3_REQUEST_TIMEOUT_MS = '1000'; + + new AWSS3Service(); + + expect(S3Client).toHaveBeenCalledWith({ + requestHandler: { connectionTimeout: 2000, requestTimeout: 1000, throwOnRequestTimeout: true }, + }); + }); + + it.each(['abc', '', '0', '-5'])('falls back to the default timeout for %p', (value) => { + process.env.S3_CONNECTION_TIMEOUT_MS = value; + process.env.S3_REQUEST_TIMEOUT_MS = value; + + new AWSS3Service(); + + expect(S3Client).toHaveBeenCalledWith(defaultTimeouts); }); it('reads the bucket name when the service is constructed, not on each call', async () => { const service = new AWSS3Service(); process.env.AWS_S3_BUCKET_NAME = 'changed-later'; - mockSend.mockResolvedValue({}); await service.deleteImage('image.png'); @@ -228,8 +228,6 @@ describe('AWSS3Service', () => { }); describe('getImageBuffer', () => { - const notFound = (props: object) => Object.assign(new Error('not found'), props); - it.each(['', undefined, null])('returns null for an empty name (%p) without calling S3', async (fileName) => { const service = new AWSS3Service(); @@ -246,10 +244,7 @@ describe('AWSS3Service', () => { expect(mockSend).toHaveBeenCalledWith({ input: { Bucket: 'vrt-bucket', Key: 'image.png' }, type: 'get' }); }); - it.each([ - ['a NoSuchKey error', { name: 'NoSuchKey' }], - ['an HTTP 404', { $metadata: { httpStatusCode: 404 } }], - ])('returns null for %s', async (_label, props) => { + it.each(notFoundCases)('returns null for %s', async (_label, props) => { const service = new AWSS3Service(); jest.spyOn((service as any).logger, 'error').mockImplementation(); mockSend.mockRejectedValue(notFound(props)); @@ -281,29 +276,41 @@ describe('AWSS3Service', () => { }); }); - describe('getImage (missing object)', () => { - it('returns undefined when the object does not exist', async () => { + describe('getImageStream', () => { + it('returns the body of the object without reading it', async () => { const service = new AWSS3Service(); - jest.spyOn((service as any).logger, 'error').mockImplementation(); - mockSend.mockRejectedValue(Object.assign(new Error('nope'), { name: 'NoSuchKey' })); + const body = Readable.from([Buffer.from('ab')]); + mockSend.mockResolvedValue({ Body: body }); - await expect(service.getImage('missing.png')).resolves.toBeUndefined(); + await expect(service.getImageStream('image.png')).resolves.toBe(body); + expect(mockSend).toHaveBeenCalledWith({ input: { Bucket: 'vrt-bucket', Key: 'image.png' }, type: 'get' }); }); - it('logs and returns undefined when the object is not a valid PNG', async () => { + it('returns null when the response has no body', async () => { const service = new AWSS3Service(); - const loggerSpy = jest.spyOn((service as any).logger, 'error').mockImplementation(); - mockSend.mockResolvedValue({ Body: { toArray: jest.fn().mockResolvedValue([Buffer.from('garbage')]) } }); - await expect(service.getImage('corrupt.png')).resolves.toBeUndefined(); - expect(loggerSpy).toHaveBeenCalledWith(expect.stringContaining('Cannot get image: corrupt.png.')); + await expect(service.getImageStream('image.png')).resolves.toBeNull(); + }); + + it.each(notFoundCases)('returns null for %s', async (_label, props) => { + const service = new AWSS3Service(); + mockSend.mockRejectedValue(notFound(props)); + + await expect(service.getImageStream('missing.png')).resolves.toBeNull(); + }); + + it('rethrows any other failure', async () => { + const service = new AWSS3Service(); + const error = Object.assign(new Error('Access Denied'), { name: 'AccessDenied' }); + mockSend.mockRejectedValue(error); + + await expect(service.getImageStream('image.png')).rejects.toBe(error); }); }); describe('copyImage', () => { it('copies the source object inside the bucket under a newly generated name', async () => { (generateNewImageName as jest.Mock).mockReturnValue('generated.baseline.png'); - mockSend.mockResolvedValue({}); const service = new AWSS3Service(); await expect(service.copyImage('baseline', 'source.png')).resolves.toBe('generated.baseline.png'); @@ -328,44 +335,6 @@ describe('AWSS3Service', () => { }); }); - describe('getImageUrl (exact parameters)', () => { - afterEach(() => { - jest.restoreAllMocks(); - }); - - it('signs for twice the window, from the start of the current window, with a matching cache header', async () => { - jest.spyOn(Date, 'now').mockReturnValue(Date.parse('2026-09-01T10:42:31.500Z')); - (getSignedUrl as jest.Mock).mockResolvedValue('https://signed-url'); - const service = new AWSS3Service(); - - await service.getImageUrl('image.png'); - - expect(URL_WINDOW_SECONDS).toBe(3600); - expect(GetObjectCommand).toHaveBeenCalledWith({ - Bucket: 'vrt-bucket', - Key: 'image.png', - ResponseCacheControl: 'private, max-age=3600', - }); - expect(getSignedUrl).toHaveBeenCalledWith(expect.objectContaining({ send: mockSend }), expect.anything(), { - expiresIn: 7200, - signingDate: new Date('2026-09-01T10:00:00.000Z'), - }); - }); - - // NOTE: current behavior — the bucket is interpolated into a template - // string here, so a missing AWS_S3_BUCKET_NAME becomes the literal - // "undefined" (the other commands pass undefined through). - it('signs against a bucket literally named "undefined" when no bucket is configured', async () => { - delete process.env.AWS_S3_BUCKET_NAME; - (getSignedUrl as jest.Mock).mockResolvedValue('https://signed-url'); - const service = new AWSS3Service(); - - await service.getImageUrl('image.png'); - - expect(GetObjectCommand).toHaveBeenCalledWith(expect.objectContaining({ Bucket: 'undefined' })); - }); - }); - describe('deleteImage', () => { it('returns false when the image name is missing', async () => { const service = new AWSS3Service(); @@ -376,7 +345,6 @@ describe('AWSS3Service', () => { it('deletes the object and returns true', async () => { const service = new AWSS3Service(); - mockSend.mockResolvedValue({}); await expect(service.deleteImage('image.png')).resolves.toBe(true); diff --git a/src/static/aws/s3.service.ts b/src/static/aws/s3.service.ts index a99a4228..2859910a 100644 --- a/src/static/aws/s3.service.ts +++ b/src/static/aws/s3.service.ts @@ -1,4 +1,3 @@ -import { PNG, PNGWithMetadata } from 'pngjs'; import { Logger } from '@nestjs/common'; import { Static } from '../static.interface'; import { @@ -11,11 +10,12 @@ import { import { Readable } from 'stream'; import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; import { generateNewImageName } from '../utils'; +import { positiveNumber } from '../../utils/env'; -// How long one pre-signed URL is reused before a new one is minted. Also the -// lifetime S3 is asked to advertise on the object itself. export const URL_WINDOW_SECONDS = 3600; +const msFromEnv = (name: string, fallback: number): number => positiveNumber(process.env[name], fallback); + export class AWSS3Service implements Static { private readonly logger: Logger = new Logger(AWSS3Service.name); private readonly AWS_S3_BUCKET_NAME = process.env.AWS_S3_BUCKET_NAME; @@ -23,7 +23,13 @@ export class AWSS3Service implements Static { private s3Client: S3Client; constructor() { - this.s3Client = new S3Client(); + this.s3Client = new S3Client({ + requestHandler: { + connectionTimeout: msFromEnv('S3_CONNECTION_TIMEOUT_MS', 5_000), + requestTimeout: msFromEnv('S3_REQUEST_TIMEOUT_MS', 60_000), + throwOnRequestTimeout: true, + }, + }); this.logger.log('AWS S3 service is being used for file storage.'); } @@ -44,19 +50,6 @@ export class AWSS3Service implements Static { } } - async getImage(fileName: string | null): Promise { - if (!fileName) return null; - try { - // the comparison pipeline treats an unreadable image as a missing - // baseline, so storage failures stay contained here - const imageBuffer = await this.getImageBuffer(fileName); - if (!imageBuffer) return undefined; - return PNG.sync.read(imageBuffer); - } catch (ex) { - this.logger.error(`Error from read : Cannot get image: ${fileName}. ${ex}`); - } - } - async getImageBuffer(fileName: string | null): Promise { if (!fileName) return null; try { @@ -66,16 +59,30 @@ export class AWSS3Service implements Static { return Buffer.concat(await stream.toArray()); } catch (ex) { this.logger.error(`Error from read : Cannot get image: ${fileName}. ${ex}`); - // only a missing object means "no image"; credentials, throttling and - // network failures have to stay errors instead of reading as absence - const error = ex as { name?: string; $metadata?: { httpStatusCode?: number } } | undefined; - if (error?.name === 'NoSuchKey' || error?.$metadata?.httpStatusCode === 404) { + if (this.isNotFound(ex)) { return null; } throw ex; } } + async getImageStream(fileName: string): Promise { + try { + const { Body } = await this.s3Client.send( + new GetObjectCommand({ Bucket: this.AWS_S3_BUCKET_NAME, Key: fileName }) + ); + return (Body as Readable | undefined) ?? null; + } catch (ex) { + if (this.isNotFound(ex)) return null; + throw ex; + } + } + + private isNotFound(ex: unknown): boolean { + const error = ex as { name?: string; $metadata?: { httpStatusCode?: number } } | undefined; + return error?.name === 'NoSuchKey' || error?.$metadata?.httpStatusCode === 404; + } + async copyImage(type: 'screenshot' | 'diff' | 'baseline', sourceImageName: string): Promise { const imageName = generateNewImageName(type); try { @@ -92,23 +99,6 @@ export class AWSS3Service implements Static { } } - /** - * A pre-signed URL for the object, stable for the length of a window. - * - * A browser caches by URL. Signing from the current instant gave every - * request a different URL, so the bytes it had just downloaded could never be - * reused: reopening a screen, or stepping back one, paid for the whole image - * again. Signing from the start of a fixed window instead makes every request - * inside that window produce the very same URL, which a cache can hit. - * - * The signature is given twice the window to live, so a URL handed out at the - * last second of one window is still good for a whole window afterwards. - * - * `ResponseCacheControl` makes S3 answer with a cache header of its own — - * without it a stable URL still leaves the browser guessing, since the bucket - * sets none. An image name is unique per upload and its bytes never change, - * so the only thing bounding the lifetime is the signature. - */ async getImageUrl(imageName: string): Promise { const command = new GetObjectCommand({ Bucket: `${this.AWS_S3_BUCKET_NAME}`, diff --git a/src/static/hdd/hdd.service.spec.ts b/src/static/hdd/hdd.service.spec.ts index 8ad62fad..358d7091 100644 --- a/src/static/hdd/hdd.service.spec.ts +++ b/src/static/hdd/hdd.service.spec.ts @@ -5,8 +5,11 @@ import { PNG } from 'pngjs'; import { HDD_IMAGE_PATH } from './constants'; import { HddService } from './hdd.service'; -// Keeps the spec out of the real imageUploads/ folder: the directory is a -// fresh temp one, still named imageUploads so the resolved paths look alike. +jest.mock('fs', () => { + const actual = jest.requireActual('fs'); + return { ...actual, mkdirSync: jest.fn(actual.mkdirSync), existsSync: jest.fn(actual.existsSync) }; +}); + jest.mock('./constants', () => ({ HDD_IMAGE_PATH: `${require('path').join(require('os').tmpdir(), `vrt-hdd-spec-${process.pid}`, 'imageUploads')}/`, })); @@ -16,10 +19,11 @@ const pngBuffer = (width = 2, height = 1) => PNG.sync.write(new PNG({ width, hei const writeImage = (name: string, data: Buffer) => fs.writeFile(path.join(imageDir, name), data); describe('HddService', () => { - const service = new HddService(); + let service: HddService; let errorSpy: jest.SpyInstance; beforeEach(() => { + service = new HddService(); errorSpy = jest.spyOn(Logger.prototype, 'error').mockImplementation(); }); @@ -39,7 +43,6 @@ describe('HddService', () => { expect(service.getImagePath('ABC123.screenshot.png')).toMatch(/imageUploads\/ABC123\.screenshot\.png$/); }); - // a name may begin with dots without climbing out of the directory it.each(['..thumbnail.png', '...png', '.hidden.png'])( 'resolves a dotted name inside the directory: %s', (imageName) => { @@ -67,12 +70,51 @@ describe('HddService', () => { expect(existsSync(imageDir)).toBe(true); }); - it('returns null for a missing image', async () => { - await expect(service.getImageBuffer('definitely-missing.png')).resolves.toBeNull(); + it('creates the image directory once per instance', () => { + (mkdirSync as jest.Mock).mockClear(); + (existsSync as jest.Mock).mockClear(); + + service.getImagePath('a.screenshot.png'); + service.getImagePath('b.screenshot.png'); + + expect(mkdirSync).toHaveBeenCalledTimes(1); + expect(existsSync).not.toHaveBeenCalled(); + }); + + it.each([ + ['getImageBuffer', (name: string) => service.getImageBuffer(name)], + ['getImageStream', (name: string) => service.getImageStream(name)], + ])('%s returns null for a missing image', async (_, read) => { + await expect(read('definitely-missing.png')).resolves.toBeNull(); }); - it('propagates a traversal attempt instead of reading the file', async () => { - await expect(service.getImageBuffer('../../etc/passwd')).rejects.toThrow(/outside of the image directory/); + it.each([ + ['getImageBuffer', (name: string) => service.getImageBuffer(name)], + ['getImageStream', (name: string) => service.getImageStream(name)], + ['copyImage', (name: string) => service.copyImage('baseline', name)], + ])('%s rejects a path outside the image directory instead of reading the file', async (_, read) => { + await expect(read('../../etc/passwd')).rejects.toThrow(/outside of the image directory/); + }); + + describe('getImageStream', () => { + const readAll = async (stream: NodeJS.ReadableStream | null) => { + const chunks: Buffer[] = []; + for await (const chunk of stream!) chunks.push(chunk as Buffer); + return Buffer.concat(chunks); + }; + + it('streams the bytes of the stored file', async () => { + const data = pngBuffer(4, 4); + await writeImage('streamed.screenshot.png', data); + + await expect(readAll(await service.getImageStream('streamed.screenshot.png'))).resolves.toEqual(data); + }); + + it('rethrows a failure other than a missing file', async () => { + await writeImage('plain-file.png', pngBuffer()); + + await expect(service.getImageStream('plain-file.png/inside.png')).rejects.toMatchObject({ code: 'ENOTDIR' }); + }); }); describe('generateNewImage', () => { @@ -90,9 +132,7 @@ describe('HddService', () => { await expect(service.getImageUrl('image.png')).resolves.toBe('/image.png'); }); - // NOTE: current behavior — unlike the other methods, the name is not - // checked for traversal here (the URL is only handed to the browser). - it('does not validate the image name', async () => { + it('builds the URL without validating the name (the controller validates it)', async () => { await expect(service.getImageUrl('../secret')).resolves.toBe('/../secret'); }); }); @@ -116,40 +156,13 @@ describe('HddService', () => { await expect(fs.readFile(path.join(imageDir, imageName), 'utf8')).resolves.toBe('HELLO'); }); - // NOTE: current behavior — HddService itself does not check the format; - // only StaticService rejects non-PNG uploads. - it('accepts a buffer that is not a PNG', async () => { + it('stores any buffer (StaticService checks the format)', async () => { const imageName = await service.saveImage('diff', Buffer.from('not a png')); await expect(fs.readFile(path.join(imageDir, imageName), 'utf8')).resolves.toBe('not a png'); }); }); - describe('getImage', () => { - it('decodes a stored PNG', async () => { - await writeImage('decodable.png', pngBuffer(3, 2)); - - const image = await service.getImage('decodable.png'); - - expect(image).toMatchObject({ width: 3, height: 2 }); - }); - - it('returns undefined for a missing image', async () => { - await expect(service.getImage('missing.png')).resolves.toBeUndefined(); - }); - - it('returns undefined for an empty name', async () => { - await expect(service.getImage('')).resolves.toBeUndefined(); - }); - - it('logs and returns undefined for a file that is not a valid PNG', async () => { - await writeImage('corrupt.png', Buffer.from('garbage')); - - await expect(service.getImage('corrupt.png')).resolves.toBeUndefined(); - expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('Cannot decode image: corrupt.png.')); - }); - }); - describe('getImageBuffer', () => { it('returns the stored bytes', async () => { await writeImage('bytes.png', Buffer.from([1, 2, 3])); @@ -183,10 +196,6 @@ describe('HddService', () => { it('rejects when the source image is missing', async () => { await expect(service.copyImage('baseline', 'missing.png')).rejects.toMatchObject({ code: 'ENOENT' }); }); - - it('rejects a source name pointing outside the image directory', async () => { - await expect(service.copyImage('baseline', '../outside.png')).rejects.toThrow(/outside of the image directory/); - }); }); describe('deleteImage', () => { @@ -201,8 +210,6 @@ describe('HddService', () => { await expect(service.deleteImage('')).resolves.toBeUndefined(); }); - // NOTE: current behavior — failures are only logged and still reported as - // a successful delete. it('logs and still returns true when the file is missing', async () => { await expect(service.deleteImage('never-existed.png')).resolves.toBe(true); expect(errorSpy).toHaveBeenCalledWith(expect.objectContaining({ code: 'ENOENT' })); diff --git a/src/static/hdd/hdd.service.ts b/src/static/hdd/hdd.service.ts index 71e14f11..5c45e865 100644 --- a/src/static/hdd/hdd.service.ts +++ b/src/static/hdd/hdd.service.ts @@ -1,7 +1,7 @@ import { Injectable, Logger } from '@nestjs/common'; import path from 'path'; -import { promises as fs, mkdirSync, existsSync } from 'fs'; -import { PNG, PNGWithMetadata } from 'pngjs'; +import { promises as fs, mkdirSync } from 'fs'; +import { Readable } from 'stream'; import { Static } from '../static.interface'; import { HDD_IMAGE_PATH } from './constants'; import { generateNewImageName } from '../utils'; @@ -9,6 +9,7 @@ import { generateNewImageName } from '../utils'; @Injectable() export class HddService implements Static { private readonly logger: Logger = new Logger(HddService.name); + private directoryReady = false; generateNewImage(type: 'screenshot' | 'diff' | 'baseline'): { imageName: string; imagePath: string } { const imageName = generateNewImageName(type); @@ -19,13 +20,12 @@ export class HddService implements Static { } getImagePath(imageName: string): string { - this.ensureDirectoryExistence(HDD_IMAGE_PATH); + if (!this.directoryReady) { + mkdirSync(HDD_IMAGE_PATH, { recursive: true }); + this.directoryReady = true; + } const root = path.resolve(HDD_IMAGE_PATH); const imagePath = path.resolve(root, imageName); - // image names reach here straight from the request, so a traversal value - // would otherwise read any file the process can see - // only a leading parent-directory step means the name climbs out: a file - // whose name merely starts with dots stays inside const relativeToRoot = path.relative(root, imagePath); const climbsOut = relativeToRoot === '..' || relativeToRoot.startsWith(`..${path.sep}`) || path.isAbsolute(relativeToRoot); @@ -41,28 +41,16 @@ export class HddService implements Static { async saveImage(type: 'screenshot' | 'diff' | 'baseline', imageBuffer: Buffer): Promise { const { imageName, imagePath } = this.generateNewImage(type); - await fs.writeFile(imagePath, new Uint8Array(imageBuffer.buffer, imageBuffer.byteOffset, imageBuffer.byteLength)); + await fs.writeFile(imagePath, imageBuffer); return imageName; } - async getImage(imageName: string | null): Promise { - const imageBuffer = await this.getImageBuffer(imageName); - if (!imageBuffer) return; - try { - return PNG.sync.read(imageBuffer); - } catch (ex) { - this.logger.error(`Cannot decode image: ${imageName}. ${ex}`); - } - } - async getImageBuffer(imageName: string | null): Promise { if (!imageName) return null; try { return await fs.readFile(this.getImagePath(imageName)); } catch (ex) { this.logger.error(`Cannot get image: ${imageName}. ${ex}`); - // an absent file is the only case that means "no image"; a permission or - // I/O failure has to stay an error rather than read as a missing image if ((ex as NodeJS.ErrnoException | undefined)?.code === 'ENOENT') { return null; } @@ -70,6 +58,16 @@ export class HddService implements Static { } } + async getImageStream(imageName: string): Promise { + const imagePath = this.getImagePath(imageName); + try { + return (await fs.open(imagePath, 'r')).createReadStream(); + } catch (ex) { + if ((ex as NodeJS.ErrnoException | undefined)?.code === 'ENOENT') return null; + throw ex; + } + } + async copyImage(type: 'screenshot' | 'diff' | 'baseline', sourceImageName: string): Promise { const { imageName, imagePath } = this.generateNewImage(type); await fs.copyFile(this.getImagePath(sourceImageName), imagePath); @@ -85,14 +83,4 @@ export class HddService implements Static { } return true; } - - private ensureDirectoryExistence(dir: string) { - const filePath = path.resolve(dir); - if (existsSync(filePath)) { - return true; - } else { - mkdirSync(dir, { recursive: true }); - this.ensureDirectoryExistence(dir); - } - } } diff --git a/src/static/static.controller.spec.ts b/src/static/static.controller.spec.ts index 99ecec66..ac0add42 100644 --- a/src/static/static.controller.spec.ts +++ b/src/static/static.controller.spec.ts @@ -1,16 +1,45 @@ import { Test, TestingModule } from '@nestjs/testing'; import { BadRequestException, Logger, NotFoundException } from '@nestjs/common'; import { Response } from 'express'; +import { PassThrough, Readable } from 'stream'; import { StaticController } from './static.controller'; import { StaticService } from './static.service'; - -const initController = async (getImageBufferMock = jest.fn(), getImageUrlMock = jest.fn()) => { +import { generateNewImageName } from './utils'; + +const IMAGE = 'ABCDEFGHJKMNPQRSTVWXYZ012345.screenshot.png'; + +const ACCEPTED_NAMES = [ + generateNewImageName('screenshot'), + generateNewImageName('diff'), + generateNewImageName('baseline'), + '1612345678901.screenshot.png', + 'V1StGXR8_Z5jdHi6B-myT.diff.png', + '0123456789ABCDEFGHJKMNPQRSTV.baseline.png', +]; + +const REJECTED_NAMES = [ + '\\evil.com', + '/\\evil.com', + 'image.png', + 'a.b.screenshot.png', + 'x.screenshot.png.exe', + '..', + '.', + '', + '../../etc/passwd', + 'nested/image.png', + '/etc/passwd', + '/example.com', + '//example.com', +]; + +const initController = async (getImageStreamMock = jest.fn(), getImageUrlMock = jest.fn()) => { const module: TestingModule = await Test.createTestingModule({ controllers: [StaticController], providers: [ { provide: StaticService, - useValue: { getImageBuffer: getImageBufferMock, getImageUrl: getImageUrlMock }, + useValue: { getImageStream: getImageStreamMock, getImageUrl: getImageUrlMock }, }, ], }).compile(); @@ -24,76 +53,81 @@ const responseMock = () => { return res as unknown as Response & { set: jest.Mock; send: jest.Mock; redirect: jest.Mock; status: jest.Mock }; }; +const streamResponse = () => { + const res = Object.assign(new PassThrough(), { set: jest.fn() }); + const chunks: Buffer[] = []; + res.on('data', (chunk: Buffer) => chunks.push(chunk)); + return { res: res as unknown as Response & { set: jest.Mock }, body: () => Buffer.concat(chunks) }; +}; + describe('download', () => { - it('answers with the image bytes', async () => { - const imageBuffer = Buffer.from([1, 2, 3]); - const getImageBufferMock = jest.fn().mockResolvedValueOnce(imageBuffer); - const controller = await initController(getImageBufferMock); - const res = responseMock(); + it('streams the image bytes', async () => { + const getImageStreamMock = jest.fn().mockResolvedValueOnce(Readable.from([Buffer.from([1, 2]), Buffer.from([3])])); + const controller = await initController(getImageStreamMock); + const { res, body } = streamResponse(); - await controller.download('image.png', res); + await controller.download(IMAGE, res); - expect(getImageBufferMock).toHaveBeenCalledWith('image.png'); + expect(getImageStreamMock).toHaveBeenCalledWith(IMAGE); expect(res.set).toHaveBeenCalledWith({ 'Content-Type': 'image/png', - 'Content-Disposition': 'attachment; filename="image.png"', + 'Content-Disposition': `attachment; filename="${IMAGE}"`, + 'Cache-Control': 'private, max-age=31536000, immutable', }); - expect(res.send).toHaveBeenCalledWith(imageBuffer); + expect(body()).toEqual(Buffer.from([1, 2, 3])); }); it('answers 404 for an image that is not there', async () => { - const getImageBufferMock = jest.fn().mockResolvedValueOnce(null); - const controller = await initController(getImageBufferMock); + const getImageStreamMock = jest.fn().mockResolvedValueOnce(null); + const controller = await initController(getImageStreamMock); + const { res } = streamResponse(); - await expect(controller.download('missing.png', responseMock())).rejects.toThrow(NotFoundException); + await expect(controller.download(IMAGE, res)).rejects.toThrow(new NotFoundException(`Image not found: ${IMAGE}`)); + expect(res.set).not.toHaveBeenCalled(); }); - // a name carrying a path could otherwise read any file the process can see - it.each(['../../etc/passwd', '..', '.', '', 'nested/image.png', '/etc/passwd'])( - 'rejects %p without touching storage', - async (fileName) => { - const getImageBufferMock = jest.fn(); - const controller = await initController(getImageBufferMock); - - await expect(controller.download(fileName, responseMock())).rejects.toThrow(BadRequestException); - expect(getImageBufferMock).not.toHaveBeenCalled(); - } - ); -}); - -describe('getUrlAndRedirect', () => { - // Without this the browser asks the API for a location on every single view. - // Storage then hands back a freshly signed, and therefore different, URL, and - // the image it already holds is dead weight — every view a fresh download. - it('lets the browser reuse the redirect it was given', async () => { - const getImageUrlMock = jest.fn().mockResolvedValueOnce('https://storage/image.png'); - const controller = await initController(jest.fn(), getImageUrlMock); - const res = responseMock(); + it('warns instead of throwing when the stream fails midway', async () => { + const warnSpy = jest.spyOn(Logger.prototype, 'warn').mockImplementation(); + const failing = new Readable({ + read() { + this.push(Buffer.from([1])); + this.destroy(new Error('disk gone')); + }, + }); + const controller = await initController(jest.fn().mockResolvedValueOnce(failing)); + const { res } = streamResponse(); - await controller.getUrlAndRedirect('image.png', res); + await expect(controller.download(IMAGE, res)).resolves.toBeUndefined(); - expect(res.set).toHaveBeenCalledWith('Cache-Control', expect.stringContaining('max-age=')); - expect(res.redirect).toHaveBeenCalledWith('https://storage/image.png'); + expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining(`Download of ${IMAGE} stopped`)); + warnSpy.mockRestore(); }); - // the redirect may not outlive the URL it points at - it('stops reusing the redirect before the signed URL can expire', async () => { - const controller = await initController(jest.fn(), jest.fn().mockResolvedValueOnce('https://storage/image.png')); - const res = responseMock(); + it.each(ACCEPTED_NAMES)('accepts the generated name %s', async (fileName) => { + const controller = await initController(jest.fn().mockResolvedValueOnce(Readable.from([Buffer.from([1])]))); + const { res } = streamResponse(); - await controller.getUrlAndRedirect('image.png', res); + await expect(controller.download(fileName, res)).resolves.toBeUndefined(); + }); + + it.each(REJECTED_NAMES)('rejects %p without touching storage', async (fileName) => { + const getImageStreamMock = jest.fn(); + const controller = await initController(getImageStreamMock); - const [, value] = res.set.mock.calls.find(([header]) => header === 'Cache-Control'); - expect(Number(/max-age=(\d+)/.exec(value)![1])).toBeLessThan(3600); + await expect(controller.download(fileName, responseMock())).rejects.toThrow(BadRequestException); + expect(getImageStreamMock).not.toHaveBeenCalled(); }); +}); - it('sets a private cache header of exactly 3000 seconds before redirecting', async () => { +describe('getUrlAndRedirect', () => { + it('lets the browser reuse the redirect for exactly 3000 seconds, short of the signed URL expiry', async () => { const controller = await initController(jest.fn(), jest.fn().mockResolvedValueOnce('https://storage/image.png')); const res = responseMock(); - await controller.getUrlAndRedirect('image.png', res); + await controller.getUrlAndRedirect(IMAGE, res); expect(res.set).toHaveBeenCalledWith('Cache-Control', 'private, max-age=3000'); + expect(res.redirect).toHaveBeenCalledWith('https://storage/image.png'); expect(res.set.mock.invocationCallOrder[0]).toBeLessThan(res.redirect.mock.invocationCallOrder[0]); }); @@ -103,30 +137,34 @@ describe('getUrlAndRedirect', () => { const controller = await initController(jest.fn(), jest.fn().mockRejectedValueOnce(error)); const res = responseMock(); - await controller.getUrlAndRedirect('image.png', res); + await controller.getUrlAndRedirect(IMAGE, res); expect(res.status).toHaveBeenCalledWith(500); expect(res.send).toHaveBeenCalledWith('Error occurred while getting the file.'); expect(res.redirect).not.toHaveBeenCalled(); expect(res.set).not.toHaveBeenCalled(); - expect(errorSpy).toHaveBeenCalledWith('Error fetching file from S3:image.png', error); + expect(errorSpy).toHaveBeenCalledWith(`Error fetching file from S3:${IMAGE}`, error); errorSpy.mockRestore(); }); - // a leading slash would come back as a protocol-relative Location such as - // '//example.com', sending the browser to another host - it.each(['/example.com', '//example.com', 'nested/image.png', '..', '.', ''])( - 'refuses %p before asking storage for a URL', - async (fileName) => { - const getImageUrlMock = jest.fn(); - const controller = await initController(jest.fn(), getImageUrlMock); - const res = responseMock(); - - await expect(controller.getUrlAndRedirect(fileName, res)).rejects.toThrow( - new BadRequestException('Invalid image name') - ); - expect(getImageUrlMock).not.toHaveBeenCalled(); - expect(res.redirect).not.toHaveBeenCalled(); - } - ); + it.each(ACCEPTED_NAMES)('accepts the generated name %s', async (fileName) => { + const controller = await initController(jest.fn(), jest.fn().mockResolvedValueOnce('https://storage/x')); + const res = responseMock(); + + await controller.getUrlAndRedirect(fileName, res); + + expect(res.redirect).toHaveBeenCalledWith('https://storage/x'); + }); + + it.each(REJECTED_NAMES)('refuses %p before asking storage for a URL', async (fileName) => { + const getImageUrlMock = jest.fn(); + const controller = await initController(jest.fn(), getImageUrlMock); + const res = responseMock(); + + await expect(controller.getUrlAndRedirect(fileName, res)).rejects.toThrow( + new BadRequestException('Invalid image name') + ); + expect(getImageUrlMock).not.toHaveBeenCalled(); + expect(res.redirect).not.toHaveBeenCalled(); + }); }); diff --git a/src/static/static.controller.ts b/src/static/static.controller.ts index 5f86d83f..59ab51dc 100644 --- a/src/static/static.controller.ts +++ b/src/static/static.controller.ts @@ -1,20 +1,15 @@ import { BadRequestException, Controller, Get, Logger, NotFoundException, Param, Res } from '@nestjs/common'; import { Response } from 'express'; import { ApiOkResponse, ApiTags } from '@nestjs/swagger'; -import { basename } from 'path'; +import { pipeline } from 'stream/promises'; import { StaticService } from './static.service'; -// Shorter than the window a signed URL is minted for (URL_WINDOW_SECONDS), so a -// cached redirect always still points at a URL that works. const REDIRECT_MAX_AGE_SECONDS = 3000; -// A stored image is always a plain file name, so anything carrying a path is -// rejected before it reaches storage — on the redirect route a name like -// '/example.com' would otherwise send the browser to another host. basename -// keeps '.' and '..' as they are, so both are named here rather than reaching -// a backend that would answer 500 for them, or ask storage for a directory. +export const IMAGE_NAME_PATTERN = /^[0-9A-Za-z_-]{1,64}\.(screenshot|diff|baseline)\.png$/; + const assertImageName = (fileName: string): void => { - if (!fileName || fileName === '.' || fileName === '..' || fileName !== basename(fileName)) { + if (!IMAGE_NAME_PATTERN.test(fileName)) { throw new BadRequestException('Invalid image name'); } }; @@ -23,6 +18,7 @@ const assertImageName = (fileName: string): void => { @Controller('images') export class StaticController { private readonly logger: Logger = new Logger(StaticController.name); + constructor(private staticService: StaticService) {} @Get('/:fileName') @@ -31,12 +27,6 @@ export class StaticController { assertImageName(fileName); try { const url = await this.staticService.getImageUrl(fileName); - // The redirect has to be reusable too, or the browser comes back here for - // a location on every single view — and a storage that signs its URLs - // hands back a different one each time, making the copy the browser - // already holds worthless. Kept well inside the signature's life so the - // redirect can never outlive what it points at. Private, because a signed - // URL is an access grant and has no business in a shared cache. res.set('Cache-Control', `private, max-age=${REDIRECT_MAX_AGE_SECONDS}`); res.redirect(url); } catch (error) { @@ -45,25 +35,24 @@ export class StaticController { } } - /** - * Serves the image bytes from this origin instead of redirecting to storage. - * Redirected pre-signed S3 URLs carry no CORS headers, so a browser `fetch` - * (used to build the bulk download zip) is blocked; an `` tag is not, - * which is why display keeps using the redirect above. - */ @Get('/:fileName/download') @ApiOkResponse() async download(@Param('fileName') fileName: string, @Res() res: Response) { assertImageName(fileName); - const imageBuffer = await this.staticService.getImageBuffer(fileName); - if (!imageBuffer) { + const stream = await this.staticService.getImageStream(fileName); + if (!stream) { throw new NotFoundException(`Image not found: ${fileName}`); } res.set({ 'Content-Type': 'image/png', 'Content-Disposition': `attachment; filename="${fileName}"`, + 'Cache-Control': 'private, max-age=31536000, immutable', }); - res.send(imageBuffer); + try { + await pipeline(stream, res); + } catch (error) { + this.logger.warn(`Download of ${fileName} stopped: ${error}`); + } } } diff --git a/src/static/static.interface.ts b/src/static/static.interface.ts index c7a38d41..492fb31f 100644 --- a/src/static/static.interface.ts +++ b/src/static/static.interface.ts @@ -1,9 +1,9 @@ -import { PNGWithMetadata } from 'pngjs'; +import { Readable } from 'stream'; export interface Static { saveImage(type: 'screenshot' | 'diff' | 'baseline', imageBuffer: Buffer): Promise; - getImage(fileName: string | null): Promise; getImageBuffer(fileName: string | null): Promise; + getImageStream(fileName: string): Promise; copyImage(type: 'screenshot' | 'diff' | 'baseline', sourceImageName: string): Promise; deleteImage(imageName: string | null): Promise; getImageUrl(imageName: string): Promise; diff --git a/src/static/static.service.spec.ts b/src/static/static.service.spec.ts index 5db2437d..e9943a9d 100644 --- a/src/static/static.service.spec.ts +++ b/src/static/static.service.spec.ts @@ -1,10 +1,13 @@ +import { BadRequestException } from '@nestjs/common'; import { Test, TestingModule } from '@nestjs/testing'; +import { Readable } from 'stream'; import { PNG } from 'pngjs'; import { StaticService } from './static.service'; import { ConfigService } from '@nestjs/config'; import { StaticModule } from './static.module'; import { StaticFactoryService } from './static.factory'; import { Static } from './static.interface'; +import { ImageTooLargeException } from './utils'; describe('StaticService', () => { let service: StaticService; @@ -35,8 +38,8 @@ describe('StaticService', () => { beforeEach(() => { backend = { saveImage: jest.fn().mockResolvedValue('saved.png'), - getImage: jest.fn().mockResolvedValue({ width: 1 }), getImageBuffer: jest.fn().mockResolvedValue(Buffer.from('bytes')), + getImageStream: jest.fn().mockResolvedValue(Readable.from([])), copyImage: jest.fn().mockResolvedValue('copied.png'), deleteImage: jest.fn().mockResolvedValue(true), getImageUrl: jest.fn().mockResolvedValue('https://storage/image.png'), @@ -65,38 +68,92 @@ describe('StaticService', () => { await expect(service.saveImage('diff', signatureOnly)).resolves.toBe('saved.png'); }); + describe('image size limit', () => { + const originalPixels = process.env.MAX_IMAGE_PIXELS; + + afterEach(() => { + if (originalPixels === undefined) { + delete process.env.MAX_IMAGE_PIXELS; + } else { + process.env.MAX_IMAGE_PIXELS = originalPixels; + } + }); + + it('rejects a screenshot above MAX_IMAGE_PIXELS with 422 before reaching the backend', async () => { + process.env.MAX_IMAGE_PIXELS = '5'; + const png = PNG.sync.write(new PNG({ width: 3, height: 2 })); + + await expect(service.saveImage('screenshot', png)).rejects.toBeInstanceOf(ImageTooLargeException); + expect(backend.saveImage).not.toHaveBeenCalled(); + }); + + it('stores a baseline or a diff of any size', async () => { + process.env.MAX_IMAGE_PIXELS = '5'; + const png = PNG.sync.write(new PNG({ width: 3, height: 2 })); + + await expect(service.saveImage('baseline', png)).resolves.toBe('saved.png'); + await expect(service.saveImage('diff', png)).resolves.toBe('saved.png'); + }); + }); + it.each([ ['an empty buffer', Buffer.alloc(0)], ['a truncated signature', PNG_SIGNATURE.subarray(0, 7)], ['a JPEG', Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0, 0, 0, 0, 0])], ])('rejects %s without reaching the backend', async (_label, buffer) => { - await expect(service.saveImage('baseline', buffer)).rejects.toThrow('Cannot parse image as PNG file'); + const result = service.saveImage('baseline', buffer); + + await expect(result).rejects.toBeInstanceOf(BadRequestException); + await expect(result).rejects.toThrow('Cannot parse image as PNG file'); expect(backend.saveImage).not.toHaveBeenCalled(); }); - it('delegates getImage', async () => { - await expect(service.getImage('a.png')).resolves.toEqual({ width: 1 }); - expect(backend.getImage).toHaveBeenCalledWith('a.png'); + it('delegates getImageStream', async () => { + const stream = Readable.from([]); + backend.getImageStream.mockResolvedValue(stream); + + await expect(service.getImageStream('a.png')).resolves.toBe(stream); + expect(backend.getImageStream).toHaveBeenCalledWith('a.png'); }); - it('delegates getImageBuffer', async () => { - await expect(service.getImageBuffer('a.png')).resolves.toEqual(Buffer.from('bytes')); - expect(backend.getImageBuffer).toHaveBeenCalledWith('a.png'); + it.each([ + ['getImageBuffer', ['a.png'], Buffer.from('bytes')], + ['copyImage', ['baseline', 'a.png'], 'copied.png'], + ['deleteImage', ['a.png'], true], + ['getImageUrl', ['a.png'], 'https://storage/image.png'], + ] as const)('delegates %s', async (method, args, result) => { + await expect((service[method] as (...a: readonly string[]) => Promise)(...args)).resolves.toEqual( + result + ); + expect(backend[method]).toHaveBeenCalledWith(...args); }); - it('delegates copyImage', async () => { - await expect(service.copyImage('baseline', 'a.png')).resolves.toBe('copied.png'); - expect(backend.copyImage).toHaveBeenCalledWith('baseline', 'a.png'); + it('deleteImages skips empty names and deletes the rest', async () => { + await service.deleteImages(['a.png', null, '', undefined, 'b.png']); + + expect(backend.deleteImage.mock.calls).toEqual([['a.png'], ['b.png']]); }); - it('delegates deleteImage', async () => { - await expect(service.deleteImage('a.png')).resolves.toBe(true); - expect(backend.deleteImage).toHaveBeenCalledWith('a.png'); + it('deleteImages accepts any iterable and caps the concurrency', async () => { + let running = 0; + let peak = 0; + backend.deleteImage.mockImplementation(async () => { + peak = Math.max(peak, ++running); + await new Promise((resolve) => setImmediate(resolve)); + running--; + return true; + }); + + await service.deleteImages(new Set(['1', '2', '3', '4', '5']), 2); + + expect(backend.deleteImage).toHaveBeenCalledTimes(5); + expect(peak).toBe(2); }); - it('delegates getImageUrl', async () => { - await expect(service.getImageUrl('a.png')).resolves.toBe('https://storage/image.png'); - expect(backend.getImageUrl).toHaveBeenCalledWith('a.png'); + it('deleteImages rejects when a delete fails', async () => { + backend.deleteImage.mockRejectedValue(new Error('disk gone')); + + await expect(service.deleteImages(['a.png'])).rejects.toThrow('disk gone'); }); it('passes backend failures through', async () => { diff --git a/src/static/static.service.ts b/src/static/static.service.ts index ce5e40ad..5eae0a5d 100644 --- a/src/static/static.service.ts +++ b/src/static/static.service.ts @@ -1,12 +1,12 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { PNGWithMetadata } from 'pngjs'; +import { BadRequestException, Injectable } from '@nestjs/common'; +import { Readable } from 'stream'; import { StaticFactoryService } from './static.factory'; -import { isPngBuffer } from './utils'; +import { assertPngWithinPixelLimit, isPngBuffer } from './utils'; import { Static } from './static.interface'; +import { mapWithConcurrency } from '../utils/map-with-concurrency'; @Injectable() export class StaticService { - private readonly logger: Logger = new Logger(StaticService.name); private readonly staticService: Static; constructor(private staticFactoryService: StaticFactoryService) { @@ -14,21 +14,23 @@ export class StaticService { } async saveImage(type: 'screenshot' | 'diff' | 'baseline', imageBuffer: Buffer): Promise { - // validate here so every storage backend rejects non-PNG uploads if (!isPngBuffer(imageBuffer)) { - throw new Error('Cannot parse image as PNG file'); + throw new BadRequestException('Cannot parse image as PNG file'); + } + if (type === 'screenshot') { + assertPngWithinPixelLimit(imageBuffer); } return this.staticService.saveImage(type, imageBuffer); } - async getImage(imageName: string | null): Promise { - return this.staticService.getImage(imageName); - } - async getImageBuffer(imageName: string | null): Promise { return this.staticService.getImageBuffer(imageName); } + async getImageStream(imageName: string): Promise { + return this.staticService.getImageStream(imageName); + } + async copyImage(type: 'screenshot' | 'diff' | 'baseline', sourceImageName: string): Promise { return this.staticService.copyImage(type, sourceImageName); } @@ -37,6 +39,11 @@ export class StaticService { return this.staticService.deleteImage(imageName); } + async deleteImages(names: Iterable, concurrency = 16): Promise { + const present = [...names].filter((name): name is string => !!name); + await mapWithConcurrency(present, concurrency, (name) => this.deleteImage(name)); + } + async getImageUrl(imageName: string): Promise { return this.staticService.getImageUrl(imageName); } diff --git a/src/static/utils.spec.ts b/src/static/utils.spec.ts index cbdeae63..0e998674 100644 --- a/src/static/utils.spec.ts +++ b/src/static/utils.spec.ts @@ -1,4 +1,21 @@ -import { generateNewImageName, isHddStaticServiceConfigured, isPngBuffer, isS3ServiceConfigured } from './utils'; +import { readFileSync } from 'fs'; +import { join } from 'path'; +import { crc32 } from 'zlib'; +import { PNG } from 'pngjs'; +import { + asBuffer, + assertPngWithinPixelLimit, + assertWithinPixelLimit, + DEFAULT_MAX_IMAGE_PIXELS, + generateNewImageName, + ImageTooLargeException, + isByteEqualWellFormedPng, + isHddStaticServiceConfigured, + isPngBuffer, + isWellFormedPng, + maxImagePixels, + readPngDimensions, +} from './utils'; describe('static utils', () => { const originalStaticService = process.env.STATIC_SERVICE; @@ -12,17 +29,14 @@ describe('static utils', () => { }); describe('storage detection from STATIC_SERVICE', () => { - // NOTE: current behavior — with STATIC_SERVICE unset both helpers answer - // true, and for an unknown value both answer false, while - // StaticFactoryService falls back to HDD in both cases. - it.each<[string | undefined, boolean, boolean]>([ - [undefined, true, true], - ['', true, true], - ['hdd', true, false], - ['s3', false, true], - ['S3', false, false], - ['gcs', false, false], - ])('STATIC_SERVICE=%p: hdd %p, s3 %p', (value, hdd, s3) => { + it.each<[string | undefined, boolean]>([ + [undefined, true], + ['', true], + ['hdd', true], + ['s3', false], + ['S3', false], + ['gcs', false], + ])('STATIC_SERVICE=%p: hdd %p', (value, hdd) => { if (value === undefined) { delete process.env.STATIC_SERVICE; } else { @@ -30,7 +44,6 @@ describe('static utils', () => { } expect(isHddStaticServiceConfigured()).toBe(hdd); - expect(isS3ServiceConfigured()).toBe(s3); }); }); @@ -52,18 +65,12 @@ describe('static utils', () => { describe('isPngBuffer', () => { const signature = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]; - it('accepts a buffer that is exactly the PNG signature', () => { - expect(isPngBuffer(Buffer.from(signature))).toBe(true); - }); - - it('accepts a buffer that starts with the PNG signature', () => { - expect(isPngBuffer(Buffer.from([...signature, 1, 2, 3]))).toBe(true); - }); - - it('accepts a signature found at the start of a buffer view', () => { - const backing = Buffer.from([0, 0, ...signature, 9]); - - expect(isPngBuffer(backing.subarray(2))).toBe(true); + it.each([ + ['a buffer that is exactly the PNG signature', Buffer.from(signature)], + ['a buffer that starts with the PNG signature', Buffer.from([...signature, 1, 2, 3])], + ['a signature found at the start of a buffer view', Buffer.from([0, 0, ...signature, 9]).subarray(2)], + ])('accepts %s', (_label, buffer) => { + expect(isPngBuffer(buffer)).toBe(true); }); it.each([ @@ -75,4 +82,242 @@ describe('static utils', () => { expect(isPngBuffer(Buffer.from(bytes))).toBe(false); }); }); + + describe('PNG header helpers', () => { + const tiny = () => PNG.sync.write(new PNG({ width: 3, height: 2 })); + const viewOf = (png: Buffer) => { + const view = new Uint8Array(png.length + 4); + view.set(png, 4); + return view.subarray(4); + }; + const real = () => readFileSync(join(__dirname, '../../test/image.png')); + const originalPixels = process.env.MAX_IMAGE_PIXELS; + + afterEach(() => { + if (originalPixels === undefined) { + delete process.env.MAX_IMAGE_PIXELS; + } else { + process.env.MAX_IMAGE_PIXELS = originalPixels; + } + }); + + const refreshCrc = (b: Buffer, offset: number) => { + const length = b.readUInt32BE(offset); + b.writeUInt32BE(crc32(b.subarray(offset + 4, offset + 8 + length)), offset + 8 + length); + }; + + const chunk = (type: string, data: Buffer) => { + const out = Buffer.alloc(12 + data.length); + out.writeUInt32BE(data.length, 0); + out.write(type, 4, 'latin1'); + data.copy(out, 8); + out.writeUInt32BE(crc32(out.subarray(4, 8 + data.length)), 8 + data.length); + return out; + }; + + describe('readPngDimensions', () => { + it('reads the size from IHDR', () => { + expect(readPngDimensions(tiny())).toStrictEqual({ width: 3, height: 2 }); + }); + + it('accepts a Uint8Array view, as it arrives after postMessage', () => { + expect(readPngDimensions(viewOf(tiny()))).toStrictEqual({ width: 3, height: 2 }); + }); + + it.each([ + ['a buffer that is only a signature-less string', () => Buffer.from('png')], + ['a buffer shorter than a header', () => tiny().subarray(0, 32)], + ['a JPEG signature', () => Buffer.concat([Buffer.from([0xff, 0xd8, 0xff, 0xe0]), Buffer.alloc(40)])], + [ + 'a first chunk that is not a 13-byte IHDR', + () => { + const png = tiny(); + png.write('IHDX', 12, 'latin1'); + return png; + }, + ], + [ + 'a zero width', + () => { + const png = tiny(); + png.writeUInt32BE(0, 16); + return png; + }, + ], + ])('returns null for %s', (_label, make) => { + expect(readPngDimensions(make())).toBeNull(); + }); + }); + + describe('asBuffer', () => { + it('returns a Buffer as is', () => { + const buffer = Buffer.from([1, 2, 3]); + + expect(asBuffer(buffer)).toBe(buffer); + }); + + it('wraps a Uint8Array view without copying or widening it', () => { + const backing = new Uint8Array([9, 1, 2, 3, 9]); + const view = backing.subarray(1, 4); + + const buffer = asBuffer(view); + + expect(Buffer.isBuffer(buffer)).toBe(true); + expect([...buffer]).toEqual([1, 2, 3]); + expect(buffer.buffer).toBe(backing.buffer); + }); + }); + + describe('isByteEqualWellFormedPng', () => { + it('accepts two identical well-formed PNGs, Buffer or Uint8Array', () => { + expect(isByteEqualWellFormedPng(tiny(), tiny())).toBe(true); + expect(isByteEqualWellFormedPng(new Uint8Array(tiny()), tiny())).toBe(true); + }); + + it('rejects PNGs that differ', () => { + const other = tiny(); + other[other.length - 20] ^= 0xff; + + expect(isByteEqualWellFormedPng(tiny(), other)).toBe(false); + }); + + it('rejects an identical pair that is not a well-formed PNG', () => { + const truncated = tiny().subarray(0, tiny().length - 5); + + expect(isByteEqualWellFormedPng(truncated, Buffer.from(truncated))).toBe(false); + }); + }); + + describe('isWellFormedPng', () => { + it.each([ + ['a generated PNG', tiny], + ['a real screenshot', real], + ])('accepts %s', (_label, make) => { + expect(isWellFormedPng(make())).toBe(true); + }); + + it('accepts a Uint8Array view', () => { + expect(isWellFormedPng(viewOf(tiny()))).toBe(true); + }); + + const withChunk = (type: string, tail = true) => { + const png = tiny(); + return Buffer.concat([ + png.subarray(0, 33), + chunk(type, Buffer.from([1, 2])), + ...(tail ? [png.subarray(33)] : []), + ]); + }; + const withHeaderByte = (position: number, value: number) => () => { + const png = tiny(); + png[position] = value; + refreshCrc(png, 8); + return png; + }; + + it.each([ + ['a file cut short by 5 bytes', () => tiny().subarray(0, tiny().length - 5)], + ['a file that ends before IEND is complete, on a chunk boundary', () => tiny().subarray(0, tiny().length - 12)], + [ + 'a flipped bit in the middle (CRC mismatch)', + () => { + const png = tiny(); + png[Math.floor(png.length / 2)] ^= 0x01; + return png; + }, + ], + ['an extra byte after IEND', () => Buffer.concat([tiny(), Buffer.from([0])])], + ['a string', () => Buffer.from('png')], + ['a bit depth PNG does not define', withHeaderByte(24, 3)], + ['a color type PNG does not define', withHeaderByte(25, 5)], + ['an unknown compression method', withHeaderByte(26, 1)], + ['an unknown filter method', withHeaderByte(27, 1)], + ['an unknown interlace method', withHeaderByte(28, 2)], + ['an unknown critical chunk', () => withChunk('ABCD')], + ['a chunk type that is not four letters', () => withChunk('ab1D')], + [ + 'a chunk that claims more data than the buffer holds', + () => { + const png = tiny(); + png.writeUInt32BE(0x7fffffff, 33); + return png; + }, + ], + ['a file without IEND', () => withChunk('abCD', false)], + ])('rejects %s', (_label, make) => { + expect(isWellFormedPng(make())).toBe(false); + }); + + it('accepts an unknown ancillary chunk with a valid CRC', () => { + expect(isWellFormedPng(withChunk('abCD'))).toBe(true); + }); + }); + + describe('maxImagePixels', () => { + it('defaults to 100 million pixels', () => { + delete process.env.MAX_IMAGE_PIXELS; + + expect(maxImagePixels()).toBe(DEFAULT_MAX_IMAGE_PIXELS); + expect(DEFAULT_MAX_IMAGE_PIXELS).toBe(100_000_000); + }); + + it.each([ + ['0', 0], + ['2500', 2500], + ['abc', DEFAULT_MAX_IMAGE_PIXELS], + ['-5', DEFAULT_MAX_IMAGE_PIXELS], + [' ', DEFAULT_MAX_IMAGE_PIXELS], + ['', DEFAULT_MAX_IMAGE_PIXELS], + ])('MAX_IMAGE_PIXELS=%p gives %p', (value, expected) => { + process.env.MAX_IMAGE_PIXELS = value; + + expect(maxImagePixels()).toBe(expected); + }); + }); + + describe('assertPngWithinPixelLimit', () => { + it('rejects a PNG above the limit with 422 and names its size', () => { + process.env.MAX_IMAGE_PIXELS = '5'; + + let error: unknown; + try { + assertPngWithinPixelLimit(tiny()); + } catch (e) { + error = e; + } + + expect(error).toBeInstanceOf(ImageTooLargeException); + expect((error as ImageTooLargeException).getStatus()).toBe(422); + expect((error as ImageTooLargeException).message).toContain('3x2'); + }); + + it('lets a PNG exactly at the limit through', () => { + process.env.MAX_IMAGE_PIXELS = '6'; + + expect(() => assertPngWithinPixelLimit(tiny())).not.toThrow(); + }); + + it('lets anything through when the limit is 0', () => { + process.env.MAX_IMAGE_PIXELS = '0'; + + expect(() => assertPngWithinPixelLimit(tiny())).not.toThrow(); + expect(() => assertWithinPixelLimit(100_000, 100_000)).not.toThrow(); + }); + + it('leaves a malformed header to the decoder', () => { + process.env.MAX_IMAGE_PIXELS = '1'; + + expect(() => assertPngWithinPixelLimit(Buffer.from('png'))).not.toThrow(); + }); + + it('rejects a tiny file whose header claims a huge canvas', () => { + delete process.env.MAX_IMAGE_PIXELS; + const png = tiny(); + png.writeUInt32BE(30000, 16); + png.writeUInt32BE(30000, 20); + + expect(() => assertPngWithinPixelLimit(png)).toThrow(ImageTooLargeException); + }); + }); + }); }); diff --git a/src/static/utils.ts b/src/static/utils.ts index 259f7954..f0d66cd3 100644 --- a/src/static/utils.ts +++ b/src/static/utils.ts @@ -1,23 +1,85 @@ +import { UnprocessableEntityException } from '@nestjs/common'; +import { crc32 } from 'zlib'; import { generateRandomKey } from '../utils/random-key'; +import { nonNegativeNumber } from '../utils/env'; export function isHddStaticServiceConfigured() { return !process.env.STATIC_SERVICE || process.env.STATIC_SERVICE === 'hdd'; } -export function isS3ServiceConfigured() { - return !process.env.STATIC_SERVICE || process.env.STATIC_SERVICE === 's3'; -} - export function generateNewImageName(type: 'screenshot' | 'diff' | 'baseline'): string { return `${generateRandomKey()}.${type}.png`; } const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); -// Signature check instead of a full decode: parsing megapixel PNGs just to -// validate them blocks the event loop on every upload. +// Signature only: fully decoding megapixel PNGs blocks the event loop on every upload export function isPngBuffer(imageBuffer: Buffer): boolean { return ( imageBuffer.length >= PNG_SIGNATURE.length && imageBuffer.subarray(0, PNG_SIGNATURE.length).equals(PNG_SIGNATURE) ); } + +export const DEFAULT_MAX_IMAGE_PIXELS = 100_000_000; + +export function maxImagePixels(): number { + return nonNegativeNumber(process.env.MAX_IMAGE_PIXELS, DEFAULT_MAX_IMAGE_PIXELS); +} + +export function asBuffer(bytes: Buffer | Uint8Array): Buffer { + return Buffer.isBuffer(bytes) ? bytes : Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength); +} + +export function readPngDimensions(buffer: Buffer | Uint8Array): { width: number; height: number } | null { + const b = asBuffer(buffer); + if (!isPngBuffer(b) || b.length < 33) return null; + if (b.readUInt32BE(8) !== 13 || b.toString('latin1', 12, 16) !== 'IHDR') return null; + const width = b.readUInt32BE(16); + const height = b.readUInt32BE(20); + return width > 0 && height > 0 ? { width, height } : null; +} + +const CRITICAL_CHUNKS = new Set(['IHDR', 'PLTE', 'IDAT', 'IEND']); + +export function isWellFormedPng(buffer: Buffer | Uint8Array): boolean { + const b = asBuffer(buffer); + if (!readPngDimensions(b)) return false; + const [bitDepth, colorType, compression, filter, interlace] = [b[24], b[25], b[26], b[27], b[28]]; + if (![1, 2, 4, 8, 16].includes(bitDepth) || ![0, 2, 3, 4, 6].includes(colorType)) return false; + if (compression !== 0 || filter !== 0 || interlace > 1) return false; + let offset = 8; + while (offset + 12 <= b.length) { + const length = b.readUInt32BE(offset); + const end = offset + 12 + length; + if (end > b.length) return false; + const type = b.toString('latin1', offset + 4, offset + 8); + if (!/^[A-Za-z]{4}$/.test(type)) return false; + if (type[0] === type[0].toUpperCase() && !CRITICAL_CHUNKS.has(type)) return false; + if (crc32(b.subarray(offset + 4, offset + 8 + length)) !== b.readUInt32BE(offset + 8 + length)) return false; + if (type === 'IEND') return end === b.length; + offset = end; + } + return false; +} + +export function isByteEqualWellFormedPng(a: Buffer | Uint8Array, b: Buffer | Uint8Array): boolean { + return asBuffer(a).equals(asBuffer(b)) && isWellFormedPng(a); +} + +export class ImageTooLargeException extends UnprocessableEntityException { + constructor(width: number, height: number, limit: number) { + super(`Image is ${width}x${height} (${width * height} pixels), above MAX_IMAGE_PIXELS=${limit}`); + } +} + +export function assertWithinPixelLimit(width: number, height: number): void { + const limit = maxImagePixels(); + if (limit > 0 && width * height > limit) { + throw new ImageTooLargeException(width, height, limit); + } +} + +export function assertPngWithinPixelLimit(buffer: Buffer | Uint8Array): void { + const size = readPngDimensions(buffer); + if (size) assertWithinPixelLimit(size.width, size.height); +} diff --git a/src/test-runs/diffResult.ts b/src/test-runs/diffResult.ts index 168fc9bb..f6617916 100644 --- a/src/test-runs/diffResult.ts +++ b/src/test-runs/diffResult.ts @@ -1,44 +1,19 @@ import { TestStatus } from '@prisma/client'; -/** - * A signature and the comparison settings it was computed under. A signature - * only means anything under its own threshold, so comparing one taken at a - * different setting quietly makes grouping worse — the settings travel with it - * so a stored one can be discarded when the project's config moves on. - */ export interface StampedSignature { threshold: number; includeAA: boolean; signature: number[]; } -// Without a baseline nothing was compared, and only diffName is set (to null). export interface DiffResult { status: TestStatus | undefined; diffName: string | null; pixelMisMatchCount: number | undefined; diffPercent: number | undefined; isSameDimension: boolean | undefined; - /** - * Optional array of analysis insights (e.g., from VLM or other AI services) - * Each string represents a distinct observation or difference - * Can be displayed as bullet points in UI - */ vlmDescription?: string; - /** - * Position-independent colour signature of the change, produced by the same - * pass that produced the diff, carrying the settings it was produced under. - * Stored on the run so the variations dialog can group a screen's locales - * without decoding their screenshots all over again. Absent when nothing - * changed, when the dimensions differ, or when the comparison was not - * pixelmatch. - */ changeSignature?: StampedSignature; - /** - * Names the small copies of the checkpoint and the diff were saved under, for - * the card grids to draw instead of the full-size files. Absent when the - * comparison produced no diff, or was not pixelmatch. - */ imageThumbnailName?: string; diffThumbnailName?: string; } diff --git a/src/test-runs/dto/create-test-request-base64.dto.ts b/src/test-runs/dto/create-test-request-base64.dto.ts index a604dc99..c4b4f210 100644 --- a/src/test-runs/dto/create-test-request-base64.dto.ts +++ b/src/test-runs/dto/create-test-request-base64.dto.ts @@ -1,11 +1,9 @@ import { ApiProperty } from '@nestjs/swagger'; -import { Transform } from 'class-transformer'; -import { IsBase64 } from 'class-validator'; +import { IsString } from 'class-validator'; import { CreateTestRequestDto } from './create-test-request.dto'; export class CreateTestRequestBase64Dto extends CreateTestRequestDto { @ApiProperty() - @Transform(({ value }) => value.replace(/(\r\n|\n|\r)/gm, '')) - @IsBase64() + @IsString() imageBase64!: string; } diff --git a/src/test-runs/dto/create-test-request-multipart.dto.ts b/src/test-runs/dto/create-test-request-multipart.dto.ts index 19322f10..96b1d45a 100644 --- a/src/test-runs/dto/create-test-request-multipart.dto.ts +++ b/src/test-runs/dto/create-test-request-multipart.dto.ts @@ -1,33 +1,54 @@ import { ApiPropertyOptional, OmitType } from '@nestjs/swagger'; import { ApiFile } from '../../shared/api-file.decorator'; import { CreateTestRequestDto } from './create-test-request.dto'; -import { Transform, Type } from 'class-transformer'; +import { plainToInstance, Transform } from 'class-transformer'; import { IgnoreAreaDto } from './ignore-area.dto'; -import { IsBoolean, IsNumber, IsOptional } from 'class-validator'; +import { ValidationOptions } from '../../shared/validation'; +import { IsArray, IsBoolean, IsDefined, IsNumber, IsObject, IsOptional, ValidateNested } from 'class-validator'; +const parseIgnoreAreas = (value: unknown): unknown => { + if (!value) { + return []; + } + if (typeof value === 'string') { + try { + value = JSON.parse(value); + } catch { + return value; + } + } + return Array.isArray(value) + ? value.map((item) => (item !== null && typeof item === 'object' ? plainToInstance(IgnoreAreaDto, item) : item)) + : value; +}; + +@ValidationOptions({ transform: true, forbidUnknownValues: true }) export class CreateTestRequestMultipartDto extends OmitType(CreateTestRequestDto, [ 'ignoreAreas', 'merge', 'diffTollerancePercent', ]) { @ApiFile() + @IsDefined() + @IsObject() image!: Express.Multer.File; @ApiPropertyOptional() @IsOptional() @IsNumber() - @Transform(({ value }) => parseFloat(value) || 0) + @Transform(({ value }) => parseFloat(value) || 0, { toClassOnly: true }) diffTollerancePercent?: number; @ApiPropertyOptional() @IsOptional() @IsBoolean() - @Transform(({ value }) => value == 'true') + @Transform(({ value }) => value === true || value === 'true', { toClassOnly: true }) merge?: boolean; @ApiPropertyOptional({ type: String }) @IsOptional() - @Type(() => String) - @Transform(({ value }) => (value ? JSON.parse(value) : [])) + @IsArray() + @ValidateNested({ each: true }) + @Transform(({ value }) => parseIgnoreAreas(value), { toClassOnly: true }) ignoreAreas?: IgnoreAreaDto[]; } diff --git a/src/test-runs/dto/create-test-request.dto.spec.ts b/src/test-runs/dto/create-test-request.dto.spec.ts index e2276049..d880164d 100644 --- a/src/test-runs/dto/create-test-request.dto.spec.ts +++ b/src/test-runs/dto/create-test-request.dto.spec.ts @@ -1,4 +1,5 @@ import { plainToInstance } from 'class-transformer'; +import { GlobalValidationPipe } from '../../shared/validation'; import { validate } from 'class-validator'; import { CreateTestRequestDto } from './create-test-request.dto'; import { CreateTestRequestBase64Dto } from './create-test-request-base64.dto'; @@ -9,23 +10,23 @@ const BUILD_ID = '146e7a8d-89f0-4565-aa2c-e61efabb0afd'; const PROJECT_ID = '3bc4a5bc-006e-4d43-8e4e-eaa132627fcc'; const area = { x: 1, y: 2, width: 3, height: 4 }; +const base = { + buildId: BUILD_ID, + projectId: PROJECT_ID, + name: 'name', + branchName: 'develop', +}; + const failedProperties = async (dto: object): Promise => (await validate(dto)).map((error) => error.property).sort(); describe('CreateTestRequestDto', () => { - const valid = { - buildId: BUILD_ID, - projectId: PROJECT_ID, - name: 'name', - branchName: 'develop', - }; - it('accepts the minimal request', async () => { - expect(await failedProperties(plainToInstance(CreateTestRequestDto, valid))).toEqual([]); + expect(await failedProperties(plainToInstance(CreateTestRequestDto, base))).toEqual([]); }); it('turns ignore areas into IgnoreAreaDto instances', () => { - const dto = plainToInstance(CreateTestRequestDto, { ...valid, ignoreAreas: [area] }); + const dto = plainToInstance(CreateTestRequestDto, { ...base, ignoreAreas: [area] }); expect(dto.ignoreAreas![0]).toBeInstanceOf(IgnoreAreaDto); expect(dto.ignoreAreas![0]).toEqual(area); @@ -33,7 +34,7 @@ describe('CreateTestRequestDto', () => { it('rejects ids that are not uuids, and malformed optional fields', async () => { const dto = plainToInstance(CreateTestRequestDto, { - ...valid, + ...base, buildId: 'not a uuid', projectId: 'not a uuid', diffTollerancePercent: '1', @@ -44,48 +45,43 @@ describe('CreateTestRequestDto', () => { expect(await failedProperties(dto)).toEqual(['buildId', 'comment', 'diffTollerancePercent', 'merge', 'projectId']); }); - it('validates every ignore area', async () => { - const dto = plainToInstance(CreateTestRequestDto, { ...valid, ignoreAreas: [area, { ...area, x: 'one' }] }); + it('baseates every ignore area', async () => { + const dto = plainToInstance(CreateTestRequestDto, { ...base, ignoreAreas: [area, { ...area, x: 'one' }] }); expect(await failedProperties(dto)).toEqual(['ignoreAreas']); }); it('rejects ignore areas that are not an array', async () => { - const dto = plainToInstance(CreateTestRequestDto, { ...valid, ignoreAreas: area }); + const dto = plainToInstance(CreateTestRequestDto, { ...base, ignoreAreas: area }); expect(await failedProperties(dto)).toContain('ignoreAreas'); }); }); describe('CreateTestRequestBase64Dto', () => { - const valid = { - buildId: BUILD_ID, - projectId: PROJECT_ID, - name: 'name', - branchName: 'develop', - }; + it('keeps the image as sent: Buffer.from ignores line breaks', async () => { + const imageBase64 = 'aGVs\r\nbG8g\nd29y\rbGQ='; + const dto = plainToInstance(CreateTestRequestBase64Dto, { ...base, imageBase64 }); + + expect(dto.imageBase64).toBe(imageBase64); + expect(await failedProperties(dto)).toEqual([]); + }); - it('strips line breaks from the image before validating it', async () => { - const dto = plainToInstance(CreateTestRequestBase64Dto, { ...valid, imageBase64: 'aGVs\r\nbG8g\nd29y\rbGQ=' }); + it('accepts any string; the controller rejects what is not a PNG', async () => { + const dto = plainToInstance(CreateTestRequestBase64Dto, { ...base, imageBase64: 'not base64!' }); - expect(dto.imageBase64).toBe('aGVsbG8gd29ybGQ='); expect(await failedProperties(dto)).toEqual([]); }); - it('rejects an image that is not base64', async () => { - const dto = plainToInstance(CreateTestRequestBase64Dto, { ...valid, imageBase64: 'not base64!' }); + it.each([[null], [5]])('rejects a missing, null or numeric image (%p) without throwing', async (imageBase64) => { + const dto = plainToInstance(CreateTestRequestBase64Dto, { ...base, imageBase64 }); expect(await failedProperties(dto)).toEqual(['imageBase64']); }); }); describe('CreateTestRequestMultipartDto', () => { - const valid = { - buildId: BUILD_ID, - projectId: PROJECT_ID, - name: 'name', - branchName: 'develop', - }; + const valid = { ...base, image: { fieldname: 'image', buffer: Buffer.from('x') } }; it.each([ ['1.5', 1.5], @@ -103,26 +99,27 @@ describe('CreateTestRequestMultipartDto', () => { ['false', false], ['TRUE', false], ['1', false], + [true, true], ])('reads merge %p from the form as %p', (value, expected) => { const dto = plainToInstance(CreateTestRequestMultipartDto, { ...valid, merge: value }); expect(dto.merge).toBe(expected); }); - it('reads ignore areas from a JSON string', () => { - const dto = plainToInstance(CreateTestRequestMultipartDto, { ...valid, ignoreAreas: JSON.stringify([area]) }); + it.each([ + ['a JSON string', JSON.stringify([area]), [area]], + ['an empty string as an empty list', '', []], + ])('reads ignore areas from %s', (_case, ignoreAreas, expected) => { + const dto = plainToInstance(CreateTestRequestMultipartDto, { ...valid, ignoreAreas }); - expect(dto.ignoreAreas).toEqual([area]); + expect(dto.ignoreAreas).toEqual(expected); }); - it('reads empty ignore areas as an empty list', () => { - const dto = plainToInstance(CreateTestRequestMultipartDto, { ...valid, ignoreAreas: '' }); - - expect(dto.ignoreAreas).toEqual([]); - }); + it('keeps ignore areas that are not JSON as text, so validation rejects them', async () => { + const dto = plainToInstance(CreateTestRequestMultipartDto, { ...valid, ignoreAreas: '[{' }); - it('throws on ignore areas that are not JSON', () => { - expect(() => plainToInstance(CreateTestRequestMultipartDto, { ...valid, ignoreAreas: '[{' })).toThrow(SyntaxError); + expect(dto.ignoreAreas).toBe('[{'); + expect(await failedProperties(dto)).toContain('ignoreAreas'); }); it('leaves absent form fields undefined', () => { @@ -135,13 +132,6 @@ describe('CreateTestRequestMultipartDto', () => { }).toEqual({ diffTollerancePercent: undefined, merge: undefined, ignoreAreas: undefined }); }); - it('reads a JSON boolean merge as false', () => { - const dto = plainToInstance(CreateTestRequestMultipartDto, { ...valid, merge: true }); - - // NOTE: current behavior — only the string 'true' counts, `true == 'true'` is false - expect(dto.merge).toBe(false); - }); - it('accepts a request with the form fields converted', async () => { const dto = plainToInstance(CreateTestRequestMultipartDto, { ...valid, @@ -152,4 +142,69 @@ describe('CreateTestRequestMultipartDto', () => { expect(await failedProperties(dto)).toEqual([]); }); + + it('rejects a request without the image', async () => { + const { image: _image, ...withoutImage } = valid; + + expect(await failedProperties(plainToInstance(CreateTestRequestMultipartDto, withoutImage))).toEqual(['image']); + }); + + it('rejects a text field named image', async () => { + const dto = plainToInstance(CreateTestRequestMultipartDto, { ...valid, image: 'text' }); + + expect(await failedProperties(dto)).toEqual(['image']); + }); + + it.each([ + ['a string coordinate', '[{"x":"1","y":1,"width":1,"height":1}]'], + ['a coordinate beyond the bounds', '[{"x":-2e6,"y":1,"width":1,"height":1}]'], + ['an object instead of an array', '{"x":1,"y":1,"width":1,"height":1}'], + ])('validates every parsed ignore area: rejects %s', async (_, ignoreAreas) => { + const dto = plainToInstance(CreateTestRequestMultipartDto, { ...valid, ignoreAreas }); + + expect(await failedProperties(dto)).toEqual(['ignoreAreas']); + }); + + it('validates every parsed ignore area: accepts fractional ones as IgnoreAreaDto', async () => { + const dto = plainToInstance(CreateTestRequestMultipartDto, { + ...valid, + ignoreAreas: '[{"x":1.5,"y":1,"width":1,"height":1}]', + }); + + expect(await failedProperties(dto)).toEqual([]); + expect(dto.ignoreAreas![0]).toBeInstanceOf(IgnoreAreaDto); + }); + + it('takes already parsed ignore areas as they are', () => { + const dto = plainToInstance(CreateTestRequestMultipartDto, { ...valid, ignoreAreas: [area] }); + + expect(dto.ignoreAreas).toEqual([area]); + expect(dto.ignoreAreas![0]).toBeInstanceOf(IgnoreAreaDto); + }); + + it('keeps the id and every other key of an area', () => { + const withExtras = { id: '1', ...area, extra: 2 }; + const dto = plainToInstance(CreateTestRequestMultipartDto, { ...valid, ignoreAreas: JSON.stringify([withExtras]) }); + + expect(dto.ignoreAreas).toEqual([withExtras]); + }); + + it('is parsed once by the global pipe, with the form values converted', async () => { + const meta = { type: 'body' as const, metatype: CreateTestRequestMultipartDto, data: '' }; + const body = { + ...valid, + merge: 'true', + diffTollerancePercent: '1.5', + ignoreAreas: '[{"id":"7","x":1.5,"y":2,"width":3,"height":4}]', + image: { fieldname: 'image', buffer: Buffer.from('png') }, + }; + + const result = (await new GlobalValidationPipe().transform(body, meta)) as CreateTestRequestMultipartDto; + + expect(result.merge).toBe(true); + expect(result.diffTollerancePercent).toBe(1.5); + expect(result.ignoreAreas).toEqual([{ id: '7', x: 1.5, y: 2, width: 3, height: 4 }]); + expect(Buffer.isBuffer(result.image.buffer)).toBe(true); + expect(result.image.buffer.toString()).toBe('png'); + }); }); diff --git a/src/test-runs/dto/create-test-request.dto.ts b/src/test-runs/dto/create-test-request.dto.ts index 1b22ac9a..211285fb 100644 --- a/src/test-runs/dto/create-test-request.dto.ts +++ b/src/test-runs/dto/create-test-request.dto.ts @@ -34,4 +34,9 @@ export class CreateTestRequestDto extends BaselineDataDto { @IsOptional() @IsString() comment?: string | null; + + @ApiPropertyOptional() + @IsOptional() + @IsString() + sourceBranch?: string; } diff --git a/src/test-runs/dto/ignore-area.dto.spec.ts b/src/test-runs/dto/ignore-area.dto.spec.ts new file mode 100644 index 00000000..8899292c --- /dev/null +++ b/src/test-runs/dto/ignore-area.dto.spec.ts @@ -0,0 +1,38 @@ +import { plainToInstance } from 'class-transformer'; +import { validate } from 'class-validator'; +import { IgnoreAreaDto } from './ignore-area.dto'; + +const area = { x: 1, y: 2, width: 3, height: 4 }; + +const failedProperties = async (dto: object): Promise => + (await validate(dto)).map((error) => error.property).sort(); + +describe('IgnoreAreaDto', () => { + it('accepts fractional and negative coordinates within the bounds', async () => { + const dto = plainToInstance(IgnoreAreaDto, { x: -10.5, y: 0.25, width: 3.3, height: 4 }); + + expect(await failedProperties(dto)).toEqual([]); + }); + + it('rejects x or y beyond a million either way', async () => { + expect(await failedProperties(plainToInstance(IgnoreAreaDto, { ...area, x: -1e6 - 1 }))).toEqual(['x']); + expect(await failedProperties(plainToInstance(IgnoreAreaDto, { ...area, y: 1e6 + 1 }))).toEqual(['y']); + }); + + it('does not bound the size', async () => { + const dto = plainToInstance(IgnoreAreaDto, { ...area, width: 1e9, height: -1e9 }); + + expect(await failedProperties(dto)).toEqual([]); + }); + + it.each([['abc'], [5]])('keeps the id the frontend sends (%p), whatever its type', async (id) => { + const dto = plainToInstance(IgnoreAreaDto, { ...area, id }); + + expect(await validate(dto, { whitelist: true })).toEqual([]); + expect(dto.id).toBe(id); + }); + + it('does not require the id', async () => { + expect(await failedProperties(plainToInstance(IgnoreAreaDto, area))).toEqual([]); + }); +}); diff --git a/src/test-runs/dto/ignore-area.dto.ts b/src/test-runs/dto/ignore-area.dto.ts index 36cc6c51..24f8bbf4 100644 --- a/src/test-runs/dto/ignore-area.dto.ts +++ b/src/test-runs/dto/ignore-area.dto.ts @@ -1,13 +1,23 @@ -import { ApiProperty } from '@nestjs/swagger'; -import { IsNumber } from 'class-validator'; +import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; +import { IsNumber, IsOptional, Max, Min } from 'class-validator'; + +export const MAX_IGNORE_AREA_COORDINATE = 1_000_000; export class IgnoreAreaDto { + @ApiPropertyOptional() + @IsOptional() + readonly id?: string; + @ApiProperty() @IsNumber() + @Min(-MAX_IGNORE_AREA_COORDINATE) + @Max(MAX_IGNORE_AREA_COORDINATE) readonly x!: number; @ApiProperty() @IsNumber() + @Min(-MAX_IGNORE_AREA_COORDINATE) + @Max(MAX_IGNORE_AREA_COORDINATE) readonly y!: number; @ApiProperty() diff --git a/src/test-runs/dto/testRun-paginated.dto.spec.ts b/src/test-runs/dto/testRun-paginated.dto.spec.ts index 3177bf6b..3720b26a 100644 --- a/src/test-runs/dto/testRun-paginated.dto.spec.ts +++ b/src/test-runs/dto/testRun-paginated.dto.spec.ts @@ -2,7 +2,6 @@ import { Pagination } from '../../shared/dto/pagination.dto'; import { PaginatedTestRunDto } from './testRun-paginated.dto'; import { TestRunDto } from './testRun.dto'; -// metadata keys @nestjs/swagger stores @ApiProperty under (not exported from the package entry point) const API_MODEL_PROPERTIES = 'swagger/apiModelProperties'; const API_MODEL_PROPERTIES_ARRAY = 'swagger/apiModelPropertiesArray'; diff --git a/src/test-runs/dto/testRun.dto.ts b/src/test-runs/dto/testRun.dto.ts index d61694ef..2662cde2 100644 --- a/src/test-runs/dto/testRun.dto.ts +++ b/src/test-runs/dto/testRun.dto.ts @@ -8,7 +8,6 @@ export class TestRunDto { buildId: string; @ApiProperty() imageName: string; - // small copies for the card grids; null on runs ingested before these existed @ApiPropertyOptional() imageThumbnailName?: string | null; @ApiPropertyOptional() diff --git a/src/test-runs/dto/testRunResult.dto.spec.ts b/src/test-runs/dto/testRunResult.dto.spec.ts index 5fcb51f1..589aee19 100644 --- a/src/test-runs/dto/testRunResult.dto.spec.ts +++ b/src/test-runs/dto/testRunResult.dto.spec.ts @@ -40,14 +40,13 @@ describe('TestRunResultDto', () => { expect(result.baselineName).toBe('baseline of the variation'); }); - it('builds the link even when the frontend url is not configured', () => { + it('renders a missing APP_FRONTEND_URL as "undefined" (startup warns about it)', () => { delete process.env.APP_FRONTEND_URL; const testRun = generateTestRun(); const testVariation = generateTestVariation({ projectId: 'project-id' }); const result = new TestRunResultDto(testRun, testVariation); - // NOTE: current behavior — a missing APP_FRONTEND_URL is rendered as the literal string "undefined" expect(result.url).toBe(`undefined/project-id?buildId=${testRun.buildId}&testId=${testRun.id}`); }); }); diff --git a/src/test-runs/dto/testRunResult.dto.ts b/src/test-runs/dto/testRunResult.dto.ts index 30742640..85fcd0c7 100644 --- a/src/test-runs/dto/testRunResult.dto.ts +++ b/src/test-runs/dto/testRunResult.dto.ts @@ -6,11 +6,11 @@ export class TestRunResultDto extends TestRunDto { @ApiPropertyOptional() pixelMisMatchCount?: number | null; @ApiPropertyOptional() - vlmDescription?: string | null; + override vlmDescription?: string | null; @ApiProperty() url: string; @ApiProperty() - baselineName: string | null; + override baselineName: string | null; constructor(testRun: TestRun, testVariation: TestVariation) { super(testRun); diff --git a/src/test-runs/dto/update-ignore-area.dto.spec.ts b/src/test-runs/dto/update-ignore-area.dto.spec.ts index 09b14441..caf9bb64 100644 --- a/src/test-runs/dto/update-ignore-area.dto.spec.ts +++ b/src/test-runs/dto/update-ignore-area.dto.spec.ts @@ -21,4 +21,10 @@ describe('UpdateIgnoreAreasDto', () => { expect(await failedProperties(dto)).toEqual(['ids', 'ignoreAreas']); }); + + it('rejects ids and areas that are not arrays', async () => { + const dto = plainToInstance(UpdateIgnoreAreasDto, { ids: 'abc', ignoreAreas: area }); + + expect(await failedProperties(dto)).toEqual(['ids', 'ignoreAreas']); + }); }); diff --git a/src/test-runs/dto/update-ignore-area.dto.ts b/src/test-runs/dto/update-ignore-area.dto.ts index 68d11932..5a072859 100644 --- a/src/test-runs/dto/update-ignore-area.dto.ts +++ b/src/test-runs/dto/update-ignore-area.dto.ts @@ -1,14 +1,16 @@ import { ApiProperty } from '@nestjs/swagger'; import { IgnoreAreaDto } from './ignore-area.dto'; import { Type } from 'class-transformer'; -import { ValidateNested, IsString } from 'class-validator'; +import { IsArray, ValidateNested, IsString } from 'class-validator'; export class UpdateIgnoreAreasDto { @ApiProperty({ type: String, isArray: true }) + @IsArray() @IsString({ each: true }) readonly ids!: string[]; @ApiProperty({ type: IgnoreAreaDto, isArray: true }) + @IsArray() @Type(() => IgnoreAreaDto) @ValidateNested({ each: true }) readonly ignoreAreas!: IgnoreAreaDto[]; diff --git a/src/test-runs/test-runs.controller.spec.ts b/src/test-runs/test-runs.controller.spec.ts index f476b687..ff863d4e 100644 --- a/src/test-runs/test-runs.controller.spec.ts +++ b/src/test-runs/test-runs.controller.spec.ts @@ -1,8 +1,16 @@ import { Test, TestingModule } from '@nestjs/testing'; -import { GUARDS_METADATA, INTERCEPTORS_METADATA, PIPES_METADATA } from '@nestjs/common/constants'; -import { NotFoundException, ValidationPipe } from '@nestjs/common'; -import { Role, TestStatus, User } from '@prisma/client'; -import { TestRunsController } from './test-runs.controller'; +import { + GUARDS_METADATA, + HTTP_CODE_METADATA, + INTERCEPTORS_METADATA, + PATH_METADATA, + PIPES_METADATA, + ROUTE_ARGS_METADATA, +} from '@nestjs/common/constants'; +import { BadRequestException, NotFoundException, ParseArrayPipe } from '@nestjs/common'; +import { PNG } from 'pngjs'; +import { Role, User } from '@prisma/client'; +import { TestRunsController, multipartLimits, parseIds, parseUuidIds } from './test-runs.controller'; import { TestRunsService } from './test-runs.service'; import { TestVariationsService } from '../test-variations/test-variations.service'; import { PrismaService } from '../prisma/prisma.service'; @@ -15,6 +23,9 @@ import { CreateTestRequestBase64Dto } from './dto/create-test-request-base64.dto import { CreateTestRequestMultipartDto } from './dto/create-test-request-multipart.dto'; import { UpdateIgnoreAreasDto } from './dto/update-ignore-area.dto'; +const png = PNG.sync.write(new PNG({ width: 1, height: 1 })); +const request = { buildId: 'build', projectId: 'project', name: 'name', branchName: 'develop' }; + const initController = async (testRunsService: Partial> = {}) => { const module: TestingModule = await Test.createTestingModule({ controllers: [TestRunsController], @@ -28,45 +39,41 @@ const initController = async (testRunsService: Partial(TestRunsController); }; +const stub = async (method: keyof TestRunsService, result?: unknown) => { + const mock = jest.fn().mockResolvedValue(result); + return { controller: await initController({ [method]: mock }), mock }; +}; + describe('TestRuns Controller', () => { - let controller: TestRunsController; const user = { id: 'user id' } as User; const ignoreAreas = [{ x: 1, y: 2, width: 3, height: 4 }]; - beforeEach(async () => { - controller = await initController(); - }); - - it('should be defined', () => { - expect(controller).toBeDefined(); + it('should be defined', async () => { + expect(await initController()).toBeDefined(); }); it('lists the test runs of a build', async () => { const list = [{ id: 'a' }, { id: 'b' }]; - const findMany = jest.fn().mockResolvedValueOnce(list); - controller = await initController({ findMany }); + const { controller, mock } = await stub('findMany', list); const result = await controller.get('build id'); - expect(findMany).toHaveBeenCalledWith('build id'); + expect(mock).toHaveBeenCalledWith('build id'); expect(result).toBe(list); }); - it('returns the details of one test run as the service gives them', async () => { + it('returns the run as stored, with its variation (kept for existing clients)', async () => { const testRun = generateTestRun(); - const findOne = jest.fn().mockResolvedValueOnce(testRun); - controller = await initController({ findOne }); + const { controller, mock } = await stub('findOne', testRun); const result = await controller.getDetails(testRun.id); - expect(findOne).toHaveBeenCalledWith(testRun.id); - // NOTE: current behavior — returned raw (no TestRunDto mapping), so it also carries testVariation etc. + expect(mock).toHaveBeenCalledWith(testRun.id); expect(result).toBe(testRun); }); it('throws NotFoundException for an unknown test run', async () => { - const findOne = jest.fn().mockResolvedValueOnce(null); - controller = await initController({ findOne }); + const { controller } = await stub('findOne', null); await expect(controller.getDetails('missing')).rejects.toThrow( new NotFoundException('Test run not found: missing') @@ -76,97 +83,78 @@ describe('TestRuns Controller', () => { describe('approveTestRun', () => { it('approves every id one after another with merge flag, no auto-approve and the current user', async () => { const order: string[] = []; - const approve = jest.fn().mockImplementation(async (id: string) => { + const { controller, mock } = await stub('approve'); + mock.mockImplementation(async (id: string) => { order.push(`start ${id}`); await new Promise((resolve) => setTimeout(resolve, 0)); order.push(`end ${id}`); }); - controller = await initController({ approve }); const result = await controller.approveTestRun(user, ['1', '2'], true); - expect(approve).toHaveBeenNthCalledWith(1, '1', true, false, user.id); - expect(approve).toHaveBeenNthCalledWith(2, '2', true, false, user.id); + expect(mock).toHaveBeenNthCalledWith(1, '1', true, false, user.id); + expect(mock).toHaveBeenNthCalledWith(2, '2', true, false, user.id); expect(order).toEqual(['start 1', 'end 1', 'start 2', 'end 2']); expect(result).toBeUndefined(); }); it('stops at the first failing id', async () => { - const approve = jest.fn().mockRejectedValueOnce(new Error('boom')); - controller = await initController({ approve }); + const { controller, mock } = await stub('approve'); + mock.mockRejectedValueOnce(new Error('boom')); await expect(controller.approveTestRun(user, ['1', '2'], false)).rejects.toThrow('boom'); - expect(approve).toHaveBeenCalledTimes(1); - expect(approve).toHaveBeenCalledWith('1', false, false, user.id); + expect(mock).toHaveBeenCalledTimes(1); + expect(mock).toHaveBeenCalledWith('1', false, false, user.id); }); it('does nothing for an empty list', async () => { - const approve = jest.fn(); - controller = await initController({ approve }); + const { controller, mock } = await stub('approve'); await controller.approveTestRun(user, [], false); - expect(approve).not.toHaveBeenCalled(); + expect(mock).not.toHaveBeenCalled(); }); }); it('returns the matching variations of a test run', async () => { const group = { variations: [{ id: 'a' }], skipped: [] as unknown[] }; - const getMatchingVariations = jest.fn().mockResolvedValueOnce(group); - controller = await initController({ getMatchingVariations }); + const { controller, mock } = await stub('getMatchingVariations', group); const result = await controller.getMatchingSiblings('a'); - expect(getMatchingVariations).toHaveBeenCalledWith('a'); + expect(mock).toHaveBeenCalledWith('a'); expect(result).toBe(group); }); - it('rejects every id by setting its status to failed', async () => { - const setStatus = jest.fn().mockResolvedValue(undefined); - controller = await initController({ setStatus }); + it.each(['reject', 'delete'] as const)('%s goes through every id one by one', async (method) => { + const { controller, mock } = await stub(method); - const result = await controller.reject(['1', '2']); + const result = await controller[method](['1', '2']); - expect(setStatus.mock.calls).toEqual([ - ['1', TestStatus.failed], - ['2', TestStatus.failed], - ]); + expect(mock.mock.calls).toEqual([['1'], ['2']]); expect(result).toBeUndefined(); }); - it('deletes every id', async () => { - const deleteMock = jest.fn().mockResolvedValue(undefined); - controller = await initController({ delete: deleteMock }); + it('revert passes the ids to the service as one request and answers 204', async () => { + const { controller, mock } = await stub('revert'); - const result = await controller.delete(['1', '2']); + const result = await controller.revert(['1', '2']); - expect(deleteMock.mock.calls).toEqual([['1'], ['2']]); + expect(mock).toHaveBeenCalledTimes(1); + expect(mock).toHaveBeenCalledWith(['1', '2']); expect(result).toBeUndefined(); + expect(Reflect.getMetadata(HTTP_CODE_METADATA, TestRunsController.prototype.revert)).toBe(204); + expect(Reflect.getMetadata(PATH_METADATA, TestRunsController.prototype.revert)).toBe('revert'); }); - it('replaces the ignore areas of every id with the same areas', async () => { - const updateIgnoreAreas = jest.fn().mockResolvedValue(undefined); - controller = await initController({ updateIgnoreAreas }); + it.each(['updateIgnoreAreas', 'addIgnoreAreas'] as const)('%s hands the same areas to every id', async (method) => { + const { controller, mock } = await stub(method); const data: UpdateIgnoreAreasDto = { ids: ['1', '2'], ignoreAreas }; - const result = await controller.updateIgnoreAreas(data); + const result = await controller[method](data); - expect(updateIgnoreAreas.mock.calls).toEqual([ - ['1', ignoreAreas], - ['2', ignoreAreas], - ]); - expect(result).toBeUndefined(); - }); - - it('adds the same ignore areas to every id', async () => { - const addIgnoreAreas = jest.fn().mockResolvedValue(undefined); - controller = await initController({ addIgnoreAreas }); - const data: UpdateIgnoreAreasDto = { ids: ['1', '2'], ignoreAreas }; - - const result = await controller.addIgnoreAreas(data); - - expect(addIgnoreAreas.mock.calls).toEqual([ + expect(mock.mock.calls).toEqual([ ['1', ignoreAreas], ['2', ignoreAreas], ]); @@ -175,54 +163,126 @@ describe('TestRuns Controller', () => { it('updates the comment of a test run and returns what the service returns', async () => { const testRun = generateTestRun({ comment: 'new' }); - const update = jest.fn().mockResolvedValueOnce(testRun); - controller = await initController({ update }); + const { controller, mock } = await stub('update', testRun); const result = await controller.update(testRun.id, { comment: 'new' }); - expect(update).toHaveBeenCalledWith(testRun.id, { comment: 'new' }); + expect(mock).toHaveBeenCalledWith(testRun.id, { comment: 'new' }); expect(result).toBe(testRun); }); - it('decodes the base64 image and posts the test run', async () => { + it('passes the request and the decoded image to the service', async () => { const response = { id: 'result' }; - const postTestRun = jest.fn().mockResolvedValueOnce(response); - controller = await initController({ postTestRun }); - const createTestRequestDto = { - buildId: 'build', - projectId: 'project', - name: 'name', - branchName: 'develop', - imageBase64: Buffer.from('png bytes').toString('base64'), - } as CreateTestRequestBase64Dto; + const { controller, mock } = await stub('postTestRun', response); + const createTestRequestDto = { ...request, imageBase64: png.toString('base64') } as CreateTestRequestBase64Dto; const result = await controller.postTestRun(createTestRequestDto); - expect(postTestRun).toHaveBeenCalledWith({ createTestRequestDto, imageBuffer: Buffer.from('png bytes') }); - // NOTE: current behavior — the whole dto, imageBase64 included, is passed on to the service - expect(postTestRun.mock.calls[0][0].createTestRequestDto.imageBase64).toBe(createTestRequestDto.imageBase64); + expect(mock).toHaveBeenCalledWith({ createTestRequestDto, imageBuffer: png }); + expect(mock.mock.calls[0][0].createTestRequestDto.imageBase64).toBe(createTestRequestDto.imageBase64); expect(result).toBe(response); }); it('posts a multipart test run with the uploaded file buffer', async () => { const response = { id: 'result' }; - const postTestRun = jest.fn().mockResolvedValueOnce(response); - controller = await initController({ postTestRun }); - const buffer = Buffer.from('png bytes'); + const { controller, mock } = await stub('postTestRun', response); const createTestRequestDto = { - buildId: 'build', - projectId: 'project', - name: 'name', - branchName: 'develop', - image: { buffer } as Express.Multer.File, + ...request, + image: { buffer: png } as Express.Multer.File, } as CreateTestRequestMultipartDto; const result = await controller.postTestRunMultipart(createTestRequestDto); - expect(postTestRun).toHaveBeenCalledWith({ createTestRequestDto, imageBuffer: buffer }); + expect(mock).toHaveBeenCalledWith({ createTestRequestDto, imageBuffer: png }); expect(result).toBe(response); }); + it.each([ + [ + 'a base64 image that is not a PNG', + (controller: TestRunsController) => + controller.postTestRun({ imageBase64: Buffer.from('GIF89a').toString('base64') } as CreateTestRequestBase64Dto), + ], + [ + 'a multipart image without a buffer', + (controller: TestRunsController) => + controller.postTestRunMultipart({ image: {} as Express.Multer.File } as CreateTestRequestMultipartDto), + ], + ])('rejects %s and posts nothing', async (_case, post) => { + const { controller, mock } = await stub('postTestRun'); + + await expect(post(controller)).rejects.toThrow(BadRequestException); + expect(mock).not.toHaveBeenCalled(); + }); + + it.each(['approveTestRun', 'reject', 'delete', 'revert'] as const)( + 'validates the bulk ids of %s as an array', + (method) => { + const args = Reflect.getMetadata(ROUTE_ARGS_METADATA, TestRunsController, method) as Record< + string, + { data?: unknown; pipes: unknown[] } + >; + + const pipes = Object.values(args).flatMap((arg) => arg.pipes); + + expect(pipes.some((pipe) => pipe instanceof ParseArrayPipe)).toBe(true); + } + ); + + it('parseIds rejects a body that is not an array and passes an array through', async () => { + const pipe = parseIds(); + + await expect(pipe.transform({ ids: [] }, { type: 'body' })).rejects.toThrow(BadRequestException); + await expect(pipe.transform(['a', 'b'], { type: 'body' })).resolves.toEqual(['a', 'b']); + }); + + it('parseUuidIds rejects ids that are not uuids and passes uuids through', () => { + const pipe = parseUuidIds(); + const id = '10fb5e02-64e0-4cf5-9f17-c00ab3c96658'; + + expect(pipe.transform([id], { type: 'body' })).toEqual([id]); + expect(() => pipe.transform([id, 'abc', '1'], { type: 'body' })).toThrow( + new BadRequestException('Not a uuid: abc, 1') + ); + }); + + it('revert validates the ids as uuids', () => { + const args = Reflect.getMetadata(ROUTE_ARGS_METADATA, TestRunsController, 'revert') as Record< + string, + { pipes: unknown[] } + >; + + expect(Object.values(args).flatMap((arg) => arg.pipes)).toHaveLength(2); + }); + + describe('multipartLimits', () => { + afterEach(() => { + delete process.env.MULTIPART_FILE_SIZE_LIMIT; + }); + + it('defaults to 100 MiB', () => { + expect(multipartLimits()).toEqual({ fileSize: 104857600 }); + }); + + it('reads the limit from the environment on every call', () => { + process.env.MULTIPART_FILE_SIZE_LIMIT = '1000'; + + expect(multipartLimits()).toEqual({ fileSize: 1000 }); + }); + + it('falls back to the default for a value that is not a number', () => { + process.env.MULTIPART_FILE_SIZE_LIMIT = 'abc'; + + expect(multipartLimits()).toEqual({ fileSize: 104857600 }); + }); + + it('falls back to the default for a negative value', () => { + process.env.MULTIPART_FILE_SIZE_LIMIT = '-1'; + + expect(multipartLimits()).toEqual({ fileSize: 104857600 }); + }); + }); + describe('route protection', () => { const guardsOf = (method: keyof TestRunsController) => Reflect.getMetadata(GUARDS_METADATA, TestRunsController.prototype[method]); @@ -237,7 +297,7 @@ describe('TestRuns Controller', () => { } ); - it.each(['approveTestRun', 'reject', 'delete', 'updateIgnoreAreas', 'addIgnoreAreas', 'update'] as const)( + it.each(['approveTestRun', 'reject', 'revert', 'delete', 'updateIgnoreAreas', 'addIgnoreAreas', 'update'] as const)( '%s needs a logged in admin or editor', (method) => { expect(guardsOf(method)).toEqual([JwtAuthGuard, RoleGuard]); @@ -250,19 +310,13 @@ describe('TestRuns Controller', () => { expect(rolesOf(method)).toEqual([Role.admin, Role.editor]); }); - it('moves the uploaded image into the body and validates the multipart request', () => { + it('moves the uploaded image into the body; the DTO carries its own validation options', () => { const method = TestRunsController.prototype.postTestRunMultipart; const interceptors = Reflect.getMetadata(INTERCEPTORS_METADATA, method); - const pipes = Reflect.getMetadata(PIPES_METADATA, method); expect(interceptors).toHaveLength(2); expect(interceptors[1]).toBe(FileToBodyInterceptor); - expect(pipes).toHaveLength(1); - expect(pipes[0]).toBeInstanceOf(ValidationPipe); - expect(pipes[0]).toMatchObject({ - isTransformEnabled: true, - validatorOptions: expect.objectContaining({ forbidUnknownValues: true }), - }); + expect(Reflect.getMetadata(PIPES_METADATA, method)).toBeUndefined(); }); }); }); diff --git a/src/test-runs/test-runs.controller.ts b/src/test-runs/test-runs.controller.ts index c9aaa69e..2505eeba 100644 --- a/src/test-runs/test-runs.controller.ts +++ b/src/test-runs/test-runs.controller.ts @@ -9,11 +9,13 @@ import { Post, ParseBoolPipe, UseInterceptors, - UsePipes, - ValidationPipe, Logger, Patch, NotFoundException, + BadRequestException, + ParseArrayPipe, + HttpCode, + PipeTransform, } from '@nestjs/common'; import { ApiTags, @@ -25,9 +27,10 @@ import { ApiConsumes, ApiBody, ApiOperation, + ApiNoContentResponse, } from '@nestjs/swagger'; import { JwtAuthGuard } from '../auth/guards/auth.guard'; -import { Role, TestRun, TestStatus, User } from '@prisma/client'; +import { Role, TestRun, User } from '@prisma/client'; import { TestRunsService } from './test-runs.service'; import { TestRunResultDto } from './dto/testRunResult.dto'; import { ApiGuard } from '../auth/guards/api.guard'; @@ -41,6 +44,35 @@ import { UpdateTestRunDto } from './dto/update-test.dto'; import { CurrentUser } from '../shared/current-user.decorator'; import { RoleGuard } from '../auth/guards/role.guard'; import { Roles } from '../shared/roles.decorator'; +import { isPngBuffer } from '../static/utils'; +import { positiveNumber } from '../utils/env'; +import { isUUID } from 'class-validator'; + +const DEFAULT_MULTIPART_FILE_SIZE_LIMIT = 100 * 1024 * 1024; + +// read per request: decorators run before ConfigModule has loaded .env +export const multipartLimits = () => ({ + fileSize: positiveNumber(process.env.MULTIPART_FILE_SIZE_LIMIT, DEFAULT_MULTIPART_FILE_SIZE_LIMIT), +}); + +export const parseIds = () => new ParseArrayPipe({ items: String }); + +export const parseUuidIds = (): PipeTransform => ({ + transform: (ids) => { + const invalid = ids.filter((id) => !isUUID(id)); + if (invalid.length) { + throw new BadRequestException(`Not a uuid: ${invalid.join(', ')}`); + } + return ids; + }, +}); + +function pngOrBadRequest(imageBuffer: unknown): Buffer { + if (!Buffer.isBuffer(imageBuffer) || !isPngBuffer(imageBuffer)) { + throw new BadRequestException('Image is not a PNG file'); + } + return imageBuffer; +} @ApiTags('test-runs') @Controller('test-runs') @@ -77,7 +109,7 @@ export class TestRunsController { @Roles(Role.admin, Role.editor) async approveTestRun( @CurrentUser() user: User, - @Body() ids: string[], + @Body(parseIds()) ids: string[], @Query('merge', new ParseBoolPipe()) merge: boolean ): Promise { this.logger.debug(`Going to approve TestRuns: ${ids}`); @@ -110,19 +142,30 @@ export class TestRunsController { @ApiBearerAuth() @UseGuards(JwtAuthGuard, RoleGuard) @Roles(Role.admin, Role.editor) - async reject(@Body() ids: string[]): Promise { + async reject(@Body(parseIds()) ids: string[]): Promise { this.logger.debug(`Going to reject TestRuns: ${ids}`); for (const id of ids) { - await this.testRunsService.setStatus(id, TestStatus.failed); + await this.testRunsService.reject(id); } } + @Post('revert') + @HttpCode(204) + @ApiNoContentResponse() + @ApiBearerAuth() + @UseGuards(JwtAuthGuard, RoleGuard) + @Roles(Role.admin, Role.editor) + revert(@Body(parseIds(), parseUuidIds()) ids: string[]): Promise { + this.logger.debug(`Going to revert TestRuns: ${ids}`); + return this.testRunsService.revert(ids); + } + @Post('delete') @ApiOkResponse({ type: undefined }) @ApiBearerAuth() @UseGuards(JwtAuthGuard, RoleGuard) @Roles(Role.admin, Role.editor) - async delete(@Body() ids: string[]): Promise { + async delete(@Body(parseIds()) ids: string[]): Promise { this.logger.debug(`Going to delete TestRuns: ${ids}`); for (const id of ids) { await this.testRunsService.delete(id); @@ -169,8 +212,8 @@ export class TestRunsController { @ApiOkResponse({ type: TestRunResultDto }) @UseGuards(ApiGuard, RoleGuard) @Roles(Role.admin, Role.editor) - postTestRun(@Body() createTestRequestDto: CreateTestRequestBase64Dto): Promise { - const imageBuffer = Buffer.from(createTestRequestDto.imageBase64, 'base64'); + async postTestRun(@Body() createTestRequestDto: CreateTestRequestBase64Dto): Promise { + const imageBuffer = pngOrBadRequest(Buffer.from(createTestRequestDto.imageBase64, 'base64')); return this.testRunsService.postTestRun({ createTestRequestDto, imageBuffer, @@ -184,10 +227,9 @@ export class TestRunsController { @ApiConsumes('multipart/form-data') @UseGuards(ApiGuard, RoleGuard) @Roles(Role.admin, Role.editor) - @UseInterceptors(FileInterceptor('image'), FileToBodyInterceptor) - @UsePipes(new ValidationPipe({ transform: true, forbidUnknownValues: true })) - postTestRunMultipart(@Body() createTestRequestDto: CreateTestRequestMultipartDto): Promise { - const imageBuffer = createTestRequestDto.image.buffer; + @UseInterceptors(FileInterceptor('image', { limits: multipartLimits }), FileToBodyInterceptor) + async postTestRunMultipart(@Body() createTestRequestDto: CreateTestRequestMultipartDto): Promise { + const imageBuffer = pngOrBadRequest(createTestRequestDto.image.buffer); return this.testRunsService.postTestRun({ createTestRequestDto, imageBuffer }); } } diff --git a/src/test-runs/test-runs.service.spec.ts b/src/test-runs/test-runs.service.spec.ts index 0a869fd7..8da59340 100644 --- a/src/test-runs/test-runs.service.spec.ts +++ b/src/test-runs/test-runs.service.spec.ts @@ -1,361 +1,361 @@ -import { NotFoundException } from '@nestjs/common'; +import { BadRequestException, ConflictException, Logger, NotFoundException } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; import { mocked } from 'jest-mock'; -import { Test, TestingModule } from '@nestjs/testing'; +import { Test } from '@nestjs/testing'; import { SIGNATURE_CONCURRENCY, TestRunsService } from './test-runs.service'; import { SIGNATURE_LENGTH } from '../compare/libs/pixelmatch/signature.core'; import { PrismaService } from '../prisma/prisma.service'; import { StaticService } from '../static/static.service'; -import { Prisma, Project, TestStatus, TestRun, TestVariation } from '@prisma/client'; +import { Baseline, Prisma, Project, TestStatus, TestRun, TestVariation } from '@prisma/client'; import { CreateTestRequestDto } from './dto/create-test-request.dto'; import { TestRunResultDto } from './dto/testRunResult.dto'; import { DiffResult } from './diffResult'; import { EventsGateway } from '../shared/events/events.gateway'; import { TestVariationsService } from '../test-variations/test-variations.service'; import { TestRunDto } from '../test-runs/dto/testRun.dto'; -import { BuildsService } from '../builds/builds.service'; import { generateBaseline, generateTestRun, generateTestVariation, TEST_PROJECT } from '../_data_'; import { getTestVariationUniqueData } from '../utils'; -import { BaselineDataDto } from '../shared/dto/baseline-data.dto'; import { CompareService } from '../compare/compare.service'; -import { UpdateTestRunDto } from './dto/update-test.dto'; jest.mock('pixelmatch'); jest.mock('./dto/testRunResult.dto'); -const initService = async ({ - testRunDeleteMock = jest.fn(), - testRunUpdateMock = jest.fn(), - testRunFindUniqueMock = jest.fn(), - testRunFindManyMock = jest.fn(), - testRunCreateMock = jest.fn(), - testRunCountMock = jest.fn(), - getImageMock = jest.fn(), - saveImageMock = jest.fn(), - deleteImageMock = jest.fn(), - eventTestRunUpdatedMock = jest.fn(), - eventTestRunCreatedMock = jest.fn(), - eventTestRunDeletedMock = jest.fn(), - eventBuildUpdatedMock = jest.fn(), - eventBuildCreatedMock = jest.fn(), - testVariationCreateMock = jest.fn(), - testVariationFindManyMock = jest.fn(), - baselineCreateMock = jest.fn(), - testVariationFindMock = jest.fn(), - testVariationUpdateMock = jest.fn(), - testVariationGetDetailsMock = jest.fn(), - testVariationFindUniqueMock = jest.fn(), - projectFindUniqueMock = jest.fn(), - compareGetDiffMock = jest.fn(), - compareGetChangeSignatureMock = jest.fn(), - compareGetComparatorMock = jest.fn(), - getImageBufferMock = jest.fn(), - copyImageMock = jest.fn(), - testVariationServiceCreateMock = jest.fn(), - testVariationServiceFindUniqueMock = jest.fn(), - testVariationAddBaselineMock = jest.fn(), -}) => { - const module: TestingModule = await Test.createTestingModule({ +const createMocks = () => { + const m = { + testRun: { + delete: jest.fn(), + findMany: jest.fn(), + findUnique: jest.fn(), + create: jest.fn(), + update: jest.fn(), + count: jest.fn(), + updateMany: jest.fn().mockResolvedValue({ count: 1 }), + }, + testVariation: { count: jest.fn() }, + baseline: { count: jest.fn().mockResolvedValue(0) }, + project: { findUnique: jest.fn() }, + build: { findUnique: jest.fn() }, + config: { get: jest.fn((_key: string, fallback?: string) => fallback) }, + static: { getImageBuffer: jest.fn(), copyImage: jest.fn(), saveImage: jest.fn(), deleteImage: jest.fn() }, + events: { testRunUpdated: jest.fn(), testRunCreated: jest.fn(), testRunDeleted: jest.fn() }, + tv: { + find: jest.fn(), + findBaselineHistory: jest.fn(), + findLatestBaselines: jest.fn(), + update: jest.fn(), + create: jest.fn(), + createOrTakeWinner: jest.fn(), + findUnique: jest.fn(), + addBaseline: jest.fn(), + }, + compare: { getDiff: jest.fn(), getChangeSignature: jest.fn(), getComparator: jest.fn() }, + tx: {} as object, + transaction: jest.fn(), + }; + m.tx = { testRun: { updateMany: m.testRun.updateMany } }; + m.transaction.mockImplementation((callback: (client: object) => unknown) => callback(m.tx)); + return m; +}; +type Mocks = ReturnType; + +const build = async () => { + const m = createMocks(); + const module = await Test.createTestingModule({ providers: [ TestRunsService, { provide: PrismaService, useValue: { - testRun: { - delete: testRunDeleteMock, - findMany: testRunFindManyMock, - findUnique: testRunFindUniqueMock, - create: testRunCreateMock, - update: testRunUpdateMock, - count: testRunCountMock, - }, - testVariation: { - create: testVariationCreateMock, - findMany: testVariationFindManyMock, - findUnique: testVariationFindUniqueMock, - }, - baseline: { - create: baselineCreateMock, - }, - project: { - findUnique: projectFindUniqueMock, - }, + $transaction: m.transaction, + testRun: m.testRun, + testVariation: m.testVariation, + baseline: m.baseline, + project: m.project, + build: m.build, }, }, + { provide: ConfigService, useValue: m.config }, { provide: StaticService, useValue: { - getImage: getImageMock, - getImageBuffer: getImageBufferMock, - copyImage: copyImageMock, - saveImage: saveImageMock, - deleteImage: deleteImageMock, - }, - }, - { - provide: EventsGateway, - useValue: { - testRunUpdated: eventTestRunUpdatedMock, - testRunCreated: eventTestRunCreatedMock, - testRunDeleted: eventTestRunDeletedMock, - buildUpdated: eventBuildUpdatedMock, - buildCreated: eventBuildCreatedMock, - }, - }, - { - provide: TestVariationsService, - useValue: { - find: testVariationFindMock, - getDetails: testVariationGetDetailsMock, - update: testVariationUpdateMock, - create: testVariationServiceCreateMock, - findUnique: testVariationServiceFindUniqueMock, - addBaseline: testVariationAddBaselineMock, - }, - }, - { - provide: BuildsService, - useValue: {}, - }, - { - provide: CompareService, - useValue: { - getDiff: compareGetDiffMock, - getChangeSignature: compareGetChangeSignatureMock, - getComparator: compareGetComparatorMock, + ...m.static, + deleteImages: async (names: Iterable) => { + for (const name of names) if (name) await m.static.deleteImage(name); + }, }, }, + { provide: EventsGateway, useValue: m.events }, + { provide: TestVariationsService, useValue: m.tv }, + { provide: CompareService, useValue: m.compare }, ], }).compile(); + return { service: module.get(TestRunsService), m }; +}; + +const prismaError = (code: string) => new Prisma.PrismaClientKnownRequestError(code, { code, clientVersion: 'test' }); - return module.get(TestRunsService); +const requestDto = (overrides: Partial = {}): CreateTestRequestDto => ({ + buildId: 'buildId', + projectId: 'projectId', + name: 'Test name', + branchName: 'develop', + ...overrides, +}); + +const okDiff: DiffResult = { + status: TestStatus.ok, + diffName: null, + pixelMisMatchCount: 0, + diffPercent: 0, + isSameDimension: true, +}; +const unresolvedDiff: DiffResult = { + status: TestStatus.unresolved, + diffName: 'diff image name', + pixelMisMatchCount: 11, + diffPercent: 22, + isSameDimension: true, }; + +const NO_VARIATION = 'No test variation found. Re-create test run'; + +const deletedNames = (m: Mocks) => m.static.deleteImage.mock.calls.map(([name]) => name).sort(); +const updatedData = (m: Mocks) => m.testRun.update.mock.calls[0][0].data; +const diffInputs = (m: Mocks) => m.compare.getDiff.mock.calls.map(([input]) => input); + describe('TestRunsService', () => { - let service: TestRunsService; const imageBuffer = Buffer.from('Image'); const ignoreAreas = [{ x: 1, y: 2, width: 10, height: 20 }]; + const saveDiff = async (diff: DiffResult | null, id = 'some id') => { + const { service, m } = await build(); + m.testRun.update.mockResolvedValueOnce(generateTestRun()); + await service.saveDiffResult(id, diff); + return m; + }; + it('findOne', async () => { - const id = 'some id'; - const testRunFindUniqueMock = jest.fn(); - service = await initService({ - testRunFindUniqueMock, + const { service, m } = await build(); + + await service.findOne('some id'); + + expect(m.testRun.findUnique).toHaveBeenCalledWith({ where: { id: 'some id' }, include: { testVariation: true } }); + }); + + describe('findOneOrThrow', () => { + it('returns the run', async () => { + const run = generateTestRun(); + const { service, m } = await build(); + m.testRun.findUnique.mockResolvedValueOnce(run); + + await expect(service.findOneOrThrow(run.id)).resolves.toBe(run); }); - service.findOne(id); + it('throws NotFoundException for an unknown run', async () => { + const { service, m } = await build(); + m.testRun.findUnique.mockResolvedValueOnce(null); - expect(testRunFindUniqueMock).toHaveBeenCalledWith({ - where: { id }, - include: { - testVariation: true, - }, + await expect(service.findOneOrThrow('missing')).rejects.toThrow( + new NotFoundException('Test run not found: missing') + ); }); }); - it('setStatus', async () => { - const testRun = { - id: 'id', - imageName: 'imageName', - diffTollerancePercent: 12, - status: TestStatus.new, - buildId: 'buildId', - testVariationId: 'testVariationId', - updatedAt: new Date(), - createdAt: new Date(), - name: 'test run name', - ignoreAreas: '[]', + describe('create', () => { + const request = requestDto({ buildId: 'b', projectId: 'p' }); + + const initCreate = async () => { + const ctx = await build(); + ctx.m.static.saveImage.mockResolvedValueOnce('image.png'); + return ctx; }; - const testRunUpdateMock = jest.fn().mockResolvedValueOnce(testRun); - const eventTestRunUpdatedMock = jest.fn(); - service = await initService({ - testRunUpdateMock, - eventTestRunUpdatedMock, + + it('create', async () => { + const createTestRequestDto = requestDto({ + os: 'OS', + browser: 'browser', + viewport: 'viewport', + device: 'device', + customTags: '', + diffTollerancePercent: undefined, + merge: true, + ignoreAreas: [{ x: 1, y: 2, width: 100, height: 200 }], + }); + const testVariation = generateTestVariation({ ignoreAreas: '[{"x":3,"y":4,"width":500,"height":600}]' }); + const testRun = generateTestRun(); + const { service, m } = await build(); + m.testRun.create.mockResolvedValueOnce(testRun); + m.static.saveImage.mockResolvedValueOnce('image name'); + + const result = await service.create({ testVariation, createTestRequestDto, imageBuffer }); + + expect(m.static.saveImage).toHaveBeenCalledWith('screenshot', imageBuffer); + expect(m.testRun.create).toHaveBeenCalledWith({ + data: { + imageName: 'image name', + testVariationId: testVariation.id, + buildId: createTestRequestDto.buildId, + projectId: createTestRequestDto.projectId, + name: testVariation.name, + browser: testVariation.browser, + device: testVariation.device, + os: testVariation.os, + viewport: testVariation.viewport, + customTags: testVariation.customTags, + baselineName: testVariation.baselineName, + ignoreAreas: testVariation.ignoreAreas, + tempIgnoreAreas: JSON.stringify(createTestRequestDto.ignoreAreas), + comment: testVariation.comment, + baselineBranchName: testVariation.branchName, + branchName: createTestRequestDto.branchName, + diffTollerancePercent: createTestRequestDto.diffTollerancePercent, + merge: createTestRequestDto.merge, + status: TestStatus.new, + }, + }); + expect(result).toBe(testRun); }); - await service.setStatus(testRun.id, TestStatus.failed); + it('prefers the comment of the request over the one of the variation and leaves tempIgnoreAreas to the database default when the request has none', async () => { + const { service, m } = await initCreate(); + m.testRun.create.mockResolvedValueOnce(generateTestRun()); - expect(testRunUpdateMock).toHaveBeenCalledWith({ - where: { id: testRun.id }, - data: { - status: TestStatus.failed, - }, + const result = await service.create({ + testVariation: generateTestVariation({ comment: 'variation comment' }), + createTestRequestDto: requestDto({ comment: 'request comment' }), + imageBuffer, + }); + + const { data } = m.testRun.create.mock.calls[0][0]; + expect(data.comment).toBe('request comment'); + expect(data.imageName).toBe('image.png'); + expect(data.tempIgnoreAreas).toBeUndefined(); + expect(m.events.testRunCreated).toHaveBeenCalledWith(result); + }); + + it('create inserts with scalar foreign keys, no nested connect', async () => { + const { service, m } = await initCreate(); + m.testRun.create.mockResolvedValueOnce(generateTestRun()); + + await service.create({ + testVariation: generateTestVariation({ id: 'v' }), + createTestRequestDto: request, + imageBuffer, + }); + + const { data } = m.testRun.create.mock.calls[0][0]; + expect(data).toMatchObject({ buildId: 'b', projectId: 'p', testVariationId: 'v' }); + expect(data.build).toBeUndefined(); + expect(data.project).toBeUndefined(); + expect(data.testVariation).toBeUndefined(); + }); + + it('a run whose build disappeared is a 404', async () => { + const { service, m } = await initCreate(); + m.testRun.create.mockRejectedValueOnce(prismaError('P2003')); + + await expect( + service.create({ testVariation: generateTestVariation(), createTestRequestDto: request, imageBuffer }) + ).rejects.toBeInstanceOf(NotFoundException); + expect(m.events.testRunCreated).not.toHaveBeenCalled(); + }); + + it('create rethrows other errors', async () => { + const failure = new Error('connection lost'); + const { service, m } = await initCreate(); + m.testRun.create.mockRejectedValueOnce(failure); + + await expect( + service.create({ testVariation: generateTestVariation(), createTestRequestDto: request, imageBuffer }) + ).rejects.toBe(failure); }); - expect(eventTestRunUpdatedMock).toHaveBeenCalledWith(testRun); }); - it('create', async () => { - const initCreateTestRequestDto: CreateTestRequestDto = { - buildId: 'buildId', - projectId: 'projectId', - name: 'Test name', - os: 'OS', - browser: 'browser', - viewport: 'viewport', - device: 'device', - customTags: '', - diffTollerancePercent: undefined, - branchName: 'develop', - merge: true, - ignoreAreas: [ - { - x: 1, - y: 2, - width: 100, - height: 200, - }, - ], - }; - const testRunWithResult = { - id: 'id', - imageName: 'imageName', - baselineName: 'baselineName', - diffTollerancePercent: 1, - ignoreAreas: '[]', - diffName: 'diffName', - status: TestStatus.unresolved, - }; + describe('calculateDiff', () => { + it('calculateDiff', async () => { + const testRun = generateTestRun(); + const diffResult = { id: 'test' }; + const { service, m } = await build(); + m.compare.getDiff.mockReturnValueOnce(diffResult); + service.saveDiffResult = jest.fn(); - const testRun = { - id: 'id', - imageName: 'imageName', - baselineName: 'baselineName', - diffTollerancePercent: 1, - ignoreAreas: '[]', - diffName: null as string | null, - }; - const testVariation = { - id: '123', - projectId: 'project Id', - name: 'Test name', - baselineName: 'baselineName', - os: 'OS', - browser: 'browser', - viewport: 'viewport', - device: 'device', - customTags: '', - ignoreAreas: '[{"x":3,"y":4,"width":500,"height":600}]', - comment: 'some comment', - createdAt: new Date(), - updatedAt: new Date(), - branchName: 'master', - }; - const diffResult: DiffResult = { - status: TestStatus.unresolved, - diffName: 'diff image name', - pixelMisMatchCount: 11, - diffPercent: 22, - isSameDimension: true, - }; - const createTestRequestDto = initCreateTestRequestDto; - const testRunCreateMock = jest.fn().mockResolvedValueOnce(testRun); - const imageName = 'image name'; - const saveImageMock = jest.fn().mockReturnValueOnce(imageName); - const image = 'image'; - const baseline = 'baseline'; - const getImageMock = jest.fn().mockReturnValueOnce(baseline).mockReturnValueOnce(image); - const eventTestRunCreatedMock = jest.fn(); - const compareGetDiffMock = jest.fn().mockReturnValueOnce(diffResult); - service = await initService({ - testRunCreateMock, - saveImageMock, - getImageMock, - eventTestRunCreatedMock, - compareGetDiffMock, - }); - - const saveDiffResultMock = jest.fn(); - service.saveDiffResult = saveDiffResultMock.mockResolvedValueOnce(testRunWithResult); - const tryAutoApproveByPastBaselines = jest.fn(); - service['tryAutoApproveByPastBaselines'] = tryAutoApproveByPastBaselines.mockResolvedValueOnce(testRunWithResult); - const tryAutoApproveByNewBaselines = jest.fn(); - service['tryAutoApproveByNewBaselines'] = tryAutoApproveByNewBaselines.mockResolvedValueOnce(testRunWithResult); - - const result = await service.create({ testVariation, createTestRequestDto, imageBuffer }); - - expect(saveImageMock).toHaveBeenCalledWith('screenshot', imageBuffer); - expect(testRunCreateMock).toHaveBeenCalledWith({ - data: { - imageName, - testVariation: { - connect: { - id: testVariation.id, - }, - }, - build: { - connect: { - id: createTestRequestDto.buildId, - }, - }, - project: { - connect: { - id: createTestRequestDto.projectId, - }, + await service.calculateDiff('projectId', testRun); + + expect(m.static.deleteImage).toHaveBeenCalledWith(testRun.diffName); + expect(m.compare.getDiff).toHaveBeenCalledWith({ + projectId: 'projectId', + data: { + image: testRun.imageName, + baseline: testRun.baselineName, + ignoreAreas: service['getAllIgnoreAreas'](testRun), + diffTollerancePercent: testRun.diffTollerancePercent, + saveDiffAsFile: true, }, - name: testVariation.name, - browser: testVariation.browser, - device: testVariation.device, - os: testVariation.os, - viewport: testVariation.viewport, - customTags: testVariation.customTags, - baselineName: testVariation.baselineName, - ignoreAreas: testVariation.ignoreAreas, - tempIgnoreAreas: JSON.stringify(createTestRequestDto.ignoreAreas), - comment: testVariation.comment, - baselineBranchName: testVariation.branchName, - branchName: createTestRequestDto.branchName, - diffTollerancePercent: createTestRequestDto.diffTollerancePercent, - merge: createTestRequestDto.merge, - status: TestStatus.new, - }, + }); + expect(service.saveDiffResult).toHaveBeenCalledWith(testRun.id, diffResult); }); - expect(result).toBe(testRun); - }); - it('calculateDiff', async () => { - const testRun: TestRun = generateTestRun(); - const testRunUpdateMock = jest.fn(); - const deleteImageMock = jest.fn(); - const diffResult = { - id: 'test', - }; - const compareGetDiffMock = jest.fn().mockReturnValueOnce(diffResult); - service = await initService({ - testRunUpdateMock, - deleteImageMock, - compareGetDiffMock, - }); - service.saveDiffResult = jest.fn(); - - await service.calculateDiff('projectId', testRun); - - expect(deleteImageMock).toHaveBeenCalledWith(testRun.diffName); - expect(compareGetDiffMock).toHaveBeenCalledWith({ - projectId: 'projectId', - data: { - image: testRun.imageName, - baseline: testRun.baselineName, - ignoreAreas: service['getAllIgnoteAreas'](testRun), - diffTollerancePercent: testRun.diffTollerancePercent, - saveDiffAsFile: true, - }, + it('compares with the permanent and temporary ignore areas together, returns the saved run and asks storage to drop the previous diff and thumbnails, missing ones included', async () => { + const permanent = { x: 1, y: 1, width: 1, height: 1 }; + const temporary = { x: 2, y: 2, width: 2, height: 2 }; + const testRun = generateTestRun({ + ignoreAreas: JSON.stringify([permanent]), + tempIgnoreAreas: JSON.stringify([temporary]), + }); + const saved = generateTestRun({ id: 'saved' }); + const { service, m } = await build(); + m.compare.getDiff.mockResolvedValueOnce({ status: TestStatus.ok }); + m.testRun.update.mockResolvedValueOnce(saved); + + const result = await service.calculateDiff('projectId', testRun); + + expect(diffInputs(m)[0].data.ignoreAreas).toEqual([permanent, temporary]); + expect(result).toBe(saved); + expect(m.static.deleteImage.mock.calls).toEqual([[testRun.diffName], [null], [null]]); + }); + + it('calculateDiff passes the bytes it is given', async () => { + const { service, m } = await build(); + m.compare.getDiff.mockResolvedValueOnce(null); + m.testRun.update.mockResolvedValueOnce({}); + + await service.calculateDiff('projectId', generateTestRun(), imageBuffer); + + expect(diffInputs(m)[0].data.imageBuffer).toBe(imageBuffer); + }); + + it('calculateDiff hands the project it is given to the comparison', async () => { + const { service, m } = await build(); + m.compare.getDiff.mockResolvedValueOnce(null); + m.testRun.update.mockResolvedValueOnce({}); + + await service.calculateDiff('projectId', generateTestRun(), imageBuffer, TEST_PROJECT); + + expect(diffInputs(m)[0].project).toBe(TEST_PROJECT); + }); + + it.each([ + ['unreadable', '{}', 'not json'], + ['null', 'null', 'null'], + ])('treats %s stored ignore areas as none', async (_case, stored, temporary) => { + const testRun = generateTestRun({ ignoreAreas: stored, tempIgnoreAreas: temporary }); + const { service, m } = await build(); + m.compare.getDiff.mockResolvedValueOnce(null); + m.testRun.update.mockResolvedValueOnce(testRun); + + await service.calculateDiff('projectId', testRun); + + expect(diffInputs(m)[0].data.ignoreAreas).toEqual([]); + expect(updatedData(m).status).toBe(TestStatus.new); }); - expect(service.saveDiffResult).toHaveBeenCalledWith(testRun.id, diffResult); }); describe('saveDiffResult', () => { - const testRun: TestRun = generateTestRun(); it('no results', async () => { - const id = 'some id'; - const testRunUpdateMock = jest.fn().mockResolvedValueOnce(testRun); - const eventTestRunUpdatedMock = jest.fn(); - service = await initService({ - testRunUpdateMock, - eventTestRunUpdatedMock, - }); + const m = await saveDiff(null); - await service.saveDiffResult(id, null); - - expect(testRunUpdateMock).toHaveBeenCalledWith({ - where: { id }, + expect(m.testRun.update).toHaveBeenCalledWith({ + where: { id: 'some id' }, data: { status: TestStatus.new, diffName: null, @@ -367,65 +367,27 @@ describe('TestRunsService', () => { diffThumbnailName: null, }, }); - expect(eventTestRunUpdatedMock).toHaveBeenCalledWith(testRun); + expect(m.events.testRunUpdated).toHaveBeenCalled(); }); - // Written here, next to the diff that decoded the screenshots anyway, so - // the variations dialog never has to decode them again. it('stores the change signature the comparison produced', async () => { - const testRunUpdateMock = jest.fn().mockResolvedValueOnce(testRun); - service = await initService({ testRunUpdateMock }); - - await service.saveDiffResult('some id', { - status: TestStatus.unresolved, - diffName: 'diff image name', - pixelMisMatchCount: 11, - diffPercent: 22, - isSameDimension: true, - changeSignature: { threshold: 0.1, includeAA: true, signature: [0.25, 0.75] }, - }); - - expect(testRunUpdateMock.mock.calls[0][0].data).toMatchObject({ - changeSignature: JSON.stringify({ threshold: 0.1, includeAA: true, signature: [0.25, 0.75] }), - }); - }); - - it('stores no signature when the comparison produced none', async () => { - const testRunUpdateMock = jest.fn().mockResolvedValueOnce(testRun); - service = await initService({ testRunUpdateMock }); + const changeSignature = { threshold: 0.1, includeAA: true, signature: [0.25, 0.75] }; - await service.saveDiffResult('some id', { - status: TestStatus.ok, - diffName: null, - pixelMisMatchCount: 0, - diffPercent: 0, - isSameDimension: true, - }); + const m = await saveDiff({ ...unresolvedDiff, changeSignature }); - expect(testRunUpdateMock.mock.calls[0][0].data).toMatchObject({ changeSignature: null }); + expect(updatedData(m)).toMatchObject({ changeSignature: JSON.stringify(changeSignature) }); }); it('with results', async () => { const diff: DiffResult = { - status: TestStatus.unresolved, - diffName: 'diff image name', - pixelMisMatchCount: 11, - diffPercent: 22, - isSameDimension: true, + ...unresolvedDiff, vlmDescription: 'VLM detected significant color differences in the header section', }; - const id = 'some id'; - const testRunUpdateMock = jest.fn().mockResolvedValueOnce(testRun); - const eventTestRunUpdatedMock = jest.fn(); - service = await initService({ - testRunUpdateMock, - eventTestRunUpdatedMock, - }); - await service.saveDiffResult(id, diff); + const m = await saveDiff(diff); - expect(testRunUpdateMock).toHaveBeenCalledWith({ - where: { id }, + expect(m.testRun.update).toHaveBeenCalledWith({ + where: { id: 'some id' }, data: { status: diff.status, diffName: diff.diffName, @@ -437,1201 +399,1547 @@ describe('TestRunsService', () => { diffThumbnailName: null, }, }); - expect(eventTestRunUpdatedMock).toHaveBeenCalledWith(testRun); + expect(m.events.testRunUpdated).toHaveBeenCalled(); + }); + + it('clears the analysis of a previous comparison when this one has none', async () => { + const m = await saveDiff(okDiff, 'id'); + + expect(updatedData(m)).toEqual({ + diffName: null, + pixelMisMatchCount: 0, + diffPercent: 0, + status: TestStatus.ok, + vlmDescription: null, + changeSignature: null, + imageThumbnailName: null, + diffThumbnailName: null, + }); }); }); it('findMany', async () => { - const buildId = 'some id'; - const testRun: TestRun = generateTestRun({ - vlmDescription: 'VLM analysis completed', - }); - const testRunFindManyMock = jest.fn().mockResolvedValueOnce([testRun]); - service = await initService({ - testRunFindManyMock, - }); + const testRun = generateTestRun({ vlmDescription: 'VLM analysis completed' }); + const { service, m } = await build(); + m.testRun.findMany.mockResolvedValueOnce([testRun]); - const result = await service.findMany(buildId); + const result = await service.findMany('some id'); - expect(testRunFindManyMock).toHaveBeenCalledWith({ - where: { buildId }, - }); + expect(m.testRun.findMany).toHaveBeenCalledWith({ where: { buildId: 'some id' } }); expect(result).toEqual([new TestRunDto(testRun)]); }); - it('delete', async () => { - const id = 'some id'; - const testRun = { - diffName: 'diffName', - imageName: 'imageName', - }; - const findOneMock = jest.fn().mockResolvedValueOnce(testRun); - const deleteImageMock = jest.fn(); - const testRunDeleteMock = jest.fn(); - const eventTestRunDeletedMock = jest.fn(); - service = await initService({ - deleteImageMock, - testRunDeleteMock, - eventTestRunDeletedMock, - }); - service.findOne = findOneMock; - - await service.delete(id); - - expect(findOneMock).toHaveBeenCalledWith(id); - expect(deleteImageMock).toHaveBeenNthCalledWith(1, testRun.diffName); - expect(deleteImageMock).toHaveBeenNthCalledWith(2, testRun.imageName); - expect(testRunDeleteMock).toHaveBeenCalledWith({ - where: { id }, - }); - expect(eventTestRunDeletedMock).toHaveBeenCalledWith(testRun); - }); + describe('delete', () => { + it('delete', async () => { + const testRun = generateTestRun({ id: 'some id', diffName: 'diffName', imageName: 'imageName' }); + const { service, m } = await build(); + m.testRun.delete.mockResolvedValueOnce(testRun); - it('delete not found', async () => { - const id = 'some id'; - const findOneMock = jest.fn().mockResolvedValueOnce(undefined); - const deleteImageMock = jest.fn(); - const testRunDeleteMock = jest.fn(); - const eventTestRunDeletedMock = jest.fn(); - service = await initService({ - deleteImageMock, - testRunDeleteMock, - eventTestRunDeletedMock, + const result = await service.delete('some id'); + + expect(m.static.deleteImage).toHaveBeenCalledWith(testRun.diffName); + expect(m.static.deleteImage).toHaveBeenCalledWith(testRun.imageName); + expect(m.testRun.delete).toHaveBeenCalledWith({ where: { id: 'some id' } }); + expect(m.events.testRunDeleted).toHaveBeenCalledWith(testRun); + expect(result).toBe(testRun); }); - service.findOne = findOneMock; - const result = await service.delete(id); + it('delete not found', async () => { + const { service, m } = await build(); + m.testRun.delete.mockRejectedValueOnce(prismaError('P2025')); - expect(deleteImageMock).not.toHaveBeenCalled(); - expect(testRunDeleteMock).not.toHaveBeenCalled(); - expect(result).toBeUndefined(); - }); + const result = await service.delete('some id'); - it('updateIgnoreAreas', async () => { - const testRun = { - id: 'testRunId', - testVariationId: 'testVariationId', - diffName: 'diffName', - imageName: 'imageName', - ignoreAreas: JSON.stringify(ignoreAreas), - }; - const testVariation = { - id: 'testVariationId', - projectId: 'someProjectId', - }; - const testRunUpdateMock = jest.fn().mockResolvedValueOnce(testRun); - const testVariationUpdateMock = jest.fn().mockResolvedValueOnce(testVariation); - service = await initService({ - testRunUpdateMock, - testVariationUpdateMock, + expect(m.static.deleteImage).not.toHaveBeenCalled(); + expect(m.events.testRunDeleted).not.toHaveBeenCalled(); + expect(result).toBeUndefined(); }); - service.calculateDiff = jest.fn(); - await service.updateIgnoreAreas(testRun.id, ignoreAreas); + it('delete removes the row before the images', async () => { + const order: string[] = []; + const testRun = generateTestRun(); + const { service, m } = await build(); + m.testRun.delete.mockImplementationOnce(async () => { + order.push('row'); + return testRun; + }); + m.static.deleteImage.mockImplementation(() => { + order.push('image'); + }); - expect(testRunUpdateMock).toHaveBeenCalledWith({ - where: { id: testRun.id }, - data: { - ignoreAreas: JSON.stringify(ignoreAreas), - }, - }); - expect(testVariationUpdateMock).toHaveBeenCalledWith(testRun.testVariationId, { - ignoreAreas: testRun.ignoreAreas, - }); - expect(service.calculateDiff).toHaveBeenCalledWith(testVariation.projectId, testRun); - }); + await service.delete(testRun.id); - it('update', async () => { - const testRun = { - id: 'testRunId', - testVariationId: 'testVariationId', - diffName: 'diffName', - imageName: 'imageName', - }; - const data: UpdateTestRunDto = { - comment: 'random comment', - }; - const testRunUpdateMock = jest.fn().mockResolvedValueOnce(testRun); - const testVariationUpdateMock = jest.fn(); - const eventTestRunUpdatedMock = jest.fn(); - service = await initService({ - testRunUpdateMock, - eventTestRunUpdatedMock, - testVariationUpdateMock, + expect(order).toEqual(['row', 'image', 'image']); }); - await service.update(testRun.id, data); + it.each([ + ['another prisma error', prismaError('P2003')], + ['any other error', new Error('connection lost')], + ['a non-prisma error carrying the P2025 code', Object.assign(new Error('gone'), { code: 'P2025' })], + ])('rethrows %s and announces nothing', async (_case, error) => { + const { service, m } = await build(); + m.testRun.delete.mockRejectedValueOnce(error); - expect(testRunUpdateMock).toHaveBeenCalledWith({ - where: { id: testRun.id }, - data: { - comment: data.comment, - }, + await expect(service.delete('id')).rejects.toBe(error); + + expect(m.events.testRunDeleted).not.toHaveBeenCalled(); + expect(m.static.deleteImage).not.toHaveBeenCalled(); }); - expect(testVariationUpdateMock).toHaveBeenCalledWith(testRun.testVariationId, data); - expect(eventTestRunUpdatedMock).toHaveBeenCalledWith(testRun); }); - it('postTestRun', async () => { - const createTestRequestDto: CreateTestRequestDto = { - buildId: 'buildId', - projectId: 'projectId', - name: 'Test name', - os: 'OS', - browser: 'browser', - viewport: 'viewport', - device: 'device', - branchName: 'develop', - customTags: 'customTags', - comment: 'new comment', - }; - const testVariation: TestVariation = { - id: '123', - projectId: 'project Id', - name: 'Test name', - baselineName: 'baselineName', - os: 'OS', - browser: 'browser', - viewport: 'viewport', - device: 'device', - customTags: '', - ignoreAreas: '[]', - comment: 'some comment', - createdAt: new Date(), - updatedAt: new Date(), - branchName: 'master', - }; - const testRun: TestRun = generateTestRun(); - const projectFindUniqueMock = jest.fn().mockResolvedValueOnce(TEST_PROJECT); - const testVariationFindMock = jest.fn().mockResolvedValueOnce(testVariation); - const testRunFindManyMock = jest.fn().mockResolvedValueOnce([testRun]); - const deleteMock = jest.fn(); - const createMock = jest.fn().mockResolvedValueOnce(testRun); - const service = await initService({ - projectFindUniqueMock, - testVariationFindMock, - testRunFindManyMock, - }); - service.delete = deleteMock; - service.create = createMock; - service.calculateDiff = jest.fn().mockResolvedValueOnce(testRun); - service['tryAutoApproveByPastBaselines'] = jest.fn().mockResolvedValueOnce(testRun); - service['tryAutoApproveByNewBaselines'] = jest.fn().mockResolvedValueOnce(testRun); - const baselineData: BaselineDataDto = { - ...getTestVariationUniqueData(createTestRequestDto), - branchName: createTestRequestDto.branchName, - }; + describe('thumbnails', () => { + const thumbnailed = (overrides: Partial = {}) => + generateTestRun({ + diffName: 'old-diff.png', + imageThumbnailName: 'old-image.thumb.png', + diffThumbnailName: 'old-diff.thumb.png', + ...overrides, + }); - await service.postTestRun({ createTestRequestDto, imageBuffer }); + it('stores the names the comparison saved them under', async () => { + const m = await saveDiff({ + ...unresolvedDiff, + diffName: 'diff.png', + imageThumbnailName: 'image.thumb.png', + diffThumbnailName: 'diff.thumb.png', + }); - expect(testVariationFindMock).toHaveBeenCalledWith(createTestRequestDto); - expect(testRunFindManyMock).toHaveBeenCalledWith({ - where: { - buildId: createTestRequestDto.buildId, - ...baselineData, - NOT: { OR: [{ status: TestStatus.approved }, { status: TestStatus.autoApproved }] }, - }, + expect(updatedData(m)).toMatchObject({ + imageThumbnailName: 'image.thumb.png', + diffThumbnailName: 'diff.thumb.png', + }); }); - expect(deleteMock).toHaveBeenCalledWith(testRun.id); - expect(createMock).toHaveBeenCalledWith({ testVariation, createTestRequestDto, imageBuffer }); - expect(service.calculateDiff).toHaveBeenCalledWith(createTestRequestDto.projectId, testRun); - expect(service['tryAutoApproveByPastBaselines']).toHaveBeenCalledWith({ testVariation, testRun }); - expect(service['tryAutoApproveByNewBaselines']).toHaveBeenCalledWith({ testVariation, testRun }); - expect(mocked(TestRunResultDto)).toHaveBeenCalledWith(testRun, testVariation); - }); - describe('tryAutoApproveByNewBaselines', () => { - it('skip if ok', async () => { - const testRun = generateTestRun({ status: TestStatus.ok }); - service = await initService({}); + it('removes the previous ones when the diff is recalculated', async () => { + const testRun = thumbnailed(); + const { service, m } = await build(); + m.compare.getDiff.mockResolvedValueOnce({ status: TestStatus.ok }); + m.testRun.update.mockResolvedValueOnce(testRun); - const result = await service['tryAutoApproveByNewBaselines']({ - testVariation: generateTestVariation(), - testRun, - }); + await service.calculateDiff('projectId', testRun); - expect(result).toBe(testRun); + expect(deletedNames(m)).toEqual(['old-diff.png', 'old-diff.thumb.png', 'old-image.thumb.png']); }); - it('should not auto approve', async () => { - const testRun = generateTestRun({ status: TestStatus.unresolved }); - const baseline = generateBaseline(); - const testRunFindManyMock = jest.fn().mockResolvedValueOnce([generateTestRun({ status: TestStatus.approved })]); - const testVariationGetDetailsMock = jest.fn().mockResolvedValueOnce(generateTestVariation({}, [baseline])); - service = await initService({ testRunFindManyMock, testVariationGetDetailsMock }); - service['shouldAutoApprove'] = jest.fn().mockResolvedValueOnce(false); + it('keeps the previous ones when the comparison fails', async () => { + const { service, m } = await build(); + m.compare.getDiff.mockRejectedValueOnce(new Error('storage is down')); - const result = await service['tryAutoApproveByNewBaselines']({ - testVariation: generateTestVariation(), - testRun, - }); + await expect(service.calculateDiff('projectId', thumbnailed())).rejects.toThrow('storage is down'); - expect(service['shouldAutoApprove']).toHaveBeenCalled(); - expect(result).toBe(testRun); + expect(m.static.deleteImage).not.toHaveBeenCalled(); }); - it('should auto approve', async () => { - const testRun = generateTestRun({ status: TestStatus.unresolved }); - const baseline = generateBaseline(); - const testRunFindManyMock = jest.fn().mockResolvedValueOnce([generateTestRun({ status: TestStatus.approved })]); - const testVariationGetDetailsMock = jest.fn().mockResolvedValueOnce(generateTestVariation({}, [baseline])); - service = await initService({ testRunFindManyMock, testVariationGetDetailsMock }); - service['shouldAutoApprove'] = jest.fn().mockResolvedValueOnce(true); - service.approve = jest.fn().mockResolvedValueOnce({ - ...testRun, - status: TestStatus.autoApproved, + it('removes them with the run they belong to', async () => { + const testRun = thumbnailed({ + imageName: 'image.png', + diffName: 'diff.png', + imageThumbnailName: 'image.thumb.png', + diffThumbnailName: 'diff.thumb.png', }); + const { service, m } = await build(); + m.testRun.findUnique.mockResolvedValueOnce(testRun); + m.testRun.delete.mockResolvedValueOnce(testRun); - const result = await service['tryAutoApproveByNewBaselines']({ - testVariation: generateTestVariation(), - testRun, - }); + await service.delete(testRun.id); - expect(result).toStrictEqual({ - ...testRun, - status: TestStatus.autoApproved, - }); + expect(deletedNames(m)).toEqual(['diff.png', 'diff.thumb.png', 'image.png', 'image.thumb.png']); }); }); - describe('tryAutoApproveByPastBaselines', () => { - it('skip if ok', async () => { - const testRun = generateTestRun({ status: TestStatus.ok }); - service = await initService({}); + describe('ignore areas and comments', () => { + const findsVariation = (m: Mocks, testVariationId: string | null = 'testVariationId') => + m.testRun.findUnique.mockResolvedValue({ testVariationId }); - const result = await service['tryAutoApproveByPastBaselines']({ - testVariation: generateTestVariation(), - testRun, + it('updateIgnoreAreas', async () => { + const testRun = { id: 'testRunId', testVariationId: 'testVariationId', ignoreAreas: JSON.stringify(ignoreAreas) }; + const { service, m } = await build(); + m.testRun.update.mockResolvedValueOnce(testRun); + m.tv.update.mockResolvedValueOnce({ id: 'testVariationId', projectId: 'someProjectId' }); + findsVariation(m); + service.calculateDiff = jest.fn(); + + await service.updateIgnoreAreas(testRun.id, ignoreAreas); + + expect(m.testRun.update).toHaveBeenCalledWith({ + where: { id: testRun.id }, + data: { ignoreAreas: JSON.stringify(ignoreAreas) }, }); + expect(m.tv.update).toHaveBeenCalledWith(testRun.testVariationId, { ignoreAreas: testRun.ignoreAreas }); + expect(service.calculateDiff).toHaveBeenCalledWith('someProjectId', testRun); + }); - expect(result).toBe(testRun); + it('returns the recalculated run', async () => { + const recalculated = generateTestRun({ id: 'recalculated' }); + const { service, m } = await build(); + m.testRun.update.mockResolvedValueOnce(generateTestRun()); + m.tv.update.mockResolvedValueOnce(generateTestVariation({ projectId: 'p' })); + findsVariation(m, 'v'); + service.calculateDiff = jest.fn().mockResolvedValueOnce(recalculated); + + expect(await service.updateIgnoreAreas('id', ignoreAreas)).toBe(recalculated); }); - it('skip if the branch the same as baseline', async () => { - const testRun = generateTestRun({ status: TestStatus.unresolved, branchName: 'a', baselineBranchName: 'a' }); - service = await initService({}); + describe('addIgnoreAreas', () => { + const newArea = { x: 5, y: 6, width: 7, height: 8 }; - const result = await service['tryAutoApproveByPastBaselines']({ - testVariation: generateTestVariation(), - testRun, - }); + it.each([ + ['appends to the existing areas', JSON.stringify(ignoreAreas), [...ignoreAreas, newArea]], + ['starts from none when they are null', 'null', [newArea]], + ])('%s', async (_case, stored, expected) => { + const updated = generateTestRun({ id: 'updated' }); + const { service, m } = await build(); + m.testRun.findUnique.mockResolvedValueOnce(generateTestRun({ ignoreAreas: stored })); + service.updateIgnoreAreas = jest.fn().mockResolvedValueOnce(updated); - expect(result).toBe(testRun); + const result = await service.addIgnoreAreas('id', [newArea]); + + expect(m.testRun.findUnique).toHaveBeenCalledWith({ where: { id: 'id' }, include: { testVariation: true } }); + expect(service.updateIgnoreAreas).toHaveBeenCalledWith('id', expected); + expect(result).toBe(updated); + }); }); - it('should not auto approve', async () => { - const testRun = generateTestRun({ status: TestStatus.unresolved }); - const baseline = generateBaseline(); - const testVariationGetDetailsMock = jest - .fn() - .mockResolvedValueOnce(generateTestVariation({}, [baseline, baseline])); - service = await initService({ testVariationGetDetailsMock }); - service['shouldAutoApprove'] = jest.fn().mockResolvedValueOnce(false); + it('update', async () => { + const testRun = { id: 'testRunId', testVariationId: 'testVariationId' }; + const { service, m } = await build(); + m.testRun.update.mockResolvedValueOnce(testRun); + findsVariation(m); - const result = await service['tryAutoApproveByPastBaselines']({ - testVariation: generateTestVariation(), - testRun, + await service.update(testRun.id, { comment: 'random comment' }); + + expect(m.testRun.update).toHaveBeenCalledWith({ + where: { id: testRun.id }, + data: { comment: 'random comment' }, }); + expect(m.tv.update).toHaveBeenCalledWith(testRun.testVariationId, { comment: 'random comment' }); + expect(m.events.testRunUpdated).toHaveBeenCalledWith(testRun); + }); - expect(service['shouldAutoApprove']).toHaveBeenCalled(); - expect(result).toBe(testRun); + it('returns the updated run', async () => { + const testRun = generateTestRun(); + const { service, m } = await build(); + m.testRun.update.mockResolvedValueOnce(testRun); + findsVariation(m, 'v'); + + expect(await service.update(testRun.id, { comment: 'c' })).toBe(testRun); }); - it('should auto approve', async () => { - const testRun = generateTestRun({ status: TestStatus.unresolved }); - const baseline = generateBaseline(); - const testVariationGetDetailsMock = jest - .fn() - .mockResolvedValueOnce(generateTestVariation({}, [baseline, baseline])); - service = await initService({ testVariationGetDetailsMock }); - service['shouldAutoApprove'] = jest.fn().mockResolvedValueOnce(true); - service.approve = jest.fn().mockResolvedValueOnce({ - ...testRun, - status: TestStatus.autoApproved, - }); + it('update hands only the comment to the variation', async () => { + const { service, m } = await build(); + m.testRun.update.mockResolvedValueOnce(generateTestRun()); + findsVariation(m, 'v'); - const result = await service['tryAutoApproveByPastBaselines']({ - testVariation: generateTestVariation(), - testRun, + await service.update('id', { comment: 'c', baselineName: 'evil.png', ignoreAreas: 'not json' } as never); + + expect(m.tv.update).toHaveBeenCalledWith('v', { comment: 'c' }); + }); + + describe.each([ + ['update', (svc: TestRunsService) => svc.update('id', { comment: 'c' })], + ['updateIgnoreAreas', (svc: TestRunsService) => svc.updateIgnoreAreas('id', ignoreAreas)], + ])('%s of a run that cannot be edited', (_name, call) => { + it('fails with a conflict before writing anything when the run has no variation', async () => { + const { service, m } = await build(); + findsVariation(m, null); + + const result = call(service); + + await expect(result).rejects.toBeInstanceOf(ConflictException); + await expect(result).rejects.toThrow(NO_VARIATION); + expect(m.testRun.update).not.toHaveBeenCalled(); }); - expect(result).toStrictEqual({ - ...testRun, - status: TestStatus.autoApproved, + it('fails with NotFoundException before writing anything for an unknown id', async () => { + const { service, m } = await build(); + m.testRun.findUnique.mockResolvedValue(null); + + await expect(call(service)).rejects.toBeInstanceOf(NotFoundException); + expect(m.testRun.update).not.toHaveBeenCalled(); }); }); }); - describe('thumbnails', () => { - it('stores the names the comparison saved them under', async () => { - const testRunUpdateMock = jest.fn().mockResolvedValueOnce(generateTestRun()); - service = await initService({ testRunUpdateMock }); + describe('postTestRun', () => { + const createTestRequestDto = requestDto(); - await service.saveDiffResult('some id', { - status: TestStatus.unresolved, - diffName: 'diff.png', - pixelMisMatchCount: 11, - diffPercent: 22, - isSameDimension: true, - imageThumbnailName: 'image.thumb.png', - diffThumbnailName: 'diff.thumb.png', - }); + const initPostService = async ({ + project = { ...TEST_PROJECT, autoApproveFeature: false }, + existingVariation = null, + previousRuns = [], + buildRow = { projectId: createTestRequestDto.projectId }, + }: { + project?: Project | null; + existingVariation?: TestVariation | null; + previousRuns?: TestRun[]; + buildRow?: { projectId: string } | null; + } = {}) => { + const testRun = generateTestRun(); + const diffed = generateTestRun({ status: TestStatus.unresolved }); + const createdVariation = generateTestVariation({ id: 'created variation' }); + const { service, m } = await build(); + m.project.findUnique.mockResolvedValue(project); + m.build.findUnique.mockResolvedValue(buildRow); + m.tv.find.mockResolvedValue(existingVariation); + m.tv.createOrTakeWinner.mockResolvedValue(createdVariation); + m.testRun.findMany.mockResolvedValue(previousRuns); + m.static.saveImage.mockResolvedValue('img.png'); + service.delete = jest.fn(); + const insert = jest.fn().mockResolvedValue(testRun); + service['insertTestRun'] = insert; + service.calculateDiff = jest.fn().mockResolvedValue(diffed); + service['tryAutoApproveByPastBaselines'] = jest.fn(); + service['tryAutoApproveByNewBaselines'] = jest.fn(); + return { service, m, testRun, diffed, createdVariation, insert }; + }; + type PostContext = Awaited>; + const post = (service: TestRunsService) => service.postTestRun({ createTestRequestDto, imageBuffer }); - expect(testRunUpdateMock.mock.calls[0][0].data).toMatchObject({ - imageThumbnailName: 'image.thumb.png', - diffThumbnailName: 'diff.thumb.png', + beforeEach(() => mocked(TestRunResultDto).mockClear()); + + it('postTestRun', async () => { + const request = requestDto({ + os: 'OS', + browser: 'browser', + viewport: 'viewport', + device: 'device', + customTags: 'customTags', + comment: 'new comment', + }); + const testVariation = generateTestVariation(); + const previousRun = generateTestRun({ id: 'previous' }); + const { service, m, insert, testRun, diffed } = await initPostService({ + project: TEST_PROJECT, + existingVariation: testVariation, + previousRuns: [previousRun], }); - }); + const afterPast = generateTestRun({ id: 'after past' }); + const afterNew = generateTestRun({ id: 'after new' }); + service['tryAutoApproveByPastBaselines'] = jest.fn().mockResolvedValueOnce(afterPast); + service['tryAutoApproveByNewBaselines'] = jest.fn().mockResolvedValueOnce(afterNew); - // Recalculating replaces the names on the row, so whatever they pointed at - // before has to go with the old diff — otherwise every ignore-area edit - // leaves two more objects nothing references. - it('removes the previous ones when the diff is recalculated', async () => { - const testRun = generateTestRun({ - diffName: 'old-diff.png', - imageThumbnailName: 'old-image.thumb.png', - diffThumbnailName: 'old-diff.thumb.png', + await service.postTestRun({ createTestRequestDto: request, imageBuffer }); + + expect(m.tv.find).toHaveBeenCalledWith(request, TEST_PROJECT.mainBranchName); + expect(m.testRun.findMany).toHaveBeenCalledWith({ + where: { + buildId: request.buildId, + ...getTestVariationUniqueData(request), + branchName: request.branchName, + NOT: { OR: [{ status: TestStatus.approved }, { status: TestStatus.autoApproved }] }, + }, }); - const deleteImageMock = jest.fn(); - service = await initService({ - deleteImageMock, - compareGetDiffMock: jest.fn().mockResolvedValueOnce({ status: TestStatus.ok }), - testRunUpdateMock: jest.fn().mockResolvedValueOnce(testRun), + expect(service.delete).toHaveBeenCalledWith(previousRun.id); + expect(insert).toHaveBeenCalledWith({ testVariation, createTestRequestDto: request, imageName: 'img.png' }); + expect(service.calculateDiff).toHaveBeenCalledWith(request.projectId, testRun, imageBuffer, TEST_PROJECT); + expect(service['tryAutoApproveByPastBaselines']).toHaveBeenCalledWith({ + testVariation, + testRun: diffed, + project: TEST_PROJECT, + imageBuffer, }); + expect(service['tryAutoApproveByNewBaselines']).toHaveBeenCalledWith({ + testVariation, + testRun: afterPast, + project: TEST_PROJECT, + imageBuffer, + }); + expect(mocked(TestRunResultDto)).toHaveBeenCalledWith(afterNew, testVariation); + }); - await service.calculateDiff('projectId', testRun); + it('creates the test variation when there is none yet', async () => { + const { service, m, testRun, diffed, createdVariation, insert } = await initPostService(); + + await post(service); - expect(deleteImageMock.mock.calls.map(([name]) => name).sort()).toEqual( - ['old-diff.png', 'old-diff.thumb.png', 'old-image.thumb.png'].sort() + expect(m.tv.createOrTakeWinner).toHaveBeenCalledWith(createTestRequestDto); + expect(m.static.saveImage).toHaveBeenCalledWith('screenshot', imageBuffer); + expect(insert).toHaveBeenCalledWith({ + testVariation: createdVariation, + createTestRequestDto, + imageName: 'img.png', + }); + expect(service.calculateDiff).toHaveBeenCalledWith( + createTestRequestDto.projectId, + testRun, + imageBuffer, + expect.objectContaining({ id: TEST_PROJECT.id }) ); + expect(mocked(TestRunResultDto)).toHaveBeenCalledWith(diffed, createdVariation); }); - // Deleting before the replacement exists means a failed comparison leaves - // the row pointing at files that are gone — the reviewer opens the run and - // sees broken pictures, with nothing to recover from. - it('keeps the previous ones when the comparison fails', async () => { - const testRun = generateTestRun({ - diffName: 'old-diff.png', - imageThumbnailName: 'old-image.thumb.png', - diffThumbnailName: 'old-diff.thumb.png', - }); - const deleteImageMock = jest.fn(); - service = await initService({ - deleteImageMock, - compareGetDiffMock: jest.fn().mockRejectedValueOnce(new Error('storage is down')), - }); + it('deletes nothing when there is no previous unapproved run', async () => { + const { service } = await initPostService({ existingVariation: generateTestVariation() }); - await expect(service.calculateDiff('projectId', testRun)).rejects.toThrow('storage is down'); + await post(service); - expect(deleteImageMock).not.toHaveBeenCalled(); + expect(service.delete).not.toHaveBeenCalled(); }); - // two extra objects per run: left behind on delete they would outlive every - // build that ever referenced them, and nothing would ever collect them - it('removes them with the run they belong to', async () => { - const testRun = generateTestRun({ - imageName: 'image.png', - diffName: 'diff.png', - imageThumbnailName: 'image.thumb.png', - diffThumbnailName: 'diff.thumb.png', - }); - const deleteImageMock = jest.fn(); - service = await initService({ - testRunFindUniqueMock: jest.fn().mockResolvedValueOnce(testRun), - testRunDeleteMock: jest.fn().mockResolvedValueOnce(testRun), - deleteImageMock, + it('only deletes the first previous unapproved run', async () => { + const { service } = await initPostService({ + existingVariation: generateTestVariation(), + previousRuns: [generateTestRun({ id: 'first' }), generateTestRun({ id: 'second' })], }); - await service.delete(testRun.id); + await post(service); - expect(deleteImageMock.mock.calls.map(([name]) => name).sort()).toEqual( - ['diff.png', 'diff.thumb.png', 'image.png', 'image.thumb.png'].sort() - ); + expect(service.delete).toHaveBeenCalledTimes(1); + expect(service.delete).toHaveBeenCalledWith('first'); }); - }); - describe('getMatchingVariations', () => { - // Full-length histograms, because a stored one is only reused when its shape - // matches what this build produces. Orthogonal, so they never match. - const histogram = (hot: number): number[] => - Array.from({ length: SIGNATURE_LENGTH }, (_unused, index) => (index === hot ? 1 : 0)); - const SAME_PALETTE = histogram(0); - const OTHER_PALETTE = histogram(SIGNATURE_LENGTH - 1); + it('skips auto approval when the project has it turned off', async () => { + const variation = generateTestVariation(); + const { service, diffed } = await initPostService({ existingVariation: variation }); - // A sibling of the reviewed screen. imageName doubles as the run's identity - // in the signature mocks below, which see buffers rather than runs. - const sibling = (id: string, overrides: Partial = {}): TestRun => - generateTestRun({ - id, - status: TestStatus.unresolved, - name: 'Screen A', - baselineName: `${id}.baseline.png`, - imageName: `${id}.screenshot.png`, - customTags: id, - diffPercent: 12, - ...overrides, + await post(service); + + expect(service['tryAutoApproveByPastBaselines']).not.toHaveBeenCalled(); + expect(service['tryAutoApproveByNewBaselines']).not.toHaveBeenCalled(); + expect(mocked(TestRunResultDto)).toHaveBeenCalledWith(diffed, variation); + }); + + it('chains the two auto approvals, each one seeing the result of the previous step', async () => { + const variation = generateTestVariation(); + const project = { ...TEST_PROJECT, autoApproveFeature: true }; + const { service, diffed } = await initPostService({ project, existingVariation: variation }); + const afterPast = generateTestRun({ id: 'after past' }); + const afterNew = generateTestRun({ id: 'after new' }); + service['tryAutoApproveByPastBaselines'] = jest.fn().mockResolvedValueOnce(afterPast); + service['tryAutoApproveByNewBaselines'] = jest.fn().mockResolvedValueOnce(afterNew); + + await post(service); + + expect(service['tryAutoApproveByPastBaselines']).toHaveBeenCalledWith({ + testVariation: variation, + testRun: diffed, + project, + imageBuffer, }); + expect(service['tryAutoApproveByNewBaselines']).toHaveBeenCalledWith({ + testVariation: variation, + testRun: afterPast, + project, + imageBuffer, + }); + expect(mocked(TestRunResultDto)).toHaveBeenCalledWith(afterNew, variation); + }); - // The pool takes PNG bytes, so the fake static service hands back the image - // name as its own content and the signature mock reads the name back out. - const initMatchingService = async ({ - testRun, - siblings, - signatureOf, - }: { - testRun: TestRun; - siblings: TestRun[]; - signatureOf: (imageName: string) => Promise | number[] | null; - }) => { - const compareGetChangeSignatureMock = jest.fn().mockImplementation(async (input) => ({ - signature: await signatureOf(input.image.toString()), - })); - const built = await initService({ - testRunFindUniqueMock: jest.fn().mockResolvedValue({ ...testRun, testVariation: generateTestVariation() }), - testRunFindManyMock: jest.fn().mockResolvedValue(siblings), - projectFindUniqueMock: jest - .fn() - .mockResolvedValue({ bulkApproveVariations: true, bulkApproveGroupBy: 'customTags' }), - getImageBufferMock: jest.fn().mockImplementation(async (name: string) => Buffer.from(name)), - compareGetChangeSignatureMock, - }); - return { service: built, compareGetChangeSignatureMock }; - }; + it('throws NotFoundException before storing anything when the project does not exist', async () => { + const { service, m } = await initPostService({ project: null, existingVariation: generateTestVariation() }); + + await expect(post(service)).rejects.toThrow( + new NotFoundException(`Project not found: ${createTestRequestDto.projectId}`) + ); + expect(m.static.saveImage).not.toHaveBeenCalled(); + }); + + describe('storing the screenshot alongside the lookups', () => { + it('stores the screenshot while the variation is being looked up', async () => { + let resolve!: (value: TestVariation) => void; + const { service, m } = await initPostService(); + m.tv.find.mockReturnValueOnce(new Promise((r) => (resolve = r))); + + const posting = post(service); + await new Promise(setImmediate); + + expect(m.static.saveImage).toHaveBeenCalledWith('screenshot', imageBuffer); + resolve(generateTestVariation()); + await posting; + }); + + it.each<[string, Parameters[0], (ctx: PostContext, failure: Error) => void]>([ + ['the variation lookup fails', {}, ({ m }, failure) => m.tv.find.mockRejectedValueOnce(failure)], + [ + 'the previous run lookup fails', + { existingVariation: generateTestVariation() }, + ({ m }, failure) => m.testRun.findMany.mockRejectedValueOnce(failure), + ], + [ + 'the previous run cannot be deleted', + { existingVariation: generateTestVariation(), previousRuns: [generateTestRun({ id: 'previous' })] }, + ({ service }, failure) => { + service.delete = jest.fn().mockRejectedValueOnce(failure); + }, + ], + ])('removes the stored screenshot when %s', async (_case, options, arrange) => { + const failure = new Error('lookup failed'); + const ctx = await initPostService(options); + arrange(ctx, failure); - it('matches same-palette, same-size siblings and skips different pattern / far larger change', async () => { - const testRun = sibling('target', { customTags: '' }); - const matching = sibling('locale-a'); - const bigger = sibling('locale-b', { diffPercent: 30 }); - const different = sibling('locale-c'); + await expect(post(ctx.service)).rejects.toBe(failure); - const built = await initMatchingService({ - testRun, - siblings: [matching, bigger, different], - signatureOf: (imageName) => (imageName.startsWith('locale-c') ? OTHER_PALETTE : SAME_PALETTE), + expect(ctx.m.static.deleteImage).toHaveBeenCalledWith('img.png'); + expect(ctx.insert).not.toHaveBeenCalled(); }); - const result = await built.service.getMatchingVariations(testRun.id); + it('passes the save error on without removing anything', async () => { + const failure = new Error('not a png'); + const { service, m, insert } = await initPostService({ existingVariation: generateTestVariation() }); + m.static.saveImage.mockRejectedValueOnce(failure); - expect(result.variations.map((variation) => variation.id)).toEqual([testRun.id, matching.id]); - expect(result.skipped.map((item) => ({ id: item.id, reason: item.reason }))).toEqual([ - { id: bigger.id, reason: 'different change size' }, - { id: different.id, reason: 'different change pattern' }, - ]); - }); + await expect(post(service)).rejects.toBe(failure); - // The whole point of storing it: a build's screenshots are fetched and - // decoded once at ingest, never again to group a screen at review time. - it('groups from the signatures stored at ingest, without decoding anything', async () => { - const signed = (id: string, signature: number[], overrides = {}) => - sibling(id, { - changeSignature: JSON.stringify({ threshold: 0.1, includeAA: true, signature }), - ...overrides, - }); - const testRun = signed('target', SAME_PALETTE, { customTags: '' }); - const matching = signed('locale-a', SAME_PALETTE); - const different = signed('locale-c', OTHER_PALETTE); + expect(m.static.deleteImage).not.toHaveBeenCalled(); + expect(insert).not.toHaveBeenCalled(); + }); - const built = await initMatchingService({ - testRun, - siblings: [matching, different], - signatureOf: () => { - throw new Error('should not be computing a signature'); - }, + it('removes the stored screenshot when the insert fails', async () => { + const failure = new Error('insert failed'); + const { service, m, insert } = await initPostService({ existingVariation: generateTestVariation() }); + insert.mockRejectedValueOnce(failure); + + await expect(post(service)).rejects.toBe(failure); + + expect(m.static.deleteImage).toHaveBeenCalledWith('img.png'); }); - const result = await built.service.getMatchingVariations(testRun.id); + it('deletes the previous unapproved run before inserting', async () => { + const order: string[] = []; + const { service, insert } = await initPostService({ + existingVariation: generateTestVariation(), + previousRuns: [generateTestRun({ id: 'previous' })], + }); + service.delete = jest.fn(async (id: string) => { + order.push(`delete ${id}`); + return undefined; + }); + insert.mockImplementationOnce(async () => { + order.push('insert'); + return generateTestRun(); + }); + + await post(service); - expect(built.compareGetChangeSignatureMock).not.toHaveBeenCalled(); - expect(result.variations.map((variation) => variation.id)).toEqual([testRun.id, matching.id]); - expect(result.skipped.map((item) => item.reason)).toEqual(['different change pattern']); + expect(order).toEqual(['delete previous', 'insert']); + }); }); - // A signature only means anything under the threshold it was computed with. - // The in-memory memo has always keyed on that; the stored one has to too, - // or changing a project's comparison config silently mixes signatures from - // two different settings and grouping quietly gets worse. - it('recomputes when the project config has moved on since ingest', async () => { - const stale = JSON.stringify({ threshold: 0.9, includeAA: false, signature: SAME_PALETTE }); - const testRun = sibling('target', { customTags: '', changeSignature: stale }); - const matching = sibling('locale-a', { changeSignature: stale }); + it('postTestRun hands the uploaded bytes to the diff', async () => { + const { service, testRun } = await initPostService({ existingVariation: generateTestVariation() }); + + await post(service); - const built = await initMatchingService({ + expect(service.calculateDiff).toHaveBeenCalledWith( + createTestRequestDto.projectId, testRun, - siblings: [matching], - signatureOf: () => SAME_PALETTE, - }); + imageBuffer, + expect.objectContaining({ id: TEST_PROJECT.id }) + ); + expect(service.calculateDiff).toHaveBeenCalledTimes(1); + }); - const result = await built.service.getMatchingVariations(testRun.id); + it('rejects an unknown build before storing anything', async () => { + const { service, m } = await initPostService({ buildRow: null, existingVariation: generateTestVariation() }); - const asked = built.compareGetChangeSignatureMock.mock.calls.map(([input]) => input.image.toString()); - expect(asked.sort()).toEqual([testRun.imageName, matching.imageName].sort()); - expect(result.variations.map((variation) => variation.id)).toEqual([testRun.id, matching.id]); + await expect(post(service)).rejects.toThrow( + new NotFoundException(`Build not found: ${createTestRequestDto.buildId}`) + ); + expect(m.static.saveImage).not.toHaveBeenCalled(); }); - // A signature is a fixed-length histogram. One of the wrong length compared - // against one of the right length gives a similarity score computed over - // undefined entries — a number, silently meaningless. The same self- - // invalidating guard as the config stamp: if the shape moves on, recompute. - it.each([ - ['the wrong length', [1, 0]], - ['values that are not numbers', new Array(SIGNATURE_LENGTH).fill('nope')], - ])('recomputes a stored signature with %s', async (_case, signature) => { - const stored = JSON.stringify({ threshold: 0.1, includeAA: true, signature }); - const testRun = sibling('target', { customTags: '', changeSignature: stored }); + describe('a build of another project', () => { + const message = `Build ${createTestRequestDto.buildId} does not belong to project ${createTestRequestDto.projectId}`; - const built = await initMatchingService({ - testRun, - siblings: [], - signatureOf: () => SAME_PALETTE, + it('is accepted with a warning by default', async () => { + const warn = jest.spyOn(Logger.prototype, 'warn').mockImplementation(); + const { service, diffed, createdVariation } = await initPostService({ buildRow: { projectId: 'other' } }); + + await post(service); + + expect(warn).toHaveBeenCalledWith(message); + expect(mocked(TestRunResultDto)).toHaveBeenCalledWith(diffed, createdVariation); + warn.mockRestore(); }); - await built.service.getMatchingVariations(testRun.id); + it('is rejected when STRICT_BUILD_PROJECT_CHECK is true', async () => { + const { service, m } = await initPostService({ buildRow: { projectId: 'other' } }); + m.config.get.mockReturnValue('TRUE'); - expect(built.compareGetChangeSignatureMock).toHaveBeenCalled(); + await expect(post(service)).rejects.toThrow(new BadRequestException(message)); + expect(m.config.get).toHaveBeenCalledWith('STRICT_BUILD_PROJECT_CHECK', 'false'); + expect(m.static.saveImage).not.toHaveBeenCalled(); + }); }); - // builds ingested before the column existed, and runs compared by something - // other than pixelmatch, still have to group - it('falls back to computing for a run with nothing stored', async () => { - const testRun = sibling('target', { - customTags: '', - changeSignature: JSON.stringify({ threshold: 0.1, includeAA: true, signature: SAME_PALETTE }), + it("fails before storing anything when the project's comparator cannot run here", async () => { + const { service, m, insert } = await initPostService({ existingVariation: generateTestVariation() }); + m.compare.getComparator.mockImplementation(() => { + throw new Error('Odiff can only be used with HDD static service.'); }); - const unsigned = sibling('locale-a'); - const built = await initMatchingService({ - testRun, - siblings: [unsigned], - signatureOf: () => SAME_PALETTE, - }); + await expect(post(service)).rejects.toThrow('Odiff can only be used with HDD static service.'); + expect(m.static.saveImage).not.toHaveBeenCalled(); + expect(insert).not.toHaveBeenCalled(); + expect(service.delete).not.toHaveBeenCalled(); + }); + }); - const result = await built.service.getMatchingVariations(testRun.id); + describe('reject', () => { + const initRejectService = async (before: TestRun | null, after?: TestRun, count = 1) => { + const ctx = await build(); + ctx.m.testRun.findUnique.mockResolvedValueOnce(before).mockResolvedValueOnce(after); + ctx.m.testRun.updateMany.mockResolvedValue({ count }); + return ctx; + }; - const asked = built.compareGetChangeSignatureMock.mock.calls.map(([input]) => input.image.toString()); - expect(asked).toEqual([unsigned.imageName]); - expect(result.variations.map((variation) => variation.id)).toEqual([testRun.id, unsigned.id]); - }); + it('reject records the previous status', async () => { + const run = generateTestRun({ status: TestStatus.unresolved }); + const rejected = { ...run, status: TestStatus.failed, statusBeforeDecision: TestStatus.unresolved }; + const { service, m } = await initRejectService(run, rejected); - it('skips a far larger change without paying for its signature', async () => { - const testRun = sibling('target', { customTags: '' }); - const bigger = sibling('locale-b', { diffPercent: 30 }); + const result = await service.reject(run.id); - const built = await initMatchingService({ - testRun, - siblings: [bigger], - signatureOf: () => SAME_PALETTE, + expect(m.testRun.updateMany).toHaveBeenCalledWith({ + where: { id: run.id, status: TestStatus.unresolved }, + data: { status: TestStatus.failed, statusBeforeDecision: TestStatus.unresolved }, }); + expect(m.events.testRunUpdated).toHaveBeenCalledWith(rejected); + expect(result).toBe(rejected); + }); - await built.service.getMatchingVariations(testRun.id); + it('rejecting a failed run changes nothing', async () => { + const run = generateTestRun({ status: TestStatus.failed }); + const { service, m } = await initRejectService(run); - const asked = built.compareGetChangeSignatureMock.mock.calls.map(([input]) => input.image.toString()); - expect(asked).toEqual([testRun.imageName]); - }); + const result = await service.reject(run.id); - it('says a sibling has no diff rather than blaming its change size', async () => { - const testRun = sibling('target', { customTags: '' }); - const unchanged = sibling('locale-b', { diffPercent: 0 }); + expect(m.testRun.updateMany).not.toHaveBeenCalled(); + expect(m.events.testRunUpdated).not.toHaveBeenCalled(); + expect(result).toBe(run); + }); - const built = await initMatchingService({ - testRun, - siblings: [unchanged], - signatureOf: () => SAME_PALETTE, - }); + it('reject of an unknown run is a 404', async () => { + const { service, m } = await initRejectService(null); - const result = await built.service.getMatchingVariations(testRun.id); + await expect(service.reject('missing')).rejects.toThrow(new NotFoundException('Test run not found: missing')); - expect(result.skipped.map((item) => item.reason)).toEqual(['no diff to match']); + expect(m.testRun.updateMany).not.toHaveBeenCalled(); }); - it('asks for the siblings signatures at once rather than one after another', async () => { - const testRun = sibling('target', { customTags: '' }); - const siblings = ['locale-a', 'locale-b', 'locale-c'].map((id) => sibling(id)); + it('a lost race emits nothing', async () => { + const run = generateTestRun({ status: TestStatus.unresolved }); + const current = { ...run, status: TestStatus.approved }; + const { service, m } = await initRejectService(run, current, 0); - let inFlight = 0; - let peakInFlight = 0; - const built = await initMatchingService({ - testRun, - siblings, - signatureOf: async () => { - inFlight += 1; - peakInFlight = Math.max(peakInFlight, inFlight); - await new Promise((resolve) => setTimeout(resolve, 0)); - inFlight -= 1; - return SAME_PALETTE; - }, + const result = await service.reject(run.id); + + expect(m.events.testRunUpdated).not.toHaveBeenCalled(); + expect(result).toBe(current); + }); + }); + + describe('revert', () => { + const approvedRun = (overrides: Partial = {}) => + generateTestRun({ + id: 'run-1', + status: TestStatus.approved, + statusBeforeDecision: TestStatus.unresolved, + branchName: 'master', + baselineBranchName: 'master', + projectId: 'project', + ...overrides, + }); + const variationOf = (overrides: Partial = {}) => + generateTestVariation({ id: 'variation', projectId: 'project', baselineName: 'new.png', ...overrides }, []); + const baselineOf = (overrides: Partial = {}) => + generateBaseline({ + id: 'baseline-1', + testRunId: 'run-1', + testVariationId: 'variation', + baselineName: 'new.png', + ...overrides, }); + const makeTx = () => ({ + testRun: { updateMany: jest.fn().mockResolvedValue({ count: 1 }), count: jest.fn().mockResolvedValue(0) }, + baseline: { + findMany: jest.fn().mockResolvedValue([]), + findUnique: jest.fn().mockResolvedValue(null), + findFirst: jest.fn().mockResolvedValue(null), + delete: jest.fn(), + count: jest.fn().mockResolvedValue(0), + }, + testVariation: { findUnique: jest.fn().mockResolvedValue(null), update: jest.fn(), delete: jest.fn() }, + }); + const claim = (run: TestRun, data: object) => ({ where: { id: run.id, status: run.status }, data }); + const restored = (status: TestStatus, extra: object = {}) => ({ status, statusBeforeDecision: null, ...extra }); + + const initRevertService = async (runs: TestRun[], counts: [number, number, number] = [0, 0, 0]) => { + const ctx = await build(); + const { m } = ctx; + const tx = makeTx(); + m.tx = tx; + m.testRun.findMany.mockResolvedValue(runs); + m.testRun.findUnique.mockImplementation((args: { where: { id: string } }) => + Promise.resolve({ ...runs.find((run) => run.id === args.where.id), testVariation: null }) + ); + m.testRun.count.mockResolvedValue(counts[0]); + m.testVariation.count.mockResolvedValue(counts[1]); + m.baseline.count.mockResolvedValue(counts[2]); + return { ...ctx, tx }; + }; + type RevertContext = Awaited>; + + const givenApproval = ({ tx }: RevertContext, previousName: string | null = 'old.png', branchName = 'master') => { + tx.baseline.findUnique.mockResolvedValue(baselineOf()); + tx.baseline.findFirst + .mockResolvedValueOnce(baselineOf()) + .mockResolvedValueOnce(previousName && baselineOf({ id: 'baseline-0', baselineName: previousName })); + tx.testVariation.findUnique.mockResolvedValue(variationOf({ branchName })); + }; + + it('reverts a rejected run to the status it was decided from', async () => { + const run = generateTestRun({ status: TestStatus.failed, statusBeforeDecision: TestStatus.ok }); + const { service, tx } = await initRevertService([run]); - await built.service.getMatchingVariations(testRun.id); + await service.revert([run.id]); - expect(peakInFlight).toBe(siblings.length + 1); + expect(tx.testRun.updateMany).toHaveBeenCalledWith(claim(run, restored(TestStatus.ok))); + expect(tx.baseline.findUnique).not.toHaveBeenCalled(); }); - it('keeps the number of images being decoded at once bounded', async () => { - const testRun = sibling('target', { customTags: '' }); - const siblings = Array.from({ length: SIGNATURE_CONCURRENCY * 3 }, (_, index) => sibling(`locale-${index}`)); + it.each([ + ['has a baseline', 'baseline.png', TestStatus.unresolved], + ['has none', null, TestStatus.new], + ])( + 'a rejected run without a recorded status falls back by its baseline: run that %s', + async (_title, baselineName, expected) => { + const run = generateTestRun({ status: TestStatus.failed, statusBeforeDecision: null, baselineName }); + const { service, tx } = await initRevertService([run]); - let inFlight = 0; - let peakInFlight = 0; - const built = await initMatchingService({ - testRun, - siblings, - signatureOf: async () => { - inFlight += 1; - peakInFlight = Math.max(peakInFlight, inFlight); - await new Promise((resolve) => setTimeout(resolve, 0)); - inFlight -= 1; - return SAME_PALETTE; - }, - }); + await service.revert([run.id]); - await built.service.getMatchingVariations(testRun.id); + expect(tx.testRun.updateMany).toHaveBeenCalledWith(claim(run, restored(expected))); + } + ); + + it('reverts a same-branch approval: deletes the baseline row, restores the previous baselineName and the status', async () => { + const run = approvedRun(); + const ctx = await initRevertService([run]); + const { service, m, tx } = ctx; + givenApproval(ctx); - expect(peakInFlight).toBe(SIGNATURE_CONCURRENCY); + await service.revert([run.id]); + + expect(tx.baseline.findUnique).toHaveBeenCalledWith({ where: { testRunId: run.id } }); + expect(tx.baseline.findFirst).toHaveBeenNthCalledWith(1, { + where: { testVariationId: 'variation' }, + orderBy: { createdAt: 'desc' }, + }); + expect(tx.baseline.delete).toHaveBeenCalledWith({ where: { id: 'baseline-1' } }); + expect(tx.testVariation.update).toHaveBeenCalledWith({ + where: { id: 'variation' }, + data: { baselineName: 'old.png' }, + }); + expect(tx.testRun.updateMany).toHaveBeenCalledWith(claim(run, restored(TestStatus.unresolved))); + expect(m.tv.findUnique).not.toHaveBeenCalled(); }); - it('reuses the signatures it already computed when the dialog is reopened', async () => { - const testRun = sibling('target', { customTags: '' }); - const matching = sibling('locale-a'); + it('reverting the first approval of a screen leaves the variation without a baseline', async () => { + const run = approvedRun(); + const ctx = await initRevertService([run]); + givenApproval(ctx, null); - const built = await initMatchingService({ - testRun, - siblings: [matching], - signatureOf: () => SAME_PALETTE, + await ctx.service.revert([run.id]); + + expect(ctx.tx.testVariation.update).toHaveBeenCalledWith({ + where: { id: 'variation' }, + data: { baselineName: null }, }); + }); - await built.service.getMatchingVariations(testRun.id); - const afterFirst = built.compareGetChangeSignatureMock.mock.calls.length; - const second = await built.service.getMatchingVariations(testRun.id); + it('reverts a merge approval into the main variation without moving the run', async () => { + const run = approvedRun({ branchName: 'feature', baselineBranchName: 'master', merge: true }); + const ctx = await initRevertService([run]); + const { tx } = ctx; + givenApproval(ctx, null); - expect(afterFirst).toBe(2); - expect(built.compareGetChangeSignatureMock).toHaveBeenCalledTimes(2); - expect(second.variations.map((variation) => variation.id)).toEqual([testRun.id, matching.id]); + await ctx.service.revert([run.id]); + + expect(tx.testVariation.update).toHaveBeenCalledWith({ + where: { id: 'variation' }, + data: { baselineName: null }, + }); + expect(tx.testRun.updateMany).toHaveBeenCalledWith(claim(run, restored(TestStatus.unresolved))); + expect(tx.testVariation.delete).not.toHaveBeenCalled(); }); - it('recomputes a signature once the run gained an ignore area', async () => { - const testRun = sibling('target', { customTags: '' }); - const matching = sibling('locale-a'); + describe('of a feature-branch approval', () => { + const featureRun = () => approvedRun({ branchName: 'feature', baselineBranchName: 'master' }); + const mainVariation = generateTestVariation({ id: 'main variation', branchName: 'master' }, []); - const built = await initMatchingService({ - testRun, - siblings: [matching], - signatureOf: () => SAME_PALETTE, + const givenFeatureApproval = (ctx: RevertContext, previousName: string | null = null) => { + givenApproval(ctx, previousName, 'feature'); + ctx.m.tv.findUnique.mockResolvedValue(mainVariation); + }; + + it('moves the run back to the main variation and deletes the feature variation it created', async () => { + const run = featureRun(); + const ctx = await initRevertService([run]); + const { service, m, tx } = ctx; + givenFeatureApproval(ctx); + + await service.revert([run.id]); + + expect(m.tv.findUnique).toHaveBeenCalledWith( + { projectId: 'project', branchName: 'master', ...getTestVariationUniqueData(run) }, + tx + ); + expect(tx.testRun.updateMany).toHaveBeenCalledWith( + claim(run, restored(TestStatus.unresolved, { testVariationId: 'main variation' })) + ); + expect(tx.testRun.count).toHaveBeenCalledWith({ where: { testVariationId: 'variation' } }); + expect(tx.baseline.count).toHaveBeenCalledWith({ where: { testVariationId: 'variation' } }); + expect(tx.testVariation.delete).toHaveBeenCalledWith({ where: { id: 'variation' } }); + expect(tx.testVariation.update).not.toHaveBeenCalled(); + }); + + it.each([ + ['other runs', 1, 0], + ['other baselines', 0, 1], + ])( + 'keeps the feature variation when it has %s, restoring its previous baselineName', + async (_title, runCount, baselineCount) => { + const run = featureRun(); + const ctx = await initRevertService([run]); + const { tx } = ctx; + givenFeatureApproval(ctx, 'old.png'); + tx.testRun.count.mockResolvedValue(runCount); + tx.baseline.count.mockResolvedValue(baselineCount); + + await ctx.service.revert([run.id]); + + expect(tx.testVariation.delete).not.toHaveBeenCalled(); + expect(tx.testVariation.update).toHaveBeenCalledWith({ + where: { id: 'variation' }, + data: { baselineName: 'old.png' }, + }); + } + ); + + it('409 when the main variation was deleted', async () => { + const run = featureRun(); + const ctx = await initRevertService([run]); + const { service, m, tx } = ctx; + givenFeatureApproval(ctx); + m.tv.findUnique.mockResolvedValue(null); + + await expect(service.revert([run.id])).rejects.toThrow(new ConflictException(NO_VARIATION)); + + expect(tx.testRun.updateMany).not.toHaveBeenCalled(); + expect(m.static.deleteImage).not.toHaveBeenCalled(); + expect(m.events.testRunUpdated).not.toHaveBeenCalled(); }); + }); - await built.service.getMatchingVariations(testRun.id); - matching.ignoreAreas = JSON.stringify([{ x: 0, y: 0, width: 10, height: 10 }]); - await built.service.getMatchingVariations(testRun.id); + it.each([ + ['a later baseline exists', () => baselineOf({ id: 'baseline-2' }), 'new.png'], + ['the variation points at another file', () => baselineOf(), 'other.png'], + ])('409 when a later approval moved the baseline on: %s', async (_title, latest, variationBaselineName) => { + const run = approvedRun(); + const { service, m, tx } = await initRevertService([run]); + tx.baseline.findUnique.mockResolvedValue(baselineOf()); + tx.baseline.findFirst.mockResolvedValue(latest()); + tx.testVariation.findUnique.mockResolvedValue(variationOf({ baselineName: variationBaselineName })); + + await expect(service.revert([run.id])).rejects.toThrow( + new ConflictException(`Test run ${run.id}: a later approval changed this baseline`) + ); - expect(built.compareGetChangeSignatureMock).toHaveBeenCalledTimes(3); + expect(tx.baseline.delete).not.toHaveBeenCalled(); + expect(tx.testVariation.update).not.toHaveBeenCalled(); + expect(tx.testRun.updateMany).not.toHaveBeenCalled(); + expect(m.static.deleteImage).not.toHaveBeenCalled(); + expect(m.events.testRunUpdated).not.toHaveBeenCalled(); }); - }); - describe('postTestRun branches', () => { - const createTestRequestDto: CreateTestRequestDto = { - buildId: 'buildId', - projectId: 'projectId', - name: 'Test name', - branchName: 'develop', - }; + it.each([TestStatus.autoApproved, TestStatus.new, TestStatus.unresolved, TestStatus.ok])( + '409 for a %s run', + async (status) => { + const run = generateTestRun({ status }); + const { service, m, tx } = await initRevertService([run]); - const initPostService = async ({ - project = { ...TEST_PROJECT, autoApproveFeature: false } as Project | null, - existingVariation = null as TestVariation | null, - previousRuns = [] as TestRun[], - }) => { - const testRun = generateTestRun(); - const diffed = generateTestRun({ status: TestStatus.unresolved }); - const createdVariation = generateTestVariation({ id: 'created variation' }); - const testVariationServiceCreateMock = jest.fn().mockResolvedValueOnce(createdVariation); - const testRunFindManyMock = jest.fn().mockResolvedValueOnce(previousRuns); - const built = await initService({ - projectFindUniqueMock: jest.fn().mockResolvedValueOnce(project), - testVariationFindMock: jest.fn().mockResolvedValueOnce(existingVariation), - testVariationServiceCreateMock, - testRunFindManyMock, - }); - built.delete = jest.fn(); - built.create = jest.fn().mockResolvedValueOnce(testRun); - built.calculateDiff = jest.fn().mockResolvedValueOnce(diffed); - built['tryAutoApproveByPastBaselines'] = jest.fn(); - built['tryAutoApproveByNewBaselines'] = jest.fn(); - return { built, testRun, diffed, createdVariation, testVariationServiceCreateMock, testRunFindManyMock }; - }; + await expect(service.revert([run.id])).rejects.toThrow( + new ConflictException(`Test run ${run.id} is ${status}: only approved or rejected runs can be reverted`) + ); - beforeEach(() => mocked(TestRunResultDto).mockClear()); + expect(tx.testRun.updateMany).not.toHaveBeenCalled(); + expect(m.events.testRunUpdated).not.toHaveBeenCalled(); + } + ); - it('creates the test variation when there is none yet', async () => { - const { built, testRun, diffed, createdVariation, testVariationServiceCreateMock } = await initPostService({}); + it('409 when the approval has no baseline row', async () => { + const run = approvedRun(); + const { service, tx } = await initRevertService([run]); - await built.postTestRun({ createTestRequestDto, imageBuffer }); + await expect(service.revert([run.id])).rejects.toThrow( + new ConflictException(`Test run ${run.id} has no baseline to revert`) + ); - expect(testVariationServiceCreateMock).toHaveBeenCalledWith({ createTestRequestDto }); - expect(built.create).toHaveBeenCalledWith({ testVariation: createdVariation, createTestRequestDto, imageBuffer }); - expect(built.calculateDiff).toHaveBeenCalledWith(createTestRequestDto.projectId, testRun); - expect(mocked(TestRunResultDto)).toHaveBeenCalledWith(diffed, createdVariation); + expect(tx.testRun.updateMany).not.toHaveBeenCalled(); }); - it('deletes nothing when there is no previous unapproved run', async () => { - const { built } = await initPostService({ existingVariation: generateTestVariation() }); + it('409 for an empty list', async () => { + const { service, m } = await initRevertService([]); - await built.postTestRun({ createTestRequestDto, imageBuffer }); + await expect(service.revert([])).rejects.toThrow(new ConflictException('Nothing to revert')); - expect(built.delete).not.toHaveBeenCalled(); + expect(m.testRun.findMany).not.toHaveBeenCalled(); + expect(m.transaction).not.toHaveBeenCalled(); }); - it('only deletes the first previous unapproved run', async () => { - const first = generateTestRun({ id: 'first' }); - const second = generateTestRun({ id: 'second' }); - const { built } = await initPostService({ - existingVariation: generateTestVariation(), - previousRuns: [first, second], - }); + it('404 for an unknown id, and then no transaction runs', async () => { + const run = generateTestRun({ status: TestStatus.failed }); + const { service, m } = await initRevertService([run]); - await built.postTestRun({ createTestRequestDto, imageBuffer }); + await expect(service.revert([run.id, 'missing 1', 'missing 2'])).rejects.toThrow( + new NotFoundException('Test run not found: missing 1, missing 2') + ); - expect(built.delete).toHaveBeenCalledTimes(1); - expect(built.delete).toHaveBeenCalledWith('first'); + expect(m.transaction).not.toHaveBeenCalled(); }); - it('skips auto approval when the project has it turned off', async () => { - const variation = generateTestVariation(); - const { built, diffed } = await initPostService({ existingVariation: variation }); + it('409 when the run changed status meanwhile', async () => { + const run = generateTestRun({ status: TestStatus.failed }); + const { service, m, tx } = await initRevertService([run]); + tx.testRun.updateMany.mockResolvedValue({ count: 0 }); - await built.postTestRun({ createTestRequestDto, imageBuffer }); + await expect(service.revert([run.id])).rejects.toThrow( + new ConflictException(`Test run ${run.id} changed while reverting, try again`) + ); - expect(built['tryAutoApproveByPastBaselines']).not.toHaveBeenCalled(); - expect(built['tryAutoApproveByNewBaselines']).not.toHaveBeenCalled(); - expect(mocked(TestRunResultDto)).toHaveBeenCalledWith(diffed, variation); + expect(m.events.testRunUpdated).not.toHaveBeenCalled(); }); - it('chains the two auto approvals, each one seeing the result of the previous step', async () => { - const variation = generateTestVariation(); - const { built, diffed } = await initPostService({ - project: { ...TEST_PROJECT, autoApproveFeature: true }, - existingVariation: variation, + describe('the freed baseline file', () => { + it.each([ + ['a run compared against it', [1, 0, 0]], + ['a variation pointing at it', [0, 1, 0]], + ['another baseline row', [0, 0, 1]], + ] as [string, [number, number, number]][])('is kept while %s', async (_title, counts) => { + const run = approvedRun(); + const ctx = await initRevertService([run], counts); + givenApproval(ctx); + + await ctx.service.revert([run.id]); + + expect(ctx.m.static.deleteImage).not.toHaveBeenCalled(); }); - const afterPast = generateTestRun({ id: 'after past' }); - const afterNew = generateTestRun({ id: 'after new' }); - built['tryAutoApproveByPastBaselines'] = jest.fn().mockResolvedValueOnce(afterPast); - built['tryAutoApproveByNewBaselines'] = jest.fn().mockResolvedValueOnce(afterNew); - await built.postTestRun({ createTestRequestDto, imageBuffer }); + it('is deleted, after the commit, when nothing references it', async () => { + const run = approvedRun({ name: 'screen' }); + const ctx = await initRevertService([run]); + const { service, m } = ctx; + givenApproval(ctx); - expect(built['tryAutoApproveByPastBaselines']).toHaveBeenCalledWith({ - testVariation: variation, - testRun: diffed, + await service.revert([run.id]); + + expect(m.testRun.count).toHaveBeenCalledWith({ where: { name: 'screen', baselineName: 'new.png' } }); + expect(m.testVariation.count).toHaveBeenCalledWith({ where: { baselineName: 'new.png' } }); + expect(m.baseline.count).toHaveBeenCalledWith({ where: { baselineName: 'new.png' } }); + expect(m.static.deleteImage).toHaveBeenCalledTimes(1); + expect(m.static.deleteImage).toHaveBeenCalledWith('new.png'); + expect(m.static.deleteImage.mock.invocationCallOrder[0]).toBeGreaterThan( + m.transaction.mock.invocationCallOrder[0] + ); }); - expect(built['tryAutoApproveByNewBaselines']).toHaveBeenCalledWith({ - testVariation: variation, - testRun: afterPast, + + it('is not touched when a rejection is reverted', async () => { + const run = generateTestRun({ status: TestStatus.failed }); + const { service, m } = await initRevertService([run]); + + await service.revert([run.id]); + + expect(m.static.deleteImage).not.toHaveBeenCalled(); }); - expect(mocked(TestRunResultDto)).toHaveBeenCalledWith(afterNew, variation); }); - it('throws NotFoundException before storing anything when the project does not exist', async () => { - const { built } = await initPostService({ project: null, existingVariation: generateTestVariation() }); + it('emits testRun_updated for every run after the commit, nothing on a 409', async () => { + const rejected = generateTestRun({ id: 'run-2', status: TestStatus.failed }); + const approved = approvedRun(); + const ctx = await initRevertService([rejected, approved]); + const { service, m } = ctx; + givenApproval(ctx); + m.testRun.findMany + .mockResolvedValueOnce([rejected, approved]) + .mockResolvedValueOnce([{ ...approved, status: TestStatus.unresolved }, rejected]); - await expect(built.postTestRun({ createTestRequestDto, imageBuffer })).rejects.toThrow( - new NotFoundException(`Project not found: ${createTestRequestDto.projectId}`) - ); - expect(built.create).not.toHaveBeenCalled(); - }); + await service.revert(['run-2', approved.id, 'run-2']); - it("fails before storing anything when the project's comparator cannot run here", async () => { - const { built } = await initPostService({ existingVariation: generateTestVariation() }); - built['compareService'].getComparator = jest.fn(() => { - throw new Error('Odiff can only be used with HDD static service.'); - }); + expect(m.testRun.findMany).toHaveBeenLastCalledWith({ where: { id: { in: ['run-2', approved.id] } } }); + expect(m.testRun.findUnique).not.toHaveBeenCalled(); + expect(m.events.testRunUpdated).toHaveBeenCalledWith( + expect.objectContaining({ id: approved.id, status: TestStatus.unresolved }) + ); + expect(m.events.testRunUpdated).toHaveBeenCalledTimes(2); + expect(m.events.testRunUpdated).toHaveBeenNthCalledWith(1, expect.objectContaining({ id: 'run-2' })); + expect(m.events.testRunUpdated).toHaveBeenNthCalledWith(2, expect.objectContaining({ id: approved.id })); + expect(m.events.testRunUpdated.mock.invocationCallOrder[0]).toBeGreaterThan( + m.transaction.mock.invocationCallOrder[0] + ); - await expect(built.postTestRun({ createTestRequestDto, imageBuffer })).rejects.toThrow( - 'Odiff can only be used with HDD static service.' + const failing = await initRevertService([generateTestRun({ status: TestStatus.new })]); + await expect(failing.service.revert(['10fb5e02-64e0-4cf5-9f17-c00ab3c96658'])).rejects.toThrow(ConflictException); + expect(failing.m.events.testRunUpdated).not.toHaveBeenCalled(); + }); + + it('reverts two approvals of one screen in one request, latest first', async () => { + const older = approvedRun({ id: 'run-1' }); + const newer = approvedRun({ id: 'run-2' }); + const baselineOlder = baselineOf({ baselineName: 'first.png', createdAt: new Date(1000) }); + const baselineNewer = baselineOf({ + id: 'baseline-2', + testRunId: 'run-2', + baselineName: 'second.png', + createdAt: new Date(2000), + }); + const { service, m, tx } = await initRevertService([older, newer]); + tx.baseline.findMany.mockResolvedValue([baselineOlder, baselineNewer]); + tx.baseline.findUnique.mockImplementation(({ where }: { where: { testRunId: string } }) => + Promise.resolve(where.testRunId === 'run-1' ? baselineOlder : baselineNewer) ); - expect(built.create).not.toHaveBeenCalled(); - expect(built.delete).not.toHaveBeenCalled(); + tx.baseline.findFirst + .mockResolvedValueOnce(baselineNewer) + .mockResolvedValueOnce(baselineOlder) + .mockResolvedValueOnce(baselineOlder) + .mockResolvedValueOnce(null); + tx.testVariation.findUnique + .mockResolvedValueOnce(variationOf({ baselineName: 'second.png' })) + .mockResolvedValueOnce(variationOf({ baselineName: 'first.png' })); + + await service.revert(['run-1', 'run-2']); + + expect(tx.baseline.findMany).toHaveBeenCalledWith({ + where: { testRunId: { in: ['run-1', 'run-2'] } }, + select: { testRunId: true, createdAt: true }, + }); + expect(tx.baseline.delete.mock.calls).toEqual([ + [{ where: { id: 'baseline-2' } }], + [{ where: { id: 'baseline-1' } }], + ]); + expect(tx.testVariation.update.mock.calls).toEqual([ + [{ where: { id: 'variation' }, data: { baselineName: 'first.png' } }], + [{ where: { id: 'variation' }, data: { baselineName: null } }], + ]); + expect(m.static.deleteImage.mock.calls).toEqual([['second.png'], ['first.png']]); }); }); describe('approve', () => { const copiedBaselineName = 'copied.baseline.png'; - const initApproveService = async (testRun: object | null) => { - const testRunFindUniqueMock = jest.fn().mockResolvedValue(testRun); - const testRunUpdateMock = jest.fn().mockResolvedValue(testRun); - const copyImageMock = jest.fn().mockResolvedValue(copiedBaselineName); - const testVariationServiceFindUniqueMock = jest.fn(); - const testVariationServiceCreateMock = jest.fn(); - const testVariationUpdateMock = jest.fn(); - const testVariationAddBaselineMock = jest.fn(); - const eventTestRunUpdatedMock = jest.fn(); - const built = await initService({ - testRunFindUniqueMock, - testRunUpdateMock, - copyImageMock, - testVariationServiceFindUniqueMock, - testVariationServiceCreateMock, - testVariationUpdateMock, - testVariationAddBaselineMock, - eventTestRunUpdatedMock, - }); - return { - built, - testRunFindUniqueMock, - testRunUpdateMock, - copyImageMock, - testVariationServiceFindUniqueMock, - testVariationServiceCreateMock, - testVariationUpdateMock, - testVariationAddBaselineMock, - eventTestRunUpdatedMock, - }; + const initApproveService = async ( + testRun: object | null, + { updateManyCount = 1, baselineCount = 0 }: { updateManyCount?: number; baselineCount?: number } = {} + ) => { + const ctx = await build(); + ctx.m.testRun.findUnique.mockResolvedValue(testRun); + ctx.m.testRun.updateMany.mockResolvedValue({ count: updateManyCount }); + ctx.m.baseline.count.mockResolvedValue(baselineCount); + ctx.m.static.copyImage.mockResolvedValue(copiedBaselineName); + return ctx; }; const mainVariation = generateTestVariation({ id: 'main variation', branchName: 'master' }); + const sameBranchRun = (overrides: Partial = {}) => ({ + ...generateTestRun({ branchName: 'master', baselineBranchName: 'master', ...overrides }), + testVariation: mainVariation, + }); + const featureRun = (overrides: Partial = {}) => ({ + ...generateTestRun({ branchName: 'feature', baselineBranchName: 'master', ...overrides }), + testVariation: mainVariation, + }); + const expectClaim = (m: Mocks, run: TestRun, status: TestStatus) => + expect(m.testRun.updateMany).toHaveBeenCalledWith({ + where: { id: run.id, status: run.status }, + data: { status, statusBeforeDecision: run.status }, + }); + const expectNoWrites = (m: Mocks) => { + expect(m.static.copyImage).not.toHaveBeenCalled(); + expect(m.tv.addBaseline).not.toHaveBeenCalled(); + expect(m.testRun.updateMany).not.toHaveBeenCalled(); + }; - it('throws when the run has no test variation', async () => { - const { built, copyImageMock } = await initApproveService({ ...generateTestRun(), testVariation: null }); + it('approve of a run without variation is a 409', async () => { + const { service, m } = await initApproveService({ ...generateTestRun(), testVariation: null }); - await expect(built.approve('id')).rejects.toThrow('No test variation found. Re-create test run'); - expect(copyImageMock).not.toHaveBeenCalled(); + const result = service.approve('id'); + + await expect(result).rejects.toBeInstanceOf(ConflictException); + await expect(result).rejects.toThrow(NO_VARIATION); + expect(m.static.copyImage).not.toHaveBeenCalled(); }); it('throws NotFoundException when the run does not exist', async () => { - const { built } = await initApproveService(null); + const { service } = await initApproveService(null); - await expect(built.approve('id')).rejects.toThrow(new NotFoundException('Test run not found: id')); + await expect(service.approve('id')).rejects.toThrow(new NotFoundException('Test run not found: id')); }); it('copies the image as the new baseline of the same-branch variation and marks the run approved', async () => { - const testRun = { - ...generateTestRun({ branchName: 'master', baselineBranchName: 'master', imageName: 'image.png' }), - testVariation: mainVariation, - }; - const mocks = await initApproveService(testRun); + const testRun = sameBranchRun({ imageName: 'image.png' }); + const { service, m } = await initApproveService(testRun); - const result = await mocks.built.approve(testRun.id, false, false, 'user id'); + const result = await service.approve(testRun.id, false, false, 'user id'); - expect(mocks.copyImageMock).toHaveBeenCalledWith('baseline', 'image.png'); - expect(mocks.testVariationServiceFindUniqueMock).not.toHaveBeenCalled(); - expect(mocks.testVariationAddBaselineMock).toHaveBeenCalledWith({ - id: mainVariation.id, - userId: 'user id', - testRunId: testRun.id, - baselineName: copiedBaselineName, - }); - expect(mocks.testRunUpdateMock).toHaveBeenCalledWith({ - where: { id: testRun.id }, - data: { status: TestStatus.approved }, - }); - expect(mocks.eventTestRunUpdatedMock).toHaveBeenCalledWith(testRun); + expect(m.static.copyImage).toHaveBeenCalledWith('baseline', 'image.png'); + expect(m.tv.findUnique).not.toHaveBeenCalled(); + expect(m.tv.addBaseline).toHaveBeenCalledWith( + { id: mainVariation.id, userId: 'user id', testRunId: testRun.id, baselineName: copiedBaselineName }, + expect.anything() + ); + expectClaim(m, testRun, TestStatus.approved); + expect(m.events.testRunUpdated).toHaveBeenCalledWith(testRun); expect(result).toBe(testRun); }); + it('claims the run and adds the baseline in one transaction', async () => { + const testRun = sameBranchRun({ status: TestStatus.failed }); + const { service, m } = await initApproveService(testRun); + + await service.approve(testRun.id); + + expectClaim(m, testRun, TestStatus.approved); + const [, tx] = m.tv.addBaseline.mock.calls[0]; + expect(tx).toHaveProperty('testRun.updateMany', m.testRun.updateMany); + expect(m.events.testRunUpdated).toHaveBeenCalledWith(testRun); + }); + + it('approve records the status it decided from', async () => { + const testRun = sameBranchRun({ status: TestStatus.unresolved }); + const { service, m } = await initApproveService(testRun); + + await service.approve(testRun.id); + + expectClaim(m, testRun, TestStatus.approved); + }); + it('creates a feature-branch variation when approving a feature-branch run against the main baseline', async () => { - const testRun = { - ...generateTestRun({ - branchName: 'feature', - baselineBranchName: 'master', - baselineName: 'old.baseline.png', - ignoreAreas: '[{"x":1}]', - comment: 'run comment', - }), - testVariation: mainVariation, - }; - const mocks = await initApproveService(testRun); + const testRun = featureRun({ + baselineName: 'old.baseline.png', + ignoreAreas: '[{"x":1}]', + comment: 'run comment', + }); + const { service, m } = await initApproveService(testRun); const created = generateTestVariation({ id: 'created feature variation', branchName: 'feature' }); const updated = generateTestVariation({ id: 'updated feature variation', branchName: 'feature' }); - mocks.testVariationServiceFindUniqueMock.mockResolvedValueOnce(null); - mocks.testVariationServiceCreateMock.mockResolvedValueOnce(created); - mocks.testVariationUpdateMock.mockResolvedValueOnce(updated); - - await mocks.built.approve(testRun.id, false, false, 'user id'); - - // NOTE: current behavior — the whole run (including the nested testVariation) is spread into findUnique - expect(mocks.testVariationServiceFindUniqueMock).toHaveBeenCalledWith({ ...testRun }); - expect(mocks.testVariationServiceCreateMock).toHaveBeenCalledWith({ - testRunId: testRun.id, - createTestRequestDto: { - projectId: testRun.projectId, - branchName: 'feature', - ...getTestVariationUniqueData(testRun), - }, - }); - // NOTE: current behavior — carries over the run's *previous* baselineName, the new copy only - // reaches the variation through addBaseline below - expect(mocks.testVariationUpdateMock).toHaveBeenCalledWith( + m.tv.findUnique.mockResolvedValueOnce(null); + m.tv.create.mockResolvedValueOnce(created); + m.tv.update.mockResolvedValueOnce(updated); + const key = { projectId: mainVariation.projectId, branchName: 'feature', ...getTestVariationUniqueData(testRun) }; + + await service.approve(testRun.id, false, false, 'user id'); + + expect(m.tv.findUnique).toHaveBeenCalledWith(key, expect.anything()); + expect(m.tv.create).toHaveBeenCalledWith({ createTestRequestDto: key }, expect.anything()); + expect(m.tv.update).toHaveBeenCalledWith( created.id, - { baselineName: 'old.baseline.png', ignoreAreas: '[{"x":1}]', comment: 'run comment' }, - testRun.id + { ignoreAreas: '[{"x":1}]', comment: 'run comment' }, + testRun.id, + expect.anything() ); - expect(mocks.testVariationAddBaselineMock).toHaveBeenCalledWith({ - id: updated.id, - userId: 'user id', - testRunId: testRun.id, - baselineName: copiedBaselineName, - }); - expect(mocks.testRunUpdateMock).toHaveBeenCalledWith({ - where: { id: testRun.id }, - data: { status: TestStatus.approved }, - }); + expect(m.tv.addBaseline).toHaveBeenCalledWith( + { id: updated.id, userId: 'user id', testRunId: testRun.id, baselineName: copiedBaselineName }, + expect.anything() + ); + expectClaim(m, testRun, TestStatus.approved); }); it('reuses the existing feature-branch variation', async () => { - const testRun = { - ...generateTestRun({ branchName: 'feature', baselineBranchName: 'master' }), - testVariation: mainVariation, - }; - const mocks = await initApproveService(testRun); + const testRun = featureRun(); + const { service, m } = await initApproveService(testRun); const existing = generateTestVariation({ id: 'existing feature variation', branchName: 'feature' }); - mocks.testVariationServiceFindUniqueMock.mockResolvedValueOnce(existing); - mocks.testVariationUpdateMock.mockResolvedValueOnce(existing); + m.tv.findUnique.mockResolvedValueOnce(existing); + m.tv.update.mockResolvedValueOnce(existing); - await mocks.built.approve(testRun.id); + await service.approve(testRun.id); - expect(mocks.testVariationServiceCreateMock).not.toHaveBeenCalled(); - expect(mocks.testVariationUpdateMock).toHaveBeenCalledWith(existing.id, expect.any(Object), testRun.id); - expect(mocks.testVariationAddBaselineMock).toHaveBeenCalledWith( - expect.objectContaining({ id: existing.id, userId: undefined }) + expect(m.tv.create).not.toHaveBeenCalled(); + expect(m.tv.update).toHaveBeenCalledWith(existing.id, expect.any(Object), testRun.id, expect.anything()); + expect(m.tv.addBaseline).toHaveBeenCalledWith( + expect.objectContaining({ id: existing.id, userId: undefined }), + expect.anything() ); }); it('adds the baseline straight to the main variation when merging', async () => { - const testRun = { - ...generateTestRun({ branchName: 'feature', baselineBranchName: 'master' }), - testVariation: mainVariation, - }; - const mocks = await initApproveService(testRun); + const testRun = featureRun(); + const { service, m } = await initApproveService(testRun); - await mocks.built.approve(testRun.id, true); + await service.approve(testRun.id, true); - expect(mocks.testVariationServiceFindUniqueMock).not.toHaveBeenCalled(); - expect(mocks.testVariationUpdateMock).not.toHaveBeenCalled(); - expect(mocks.testVariationAddBaselineMock).toHaveBeenCalledWith( - expect.objectContaining({ id: mainVariation.id, baselineName: copiedBaselineName }) + expect(m.tv.findUnique).not.toHaveBeenCalled(); + expect(m.tv.update).not.toHaveBeenCalled(); + expect(m.tv.addBaseline).toHaveBeenCalledWith( + expect.objectContaining({ id: mainVariation.id, baselineName: copiedBaselineName }), + expect.anything() ); - expect(mocks.testRunUpdateMock).toHaveBeenCalledWith({ - where: { id: testRun.id }, - data: { status: TestStatus.approved }, - }); + expectClaim(m, testRun, TestStatus.approved); }); - it('auto approves a feature-branch run without touching any variation', async () => { - const testRun = { - ...generateTestRun({ branchName: 'feature', baselineBranchName: 'master' }), - testVariation: mainVariation, - }; - const mocks = await initApproveService(testRun); + it('auto approving a feature-branch run copies no baseline image', async () => { + const testRun = featureRun(); + const { service, m } = await initApproveService(testRun); - await mocks.built.approve(testRun.id, false, true); + await service.approve(testRun.id, false, true); - // NOTE: current behavior — the baseline image is still copied even though nothing references it - expect(mocks.copyImageMock).toHaveBeenCalledWith('baseline', testRun.imageName); - expect(mocks.testVariationServiceFindUniqueMock).not.toHaveBeenCalled(); - expect(mocks.testVariationAddBaselineMock).not.toHaveBeenCalled(); - expect(mocks.testRunUpdateMock).toHaveBeenCalledWith({ - where: { id: testRun.id }, - data: { status: TestStatus.autoApproved }, - }); + expect(m.static.copyImage).not.toHaveBeenCalled(); + expect(m.tv.findUnique).not.toHaveBeenCalled(); + expect(m.tv.addBaseline).not.toHaveBeenCalled(); + expectClaim(m, testRun, TestStatus.autoApproved); }); it('auto approves a same-branch run by adding the baseline', async () => { - const testRun = { - ...generateTestRun({ branchName: 'master', baselineBranchName: 'master' }), - testVariation: mainVariation, - }; - const mocks = await initApproveService(testRun); + const testRun = sameBranchRun(); + const { service, m } = await initApproveService(testRun); - await mocks.built.approve(testRun.id, false, true); + await service.approve(testRun.id, false, true); - expect(mocks.testVariationAddBaselineMock).toHaveBeenCalledWith({ - id: mainVariation.id, - userId: undefined, - testRunId: testRun.id, - baselineName: copiedBaselineName, - }); - expect(mocks.testRunUpdateMock).toHaveBeenCalledWith({ - where: { id: testRun.id }, - data: { status: TestStatus.autoApproved }, - }); + expect(m.tv.addBaseline).toHaveBeenCalledWith( + { id: mainVariation.id, userId: undefined, testRunId: testRun.id, baselineName: copiedBaselineName }, + expect.anything() + ); + expectClaim(m, testRun, TestStatus.autoApproved); }); - }); - describe('create details', () => { - it('prefers the comment of the request over the one of the variation', async () => { - const testRunCreateMock = jest.fn().mockResolvedValueOnce(generateTestRun()); - const eventTestRunCreatedMock = jest.fn(); - service = await initService({ - testRunCreateMock, - eventTestRunCreatedMock, - saveImageMock: jest.fn().mockResolvedValueOnce('image.png'), - }); + it('re-approving an already approved feature-branch run changes nothing', async () => { + const testRun = featureRun({ status: TestStatus.approved }); + const { service, m } = await initApproveService(testRun); - const result = await service.create({ - testVariation: generateTestVariation({ comment: 'variation comment' }), - createTestRequestDto: { - buildId: 'buildId', - projectId: 'projectId', - name: 'Test name', - branchName: 'develop', - comment: 'request comment', - }, - imageBuffer, - }); + const result = await service.approve(testRun.id); - const { data } = testRunCreateMock.mock.calls[0][0]; - expect(data.comment).toBe('request comment'); - expect(data.imageName).toBe('image.png'); - // NOTE: current behavior — absent ignore areas are stored as undefined (JSON.stringify(undefined)) - expect(data.tempIgnoreAreas).toBeUndefined(); - expect(eventTestRunCreatedMock).toHaveBeenCalledWith(result); + expect(result).toBe(testRun); + expectNoWrites(m); + expect(m.tv.update).not.toHaveBeenCalled(); + expect(m.events.testRunUpdated).not.toHaveBeenCalled(); }); - }); - describe('calculateDiff details', () => { - it('compares with the permanent and temporary ignore areas together and returns the saved run', async () => { - const permanent = { x: 1, y: 1, width: 1, height: 1 }; - const temporary = { x: 2, y: 2, width: 2, height: 2 }; - const testRun = generateTestRun({ - ignoreAreas: JSON.stringify([permanent]), - tempIgnoreAreas: JSON.stringify([temporary]), - }); - const saved = generateTestRun({ id: 'saved' }); - const compareGetDiffMock = jest.fn().mockResolvedValueOnce({ status: TestStatus.ok }); - const deleteImageMock = jest.fn(); - service = await initService({ - compareGetDiffMock, - deleteImageMock, - testRunUpdateMock: jest.fn().mockResolvedValueOnce(saved), - }); + it('auto approving an already auto approved run changes nothing', async () => { + const testRun = featureRun({ status: TestStatus.autoApproved }); + const { service, m } = await initApproveService(testRun); + + const result = await service.approve(testRun.id, false, true); + + expect(result).toBe(testRun); + expect(m.baseline.count).not.toHaveBeenCalled(); + expect(m.static.copyImage).not.toHaveBeenCalled(); + expect(m.testRun.updateMany).not.toHaveBeenCalled(); + }); + + it('manually approving an autoApproved run that already has a baseline is a no-op', async () => { + const testRun = sameBranchRun({ status: TestStatus.autoApproved }); + const { service, m } = await initApproveService(testRun, { baselineCount: 1 }); + + const result = await service.approve(testRun.id); + + expect(m.baseline.count).toHaveBeenCalledWith({ where: { testRunId: testRun.id } }); + expect(result).toBe(testRun); + expectNoWrites(m); + }); + + it('manually approving an autoApproved feature-branch run without a baseline approves it', async () => { + const testRun = featureRun({ status: TestStatus.autoApproved }); + const { service, m } = await initApproveService(testRun, { baselineCount: 0 }); + m.tv.findUnique.mockResolvedValueOnce(null); + m.tv.create.mockResolvedValueOnce(generateTestVariation({ id: 'feature variation' })); + m.tv.update.mockResolvedValueOnce(generateTestVariation({ id: 'feature variation' })); + + await service.approve(testRun.id); + + expectClaim(m, testRun, TestStatus.approved); + expect(m.tv.addBaseline).toHaveBeenCalledWith( + expect.objectContaining({ id: 'feature variation', baselineName: copiedBaselineName }), + expect.anything() + ); + }); + + it('a lost claim deletes the copy and changes nothing', async () => { + const testRun = sameBranchRun(); + const { service, m } = await initApproveService(testRun, { updateManyCount: 0 }); + + const result = await service.approve(testRun.id); + + expect(m.tv.addBaseline).not.toHaveBeenCalled(); + expect(m.static.deleteImage).toHaveBeenCalledWith(copiedBaselineName); + expect(m.events.testRunUpdated).not.toHaveBeenCalled(); + expect(result).toBe(testRun); + }); + + it('a lost claim returns the run it read when the run is gone meanwhile', async () => { + const testRun = sameBranchRun(); + const { service, m } = await initApproveService(testRun, { updateManyCount: 0 }); + m.testRun.findUnique.mockResolvedValueOnce(testRun).mockResolvedValueOnce(null); + + expect(await service.approve(testRun.id)).toBe(testRun); + }); - const result = await service.calculateDiff('projectId', testRun); + it('deletes the copy when the transaction fails', async () => { + const testRun = sameBranchRun(); + const { service, m } = await initApproveService(testRun); + m.tv.addBaseline.mockRejectedValueOnce(new Error('db')); - expect(compareGetDiffMock.mock.calls[0][0].data.ignoreAreas).toEqual([permanent, temporary]); - expect(result).toBe(saved); - // NOTE: current behavior — deleteImage is also asked to delete the missing (null) thumbnails - expect(deleteImageMock.mock.calls).toEqual([[testRun.diffName], [null], [null]]); + await expect(service.approve(testRun.id)).rejects.toThrow('db'); + + expect(m.static.deleteImage).toHaveBeenCalledWith(copiedBaselineName); + expect(m.events.testRunUpdated).not.toHaveBeenCalled(); }); - it('treats null ignore areas as none', async () => { - const testRun = generateTestRun({ ignoreAreas: 'null', tempIgnoreAreas: 'null' }); - const compareGetDiffMock = jest.fn().mockResolvedValueOnce(null); - const testRunUpdateMock = jest.fn().mockResolvedValueOnce(testRun); - service = await initService({ compareGetDiffMock, testRunUpdateMock }); + it('retries once when a concurrent approval created the feature variation', async () => { + const testRun = featureRun(); + const { service, m } = await initApproveService(testRun); + m.static.copyImage.mockResolvedValueOnce('copy1.png').mockResolvedValueOnce('copy2.png'); + const winner = generateTestVariation({ id: 'winner', branchName: 'feature' }); + m.tv.findUnique.mockResolvedValueOnce(null).mockResolvedValueOnce(winner); + m.tv.create.mockRejectedValueOnce(prismaError('P2002')); + m.tv.update.mockResolvedValueOnce(winner); - await service.calculateDiff('projectId', testRun); + const result = await service.approve(testRun.id); - expect(compareGetDiffMock.mock.calls[0][0].data.ignoreAreas).toEqual([]); - expect(testRunUpdateMock.mock.calls[0][0].data.status).toBe(TestStatus.new); + expect(result).toBe(testRun); + expect(m.static.deleteImage).toHaveBeenCalledWith('copy1.png'); + expect(m.tv.addBaseline).toHaveBeenCalledTimes(1); + expect(m.tv.addBaseline).toHaveBeenCalledWith( + expect.objectContaining({ id: winner.id, baselineName: 'copy2.png' }), + expect.anything() + ); }); - }); - describe('saveDiffResult details', () => { - it('clears the analysis of a previous comparison when this one has none', async () => { - const testRunUpdateMock = jest.fn().mockResolvedValueOnce(generateTestRun()); - service = await initService({ testRunUpdateMock }); + it('rethrows a second P2002', async () => { + const testRun = featureRun(); + const { service, m } = await initApproveService(testRun); + m.tv.findUnique.mockResolvedValue(null); + m.tv.create.mockRejectedValue(prismaError('P2002')); - await service.saveDiffResult('id', { - status: TestStatus.ok, - diffName: null, - pixelMisMatchCount: 0, - diffPercent: 0, - isSameDimension: true, - }); + await expect(service.approve(testRun.id)).rejects.toMatchObject({ code: 'P2002' }); - expect(testRunUpdateMock.mock.calls[0][0].data).toEqual({ - diffName: null, - pixelMisMatchCount: 0, - diffPercent: 0, - status: TestStatus.ok, - vlmDescription: null, - changeSignature: null, - imageThumbnailName: null, - diffThumbnailName: null, - }); + expect(m.tv.create).toHaveBeenCalledTimes(2); + expect(m.tv.addBaseline).not.toHaveBeenCalled(); }); }); - describe('delete failures', () => { - const initDeleteService = async (deleteError: unknown) => { - const testRun = generateTestRun(); - const eventTestRunDeletedMock = jest.fn(); - const built = await initService({ - testRunFindUniqueMock: jest.fn().mockResolvedValueOnce(testRun), - testRunDeleteMock: jest.fn().mockRejectedValueOnce(deleteError), - eventTestRunDeletedMock, + describe('auto approval', () => { + const tryNew = (service: TestRunsService, testRun: TestRun, testVariation = generateTestVariation()) => + service['tryAutoApproveByNewBaselines']({ testVariation, testRun, project: TEST_PROJECT }); + const tryPast = (service: TestRunsService, testRun: TestRun, testVariation = generateTestVariation()) => + service['tryAutoApproveByPastBaselines']({ testVariation, testRun, project: TEST_PROJECT }); + const imageBytes = Buffer.from('x'); + + describe.each([ + [ + 'tryAutoApproveByNewBaselines', + tryNew, + (m: Mocks) => { + m.testRun.findMany.mockResolvedValueOnce([{ testVariationId: 'other variation' }]); + m.tv.findLatestBaselines.mockResolvedValueOnce([generateBaseline()]); + }, + ], + [ + 'tryAutoApproveByPastBaselines', + tryPast, + (m: Mocks) => m.tv.findBaselineHistory.mockResolvedValueOnce([generateBaseline(), generateBaseline()]), + ], + ])('%s', (_name, run, arrange) => { + it('skip if ok', async () => { + const testRun = generateTestRun({ status: TestStatus.ok }); + const { service } = await build(); + + expect(await run(service, testRun)).toBe(testRun); }); - return { built, testRun, eventTestRunDeletedMock }; - }; - it('treats a run that is already gone as deleted, without announcing it', async () => { - const { built, testRun, eventTestRunDeletedMock } = await initDeleteService( - new Prisma.PrismaClientKnownRequestError('gone', { code: 'P2025', clientVersion: 'test' }) - ); + it('should not auto approve', async () => { + const testRun = generateTestRun({ status: TestStatus.unresolved }); + const { service, m } = await build(); + arrange(m); + service['shouldAutoApprove'] = jest.fn().mockResolvedValueOnce(false); - const result = await built.delete(testRun.id); + const result = await run(service, testRun); - expect(result).toBeUndefined(); - expect(eventTestRunDeletedMock).not.toHaveBeenCalled(); + expect(service['shouldAutoApprove']).toHaveBeenCalled(); + expect(result).toBe(testRun); + }); + + it('should auto approve', async () => { + const testRun = generateTestRun({ status: TestStatus.unresolved }); + const { service, m } = await build(); + arrange(m); + service['shouldAutoApprove'] = jest.fn().mockResolvedValueOnce(true); + service.approve = jest.fn().mockResolvedValueOnce({ ...testRun, status: TestStatus.autoApproved }); + + const result = await run(service, testRun); + + expect(result).toStrictEqual({ ...testRun, status: TestStatus.autoApproved }); + }); }); - it.each([ - [ - 'another prisma error', - new Prisma.PrismaClientKnownRequestError('fk', { code: 'P2003', clientVersion: 'test' }), - ], - ['any other error', new Error('connection lost')], - ['a non-prisma error carrying the P2025 code', Object.assign(new Error('gone'), { code: 'P2025' })], - ])('swallows %s and reports the run as deleted', async (_case, error) => { - const { built, testRun, eventTestRunDeletedMock } = await initDeleteService(error); + it('does not look again once the past baselines approved the run', async () => { + const testRun = generateTestRun({ status: TestStatus.autoApproved }); + const { service, m } = await build(); - const result = await built.delete(testRun.id); + const result = await tryNew(service, testRun); - // NOTE: current behavior — every delete failure other than P2025 is silently ignored and the run is - // announced as deleted although it may still exist + expect(m.testRun.findMany).not.toHaveBeenCalled(); expect(result).toBe(testRun); - expect(eventTestRunDeletedMock).toHaveBeenCalledWith(testRun); }); - }); - describe('addIgnoreAreas', () => { - const newArea = { x: 5, y: 6, width: 7, height: 8 }; - - it.each([ - ['appends to the existing areas', JSON.stringify(ignoreAreas), [...ignoreAreas, newArea]], - ['starts from none when they are null', 'null', [newArea]], - ])('%s', async (_case, stored, expected) => { - const testRunFindUniqueMock = jest.fn().mockResolvedValueOnce(generateTestRun({ ignoreAreas: stored })); - service = await initService({ testRunFindUniqueMock }); - const updated = generateTestRun({ id: 'updated' }); - service.updateIgnoreAreas = jest.fn().mockResolvedValueOnce(updated); + it('skip if the branch the same as baseline', async () => { + const testRun = generateTestRun({ status: TestStatus.unresolved, branchName: 'a', baselineBranchName: 'a' }); + const { service } = await build(); + + expect(await tryPast(service, testRun)).toBe(testRun); + }); + + describe('details', () => { + it('by past baselines skips the latest baseline and approves on the first older one that matches', async () => { + const testRun = generateTestRun({ status: TestStatus.unresolved, branchName: 'feature' }); + const testVariation = generateTestVariation({ id: 'variation', projectId: 'project' }); + const { service, m } = await build(); + m.tv.findBaselineHistory.mockResolvedValueOnce( + ['latest', 'older', 'oldest'].map((name) => generateBaseline({ id: name, baselineName: `${name}.png` })) + ); + m.compare.getDiff.mockResolvedValueOnce({ status: TestStatus.ok }); + m.static.getImageBuffer.mockResolvedValue(imageBytes); + const approved = generateTestRun({ status: TestStatus.autoApproved }); + service.approve = jest.fn().mockResolvedValueOnce(approved); + + const result = await tryPast(service, testRun, testVariation); + + expect(m.tv.findBaselineHistory).toHaveBeenCalledWith('variation'); + expect(m.compare.getDiff).toHaveBeenCalledTimes(1); + expect(diffInputs(m)[0].data.baseline).toBe('older.png'); + expect(service.approve).toHaveBeenCalledWith(testRun.id, false, true); + expect(result).toBe(approved); + }); + + it('by past baselines has nothing to try with a single baseline', async () => { + const testRun = generateTestRun({ status: TestStatus.unresolved, branchName: 'feature' }); + const { service, m } = await build(); + m.tv.findBaselineHistory.mockResolvedValueOnce([generateBaseline()]); + service['shouldAutoApprove'] = jest.fn(); + + const result = await tryPast(service, testRun); + + expect(service['shouldAutoApprove']).not.toHaveBeenCalled(); + expect(result).toBe(testRun); + }); + + it('by new baselines looks for approved runs against the same baseline and compares with their latest baseline', async () => { + const testRun = generateTestRun({ status: TestStatus.unresolved }); + const testVariation = generateTestVariation({ id: 'variation', projectId: 'project', baselineName: 'b.png' }); + const { service, m } = await build(); + m.testRun.findMany.mockResolvedValueOnce([{ testVariationId: 'other variation' }]); + m.tv.findLatestBaselines.mockResolvedValueOnce([ + generateBaseline({ id: 'latest', baselineName: 'latest.png' }), + ]); + m.compare.getDiff.mockResolvedValueOnce({ status: TestStatus.ok }); + m.static.getImageBuffer.mockResolvedValue(imageBytes); + service.approve = jest.fn().mockResolvedValueOnce(testRun); + + await tryNew(service, testRun, testVariation); + + expect(m.testRun.findMany).toHaveBeenCalledWith({ + where: { + ...getTestVariationUniqueData(testVariation), + baselineName: 'b.png', + status: TestStatus.approved, + testVariation: { projectId: 'project' }, + }, + select: { testVariationId: true }, + }); + expect(m.tv.findLatestBaselines).toHaveBeenCalledWith(['other variation']); + expect(diffInputs(m)[0].data.baseline).toBe('latest.png'); + expect(service.approve).toHaveBeenCalledWith(testRun.id, false, true); + }); - const result = await service.addIgnoreAreas('id', [newArea]); + it('by new baselines asks once per variation, however many runs it approved', async () => { + const { service, m } = await build(); + m.testRun.findMany.mockResolvedValueOnce( + ['v1', 'v2', 'v1', null].map((testVariationId) => ({ testVariationId })) + ); + m.tv.findLatestBaselines.mockResolvedValueOnce([]); - expect(testRunFindUniqueMock).toHaveBeenCalledWith({ where: { id: 'id' }, include: { testVariation: true } }); - expect(service.updateIgnoreAreas).toHaveBeenCalledWith('id', expected); - expect(result).toBe(updated); - }); - }); + await tryNew(service, generateTestRun({ status: TestStatus.unresolved })); - describe('updateIgnoreAreas details', () => { - it('returns the recalculated run', async () => { - const testRun = generateTestRun(); - const recalculated = generateTestRun({ id: 'recalculated' }); - service = await initService({ - testRunUpdateMock: jest.fn().mockResolvedValueOnce(testRun), - testVariationUpdateMock: jest.fn().mockResolvedValueOnce(generateTestVariation({ projectId: 'p' })), + expect(m.tv.findLatestBaselines).toHaveBeenCalledWith(['v1', 'v2']); }); - service.calculateDiff = jest.fn().mockResolvedValueOnce(recalculated); - expect(await service.updateIgnoreAreas(testRun.id, ignoreAreas)).toBe(recalculated); + it.each([ + ['skips a variation without baselines instead of failing', [{ testVariationId: 'v1' }], ['v1']], + ['keeps the run when nothing was approved before', [], []], + ])('by new baselines %s', async (_case, approvedRuns, variationIds) => { + const testRun = generateTestRun({ status: TestStatus.unresolved }); + const { service, m } = await build(); + m.testRun.findMany.mockResolvedValueOnce(approvedRuns); + m.tv.findLatestBaselines.mockResolvedValueOnce([]); + + const result = await tryNew(service, testRun); + + expect(m.tv.findLatestBaselines).toHaveBeenCalledWith(variationIds); + expect(m.compare.getDiff).not.toHaveBeenCalled(); + expect(result).toBe(testRun); + }); }); - }); - describe('update details', () => { - it('returns the updated run', async () => { - const testRun = generateTestRun(); - service = await initService({ testRunUpdateMock: jest.fn().mockResolvedValueOnce(testRun) }); + describe('decides as before', () => { + const legacyPast = (history: string[], ok: Set) => history.slice(1).some((name) => ok.has(name)); + const legacyNew = (runVariations: string[], newest: Record, ok: Set) => + runVariations.some((id) => ok.has(newest[id])); - expect(await service.update(testRun.id, { comment: 'c' })).toBe(testRun); + const testRun = generateTestRun({ status: TestStatus.unresolved, branchName: 'feature' }); + const testVariation = generateTestVariation({ id: 'variation', projectId: 'project' }); + + const initWith = async (ok: Set, arrange: (m: Mocks) => void) => { + const { service, m } = await build(); + arrange(m); + m.compare.getDiff.mockImplementation(async ({ data }) => ({ + status: ok.has(data.baseline) ? TestStatus.ok : TestStatus.unresolved, + })); + m.static.getImageBuffer.mockResolvedValue(Buffer.from('image')); + service.approve = jest.fn().mockResolvedValue(generateTestRun({ status: TestStatus.autoApproved })); + return { service, compared: () => diffInputs(m).map(({ data }) => data.baseline as string) }; + }; + + it.each<[string, string[], string[]]>([ + ['no history', [], []], + ['a single baseline that would match', ['a'], ['a']], + ['an older baseline that matches', ['a', 'b'], ['b']], + ['a repeated name before a match', ['a', 'b', 'b', 'c'], ['c']], + ['a repeated name without a match', ['a', 'b', 'b'], []], + ['the newest name repeated in the past', ['a', 'a'], ['a']], + ['a match on the newest baseline only', ['a', 'b', 'c'], ['a']], + ])('by past baselines: %s', async (_name, history, matching) => { + const ok = new Set(matching); + const { service, compared } = await initWith(ok, (m) => + m.tv.findBaselineHistory.mockResolvedValue( + history.map((baselineName, i) => generateBaseline({ id: `b${i}`, baselineName })) + ) + ); + + await tryPast(service, testRun, testVariation); + + expect(jest.mocked(service.approve).mock.calls.length > 0).toBe(legacyPast(history, ok)); + expect(new Set(compared()).size).toBe(compared().length); + expect(compared().length).toBeLessThanOrEqual(new Set(history.slice(1)).size); + }); + + it.each<[string, string[], Record, string[]]>([ + ['two runs of one variation, a match on the other', ['v1', 'v1', 'v2'], { v1: 'x', v2: 'y' }, ['y']], + ['two variations with the same newest baseline, no match', ['v1', 'v2'], { v1: 'x', v2: 'x' }, []], + ['no approved runs', [], {}, []], + ])('by new baselines: %s', async (_name, runVariations, newest, matching) => { + const ok = new Set(matching); + const { service, compared } = await initWith(ok, (m) => { + m.testRun.findMany.mockResolvedValue(runVariations.map((testVariationId) => ({ testVariationId }))); + m.tv.findLatestBaselines.mockImplementation(async (ids: string[]) => + ids.flatMap((id) => (newest[id] ? [generateBaseline({ id: `b-${id}`, baselineName: newest[id] })] : [])) + ); + }); + + await tryNew(service, testRun, testVariation); + + expect(jest.mocked(service.approve).mock.calls.length > 0).toBe(legacyNew(runVariations, newest, ok)); + expect(new Set(compared()).size).toBe(compared().length); + expect(compared().length).toBeLessThanOrEqual(new Set(runVariations.map((id) => newest[id])).size); + }); }); }); @@ -1642,17 +1950,27 @@ describe('TestRunsService', () => { diffTollerancePercent: 3, }); const baseline = generateBaseline({ baselineName: 'past.baseline.png' }); + const project = { ...TEST_PROJECT, id: 'projectId' }; + const bytes = Buffer.from('image bytes'); it('compares the run against the given baseline without saving a diff, and approves an ok result', async () => { - const compareGetDiffMock = jest.fn().mockResolvedValueOnce({ status: TestStatus.ok }); - service = await initService({ compareGetDiffMock }); + const { service, m } = await build(); + m.compare.getDiff.mockResolvedValueOnce({ status: TestStatus.ok }); - const result = await service['shouldAutoApprove']({ projectId: 'projectId', baseline, testRun }); + const result = await service['shouldAutoApprove']({ + project, + baseline, + testRun, + imageBuffer: bytes, + ignoreAreas, + }); - expect(compareGetDiffMock).toHaveBeenCalledWith({ + expect(m.compare.getDiff).toHaveBeenCalledWith({ projectId: 'projectId', + project, data: { image: 'image.png', + imageBuffer: bytes, baseline: 'past.baseline.png', ignoreAreas, diffTollerancePercent: 3, @@ -1663,104 +1981,117 @@ describe('TestRunsService', () => { }); it('does not approve any other result', async () => { - service = await initService({ - compareGetDiffMock: jest.fn().mockResolvedValueOnce({ status: TestStatus.unresolved }), - }); - - const result = await service['shouldAutoApprove']({ projectId: 'projectId', baseline, testRun }); + const { service, m } = await build(); + m.compare.getDiff.mockResolvedValueOnce({ status: TestStatus.unresolved }); - // NOTE: current behavior — resolves to undefined, not false - expect(result).toBeUndefined(); + expect(await service['shouldAutoApprove']({ project, baseline, testRun, ignoreAreas })).toBe(false); }); }); - describe('auto approval details', () => { - it('by past baselines skips the latest baseline and approves on the first older one that matches', async () => { - const testRun = generateTestRun({ status: TestStatus.unresolved, branchName: 'feature' }); - const latest = generateBaseline({ id: 'latest' }); - const older = generateBaseline({ id: 'older' }); - const oldest = generateBaseline({ id: 'oldest' }); - const testVariation = generateTestVariation({ id: 'variation', projectId: 'project' }); - const testVariationGetDetailsMock = jest - .fn() - .mockResolvedValueOnce(generateTestVariation({}, [latest, older, oldest])); - service = await initService({ testVariationGetDetailsMock }); - service['shouldAutoApprove'] = jest.fn().mockResolvedValueOnce(true); - const approved = generateTestRun({ status: TestStatus.autoApproved }); - service.approve = jest.fn().mockResolvedValueOnce(approved); + describe('autoApproveOnFirstMatch', () => { + const testRun = generateTestRun({ + status: TestStatus.unresolved, + imageName: 'image.png', + ignoreAreas: JSON.stringify(ignoreAreas), + diffTollerancePercent: 3, + }); + const ref = (baselineName: string) => ({ id: `id of ${baselineName}`, baselineName }); + const refs = (...names: string[]) => names.map(ref); + const firstMatch = (service: TestRunsService, baselines: ReturnType, uploaded?: Buffer) => + service['autoApproveOnFirstMatch'](TEST_PROJECT, testRun, baselines, uploaded); + const initFirstMatch = async () => { + const ctx = await build(); + ctx.m.compare.getDiff.mockResolvedValue({ status: TestStatus.unresolved }); + return ctx; + }; + const expectBytes = (m: Mocks, times: number, expected: Buffer | undefined) => { + expect(m.compare.getDiff).toHaveBeenCalledTimes(times); + diffInputs(m).forEach(({ data }) => expect(data.imageBuffer).toBe(expected)); + }; - const result = await service['tryAutoApproveByPastBaselines']({ testVariation, testRun }); + it('compares each baseline name once', async () => { + const { service, m } = await initFirstMatch(); + m.static.getImageBuffer.mockResolvedValue(Buffer.from('x')); - expect(testVariationGetDetailsMock).toHaveBeenCalledWith('variation'); - expect(service['shouldAutoApprove']).toHaveBeenCalledTimes(1); - expect(service['shouldAutoApprove']).toHaveBeenCalledWith({ projectId: 'project', baseline: older, testRun }); - expect(service.approve).toHaveBeenCalledWith(testRun.id, false, true); - expect(result).toBe(approved); + const result = await firstMatch(service, refs('a', 'b', 'a', 'c')); + + expect(diffInputs(m).map(({ data }) => data.baseline)).toStrictEqual(['a', 'b', 'c']); + expect(result).toBe(testRun); }); - it('by past baselines has nothing to try with a single baseline', async () => { - const testRun = generateTestRun({ status: TestStatus.unresolved, branchName: 'feature' }); - service = await initService({ - testVariationGetDetailsMock: jest.fn().mockResolvedValueOnce(generateTestVariation({}, [generateBaseline()])), - }); - service['shouldAutoApprove'] = jest.fn(); + it('reads the run image once for all candidates', async () => { + const bytes = Buffer.from('image bytes'); + const { service, m } = await initFirstMatch(); + m.static.getImageBuffer.mockResolvedValue(bytes); - const result = await service['tryAutoApproveByPastBaselines']({ - testVariation: generateTestVariation(), - testRun, - }); + await firstMatch(service, refs('a', 'b', 'c')); + + expect(m.static.getImageBuffer).toHaveBeenCalledTimes(1); + expect(m.static.getImageBuffer).toHaveBeenCalledWith('image.png'); + expectBytes(m, 3, bytes); + }); + + it('uses the uploaded bytes without reading the storage', async () => { + const uploaded = Buffer.from('uploaded bytes'); + const { service, m } = await initFirstMatch(); + + await firstMatch(service, refs('a', 'b'), uploaded); + + expect(m.static.getImageBuffer).not.toHaveBeenCalled(); + expectBytes(m, 2, uploaded); + }); + + it('lets each comparison read the image when reading it up front fails', async () => { + const { service, m } = await initFirstMatch(); + m.static.getImageBuffer.mockRejectedValueOnce(new Error('s3')); - expect(service['shouldAutoApprove']).not.toHaveBeenCalled(); + const result = await firstMatch(service, refs('a', 'b')); + + expectBytes(m, 2, undefined); expect(result).toBe(testRun); }); - it('by new baselines looks for approved runs against the same baseline and compares with their latest baseline', async () => { - const testRun = generateTestRun({ status: TestStatus.unresolved }); - const testVariation = generateTestVariation({ id: 'variation', projectId: 'project', baselineName: 'b.png' }); - const approvedRun = generateTestRun({ status: TestStatus.approved, testVariationId: 'other variation' }); - const latest = generateBaseline({ id: 'latest' }); - const testRunFindManyMock = jest.fn().mockResolvedValueOnce([approvedRun]); - const testVariationGetDetailsMock = jest - .fn() - .mockResolvedValueOnce(generateTestVariation({}, [latest, generateBaseline({ id: 'older' })])); - service = await initService({ testRunFindManyMock, testVariationGetDetailsMock }); - service['shouldAutoApprove'] = jest.fn().mockResolvedValueOnce(true); - service.approve = jest.fn().mockResolvedValueOnce(testRun); + it('lets each comparison read the image when the storage has none', async () => { + const { service, m } = await initFirstMatch(); + m.static.getImageBuffer.mockResolvedValue(null); - await service['tryAutoApproveByNewBaselines']({ testVariation, testRun }); + await firstMatch(service, refs('a')); - expect(testRunFindManyMock).toHaveBeenCalledWith({ - where: { - ...getTestVariationUniqueData(testVariation), - baselineName: 'b.png', - status: TestStatus.approved, - testVariation: { projectId: 'project' }, - }, - }); - expect(testVariationGetDetailsMock).toHaveBeenCalledWith('other variation'); - expect(service['shouldAutoApprove']).toHaveBeenCalledWith({ projectId: 'project', baseline: latest, testRun }); - expect(service.approve).toHaveBeenCalledWith(testRun.id, false, true); + expect(diffInputs(m)[0].data.imageBuffer).toBeUndefined(); }); - it('by new baselines keeps the run when nothing was approved before', async () => { - const testRun = generateTestRun({ status: TestStatus.unresolved }); - service = await initService({ testRunFindManyMock: jest.fn().mockResolvedValueOnce([]) }); - service['shouldAutoApprove'] = jest.fn(); + it('reads nothing without candidates', async () => { + const { service, m } = await build(); - const result = await service['tryAutoApproveByNewBaselines']({ - testVariation: generateTestVariation(), - testRun, - }); + const result = await firstMatch(service, []); - expect(service['shouldAutoApprove']).not.toHaveBeenCalled(); + expect(m.static.getImageBuffer).not.toHaveBeenCalled(); + expect(m.compare.getDiff).not.toHaveBeenCalled(); expect(result).toBe(testRun); }); + + it('stops at the first match and approves', async () => { + const { service, m } = await build(); + m.compare.getDiff.mockImplementation(async ({ data }) => ({ + status: data.baseline === 'b' ? TestStatus.ok : TestStatus.unresolved, + })); + m.static.getImageBuffer.mockResolvedValue(Buffer.from('x')); + const approved = generateTestRun({ status: TestStatus.autoApproved }); + service.approve = jest.fn().mockResolvedValueOnce(approved); + + const result = await firstMatch(service, refs('a', 'b', 'c')); + + expect(m.compare.getDiff).toHaveBeenCalledTimes(2); + expect(service.approve).toHaveBeenCalledWith(testRun.id, false, true); + expect(result).toBe(approved); + }); }); - describe('getMatchingVariations edge cases', () => { + describe('getMatchingVariations', () => { const histogram = (hot: number): number[] => Array.from({ length: SIGNATURE_LENGTH }, (_unused, index) => (index === hot ? 1 : 0)); const SAME_PALETTE = histogram(0); + const OTHER_PALETTE = histogram(SIGNATURE_LENGTH - 1); const sibling = (id: string, overrides: Partial = {}): TestRun => generateTestRun({ @@ -1773,76 +2104,218 @@ describe('TestRunsService', () => { diffPercent: 12, ...overrides, }); + const target = (overrides: Partial = {}) => sibling('target', { customTags: '', ...overrides }); + const stamped = (signature: unknown[], stamp = { threshold: 0.1, includeAA: true }) => + JSON.stringify({ ...stamp, signature }); - const initEdgeService = async ({ + type Result = Awaited>; + const variationIds = (result: Result) => result.variations.map((variation) => variation.id); + const skippedReasons = (result: Result) => result.skipped.map((item) => item.reason); + const skippedPairs = (result: Result) => result.skipped.map((item) => [item.id, item.reason]); + + const initMatching = async ({ testRun, siblings = [], - project = { bulkApproveVariations: true, bulkApproveGroupBy: 'customTags', imageComparisonConfig: null } as - object | null, - signatureOf = (() => SAME_PALETTE) as (imageName: string) => Promise | number[] | null, - getImageBufferMock = jest.fn().mockImplementation(async (name: string) => Buffer.from(name)), + project = { bulkApproveVariations: true, bulkApproveGroupBy: 'customTags', imageComparisonConfig: null }, + signatureOf = () => SAME_PALETTE, + imageBufferOf = async (name: string) => Buffer.from(name), }: { testRun: TestRun | null; siblings?: TestRun[]; project?: object | null; signatureOf?: (imageName: string) => Promise | number[] | null; - getImageBufferMock?: jest.Mock; + imageBufferOf?: (name: string) => Promise; }) => { - const compareGetChangeSignatureMock = jest.fn().mockImplementation(async (input) => ({ + const { service, m } = await build(); + m.compare.getChangeSignature.mockImplementation(async (input) => ({ signature: await signatureOf(input.image.toString()), })); - const projectFindUniqueMock = jest.fn().mockResolvedValue(project); - const testRunFindManyMock = jest.fn().mockResolvedValue(siblings); - const built = await initService({ - testRunFindUniqueMock: jest.fn().mockResolvedValue(testRun), - testRunFindManyMock, - projectFindUniqueMock, - getImageBufferMock, - compareGetChangeSignatureMock, - }); - return { built, compareGetChangeSignatureMock, projectFindUniqueMock, testRunFindManyMock }; + m.testRun.findUnique.mockResolvedValue(testRun); + m.testRun.findMany.mockResolvedValue(siblings); + m.project.findUnique.mockResolvedValue(project); + m.static.getImageBuffer.mockImplementation(imageBufferOf); + return { + service, + m, + matches: () => service.getMatchingVariations(testRun?.id ?? 'missing'), + signed: () => m.compare.getChangeSignature.mock.calls.map(([input]) => input.image.toString()), + }; + }; + + const trackConcurrency = () => { + let inFlight = 0; + let peak = 0; + return { + peak: () => peak, + signatureOf: async () => { + inFlight += 1; + peak = Math.max(peak, inFlight); + await new Promise((resolve) => setTimeout(resolve, 0)); + inFlight -= 1; + return SAME_PALETTE; + }, + }; }; + it('matches same-palette, same-size siblings and skips different pattern / far larger change', async () => { + const testRun = target(); + const matching = sibling('locale-a'); + const bigger = sibling('locale-b', { diffPercent: 30 }); + const different = sibling('locale-c'); + const { matches } = await initMatching({ + testRun, + siblings: [matching, bigger, different], + signatureOf: (imageName) => (imageName.startsWith('locale-c') ? OTHER_PALETTE : SAME_PALETTE), + }); + + const result = await matches(); + + expect(variationIds(result)).toEqual([testRun.id, matching.id]); + expect(result.skipped.map((item) => ({ id: item.id, reason: item.reason }))).toEqual([ + { id: bigger.id, reason: 'different change size' }, + { id: different.id, reason: 'different change pattern' }, + ]); + }); + + it('groups from the signatures stored at ingest, without decoding anything', async () => { + const signed = (id: string, signature: number[], overrides = {}) => + sibling(id, { changeSignature: stamped(signature), ...overrides }); + const testRun = signed('target', SAME_PALETTE, { customTags: '' }); + const matching = signed('locale-a', SAME_PALETTE); + const different = signed('locale-c', OTHER_PALETTE); + const ctx = await initMatching({ + testRun, + siblings: [matching, different], + signatureOf: () => { + throw new Error('should not be computing a signature'); + }, + }); + + const result = await ctx.matches(); + + expect(ctx.m.compare.getChangeSignature).not.toHaveBeenCalled(); + expect(variationIds(result)).toEqual([testRun.id, matching.id]); + expect(skippedReasons(result)).toEqual(['different change pattern']); + }); + + it('recomputes when the project config has moved on since ingest', async () => { + const stale = stamped(SAME_PALETTE, { threshold: 0.9, includeAA: false }); + const testRun = target({ changeSignature: stale }); + const matching = sibling('locale-a', { changeSignature: stale }); + const ctx = await initMatching({ testRun, siblings: [matching] }); + + const result = await ctx.matches(); + + expect(ctx.signed().sort()).toEqual([testRun.imageName, matching.imageName].sort()); + expect(variationIds(result)).toEqual([testRun.id, matching.id]); + }); + + it('falls back to computing for a run with nothing stored', async () => { + const testRun = target({ changeSignature: stamped(SAME_PALETTE) }); + const unsigned = sibling('locale-a'); + const ctx = await initMatching({ testRun, siblings: [unsigned] }); + + const result = await ctx.matches(); + + expect(ctx.signed()).toEqual([unsigned.imageName]); + expect(variationIds(result)).toEqual([testRun.id, unsigned.id]); + }); + + it('skips a far larger change without paying for its signature', async () => { + const testRun = target(); + const ctx = await initMatching({ testRun, siblings: [sibling('locale-b', { diffPercent: 30 })] }); + + await ctx.matches(); + + expect(ctx.signed()).toEqual([testRun.imageName]); + }); + + it('says a sibling has no diff rather than blaming its change size', async () => { + const ctx = await initMatching({ testRun: target(), siblings: [sibling('locale-b', { diffPercent: 0 })] }); + + expect(skippedReasons(await ctx.matches())).toEqual(['no diff to match']); + }); + + it('asks for the siblings signatures at once rather than one after another', async () => { + const siblings = ['locale-a', 'locale-b', 'locale-c'].map((id) => sibling(id)); + const tracked = trackConcurrency(); + const ctx = await initMatching({ testRun: target(), siblings, signatureOf: tracked.signatureOf }); + + await ctx.matches(); + + expect(tracked.peak()).toBe(siblings.length + 1); + }); + + it('keeps the number of images being decoded at once bounded', async () => { + const siblings = Array.from({ length: SIGNATURE_CONCURRENCY * 3 }, (_, index) => sibling(`locale-${index}`)); + const tracked = trackConcurrency(); + const ctx = await initMatching({ testRun: target(), siblings, signatureOf: tracked.signatureOf }); + + await ctx.matches(); + + expect(tracked.peak()).toBe(SIGNATURE_CONCURRENCY); + }); + + it('reuses the signatures it already computed when the dialog is reopened', async () => { + const testRun = target(); + const matching = sibling('locale-a'); + const ctx = await initMatching({ testRun, siblings: [matching] }); + + await ctx.matches(); + const afterFirst = ctx.m.compare.getChangeSignature.mock.calls.length; + const second = await ctx.matches(); + + expect(afterFirst).toBe(2); + expect(ctx.m.compare.getChangeSignature).toHaveBeenCalledTimes(2); + expect(variationIds(second)).toEqual([testRun.id, matching.id]); + }); + + it('recomputes a signature once the run gained an ignore area', async () => { + const matching = sibling('locale-a'); + const ctx = await initMatching({ testRun: target(), siblings: [matching] }); + + await ctx.matches(); + matching.ignoreAreas = JSON.stringify([{ x: 0, y: 0, width: 10, height: 10 }]); + await ctx.matches(); + + expect(ctx.m.compare.getChangeSignature).toHaveBeenCalledTimes(3); + }); + it('throws for an unknown test run', async () => { - const { built } = await initEdgeService({ testRun: null }); + const ctx = await initMatching({ testRun: null }); - await expect(built.getMatchingVariations('missing')).rejects.toThrow( - new NotFoundException('Test run not found: missing') - ); + await expect(ctx.matches()).rejects.toThrow(new NotFoundException('Test run not found: missing')); }); it('returns only the run itself when the run has no project', async () => { const testRun = sibling('target', { projectId: null }); - const { built, projectFindUniqueMock, testRunFindManyMock } = await initEdgeService({ testRun }); + const ctx = await initMatching({ testRun }); - const result = await built.getMatchingVariations(testRun.id); + const result = await ctx.matches(); - expect(projectFindUniqueMock).not.toHaveBeenCalled(); - expect(testRunFindManyMock).not.toHaveBeenCalled(); + expect(ctx.m.project.findUnique).not.toHaveBeenCalled(); + expect(ctx.m.testRun.findMany).not.toHaveBeenCalled(); expect(result).toEqual({ variations: [new TestRunDto(testRun)], skipped: [] }); }); it('returns only the run itself when the project has bulk approval turned off', async () => { const testRun = sibling('target'); - const { built, projectFindUniqueMock, testRunFindManyMock } = await initEdgeService({ - testRun, - project: { bulkApproveVariations: false }, - }); + const ctx = await initMatching({ testRun, project: { bulkApproveVariations: false } }); - const result = await built.getMatchingVariations(testRun.id); + const result = await ctx.matches(); - expect(projectFindUniqueMock).toHaveBeenCalledWith({ where: { id: testRun.projectId } }); - expect(testRunFindManyMock).not.toHaveBeenCalled(); + expect(ctx.m.project.findUnique).toHaveBeenCalledWith({ where: { id: testRun.projectId } }); + expect(ctx.m.testRun.findMany).not.toHaveBeenCalled(); expect(result).toEqual({ variations: [new TestRunDto(testRun)], skipped: [] }); }); it('looks for unresolved and new siblings that differ only in customTags by default', async () => { const testRun = sibling('target', { os: 'linux', device: null, browser: 'chrome', viewport: '800x600' }); - const { built, testRunFindManyMock } = await initEdgeService({ testRun }); + const ctx = await initMatching({ testRun }); - await built.getMatchingVariations(testRun.id); + await ctx.matches(); - expect(testRunFindManyMock).toHaveBeenCalledWith({ + expect(ctx.m.testRun.findMany).toHaveBeenCalledWith({ where: { id: { not: testRun.id }, buildId: testRun.buildId, @@ -1862,25 +2335,24 @@ describe('TestRunsService', () => { ['viewport', { customTags: 'target', os: null, device: null, browser: 'chromium' }], ['an unknown axis', { os: null, device: null, browser: 'chromium', viewport: '1800x1600' }], ])('groups across %s when the project says so', async (groupBy, fixedAxes) => { - const testRun = sibling('target'); - const { built, testRunFindManyMock } = await initEdgeService({ - testRun, + const ctx = await initMatching({ + testRun: sibling('target'), project: { bulkApproveVariations: true, bulkApproveGroupBy: groupBy === 'an unknown axis' ? 'locale' : groupBy, }, }); - await built.getMatchingVariations(testRun.id); + await ctx.matches(); - const { where } = testRunFindManyMock.mock.calls[0][0]; + const { where } = ctx.m.testRun.findMany.mock.calls[0][0]; const axes = ['customTags', 'os', 'device', 'browser', 'viewport'].filter((axis) => axis in where); expect(Object.fromEntries(axes.map((axis) => [axis, where[axis]]))).toEqual(fixedAxes); }); it('signs with the threshold and anti-aliasing setting of the project config', async () => { const testRun = sibling('target', { ignoreAreas: JSON.stringify(ignoreAreas) }); - const { built, compareGetChangeSignatureMock } = await initEdgeService({ + const ctx = await initMatching({ testRun, project: { bulkApproveVariations: true, @@ -1888,9 +2360,9 @@ describe('TestRunsService', () => { }, }); - await built.getMatchingVariations(testRun.id); + await ctx.matches(); - expect(compareGetChangeSignatureMock).toHaveBeenCalledWith({ + expect(ctx.m.compare.getChangeSignature).toHaveBeenCalledWith({ baseline: Buffer.from(testRun.baselineName!), image: Buffer.from(testRun.imageName), ignoreAreas, @@ -1903,36 +2375,29 @@ describe('TestRunsService', () => { const testRun = sibling('target', { baselineName: null }); const unchanged = sibling('locale-a', { diffPercent: 0 }); const candidate = sibling('locale-b'); - const { built, compareGetChangeSignatureMock } = await initEdgeService({ - testRun, - siblings: [unchanged, candidate], - }); + const ctx = await initMatching({ testRun, siblings: [unchanged, candidate] }); - const result = await built.getMatchingVariations(testRun.id); + const result = await ctx.matches(); - expect(result.variations.map((variation) => variation.id)).toEqual([testRun.id]); - expect(result.skipped.map((item) => [item.id, item.reason])).toEqual([ + expect(variationIds(result)).toEqual([testRun.id]); + expect(skippedPairs(result)).toEqual([ [unchanged.id, 'no reference diff'], [candidate.id, 'no reference diff'], ]); - // NOTE: current behavior — the candidate is still signed although there is nothing to match it against - expect(compareGetChangeSignatureMock.mock.calls.map(([input]) => input.image.toString())).toEqual([ - candidate.imageName, - ]); + expect(ctx.signed()).toEqual([candidate.imageName]); }); it('treats a missing screenshot as no reference diff', async () => { - const testRun = sibling('target'); - const { built, compareGetChangeSignatureMock } = await initEdgeService({ - testRun, + const ctx = await initMatching({ + testRun: sibling('target'), siblings: [sibling('locale-a')], - getImageBufferMock: jest.fn().mockResolvedValue(null), + imageBufferOf: async () => null, }); - const result = await built.getMatchingVariations(testRun.id); + const result = await ctx.matches(); - expect(compareGetChangeSignatureMock).not.toHaveBeenCalled(); - expect(result.skipped.map((item) => item.reason)).toEqual(['no reference diff']); + expect(ctx.m.compare.getChangeSignature).not.toHaveBeenCalled(); + expect(skippedReasons(result)).toEqual(['no reference diff']); }); it('skips a sibling that has nothing to compare with no diff to match', async () => { @@ -1940,35 +2405,34 @@ describe('TestRunsService', () => { const noBaseline = sibling('locale-a', { baselineName: null }); const noSignature = sibling('locale-b'); const matching = sibling('locale-c'); - const { built } = await initEdgeService({ + const ctx = await initMatching({ testRun, siblings: [noBaseline, noSignature, matching], signatureOf: (imageName) => (imageName.startsWith('locale-b') ? null : SAME_PALETTE), }); - const result = await built.getMatchingVariations(testRun.id); + const result = await ctx.matches(); - expect(result.variations.map((variation) => variation.id)).toEqual([testRun.id, matching.id]); - expect(result.skipped.map((item) => [item.id, item.reason])).toEqual([ + expect(variationIds(result)).toEqual([testRun.id, matching.id]); + expect(skippedPairs(result)).toEqual([ [noBaseline.id, 'no diff to match'], [noSignature.id, 'no diff to match'], ]); }); it('lists skipped siblings in the order they were found, whichever check skipped them', async () => { - const testRun = sibling('target'); const otherPattern = sibling('locale-a'); const unchanged = sibling('locale-b', { diffPercent: 0 }); const bigger = sibling('locale-c', { diffPercent: 100 }); - const { built } = await initEdgeService({ - testRun, + const ctx = await initMatching({ + testRun: sibling('target'), siblings: [otherPattern, unchanged, bigger], - signatureOf: (imageName) => (imageName.startsWith('locale-a') ? histogram(SIGNATURE_LENGTH - 1) : SAME_PALETTE), + signatureOf: (imageName) => (imageName.startsWith('locale-a') ? OTHER_PALETTE : SAME_PALETTE), }); - const result = await built.getMatchingVariations(testRun.id); + const result = await ctx.matches(); - expect(result.skipped.map((item) => [item.id, item.reason])).toEqual([ + expect(skippedPairs(result)).toEqual([ [otherPattern.id, 'different change pattern'], [unchanged.id, 'no diff to match'], [bigger.id, 'different change size'], @@ -1986,72 +2450,48 @@ describe('TestRunsService', () => { ])('with the reviewed change at %p%% a sibling at %p%% %s', async (reference, size, outcome) => { const testRun = sibling('target', { diffPercent: reference }); const other = sibling('locale-a', { diffPercent: size }); - const { built } = await initEdgeService({ testRun, siblings: [other] }); + const ctx = await initMatching({ testRun, siblings: [other] }); - const result = await built.getMatchingVariations(testRun.id); + const result = await ctx.matches(); if (outcome === 'matches') { - expect(result.variations.map((variation) => variation.id)).toEqual([testRun.id, other.id]); + expect(variationIds(result)).toEqual([testRun.id, other.id]); } else { - expect(result.skipped.map((item) => item.reason)).toEqual([outcome]); + expect(skippedReasons(result)).toEqual([outcome]); } }); it('recomputes and warns when the stored signature cannot be read', async () => { - const testRun = sibling('target', { changeSignature: '{not json' }); - const { built, compareGetChangeSignatureMock } = await initEdgeService({ testRun }); - const warn = jest.spyOn(built['logger'], 'warn').mockImplementation(() => undefined); + const ctx = await initMatching({ testRun: sibling('target', { changeSignature: '{not json' }) }); + const warn = jest.spyOn(ctx.service['logger'], 'warn').mockImplementation(() => undefined); - await built.getMatchingVariations(testRun.id); + await ctx.matches(); expect(warn).toHaveBeenCalledWith(expect.stringContaining('Ignoring unreadable stored change signature')); - expect(compareGetChangeSignatureMock).toHaveBeenCalledTimes(1); - }); - - it('recomputes a stored signature that has no histogram at all', async () => { - const testRun = sibling('target', { changeSignature: 'null' }); - const { built, compareGetChangeSignatureMock } = await initEdgeService({ testRun }); - - await built.getMatchingVariations(testRun.id); - - expect(compareGetChangeSignatureMock).toHaveBeenCalledTimes(1); + expect(ctx.m.compare.getChangeSignature).toHaveBeenCalledTimes(1); }); - // the default project config is threshold 0.1 with antialiasing ignored - const stamped = (signature: unknown[], stamp = { threshold: 0.1, includeAA: true }) => - JSON.stringify({ ...stamp, signature }); - it('reuses a stored signature stamped with the current config', async () => { - const testRun = sibling('target', { changeSignature: stamped(SAME_PALETTE) }); - const { built, compareGetChangeSignatureMock } = await initEdgeService({ testRun }); - - await built.getMatchingVariations(testRun.id); - - expect(compareGetChangeSignatureMock).not.toHaveBeenCalled(); - }); - - it.each([ - ['only the threshold changed', { threshold: 0.2, includeAA: true }], - ['only the antialiasing setting changed', { threshold: 0.1, includeAA: false }], - ])('recomputes a stored signature when %s', async (_case, stamp) => { - const testRun = sibling('target', { changeSignature: stamped(SAME_PALETTE, stamp) }); - const { built, compareGetChangeSignatureMock } = await initEdgeService({ testRun }); + const ctx = await initMatching({ testRun: sibling('target', { changeSignature: stamped(SAME_PALETTE) }) }); - await built.getMatchingVariations(testRun.id); + await ctx.matches(); - expect(compareGetChangeSignatureMock).toHaveBeenCalledTimes(1); + expect(ctx.m.compare.getChangeSignature).not.toHaveBeenCalled(); }); it.each([ + ['only the threshold changed', stamped(SAME_PALETTE, { threshold: 0.2, includeAA: true })], + ['only the antialiasing setting changed', stamped(SAME_PALETTE, { threshold: 0.1, includeAA: false })], + ['the wrong length', stamped([1, 0])], + ['values that are not numbers', stamped(new Array(SIGNATURE_LENGTH).fill('nope'))], ['a single value that is not a number', stamped([1, ...new Array(SIGNATURE_LENGTH - 1).fill('nope')])], ['a value that is not finite', stamped(SAME_PALETTE).replace('[1,', '[1e999,')], - ])('recomputes a stored signature with %s', async (_case, changeSignature) => { - const testRun = sibling('target', { changeSignature }); - const { built, compareGetChangeSignatureMock } = await initEdgeService({ testRun }); + ])('recomputes a stored signature when %s', async (_case, changeSignature) => { + const ctx = await initMatching({ testRun: sibling('target', { changeSignature }) }); - await built.getMatchingVariations(testRun.id); + await ctx.matches(); - expect(compareGetChangeSignatureMock).toHaveBeenCalledTimes(1); + expect(ctx.m.compare.getChangeSignature).toHaveBeenCalledTimes(1); }); it.each([ @@ -2059,20 +2499,19 @@ describe('TestRunsService', () => { ['an empty stored signature', ''], ['a stored json null', 'null'], ])('computes quietly for %s', async (_case, changeSignature) => { - const testRun = sibling('target', { changeSignature }); - const { built, compareGetChangeSignatureMock } = await initEdgeService({ testRun }); - const warn = jest.spyOn(built['logger'], 'warn').mockImplementation(() => undefined); + const ctx = await initMatching({ testRun: sibling('target', { changeSignature }) }); + const warn = jest.spyOn(ctx.service['logger'], 'warn').mockImplementation(() => undefined); - await built.getMatchingVariations(testRun.id); + await ctx.matches(); expect(warn).not.toHaveBeenCalled(); - expect(compareGetChangeSignatureMock).toHaveBeenCalledTimes(1); + expect(ctx.m.compare.getChangeSignature).toHaveBeenCalledTimes(1); }); it('does not remember a failed signature', async () => { const testRun = sibling('target'); let calls = 0; - const { built, compareGetChangeSignatureMock } = await initEdgeService({ + const ctx = await initMatching({ testRun, signatureOf: () => { calls += 1; @@ -2083,61 +2522,53 @@ describe('TestRunsService', () => { }, }); - await expect(built.getMatchingVariations(testRun.id)).rejects.toThrow('worker crashed'); - const result = await built.getMatchingVariations(testRun.id); + await expect(ctx.matches()).rejects.toThrow('worker crashed'); + const result = await ctx.matches(); - expect(compareGetChangeSignatureMock).toHaveBeenCalledTimes(2); - expect(result.variations.map((variation) => variation.id)).toEqual([testRun.id]); + expect(ctx.m.compare.getChangeSignature).toHaveBeenCalledTimes(2); + expect(variationIds(result)).toEqual([testRun.id]); }); }); describe('signature cache', () => { - beforeEach(async () => { - service = await initService({}); - }); - - it('forgets the oldest entry once it holds more than 2000', () => { + const filledCache = async (size = 2000) => { + const { service } = await build(); const cache = service['signatureCache']; - for (let index = 0; index <= 2000; index++) { + for (let index = 0; index < size; index++) { cache.set(`key ${index}`, Promise.resolve([index])); } + return cache; + }; + + it('forgets the oldest entry once it holds more than 2000', async () => { + const cache = await filledCache(2001); expect(cache.get('key 0')).toBeUndefined(); expect(cache.get('key 1')).toBeDefined(); expect(cache.get('key 2000')).toBeDefined(); }); - it('does not take up room for a key that was looked up but never set', () => { - const cache = service['signatureCache']; - for (let index = 0; index < 2000; index++) { - cache.set(`key ${index}`, Promise.resolve([index])); - } + it('does not take up room for a key that was looked up but never set', async () => { + const cache = await filledCache(); expect(cache.get('absent')).toBeUndefined(); - // replacing an existing key keeps the size, so nothing is evicted cache.set('key 0', Promise.resolve([0])); expect(cache.get('key 1')).toBeDefined(); }); - it('keeps an entry that was read recently', () => { - const cache = service['signatureCache']; - for (let index = 0; index < 2000; index++) { - cache.set(`key ${index}`, Promise.resolve([index])); - } + it('keeps an entry that was read recently', async () => { + const cache = await filledCache(); - cache.get('key 0'); + void cache.get('key 0'); cache.set('one more', Promise.resolve([])); expect(cache.get('key 0')).toBeDefined(); expect(cache.get('key 1')).toBeUndefined(); }); - it('moves a re-set entry to the back', () => { - const cache = service['signatureCache']; - for (let index = 0; index < 2000; index++) { - cache.set(`key ${index}`, Promise.resolve([index])); - } + it('moves a re-set entry to the back', async () => { + const cache = await filledCache(); const replacement = Promise.resolve([42]); cache.set('key 0', replacement); diff --git a/src/test-runs/test-runs.service.ts b/src/test-runs/test-runs.service.ts index 03e042f8..4fd904cd 100644 --- a/src/test-runs/test-runs.service.ts +++ b/src/test-runs/test-runs.service.ts @@ -1,15 +1,27 @@ -import { forwardRef, Inject, Injectable, Logger, NotFoundException } from '@nestjs/common'; +import { + BadRequestException, + ConflictException, + forwardRef, + Inject, + Injectable, + Logger, + NotFoundException, +} from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; import { CreateTestRequestDto } from './dto/create-test-request.dto'; import { IgnoreAreaDto } from './dto/ignore-area.dto'; import { StaticService } from '../static/static.service'; import { PrismaService } from '../prisma/prisma.service'; -import { Baseline, Prisma, TestRun, TestStatus, TestVariation } from '@prisma/client'; +import { isPrismaError } from '../prisma/prisma-errors'; +import { Prisma, Project, TestRun, TestStatus, TestVariation } from '@prisma/client'; import { DiffResult, StampedSignature } from './diffResult'; import { EventsGateway } from '../shared/events/events.gateway'; import { TestRunResultDto } from '../test-runs/dto/testRunResult.dto'; -import { TestVariationsService } from '../test-variations/test-variations.service'; +import { BaselineRef, TestVariationsService } from '../test-variations/test-variations.service'; import { TestRunDto } from './dto/testRun.dto'; -import { getTestVariationUniqueData } from '../utils'; +import { getTestVariationUniqueData, parseIgnoreAreas, testRunImageNames, variationKey } from '../utils'; +import { BoundedCache } from '../utils/bounded-cache'; +import { mapWithConcurrency } from '../utils/map-with-concurrency'; import { CompareService } from '../compare/compare.service'; import { UpdateTestRunDto } from './dto/update-test.dto'; import { parseConfig } from '../compare/utils'; @@ -29,7 +41,8 @@ export class TestRunsService { private prismaService: PrismaService, private staticService: StaticService, private compareService: CompareService, - private eventsGateway: EventsGateway + private eventsGateway: EventsGateway, + private configService: ConfigService ) {} async findMany(buildId: string): Promise { @@ -39,12 +52,7 @@ export class TestRunsService { return list.map((item) => new TestRunDto(item)); } - async findOne(id: string): Promise< - | (TestRun & { - testVariation: TestVariation | null; - }) - | null - > { + async findOne(id: string): Promise { return this.prismaService.testRun.findUnique({ where: { id }, include: { @@ -53,6 +61,14 @@ export class TestRunsService { }); } + async findOneOrThrow(id: string): Promise { + const testRun = await this.findOne(id); + if (!testRun) { + throw testRunNotFound(id); + } + return testRun; + } + async postTestRun({ createTestRequestDto, imageBuffer, @@ -60,138 +76,321 @@ export class TestRunsService { createTestRequestDto: CreateTestRequestDto; imageBuffer: Buffer; }): Promise { - const project = await this.prismaService.project.findUnique({ where: { id: createTestRequestDto.projectId } }); + const [project, build] = await Promise.all([ + this.prismaService.project.findUnique({ where: { id: createTestRequestDto.projectId } }), + this.prismaService.build.findUnique({ where: { id: createTestRequestDto.buildId }, select: { projectId: true } }), + ]); if (!project) { throw new NotFoundException(`Project not found: ${createTestRequestDto.projectId}`); } - // A comparator that cannot run on this server (odiff without HDD storage) - // fails the request here, before the run is stored: afterwards it would - // stay in the build as "new", never to get a verdict. + if (!build) { + throw new NotFoundException(`Build not found: ${createTestRequestDto.buildId}`); + } + if (build.projectId !== createTestRequestDto.projectId) { + this.rejectForeignBuild(createTestRequestDto); + } this.compareService.getComparator(project.imageComparison); - let testVariation = await this.testVariationService.find(createTestRequestDto); - // creates variatioin if does not exist - if (!testVariation) { - testVariation = await this.testVariationService.create({ - createTestRequestDto, - }); + // allSettled, not all: a rejection must not leave the other promises unhandled or the screenshot orphaned + const [savedImage, foundVariation, previousRuns] = await Promise.allSettled([ + this.staticService.saveImage('screenshot', imageBuffer), + this.findOrCreateVariation(createTestRequestDto, project.mainBranchName), + this.prismaService.testRun.findMany({ + where: { + buildId: createTestRequestDto.buildId, + branchName: createTestRequestDto.branchName, + ...getTestVariationUniqueData(createTestRequestDto), + NOT: { OR: [{ status: TestStatus.approved }, { status: TestStatus.autoApproved }] }, + }, + }), + ]); + if ( + savedImage.status === 'rejected' || + foundVariation.status === 'rejected' || + previousRuns.status === 'rejected' + ) { + if (savedImage.status === 'fulfilled') { + await this.staticService.deleteImage(savedImage.value); + } + throw [savedImage, foundVariation, previousRuns].find((result) => result.status === 'rejected')!.reason; } + const imageName = savedImage.value; + const testVariation = foundVariation.value; + const [previousTestRun] = previousRuns.value; - // delete previous test run if exists - const [previousTestRun] = await this.prismaService.testRun.findMany({ - where: { - buildId: createTestRequestDto.buildId, - branchName: createTestRequestDto.branchName, - ...getTestVariationUniqueData(createTestRequestDto), - NOT: { OR: [{ status: TestStatus.approved }, { status: TestStatus.autoApproved }] }, - }, - }); - if (!!previousTestRun) { - await this.delete(previousTestRun.id); + let testRun: TestRun; + try { + if (previousTestRun) { + await this.delete(previousTestRun.id); + } + testRun = await this.insertTestRun({ testVariation, createTestRequestDto, imageName }); + } catch (e) { + await this.staticService.deleteImage(imageName); + throw e; } + let testRunWithResult = await this.calculateDiff(createTestRequestDto.projectId, testRun, imageBuffer, project); - // create test run result - const testRun = await this.create({ testVariation, createTestRequestDto, imageBuffer }); - - // calculate diff - let testRunWithResult = await this.calculateDiff(createTestRequestDto.projectId, testRun); - - // try auto approve if (project.autoApproveFeature) { - testRunWithResult = await this.tryAutoApproveByPastBaselines({ testVariation, testRun: testRunWithResult }); - testRunWithResult = await this.tryAutoApproveByNewBaselines({ testVariation, testRun: testRunWithResult }); + testRunWithResult = await this.tryAutoApproveByPastBaselines({ + testVariation, + testRun: testRunWithResult, + project, + imageBuffer, + }); + testRunWithResult = await this.tryAutoApproveByNewBaselines({ + testVariation, + testRun: testRunWithResult, + project, + imageBuffer, + }); } return new TestRunResultDto(testRunWithResult, testVariation); } - /** - * Confirm difference for testRun - */ + private async findOrCreateVariation( + createTestRequestDto: CreateTestRequestDto, + mainBranchName: string + ): Promise { + return ( + (await this.testVariationService.find(createTestRequestDto, mainBranchName)) ?? + (await this.testVariationService.createOrTakeWinner(createTestRequestDto)) + ); + } + + private rejectForeignBuild({ buildId, projectId }: CreateTestRequestDto): void { + const message = `Build ${buildId} does not belong to project ${projectId}`; + if (this.configService.get('STRICT_BUILD_PROJECT_CHECK', 'false').toLowerCase() === 'true') { + throw new BadRequestException(message); + } + this.logger.warn(message); + } + async approve(id: string, merge = false, autoApprove = false, userId?: string): Promise { - this.logger.log(`Approving testRun: ${id} merge: ${merge} autoApprove: ${autoApprove}`); - const testRun = await this.findOne(id); - if (!testRun) { - throw new NotFoundException(`Test run not found: ${id}`); + try { + return await this.approveOnce(id, merge, autoApprove, userId); + } catch (e) { + if (isPrismaError(e, 'P2002')) { + return this.approveOnce(id, merge, autoApprove, userId); + } + throw e; } - let { testVariation } = testRun; + } + + private async approveOnce(id: string, merge: boolean, autoApprove: boolean, userId?: string): Promise { + this.logger.log(`Approving testRun: ${id} merge: ${merge} autoApprove: ${autoApprove}`); + const testRun = await this.findOneOrThrow(id); + const { testVariation } = testRun; if (!testVariation) { - throw new Error('No test variation found. Re-create test run'); - } - - // save new baseline as a byte-for-byte copy — decoding and re-encoding the - // PNG here blocked the event loop for every approved run - const baselineName = await this.staticService.copyImage('baseline', testRun.imageName); - - if (testRun.baselineBranchName !== testRun.branchName && !merge && !autoApprove) { - // replace main branch with feature branch test variation - // looked up by the run's own columns, which it copied from its variation - const featureBranchTestVariation = await this.testVariationService.findUnique( - testRun as Parameters[0] - ); - - if (!featureBranchTestVariation) { - testVariation = await this.testVariationService.create({ - testRunId: id, - createTestRequestDto: { - projectId: testRun.projectId!, - branchName: testRun.branchName, - ...getTestVariationUniqueData(testRun), + throw new ConflictException(NO_TEST_VARIATION); + } + if (await this.isAlreadyApproved(testRun, autoApprove)) { + return testRun; + } + + const toFeatureBranch = testRun.baselineBranchName !== testRun.branchName && !merge && !autoApprove; + const addsBaseline = !autoApprove || testRun.baselineBranchName === testRun.branchName; + const baselineName = addsBaseline ? await this.staticService.copyImage('baseline', testRun.imageName) : null; + + let claimed: boolean; + try { + claimed = await this.prismaService.$transaction(async (tx) => { + const { count } = await tx.testRun.updateMany({ + where: { id, status: testRun.status }, + data: { + status: autoApprove ? TestStatus.autoApproved : TestStatus.approved, + statusBeforeDecision: testRun.status, }, }); - } else { - testVariation = featureBranchTestVariation; + if (count === 0) { + return false; + } + let target: TestVariation = testVariation; + if (toFeatureBranch) { + const key = variationKey(testVariation.projectId, testRun.branchName, testRun); + const featureBranchTestVariation = + (await this.testVariationService.findUnique(key, tx)) ?? + (await this.testVariationService.create({ createTestRequestDto: key }, tx)); + target = await this.testVariationService.update( + featureBranchTestVariation.id, + { ignoreAreas: testRun.ignoreAreas, comment: testRun.comment }, + id, + tx + ); + } + if (baselineName) { + await this.testVariationService.addBaseline({ id: target.id, userId, testRunId: id, baselineName }, tx); + } + return true; + }); + } catch (e) { + await this.staticService.deleteImage(baselineName); + throw e; + } + if (!claimed) { + await this.staticService.deleteImage(baselineName); + return (await this.findOne(id)) ?? testRun; + } + + const approved = (await this.findOne(id))!; + this.eventsGateway.testRunUpdated(approved); + return approved; + } + + private async isAlreadyApproved(testRun: TestRun, autoApprove: boolean): Promise { + if (testRun.status === TestStatus.approved) { + return true; + } + if (testRun.status !== TestStatus.autoApproved) { + return false; + } + return autoApprove || (await this.prismaService.baseline.count({ where: { testRunId: testRun.id } })) > 0; + } + + async reject(id: string): Promise { + const testRun = await this.prismaService.testRun.findUnique({ where: { id } }); + if (!testRun) { + throw testRunNotFound(id); + } + if (testRun.status === TestStatus.failed) { + return testRun; + } + const { count } = await this.prismaService.testRun.updateMany({ + where: { id, status: testRun.status }, + data: { status: TestStatus.failed, statusBeforeDecision: testRun.status }, + }); + const current = (await this.prismaService.testRun.findUnique({ where: { id } }))!; + if (count > 0) { + this.eventsGateway.testRunUpdated(current); + } + return current; + } + + async revert(ids: string[]): Promise { + const uniqueIds = [...new Set(ids)]; + if (uniqueIds.length === 0) { + throw new ConflictException('Nothing to revert'); + } + const found = await this.prismaService.testRun.findMany({ where: { id: { in: uniqueIds } } }); + const byId = new Map(found.map((run) => [run.id, run])); + const missing = uniqueIds.filter((id) => !byId.has(id)); + if (missing.length) { + throw testRunNotFound(missing.join(', ')); + } + const runs = uniqueIds.map((id) => byId.get(id)!); + + const freedBaselines = await this.prismaService.$transaction(async (tx) => { + const freed: { name: string; baselineName: string }[] = []; + for (const run of await this.orderForRevert(runs, tx)) { + if (run.status === TestStatus.failed) { + await this.restoreStatus(run, {}, tx); + } else if (run.status === TestStatus.approved) { + freed.push({ name: run.name, baselineName: await this.revertApproval(run, tx) }); + } else { + throw new ConflictException( + `Test run ${run.id} is ${run.status}: only approved or rejected runs can be reverted` + ); + } } + return freed; + }); - // carry over data from testRun - testVariation = await this.testVariationService.update( - testVariation.id, - { - baselineName: testRun.baselineName, - ignoreAreas: testRun.ignoreAreas, - comment: testRun.comment, - }, - testRun.id - ); + for (const { name, baselineName } of freedBaselines) { + if (!(await this.isBaselineReferenced(name, baselineName))) { + await this.staticService.deleteImage(baselineName); + } + } + const reverted = await this.prismaService.testRun.findMany({ where: { id: { in: uniqueIds } } }); + const revertedById = new Map(reverted.map((run) => [run.id, run])); + for (const id of uniqueIds) { + this.eventsGateway.testRunUpdated(revertedById.get(id)!); } + } - if (!autoApprove || (autoApprove && testRun.baselineBranchName === testRun.branchName)) { - // add baseline - await this.testVariationService.addBaseline({ - id: testVariation.id, - userId, - testRunId: testRun.id, - baselineName, - }); + private async orderForRevert(runs: TestRun[], tx: Prisma.TransactionClient): Promise { + const approved = runs.filter((run) => run.status === TestStatus.approved); + const baselines = await tx.baseline.findMany({ + where: { testRunId: { in: approved.map((run) => run.id) } }, + select: { testRunId: true, createdAt: true }, + }); + const approvedAt = new Map(baselines.map((baseline) => [baseline.testRunId, baseline.createdAt.getTime()])); + approved.sort((a, b) => (approvedAt.get(b.id) ?? 0) - (approvedAt.get(a.id) ?? 0)); + return [...approved, ...runs.filter((run) => run.status !== TestStatus.approved)]; + } + + private async restoreStatus( + run: TestRun, + extra: { testVariationId?: string }, + tx: Prisma.TransactionClient + ): Promise { + const { count } = await tx.testRun.updateMany({ + where: { id: run.id, status: run.status }, + data: { + status: run.statusBeforeDecision ?? (run.baselineName ? TestStatus.unresolved : TestStatus.new), + statusBeforeDecision: null, + ...extra, + }, + }); + if (count === 0) { + throw new ConflictException(`Test run ${run.id} changed while reverting, try again`); } + } + + private async revertApproval(run: TestRun, tx: Prisma.TransactionClient): Promise { + const baseline = await tx.baseline.findUnique({ where: { testRunId: run.id } }); + if (!baseline) { + throw new ConflictException(`Test run ${run.id} has no baseline to revert`); + } + const latest = await latestBaseline(tx, baseline.testVariationId); + const variation = await tx.testVariation.findUnique({ where: { id: baseline.testVariationId } }); + if (latest?.id !== baseline.id || variation?.baselineName !== baseline.baselineName) { + throw new ConflictException(`Test run ${run.id}: a later approval changed this baseline`); + } + await tx.baseline.delete({ where: { id: baseline.id } }); + const previous = await latestBaseline(tx, variation.id); + const restoreBaselineName = () => + tx.testVariation.update({ where: { id: variation.id }, data: { baselineName: previous?.baselineName ?? null } }); + + const wasFeatureBranchApproval = + variation.branchName === run.branchName && run.baselineBranchName !== run.branchName; + if (!wasFeatureBranchApproval) { + await restoreBaselineName(); + await this.restoreStatus(run, {}, tx); + return baseline.baselineName; + } + + const mainVariation = await this.testVariationService.findUnique( + variationKey(variation.projectId, run.baselineBranchName!, run), + tx + ); + if (!mainVariation) { + throw new ConflictException(NO_TEST_VARIATION); + } + await this.restoreStatus(run, { testVariationId: mainVariation.id }, tx); + const [baselines, testRuns] = await Promise.all([ + tx.baseline.count({ where: { testVariationId: variation.id } }), + tx.testRun.count({ where: { testVariationId: variation.id } }), + ]); + if (baselines === 0 && testRuns === 0) { + await tx.testVariation.delete({ where: { id: variation.id } }); + } else { + await restoreBaselineName(); + } + return baseline.baselineName; + } + + private async isBaselineReferenced(name: string, baselineName: string): Promise { + const [runs, variations, baselines] = await Promise.all([ + this.prismaService.testRun.count({ where: { name, baselineName } }), + this.prismaService.testVariation.count({ where: { baselineName } }), + this.prismaService.baseline.count({ where: { baselineName } }), + ]); + return runs + variations + baselines > 0; + } - // update status - const status = autoApprove ? TestStatus.autoApproved : TestStatus.approved; - return this.setStatus(id, status); - } - - /** - * Finds sibling variations of a test run in the same build — same screen - * (name) and same test-variation axes except the one configured on the project - * as `bulkApproveGroupBy` (customTags by default, i.e. per-locale screenshots). - * The feature is opt-in per project via `bulkApproveVariations`. - * - * Matching uses a position-independent color signature of the changed pixels - * (which colors appeared/changed), so it survives per-locale layout shifts — - * e.g. a title wrapping to a different number of lines pushes the whole screen - * down, but the same change (a moved selection, a recolored button) still - * produces the same signature. A loose guard on change size (diffPercent) also - * skips variations whose change area is far larger/smaller — a same-palette but - * additional/different change — while tolerating per-locale text reflow. - * Siblings that don't match, or have no diff to compare, are reported as - * skipped. Used by {@link getMatchingVariations}. - */ private async findMatchingSiblings( id: string ): Promise<{ testRun: TestRun; matching: TestRun[]; skipped: SkippedSibling[] }> { - const testRun = await this.findOne(id); - if (!testRun) { - throw new NotFoundException(`Test run not found: ${id}`); - } + const testRun = await this.findOneOrThrow(id); const project = testRun.projectId ? await this.prismaService.project.findUnique({ where: { id: testRun.projectId } }) @@ -200,18 +399,13 @@ export class TestRunsService { return { testRun, matching: [], skipped: [] }; } - // Mirror the project's diff config so the change signature classifies the - // same way the normal pixelmatch diff does, rather than a hardcoded threshold. - const config: PixelmatchConfig = { - ...DEFAULT_CONFIG, - ...parseConfig(project.imageComparisonConfig, DEFAULT_CONFIG, this.logger), - }; + const config: PixelmatchConfig = parseConfig(project.imageComparisonConfig, DEFAULT_CONFIG, this.logger); const groupBy = resolveGroupByAxis(project.bulkApproveGroupBy); const fixedAxes: Record = {}; for (const axis of GROUP_BY_AXES) { if (axis !== groupBy) { - fixedAxes[axis] = (testRun as unknown as Record)[axis] ?? null; + fixedAxes[axis] = testRun[axis] ?? null; } } @@ -226,73 +420,46 @@ export class TestRunsService { }, }); - const matching: TestRun[] = []; - const skipped: SkippedSibling[] = []; - - // Comparing change sizes is free, computing a signature costs two decoded - // screenshots — so the cheap test goes first and takes those siblings out - // of the expensive pass entirely. - const candidates: TestRun[] = []; - for (const sibling of siblings) { - if (!sibling.diffPercent) { - // Nothing changed on this one, so there is no change to recognise — - // the same verdict the signature pass would have reached, for free. - skipped.push({ run: sibling, reason: 'no diff to match' }); - } else if (!magnitudesSimilar(testRun.diffPercent, sibling.diffPercent)) { - skipped.push({ run: sibling, reason: 'different change size' }); - } else { - candidates.push(sibling); - } - } - - // The reviewed run and every remaining sibling are signed in one bounded - // fan-out across the worker pool. Signing them one after another is what - // made this dialog take tens of seconds on a build with many locales. + const candidates = siblings.filter( + (sibling) => !!sibling.diffPercent && magnitudesSimilar(testRun.diffPercent, sibling.diffPercent) + ); const [referenceSignature, ...candidateSignatures] = await mapWithConcurrency( [testRun, ...candidates], SIGNATURE_CONCURRENCY, (run) => this.getChangeSignature(run, config) ); - // Nothing to match against: every sibling goes to manual review, whatever - // the cheap pass made of it. if (!referenceSignature) { return { testRun, - matching, + matching: [], skipped: siblings.map((run) => ({ run, reason: 'no reference diff' })), }; } - candidates.forEach((sibling, index) => { - const signature = candidateSignatures[index]; - if (!signature) { - skipped.push({ run: sibling, reason: 'no diff to match' }); + const signatures = new Map(candidates.map((sibling, index) => [sibling.id, candidateSignatures[index]] as const)); + const matching: TestRun[] = []; + const skipped: SkippedSibling[] = []; + for (const sibling of siblings) { + const signature = signatures.get(sibling.id); + let reason: string | null = null; + if (!sibling.diffPercent || signature === null) { + reason = 'no diff to match'; + } else if (signature === undefined) { + reason = 'different change size'; } else if (!signaturesMatch(referenceSignature, signature)) { - skipped.push({ run: sibling, reason: 'different change pattern' }); + reason = 'different change pattern'; + } + if (reason) { + skipped.push({ run: sibling, reason }); } else { matching.push(sibling); } - }); - - // The cheap pass ran first, so restore the order the siblings came in — - // the dialog lists them as one group and its order should not depend on - // which test rejected a sibling. - const orderOf = new Map(siblings.map((sibling, index) => [sibling.id, index] as const)); - skipped.sort((a, b) => orderOf.get(a.run.id)! - orderOf.get(b.run.id)!); + } return { testRun, matching, skipped }; } - /** - * Returns the group of variations for the reviewed run — the run itself plus - * the sibling variations whose change matches it — for the reviewer to confirm - * before a bulk approve/reject. Nothing is mutated here: the actual action is - * driven by the reviewer's explicit selection (regular approve/reject of the - * chosen ids), so a regression that merely looks similar can never be approved - * without a human seeing it. `skipped` lists siblings left out of the group - * (different/additional change, or no diff to compare). - */ async getMatchingVariations( id: string ): Promise<{ variations: TestRunDto[]; skipped: Array }> { @@ -302,19 +469,6 @@ export class TestRunsService { return { variations, skipped: skippedDto }; } - /** - * Position-independent color signature of the change between a test run's - * baseline and image. Null when there is no baseline, dimensions differ, or - * nothing changed. - * - * Normally this was written at ingest, beside the diff that had already - * decoded both screenshots, and grouping a screen costs nothing but the row - * it is read from. The rest of this is the fallback for runs that carry no - * stored signature — ingested before the column existed, or compared by - * something other than pixelmatch: their bytes go to the worker pool - * undecoded, and the result is memoized so reopening the dialog on an old - * build does not pay for the same decodes twice. - */ private async getChangeSignature(testRun: TestRun, config: PixelmatchConfig): Promise { const stored = parseStoredSignature(testRun.changeSignature, config, this.logger); if (stored) { @@ -323,9 +477,7 @@ export class TestRunsService { if (!testRun.baselineName) { return null; } - const ignoreAreas = this.getAllIgnoteAreas(testRun); - // Image names are unique per upload, so only the ignore areas and the - // project's diff config can change a signature under a stable pair. + const ignoreAreas = this.getAllIgnoreAreas(testRun); const key = [ testRun.baselineName, testRun.imageName, @@ -340,7 +492,6 @@ export class TestRunsService { const pending = this.computeChangeSignature(testRun, ignoreAreas, config); this.signatureCache.set(key, pending); - // A failed read must not be remembered as "no signature" forever. pending.catch(() => this.signatureCache.delete(key)); return pending; } @@ -367,75 +518,43 @@ export class TestRunsService { return signature; } - async setStatus(id: string, status: TestStatus): Promise { + async saveDiffResult(id: string, diffResult: DiffResult | null): Promise { const testRun = await this.prismaService.testRun.update({ where: { id }, data: { - status, + diffName: diffResult && diffResult.diffName, + pixelMisMatchCount: diffResult && diffResult.pixelMisMatchCount, + diffPercent: diffResult && diffResult.diffPercent, + status: diffResult ? diffResult.status : TestStatus.new, + // null, not undefined: Prisma skips undefined and would keep the previous analysis + vlmDescription: diffResult?.vlmDescription ?? null, + changeSignature: diffResult?.changeSignature ? JSON.stringify(diffResult.changeSignature) : null, + imageThumbnailName: diffResult?.imageThumbnailName ?? null, + diffThumbnailName: diffResult?.diffThumbnailName ?? null, }, }); - this.eventsGateway.testRunUpdated(testRun); - return (await this.findOne(id))!; - } - - // Nullable because it genuinely is: a comparison that produced nothing at all - // clears the row back to "new". The body has always handled that; the - // signature just did not say so. - async saveDiffResult(id: string, diffResult: DiffResult | null): Promise { - return this.prismaService.testRun - .update({ - where: { id }, - data: { - diffName: diffResult && diffResult.diffName, - pixelMisMatchCount: diffResult && diffResult.pixelMisMatchCount, - diffPercent: diffResult && diffResult.diffPercent, - status: diffResult ? diffResult.status : TestStatus.new, - // null, not undefined: Prisma skips undefined, which would leave the - // previous comparison's analysis on a run recomputed without one — - // e.g. one the rendering-noise filter now passes before VLM is asked - vlmDescription: diffResult?.vlmDescription ?? null, - // Always written, never merged: a recomputed diff — after the - // reviewer edits the ignore areas, say — must not leave the previous - // signature behind describing a change that no longer exists. - changeSignature: diffResult?.changeSignature ? JSON.stringify(diffResult.changeSignature) : null, - // Overwritten together with the diff they were made from, so a - // recomputed comparison never leaves a picture of the old change. - imageThumbnailName: diffResult?.imageThumbnailName ?? null, - diffThumbnailName: diffResult?.diffThumbnailName ?? null, - }, - }) - .then((testRun) => { - this.eventsGateway.testRunUpdated(testRun); - return testRun; - }); + return testRun; } - async calculateDiff(projectId: string, testRun: TestRun): Promise { - // The recomputed result replaces all three names on the row, so the old - // pictures go with it — leaving the thumbnails behind would strand two more - // objects on every ignore-area edit. - // - // They go *after* the new result is persisted, not before. Deleting first - // meant a comparison that failed left the row pointing at files that were - // already gone: the reviewer opens the run and finds broken pictures, with - // nothing to fall back on. Deleting late can only ever leak bytes, which is - // the cheaper of the two failures by a distance. + async calculateDiff(projectId: string, testRun: TestRun, imageBuffer?: Buffer, project?: Project): Promise { const previous = [testRun.diffName, testRun.imageThumbnailName, testRun.diffThumbnailName]; const diffResult = await this.compareService.getDiff({ projectId, + project, data: { image: testRun.imageName, + imageBuffer, baseline: testRun.baselineName, - ignoreAreas: this.getAllIgnoteAreas(testRun), + ignoreAreas: this.getAllIgnoreAreas(testRun), diffTollerancePercent: testRun.diffTollerancePercent, saveDiffAsFile: true, }, }); const saved = await this.saveDiffResult(testRun.id, diffResult); - previous.forEach((name) => this.staticService.deleteImage(name)); + for (const name of previous) void this.staticService.deleteImage(name); return saved; } @@ -448,163 +567,147 @@ export class TestRunsService { createTestRequestDto: CreateTestRequestDto; imageBuffer: Buffer; }): Promise { - // save image const imageName = await this.staticService.saveImage('screenshot', imageBuffer); + return this.insertTestRun({ testVariation, createTestRequestDto, imageName }); + } - const testRun = await this.prismaService.testRun.create({ - data: { - imageName, - testVariation: { - connect: { - id: testVariation.id, - }, - }, - build: { - connect: { - id: createTestRequestDto.buildId, - }, - }, - project: { - connect: { - id: createTestRequestDto.projectId, - }, + private async insertTestRun({ + testVariation, + createTestRequestDto, + imageName, + }: { + testVariation: TestVariation; + createTestRequestDto: CreateTestRequestDto; + imageName: string; + }): Promise { + let testRun: TestRun; + try { + testRun = await this.prismaService.testRun.create({ + data: { + imageName, + testVariationId: testVariation.id, + buildId: createTestRequestDto.buildId, + projectId: createTestRequestDto.projectId, + ...getTestVariationUniqueData(testVariation), + baselineName: testVariation.baselineName, + baselineBranchName: testVariation.branchName, + ignoreAreas: testVariation.ignoreAreas, + tempIgnoreAreas: JSON.stringify(createTestRequestDto.ignoreAreas), + comment: createTestRequestDto.comment || testVariation.comment, + diffTollerancePercent: createTestRequestDto.diffTollerancePercent, + branchName: createTestRequestDto.branchName, + merge: createTestRequestDto.merge, + status: TestStatus.new, }, - ...getTestVariationUniqueData(testVariation), - baselineName: testVariation.baselineName, - baselineBranchName: testVariation.branchName, - ignoreAreas: testVariation.ignoreAreas, - tempIgnoreAreas: JSON.stringify(createTestRequestDto.ignoreAreas), - comment: createTestRequestDto.comment || testVariation.comment, - diffTollerancePercent: createTestRequestDto.diffTollerancePercent, - branchName: createTestRequestDto.branchName, - merge: createTestRequestDto.merge, - status: TestStatus.new, - }, - }); - + }); + } catch (e) { + if (isPrismaError(e, 'P2003')) { + throw new NotFoundException( + `Build, project or test variation not found for test run ${createTestRequestDto.name}` + ); + } + throw e; + } this.eventsGateway.testRunCreated(testRun); return testRun; } async delete(id: string): Promise { this.logger.debug(`Going to remove TestRun ${id}`); - const testRun = await this.findOne(id); - - if (!testRun) { - this.logger.warn(`TestRun not found ${id}`); - return; - } - - await Promise.all([ - this.staticService.deleteImage(testRun.diffName), - this.staticService.deleteImage(testRun.imageName), - // left behind, these would outlive every build that referenced them and - // nothing would ever collect them - this.staticService.deleteImage(testRun.imageThumbnailName), - this.staticService.deleteImage(testRun.diffThumbnailName), - ]); - + let testRun: TestRun; try { - await this.prismaService.testRun.delete({ where: { id } }); + testRun = await this.prismaService.testRun.delete({ where: { id } }); } catch (e) { - if (e instanceof Prisma.PrismaClientKnownRequestError) { - // workaround https://github.com/Visual-Regression-Tracker/Visual-Regression-Tracker/issues/435 - if (e.code === 'P2025') { - this.logger.warn(`TestRun already deleted ${id}`); - return; - } + // workaround https://github.com/Visual-Regression-Tracker/Visual-Regression-Tracker/issues/435 + if (isPrismaError(e, 'P2025')) { + this.logger.warn(`TestRun not found ${id}`); + return; } + throw e; } - + await this.staticService.deleteImages(testRunImageNames(testRun)); this.logger.log(`TestRun deleted ${id}`); this.eventsGateway.testRunDeleted(testRun); return testRun; } async updateIgnoreAreas(id: string, ignoreAreas: IgnoreAreaDto[]): Promise { - return this.prismaService.testRun - .update({ - where: { id }, - data: { - ignoreAreas: JSON.stringify(ignoreAreas), - }, - }) - .then(async (testRun: TestRun) => { - const testVariation = await this.testVariationService.update(testRun.testVariationId!, { - ignoreAreas: testRun.ignoreAreas, - }); - return this.calculateDiff(testVariation.projectId, testRun); - }); + const testVariationId = await this.getTestVariationId(id); + const testRun = await this.prismaService.testRun.update({ + where: { id }, + data: { + ignoreAreas: JSON.stringify(ignoreAreas), + }, + }); + const testVariation = await this.testVariationService.update(testVariationId, { + ignoreAreas: testRun.ignoreAreas, + }); + return this.calculateDiff(testVariation.projectId, testRun); } async addIgnoreAreas(id: string, ignoreAreas: IgnoreAreaDto[]): Promise { - const testRun = await this.findOne(id); - if (!testRun) { - throw new NotFoundException(`Test run not found: ${id}`); - } - const oldIgnoreAreas: IgnoreAreaDto[] = JSON.parse(testRun.ignoreAreas) ?? []; - return this.updateIgnoreAreas(id, oldIgnoreAreas.concat(ignoreAreas)); + const testRun = await this.findOneOrThrow(id); + return this.updateIgnoreAreas(id, parseIgnoreAreas(testRun.ignoreAreas).concat(ignoreAreas)); } async update(id: string, data: UpdateTestRunDto): Promise { - return this.prismaService.testRun - .update({ - where: { id }, - data: { - comment: data.comment, - }, - }) - .then(async (testRun) => { - await this.testVariationService.update(testRun.testVariationId!, data); - this.eventsGateway.testRunUpdated(testRun); - return testRun; - }); + const testVariationId = await this.getTestVariationId(id); + const testRun = await this.prismaService.testRun.update({ + where: { id }, + data: { + comment: data.comment, + }, + }); + await this.testVariationService.update(testVariationId, { comment: data.comment }); + this.eventsGateway.testRunUpdated(testRun); + return testRun; } - private getAllIgnoteAreas(testRun: TestRun): IgnoreAreaDto[] { - const ignoreAreas: IgnoreAreaDto[] = JSON.parse(testRun.ignoreAreas) ?? []; - const tempIgnoreAreas: IgnoreAreaDto[] = JSON.parse(testRun.tempIgnoreAreas) ?? []; - return ignoreAreas.concat(tempIgnoreAreas); + private async getTestVariationId(id: string): Promise { + const testRun = await this.prismaService.testRun.findUnique({ + where: { id }, + select: { testVariationId: true }, + }); + if (!testRun) { + throw testRunNotFound(id); + } + if (!testRun.testVariationId) { + throw new ConflictException(NO_TEST_VARIATION); + } + return testRun.testVariationId; + } + + private getAllIgnoreAreas(testRun: TestRun): IgnoreAreaDto[] { + return parseIgnoreAreas(testRun.ignoreAreas).concat(parseIgnoreAreas(testRun.tempIgnoreAreas)); } - /** - * Reason: not rebased code from feature branch is compared agains new main branch baseline thus diff is expected - * Tries to find past baseline in main branch and autoApprove in case matched - * @param testVariation - * @param testRun - */ - private async tryAutoApproveByPastBaselines({ testRun, testVariation }: AutoApproveProps): Promise { + private async tryAutoApproveByPastBaselines({ + testRun, + testVariation, + project, + imageBuffer, + }: AutoApproveProps): Promise { if (testRun.status === TestStatus.ok || testRun.branchName === testRun.baselineBranchName) { return testRun; } this.logger.log(`Try AutoApproveByPastBaselines testRun: ${testRun.id}`); - const testVariationHistory = (await this.testVariationService.getDetails(testVariation.id))!; - // skip first baseline as it was used by default in general flow - for (const baseline of testVariationHistory.baselines.slice(1)) { - if (await this.shouldAutoApprove({ projectId: testVariation.projectId, baseline, testRun })) { - return this.approve(testRun.id, false, true); - } - } - - return testRun; + const [, ...pastBaselines] = await this.testVariationService.findBaselineHistory(testVariation.id); + return this.autoApproveOnFirstMatch(project, testRun, pastBaselines, imageBuffer); } - /** - * Reason: branch got another one merged thus diff is expected - * Tries to find latest baseline in test variation - * that has already approved test agains the same baseline image - * and autoApprove in case matched - * @param testVariation - * @param testRun - */ - private async tryAutoApproveByNewBaselines({ testVariation, testRun }: AutoApproveProps): Promise { - if (testRun.status === TestStatus.ok) { + private async tryAutoApproveByNewBaselines({ + testVariation, + testRun, + project, + imageBuffer, + }: AutoApproveProps): Promise { + if (testRun.status === TestStatus.ok || testRun.status === TestStatus.autoApproved) { return testRun; } this.logger.log(`Try AutoApproveByNewBaselines testRun: ${testRun.id}`); - const alreadyApprovedTestRuns: TestRun[] = await this.prismaService.testRun.findMany({ + const alreadyApprovedTestRuns = await this.prismaService.testRun.findMany({ where: { ...getTestVariationUniqueData(testVariation), baselineName: testVariation.baselineName, @@ -613,14 +716,37 @@ export class TestRunsService { projectId: testVariation.projectId, }, }, + select: { testVariationId: true }, }); + const testVariationIds = [ + ...new Set(alreadyApprovedTestRuns.flatMap(({ testVariationId }) => (testVariationId ? [testVariationId] : []))), + ]; + const latestBaselines = await this.testVariationService.findLatestBaselines(testVariationIds); + return this.autoApproveOnFirstMatch(project, testRun, latestBaselines, imageBuffer); + } - for (const approvedTestRun of alreadyApprovedTestRuns) { - // the query only matched runs whose variation exists - const approvedTestVariation = (await this.testVariationService.getDetails(approvedTestRun.testVariationId!))!; - const baseline = approvedTestVariation.baselines.shift()!; - - if (await this.shouldAutoApprove({ projectId: testVariation.projectId, baseline, testRun })) { + private async autoApproveOnFirstMatch( + project: Project, + testRun: TestRun, + baselines: BaselineRef[], + uploaded?: Buffer + ): Promise { + const seen = new Set(); + const candidates = baselines.filter(({ baselineName }) => !seen.has(baselineName) && !!seen.add(baselineName)); + if (candidates.length === 0) { + return testRun; + } + let imageBuffer = uploaded; + if (!imageBuffer) { + try { + imageBuffer = (await this.staticService.getImageBuffer(testRun.imageName)) ?? undefined; + } catch { + imageBuffer = undefined; + } + } + const ignoreAreas = this.getAllIgnoreAreas(testRun); + for (const baseline of candidates) { + if (await this.shouldAutoApprove({ project, baseline, testRun, imageBuffer, ignoreAreas })) { return this.approve(testRun.id, false, true); } } @@ -629,35 +755,44 @@ export class TestRunsService { } private async shouldAutoApprove({ - projectId, + project, baseline, testRun, + imageBuffer, + ignoreAreas, }: { - projectId: string; - baseline: Baseline; + project: Project; + baseline: BaselineRef; testRun: TestRun; - }): Promise { + imageBuffer?: Buffer; + ignoreAreas: IgnoreAreaDto[]; + }): Promise { const diffResult = await this.compareService.getDiff({ - projectId, + projectId: project.id, + project, data: { image: testRun.imageName, + imageBuffer, baseline: baseline.baselineName, - ignoreAreas: this.getAllIgnoteAreas(testRun), + ignoreAreas, diffTollerancePercent: testRun.diffTollerancePercent, saveDiffAsFile: false, }, }); - if (diffResult.status === TestStatus.ok) { - this.logger.log(`TestRun ${testRun.id} could be auto approved based on Baseline ${baseline.id}`); - return true; + if (diffResult.status !== TestStatus.ok) { + return false; } + this.logger.log(`TestRun ${testRun.id} could be auto approved based on Baseline ${baseline.id}`); + return true; } } interface AutoApproveProps { testVariation: TestVariation; testRun: TestRun; + project: Project; + imageBuffer?: Buffer; } interface SkippedSibling { @@ -665,84 +800,25 @@ interface SkippedSibling { reason: string; } -// Test-variation axes a project may bulk-approve across (the one that varies -// within a group). customTags (e.g. locales) is the default. +type TestRunWithVariation = TestRun & { testVariation: TestVariation | null }; + +const testRunNotFound = (id: string) => new NotFoundException(`Test run not found: ${id}`); + +const latestBaseline = (tx: Prisma.TransactionClient, testVariationId: string) => + tx.baseline.findFirst({ where: { testVariationId }, orderBy: { createdAt: 'desc' } }); + +const NO_TEST_VARIATION = 'No test variation found. Re-create test run'; + const GROUP_BY_AXES = ['customTags', 'os', 'device', 'browser', 'viewport'] as const; function resolveGroupByAxis(value: string | null | undefined): string { return value && (GROUP_BY_AXES as readonly string[]).includes(value) ? value : 'customTags'; } -// How many sibling screenshots may be in the worker pool at once for one -// variations dialog. The pool queue is shared with build ingestion, and every -// queued job holds two image buffers, so the fan-out stays bounded rather than -// handing the pool a job per locale. export const SIGNATURE_CONCURRENCY = 8; -// Signatures kept across requests, keyed by the image pair and diff settings. -// Roughly one build's worth of screens, at 64 floats each. const SIGNATURE_CACHE_SIZE = 2000; -/** - * Insertion-ordered cache with a hard size limit: reviewing build after build - * would otherwise grow the signature memo without bound. Reinserting a key - * refreshes its position, so what the reviewer keeps coming back to survives. - */ -class BoundedCache { - private readonly entries = new Map(); - - constructor(private readonly limit: number) {} - - get(key: string): T | undefined { - const value = this.entries.get(key); - if (value !== undefined) { - this.entries.delete(key); - this.entries.set(key, value); - } - return value; - } - - set(key: string, value: T): void { - this.entries.delete(key); - this.entries.set(key, value); - while (this.entries.size > this.limit) { - // the map is not empty while it is over the limit - this.entries.delete(this.entries.keys().next().value!); - } - } - - delete(key: string): void { - this.entries.delete(key); - } -} - -/** - * Runs `task` over every item with at most `limit` in flight, keeping the - * results in the items' order. - */ -async function mapWithConcurrency(items: T[], limit: number, task: (item: T) => Promise): Promise { - const results = new Array(items.length); - let next = 0; - const worker = async (): Promise => { - while (next < items.length) { - const index = next++; - results[index] = await task(items[index]); - } - }; - await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker)); - return results; -} - -/** - * The signature stored on a run, if it is still usable. Null — meaning - * "recompute" — when there is none, when it cannot be read, or when the - * project's comparison settings have moved on since it was written. - * - * That last case is the point. Comparing a signature taken at one threshold - * against a sibling's taken at another quietly makes grouping worse, and the - * reviewer has no way to tell why. The in-memory memo has always keyed on the - * same settings; this is the stored equivalent. - */ function parseStoredSignature( stored: string | null | undefined, config: PixelmatchConfig, @@ -753,9 +829,6 @@ function parseStoredSignature( } try { const parsed: StampedSignature = JSON.parse(stored); - // Shape first: a histogram of the wrong length, or one carrying anything - // that is not a number, compares against a current signature to produce a - // score that looks fine and means nothing. Recomputing is always safe. const wellFormed = Array.isArray(parsed?.signature) && parsed.signature.length === SIGNATURE_LENGTH && @@ -771,9 +844,6 @@ function parseStoredSignature( } } -// Same palette but a much larger/smaller change area signals a different or -// additional change (an extra element changed too), not just per-locale text -// reflow — which stays well under this ratio. Such variations go to manual review. const MATCH_MAGNITUDE_RATIO = 2; function magnitudesSimilar(a: number | null, b: number | null): boolean { diff --git a/src/test-variations/test-variations.controller.ts b/src/test-variations/test-variations.controller.ts index b3e42fd6..2a22e512 100644 --- a/src/test-variations/test-variations.controller.ts +++ b/src/test-variations/test-variations.controller.ts @@ -35,7 +35,7 @@ export class TestVariationsController { @ApiOkResponse({ type: TestVariationDto }) @ApiBearerAuth() @UseGuards(JwtAuthGuard) - getDetails(@Param('id', new ParseUUIDPipe()) id: string): Promise { + getDetails(@Param('id', new ParseUUIDPipe()) id: string): Promise { return this.testVariations.getDetails(id); } diff --git a/src/test-variations/test-variations.service.spec.ts b/src/test-variations/test-variations.service.spec.ts index 25324f5f..2e54edbd 100644 --- a/src/test-variations/test-variations.service.spec.ts +++ b/src/test-variations/test-variations.service.spec.ts @@ -1,101 +1,89 @@ import { NotFoundException } from '@nestjs/common'; -import { Test, TestingModule } from '@nestjs/testing'; +import { Test } from '@nestjs/testing'; import { TestVariationsService } from './test-variations.service'; import { PrismaService } from '../prisma/prisma.service'; -import { CreateTestRequestDto } from '../test-runs/dto/create-test-request.dto'; import { StaticService } from '../static/static.service'; -import { TestVariation, Baseline, Project, Build } from '@prisma/client'; +import { Prisma } from '@prisma/client'; import { PNG } from 'pngjs'; import { BuildsService } from '../builds/builds.service'; import { TestRunsService } from '../test-runs/test-runs.service'; import { generateTestVariation, TEST_PROJECT } from '../_data_'; -import { TestVariationUpdateDto } from './dto/test-variation-update.dto'; - -const initModule = async ({ - imageDeleteMock = jest.fn(), - getImageBufferMock = jest.fn(), - variationfindUniqueMock = jest.fn, - variationFindManyMock = jest.fn().mockReturnValue([]), - variationCreateMock = jest.fn(), - variationUpdateMock = jest.fn(), - variationDeleteMock = jest.fn(), - baselineDeleteMock = jest.fn(), - projectFindUniqueMock = jest.fn(), - buildFindOrCreateMock = jest.fn(), - buildUpdateMock = jest.fn(), - testRunCreateMock = jest.fn(), - testRunFindMany = jest.fn(), - testRunDeleteMock = jest.fn(), - testRuncalCulateDiffMock = jest.fn(), - $executeRawMock = jest.fn(), -}) => { - const module: TestingModule = await Test.createTestingModule({ + +const build = async () => { + const m = { + static: { getImageBuffer: jest.fn(), deleteImage: jest.fn() }, + builds: { findOrCreate: jest.fn(), update: jest.fn() }, + testRuns: { create: jest.fn(), calculateDiff: jest.fn() }, + prisma: { + $executeRaw: jest.fn(), + $transaction: jest.fn((operations: Promise[]) => Promise.all(operations)), + testVariation: { + findUnique: jest.fn(), + findMany: jest.fn().mockReturnValue([]), + create: jest.fn(), + update: jest.fn(), + delete: jest.fn(), + }, + baseline: { findMany: jest.fn().mockResolvedValue([]), deleteMany: jest.fn() }, + project: { findUnique: jest.fn() }, + }, + }; + const module = await Test.createTestingModule({ providers: [ TestVariationsService, { provide: StaticService, useValue: { - getImageBuffer: getImageBufferMock, - deleteImage: imageDeleteMock, - }, - }, - { - provide: BuildsService, - useValue: { - findOrCreate: buildFindOrCreateMock, - update: buildUpdateMock, - }, - }, - { - provide: TestRunsService, - useValue: { - delete: testRunDeleteMock, - create: testRunCreateMock, - calculateDiff: testRuncalCulateDiffMock, - }, - }, - { - provide: PrismaService, - useValue: { - $executeRaw: $executeRawMock, - testVariation: { - findUnique: variationfindUniqueMock, - findMany: variationFindManyMock, - create: variationCreateMock, - update: variationUpdateMock, - delete: variationDeleteMock, - }, - baseline: { - delete: baselineDeleteMock, - }, - project: { - findUnique: projectFindUniqueMock, - }, - testRun: { - findMany: testRunFindMany, + ...m.static, + deleteImages: async (names: Iterable) => { + for (const name of names) if (name) await m.static.deleteImage(name); }, }, }, + { provide: BuildsService, useValue: m.builds }, + { provide: TestRunsService, useValue: m.testRuns }, + { provide: PrismaService, useValue: m.prisma }, ], }).compile(); + return { service: module.get(TestVariationsService), m }; +}; - return module.get(TestVariationsService); +const baselineData = { + name: 'Test name', + os: 'OS', + browser: 'browser', + viewport: 'viewport', + device: 'device', + customTags: 'tag', }; +const findRequest = (overrides: object = {}) => ({ + ...baselineData, + projectId: TEST_PROJECT.id, + branchName: 'develop', + ...overrides, +}); +const keyOf = (variation: typeof baselineData & { projectId: string }, branchName: string) => ({ + name: variation.name, + os: variation.os, + device: variation.device, + browser: variation.browser, + viewport: variation.viewport, + customTags: variation.customTags, + branchName, + projectId: variation.projectId, +}); +const prismaError = (code: string) => new Prisma.PrismaClientKnownRequestError(code, { code, clientVersion: 'test' }); describe('TestVariationsService', () => { - let service: TestVariationsService; - const projectMock = TEST_PROJECT; - describe('getDetails', () => { it('can find one', async () => { - const id = 'test id'; - const variationfindUniqueMock = jest.fn().mockResolvedValueOnce(generateTestVariation({ id }, [])); - service = await initModule({ variationfindUniqueMock }); + const { service, m } = await build(); + m.prisma.testVariation.findUnique.mockResolvedValueOnce(generateTestVariation({ id: 'test id' }, [])); - await service.getDetails(id); + await service.getDetails('test id'); - expect(variationfindUniqueMock).toHaveBeenCalledWith({ - where: { id }, + expect(m.prisma.testVariation.findUnique).toHaveBeenCalledWith({ + where: { id: 'test id' }, include: { baselines: { include: { @@ -111,8 +99,8 @@ describe('TestVariationsService', () => { }); it('throws NotFoundException for an unknown id', async () => { - const variationfindUniqueMock = jest.fn().mockResolvedValueOnce(null); - service = await initModule({ variationfindUniqueMock }); + const { service, m } = await build(); + m.prisma.testVariation.findUnique.mockResolvedValueOnce(null); await expect(service.getDetails('missing')).rejects.toThrow( new NotFoundException('Test variation not found: missing') @@ -120,535 +108,161 @@ describe('TestVariationsService', () => { }); }); - describe('find', () => { - it('can find by main branch', async () => { - const createRequest: CreateTestRequestDto = { - buildId: 'buildId', - projectId: projectMock.id, - name: 'Test name', - os: 'OS', - browser: 'browser', - viewport: 'viewport', - device: 'device', - customTags: '', - branchName: 'develop', - }; - - const variationMock: TestVariation = { - id: '123', - projectId: projectMock.id, - name: 'Test name', - baselineName: 'baselineName', - os: 'OS', - browser: 'browser', - viewport: 'viewport', - device: 'device', - customTags: '', - ignoreAreas: '[]', - comment: 'some comment', - branchName: 'develop', - createdAt: new Date(), - updatedAt: new Date(), - }; - const projectFindUniqueMock = jest.fn().mockReturnValueOnce(projectMock); - service = await initModule({ projectFindUniqueMock }); - service.findUnique = jest.fn().mockResolvedValueOnce(variationMock).mockResolvedValueOnce(undefined); - - const result = await service.find(createRequest); - - expect(projectFindUniqueMock).toHaveBeenCalledWith({ where: { id: createRequest.projectId } }); - expect(service.findUnique).toHaveBeenNthCalledWith(1, { - name: createRequest.name, - projectId: createRequest.projectId, - os: createRequest.os, - browser: createRequest.browser, - viewport: createRequest.viewport, - device: createRequest.device, - customTags: createRequest.customTags, - branchName: projectMock.mainBranchName, + describe('findBaselineHistory', () => { + it('lists id and name of every baseline, newest first', async () => { + const history = [ + { id: 'b3', baselineName: 'c.png' }, + { id: 'b2', baselineName: 'b.png' }, + ]; + const { service, m } = await build(); + m.prisma.baseline.findMany.mockResolvedValueOnce(history); + + const result = await service.findBaselineHistory('variationId'); + + expect(m.prisma.baseline.findMany).toHaveBeenCalledWith({ + where: { testVariationId: 'variationId' }, + orderBy: { createdAt: 'desc' }, + select: { id: true, baselineName: true }, }); - expect(service.findUnique).toHaveBeenNthCalledWith(2, { - name: createRequest.name, - projectId: createRequest.projectId, - os: createRequest.os, - browser: createRequest.browser, - viewport: createRequest.viewport, - device: createRequest.device, - customTags: createRequest.customTags, - branchName: createRequest.branchName, - }); - expect(result).toBe(variationMock); - }); - - it('can find by current branch', async () => { - const createRequest: CreateTestRequestDto = { - buildId: 'buildId', - projectId: projectMock.id, - name: 'Test name', - os: 'OS', - browser: 'browser', - viewport: 'viewport', - device: 'device', - customTags: '', - branchName: 'develop', - }; - - const variationMock: TestVariation = { - id: '123', - projectId: projectMock.id, - name: 'Test name', - baselineName: 'baselineName', - os: 'OS', - browser: 'browser', - viewport: 'viewport', - device: 'device', - customTags: '', - ignoreAreas: '[]', - comment: 'some comment', - branchName: 'develop', - createdAt: new Date(), - updatedAt: new Date(), - }; - const projectFindUniqueMock = jest.fn().mockReturnValueOnce(projectMock); - service = await initModule({ projectFindUniqueMock }); - service.findUnique = jest.fn().mockResolvedValueOnce(undefined).mockResolvedValueOnce(variationMock); - - const result = await service.find(createRequest); - - expect(projectFindUniqueMock).toHaveBeenCalledWith({ where: { id: createRequest.projectId } }); - expect(service.findUnique).toHaveBeenNthCalledWith(1, { - name: createRequest.name, - projectId: createRequest.projectId, - os: createRequest.os, - browser: createRequest.browser, - viewport: createRequest.viewport, - device: createRequest.device, - customTags: createRequest.customTags, - branchName: projectMock.mainBranchName, - }); - expect(service.findUnique).toHaveBeenNthCalledWith(2, { - name: createRequest.name, - projectId: createRequest.projectId, - os: createRequest.os, - browser: createRequest.browser, - viewport: createRequest.viewport, - device: createRequest.device, - customTags: createRequest.customTags, - branchName: createRequest.branchName, - }); - expect(result).toBe(variationMock); - }); - - it('can find by current branch but main branch is more relevant', async () => { - const createRequest: CreateTestRequestDto = { - buildId: 'buildId', - projectId: projectMock.id, - name: 'Test name', - os: 'OS', - browser: 'browser', - viewport: 'viewport', - device: 'device', - customTags: '', - branchName: 'develop', - }; - - const variationMainMock: TestVariation = { - id: '123', - projectId: projectMock.id, - name: 'Test name', - baselineName: 'baselineName', - os: 'OS', - browser: 'browser', - viewport: 'viewport', - device: 'device', - customTags: '', - ignoreAreas: '[]', - comment: 'some comment', - branchName: 'master', - createdAt: new Date(), - updatedAt: new Date(), - }; - const variationFeatureMock: TestVariation = { - id: '123', - projectId: projectMock.id, - name: 'Test name', - baselineName: 'baselineName', - os: 'OS', - browser: 'browser', - viewport: 'viewport', - device: 'device', - customTags: '', - ignoreAreas: '[]', - comment: 'some comment', - branchName: 'develop', - createdAt: new Date(), - updatedAt: new Date(variationMainMock.updatedAt.getDate() - 1), - }; - const projectFindUniqueMock = jest.fn().mockReturnValueOnce(projectMock); - service = await initModule({ projectFindUniqueMock }); - service.findUnique = jest - .fn() - .mockResolvedValueOnce(variationMainMock) - .mockResolvedValueOnce(variationFeatureMock); - - const result = await service.find(createRequest); - - expect(projectFindUniqueMock).toHaveBeenCalledWith({ where: { id: createRequest.projectId } }); - expect(service.findUnique).toHaveBeenNthCalledWith(1, { - name: createRequest.name, - projectId: createRequest.projectId, - os: createRequest.os, - browser: createRequest.browser, - viewport: createRequest.viewport, - device: createRequest.device, - customTags: createRequest.customTags, - branchName: projectMock.mainBranchName, - }); - expect(service.findUnique).toHaveBeenNthCalledWith(2, { - name: createRequest.name, - projectId: createRequest.projectId, - os: createRequest.os, - browser: createRequest.browser, - viewport: createRequest.viewport, - device: createRequest.device, - customTags: createRequest.customTags, - branchName: createRequest.branchName, - }); - expect(result).toBe(variationMainMock); + expect(result).toBe(history); }); }); - it('create', async () => { - const createRequest: CreateTestRequestDto = { - buildId: 'buildId', - projectId: projectMock.id, - name: 'Test name', - os: 'OS', - browser: 'browser', - viewport: 'viewport', - device: 'device', - customTags: '', - branchName: 'develop', - }; + describe('findLatestBaselines', () => { + const row = (id: string) => ({ id, baselineName: `${id}.png`, testVariationId: id.split('-')[0] }); - const projectFindUniqueMock = jest.fn().mockReturnValueOnce(projectMock); - const variationCreateMock = jest.fn(); - service = await initModule({ projectFindUniqueMock, variationCreateMock }); - service.findUnique = jest.fn().mockResolvedValueOnce(undefined).mockResolvedValueOnce(undefined); - - await service.create({ - testRunId: 'testRunId', - createTestRequestDto: createRequest, - }); - - expect(variationCreateMock).toHaveBeenCalledWith({ - data: { - name: createRequest.name, - os: createRequest.os, - browser: createRequest.browser, - viewport: createRequest.viewport, - device: createRequest.device, - customTags: createRequest.customTags, - branchName: createRequest.branchName, - testRuns: { connect: { id: 'testRunId' } }, - project: { - connect: { - id: createRequest.projectId, - }, - }, - }, + it('takes the newest baseline of each variation in the given order', async () => { + const { service, m } = await build(); + m.prisma.baseline.findMany.mockResolvedValueOnce(['v2-new', 'v1-new', 'v2-old', 'v1-old'].map(row)); + + const result = await service.findLatestBaselines(['v1', 'v2']); + + expect(m.prisma.baseline.findMany).toHaveBeenCalledWith({ + where: { testVariationId: { in: ['v1', 'v2'] } }, + orderBy: { createdAt: 'desc' }, + select: { id: true, baselineName: true, testVariationId: true }, + }); + expect(result).toStrictEqual([ + { id: 'v1-new', baselineName: 'v1-new.png' }, + { id: 'v2-new', baselineName: 'v2-new.png' }, + ]); }); - }); - it('update', async () => { - const id = 'some id'; - const data: TestVariationUpdateDto = { - baselineName: 'baselineName', - comment: 'random comment', - ignoreAreas: '[]', - }; - const variationUpdateMock = jest.fn(); - service = await initModule({ - variationUpdateMock, + it('leaves out variations without baselines', async () => { + const { service, m } = await build(); + m.prisma.baseline.findMany.mockResolvedValueOnce([row('v2-new')]); + + expect(await service.findLatestBaselines(['v1', 'v2'])).toStrictEqual([ + { id: 'v2-new', baselineName: 'v2-new.png' }, + ]); }); - await service.update(id, data); + it('does not query for no ids', async () => { + const { service, m } = await build(); - expect(variationUpdateMock).toHaveBeenCalledWith({ - where: { id }, - data, + expect(await service.findLatestBaselines([])).toStrictEqual([]); + expect(m.prisma.baseline.findMany).not.toHaveBeenCalled(); }); }); - it('merge', async () => { - const fromBranch = 'develop'; - const targetBranch = 'test'; - const project: Project = TEST_PROJECT; - const build: Build = { - id: 'a9385fc1-884d-4f9f-915e-40da0e7773d5', - ciBuildId: 'ciBuildId', - number: null, - branchName: targetBranch, - status: null, - projectId: project.id, - updatedAt: new Date(), - createdAt: new Date(), - userId: null, - isRunning: true, - }; - const testVariation: TestVariation = { - id: '123', - projectId: project.id, - name: 'Test name', - baselineName: 'baselineName', - os: 'OS', - browser: 'browser', - viewport: 'viewport', - device: 'device', - customTags: '', - ignoreAreas: '[]', - comment: 'some comment', - createdAt: new Date(), - updatedAt: new Date(), - branchName: fromBranch, - }; - const testVariationSecond: TestVariation = { - id: '123', - projectId: project.id, - name: 'Test name second', - baselineName: 'baselineName', - os: 'OS', - browser: 'browser', - viewport: 'viewport', - device: 'device', - customTags: '', - ignoreAreas: '[]', - comment: 'some comment', - createdAt: new Date(), - updatedAt: new Date(), - branchName: fromBranch, - }; - const testVariationNoBaseline: TestVariation = { - id: '123', - projectId: project.id, - name: 'Test name', - baselineName: null, - os: 'OS', - browser: 'browser', - viewport: 'viewport', - device: 'device', - customTags: '', - ignoreAreas: '[]', - comment: 'some comment', - createdAt: new Date(), - updatedAt: new Date(), - branchName: fromBranch, - }; - const testVariationSourceBranch: TestVariation = { - id: '123', - projectId: project.id, - name: 'Test name', - baselineName: 'baselineName', - os: 'OS', - browser: 'browser', - viewport: 'viewport', - device: 'device', - customTags: '', - ignoreAreas: '[]', - comment: 'some comment', - createdAt: new Date(), - updatedAt: new Date(), - branchName: fromBranch, - }; - const testVariationTargetBranch: TestVariation = { - ...testVariationSourceBranch, - branchName: targetBranch, + describe('find', () => { + const initFind = async (...found: unknown[]) => { + const ctx = await build(); + ctx.service.findUnique = jest.fn(); + found.forEach((variation) => jest.mocked(ctx.service.findUnique).mockResolvedValueOnce(variation as never)); + return ctx; }; - const buildFindOrCreateMock = jest.fn().mockResolvedValueOnce(build); - const variationFindManyMock = jest - .fn() - .mockResolvedValueOnce([testVariation, testVariationSecond, testVariationNoBaseline]); - const image = new PNG({ - width: 10, - height: 10, - }); - const imageBuffer = PNG.sync.write(image); - const getImageBufferMock = jest - .fn() - .mockReturnValueOnce(imageBuffer) - .mockReturnValueOnce(imageBuffer) - .mockReturnValueOnce(null); - const testRunCreateMock = jest.fn(); - const buildUpdateMock = jest.fn(); - const testRuncalCulateDiffMock = jest.fn(); - const service = await initModule({ - buildFindOrCreateMock, - buildUpdateMock, - testRunCreateMock, - testRuncalCulateDiffMock, - variationFindManyMock, - getImageBufferMock, - }); - // neither screen has a variation on the target branch yet - service.findUnique = jest.fn().mockResolvedValueOnce(null).mockResolvedValueOnce(null); - service.create = jest - .fn() - .mockResolvedValueOnce(testVariationTargetBranch) - .mockResolvedValueOnce(testVariationTargetBranch); - - await service.merge(project.id, fromBranch, targetBranch); - - expect(buildFindOrCreateMock).toHaveBeenCalledWith({ - branchName: targetBranch, - projectId: project.id, - }); - expect(service.findUnique).toHaveBeenNthCalledWith(1, { - name: testVariation.name, - os: testVariation.os, - device: testVariation.device, - browser: testVariation.browser, - viewport: testVariation.viewport, - customTags: testVariation.customTags, - branchName: targetBranch, - projectId: project.id, - }); - expect(service.findUnique).toHaveBeenNthCalledWith(2, { - name: testVariationSecond.name, - os: testVariationSecond.os, - device: testVariationSecond.device, - browser: testVariationSecond.browser, - viewport: testVariationSecond.viewport, - customTags: testVariationSecond.customTags, - branchName: targetBranch, - projectId: project.id, - }); - expect(testRunCreateMock).toHaveBeenNthCalledWith(1, { - testVariation: testVariationTargetBranch, - createTestRequestDto: { - ...testVariation, - branchName: targetBranch, - buildId: build.id, - diffTollerancePercent: 0, - merge: true, - ignoreAreas: JSON.parse(testVariation.ignoreAreas), - }, - imageBuffer, - }); - expect(testRunCreateMock).toHaveBeenNthCalledWith(2, { - testVariation: testVariationTargetBranch, - createTestRequestDto: { - ...testVariationSecond, - branchName: targetBranch, - buildId: build.id, - diffTollerancePercent: 0, - merge: true, - ignoreAreas: JSON.parse(testVariationSecond.ignoreAreas), - }, - imageBuffer, + const main = generateTestVariation({ branchName: 'master', updatedAt: new Date(2000) }); + const current = generateTestVariation({ branchName: 'develop', updatedAt: new Date(1000) }); + + it.each([ + ['the main branch', [main, undefined], main], + ['the current branch', [undefined, current], current], + ['the current branch but main branch is more relevant', [main, current], main], + ])('can find by %s', async (_case, [mainResult, currentResult], expected) => { + const request = findRequest({ projectId: TEST_PROJECT.id }); + const { service, m } = await initFind(mainResult, currentResult); + m.prisma.project.findUnique.mockReturnValueOnce(TEST_PROJECT); + + const result = await service.find(request); + + expect(m.prisma.project.findUnique).toHaveBeenCalledWith({ where: { id: request.projectId } }); + expect(service.findUnique).toHaveBeenNthCalledWith(1, keyOf(request, TEST_PROJECT.mainBranchName)); + expect(service.findUnique).toHaveBeenNthCalledWith(2, keyOf(request, request.branchName)); + expect(result).toBe(expected); }); - expect(testRunCreateMock).toHaveBeenCalledTimes(2); - expect(buildUpdateMock).toHaveBeenCalledWith(build.id, { isRunning: false }); - }); - it('delete', async () => { - const testRunId = 'test run id'; - const testVariationId = 'test variation id'; - const variation: TestVariation & { - baselines: Baseline[]; - } = { - id: testVariationId, - projectId: 'project Id', - name: 'Test name', - baselineName: 'baselineName', - os: 'OS', - browser: 'browser', - viewport: 'viewport', - device: 'device', - customTags: '', - ignoreAreas: '[]', - comment: 'some comment', - branchName: 'develop', - createdAt: new Date(), - updatedAt: new Date(), - baselines: [ - { - id: 'baseline id 1', - baselineName: 'image name 1', - testVariationId: testVariationId, - testRunId: testRunId, - userId: 'userId', - createdAt: new Date(), - updatedAt: new Date(), - }, - ], - }; + it('returns undefined when neither branch has the variation', async () => { + const { service, m } = await initFind(null, null); + m.prisma.project.findUnique.mockResolvedValueOnce(TEST_PROJECT); - const $executeRawMock = jest.fn(); - const variationDeleteMock = jest.fn(); - const getDetailsMock = jest.fn().mockResolvedValueOnce(variation); - const deleteBaselineMock = jest.fn().mockResolvedValueOnce(variation.baselines[0]); - const service = await initModule({ - variationDeleteMock, - $executeRawMock, + await expect(service.find(findRequest())).resolves.toBeUndefined(); }); - service.getDetails = getDetailsMock; - service.deleteBaseline = deleteBaselineMock; - await service.delete(testVariationId); + it('only looks on the main branch when the build runs on it', async () => { + const variation = generateTestVariation({ branchName: TEST_PROJECT.mainBranchName }); + const { service, m } = await initFind(variation); + m.prisma.project.findUnique.mockResolvedValueOnce(TEST_PROJECT); + const request = findRequest({ branchName: TEST_PROJECT.mainBranchName }); - expect(service.getDetails).toHaveBeenCalledWith(testVariationId); - expect(service.deleteBaseline).toHaveBeenCalledWith(variation.baselines[0]); - expect($executeRawMock).toHaveBeenCalled(); - expect(variationDeleteMock).toHaveBeenCalledWith({ - where: { id: testVariationId }, + const result = await service.find(request); + + expect(result).toBe(variation); + expect(service.findUnique).toHaveBeenCalledTimes(1); + expect(service.findUnique).toHaveBeenCalledWith(request); }); - }); - it('deleteBaseline', async () => { - const baseline: Baseline = { - id: 'baseline id 1', - baselineName: 'image name 1', - testVariationId: 'test variation id', - testRunId: 'test run id 1', - userId: 'userId', - createdAt: new Date(), - updatedAt: new Date(), - }; + it('uses sourceBranch instead of the project main branch when given', async () => { + const { service, m } = await initFind(); + + await service.find(findRequest({ branchName: 'feature', sourceBranch: 'release' })); - const baselineDeleteMock = jest.fn(); - const imageDeleteMock = jest.fn(); - const service = await initModule({ - baselineDeleteMock, - imageDeleteMock, + expect(m.prisma.project.findUnique).not.toHaveBeenCalled(); + expect(service.findUnique).toHaveBeenNthCalledWith(1, expect.objectContaining({ branchName: 'release' })); + expect(service.findUnique).toHaveBeenNthCalledWith(2, expect.objectContaining({ branchName: 'feature' })); }); - await service.deleteBaseline(baseline); + it('takes the main branch from the caller instead of reading the project', async () => { + const { service, m } = await initFind(); - expect(imageDeleteMock).toHaveBeenCalledWith(baseline.baselineName); - expect(baselineDeleteMock).toHaveBeenCalledWith({ - where: { id: baseline.id }, + await service.find(findRequest({ branchName: 'feature' }), 'trunk'); + + expect(m.prisma.project.findUnique).not.toHaveBeenCalled(); + expect(service.findUnique).toHaveBeenNthCalledWith(1, expect.objectContaining({ branchName: 'trunk' })); + expect(service.findUnique).toHaveBeenNthCalledWith(2, expect.objectContaining({ branchName: 'feature' })); }); - }); - describe('characterization', () => { - const baselineData = { - name: 'Test name', - os: 'OS', - browser: 'browser', - viewport: 'viewport', - device: 'device', - customTags: 'tag', - }; + it('reads the project main branch when neither sourceBranch nor the caller gives one', async () => { + const { service, m } = await initFind(); + m.prisma.project.findUnique.mockResolvedValueOnce({ ...TEST_PROJECT, mainBranchName: 'trunk' }); + + await service.find(findRequest({ branchName: 'feature' })); + + expect(m.prisma.project.findUnique).toHaveBeenCalledWith({ where: { id: TEST_PROJECT.id } }); + expect(service.findUnique).toHaveBeenNthCalledWith(1, expect.objectContaining({ branchName: 'trunk' })); + }); + + it('keeps the current branch variation when it is as new as the main branch one', async () => { + const updatedAt = new Date('2024-01-01'); + const sameAgeMain = generateTestVariation({ id: 'main', updatedAt }); + const sameAgeCurrent = generateTestVariation({ id: 'current', updatedAt }); + const { service, m } = await initFind(sameAgeMain, sameAgeCurrent); + m.prisma.project.findUnique.mockResolvedValueOnce(TEST_PROJECT); + + await expect(service.find(findRequest())).resolves.toBe(sameAgeCurrent); + }); + }); - it('findUnique looks the variation up by its full composite key', async () => { + describe('findUnique', () => { + it('looks the variation up by its full composite key', async () => { const variation = generateTestVariation(); - const variationfindUniqueMock = jest.fn().mockResolvedValueOnce(variation); - service = await initModule({ variationfindUniqueMock }); + const { service, m } = await build(); + m.prisma.testVariation.findUnique.mockResolvedValueOnce(variation); const result = await service.findUnique({ ...baselineData, projectId: 'projectId', branchName: 'develop' }); expect(result).toBe(variation); - expect(variationfindUniqueMock).toHaveBeenCalledWith({ + expect(m.prisma.testVariation.findUnique).toHaveBeenCalledWith({ where: { projectId_name_browser_device_os_viewport_customTags_branchName: { projectId: 'projectId', @@ -658,74 +272,54 @@ describe('TestVariationsService', () => { }, }); }); + }); - describe('find', () => { - it('returns undefined when neither branch has the variation', async () => { - const projectFindUniqueMock = jest.fn().mockResolvedValueOnce(TEST_PROJECT); - service = await initModule({ projectFindUniqueMock }); - service.findUnique = jest.fn().mockResolvedValue(null); + describe('create', () => { + it('create stores the project, branch and screen', async () => { + const { service, m } = await build(); - // NOTE: current behavior — declared as `TestVariation | null` but resolves undefined - await expect( - service.find({ ...baselineData, projectId: TEST_PROJECT.id, branchName: 'develop' }) - ).resolves.toBeUndefined(); + await service.create({ + createTestRequestDto: { ...baselineData, projectId: 'projectId', branchName: 'develop' }, }); - it('only looks on the main branch when the build runs on it', async () => { - const variation = generateTestVariation({ branchName: TEST_PROJECT.mainBranchName }); - const projectFindUniqueMock = jest.fn().mockResolvedValueOnce(TEST_PROJECT); - service = await initModule({ projectFindUniqueMock }); - service.findUnique = jest.fn().mockResolvedValueOnce(variation); - - const result = await service.find({ - ...baselineData, - projectId: TEST_PROJECT.id, - branchName: TEST_PROJECT.mainBranchName, - }); - - expect(result).toBe(variation); - expect(service.findUnique).toHaveBeenCalledTimes(1); - expect(service.findUnique).toHaveBeenCalledWith({ - ...baselineData, - projectId: TEST_PROJECT.id, - branchName: TEST_PROJECT.mainBranchName, - }); + expect(m.prisma.testVariation.create.mock.calls[0][0].data).toStrictEqual({ + project: { connect: { id: 'projectId' } }, + branchName: 'develop', + ...baselineData, }); + }); - it('uses sourceBranch instead of the project main branch when given', async () => { - const projectFindUniqueMock = jest.fn().mockResolvedValueOnce(TEST_PROJECT); - service = await initModule({ projectFindUniqueMock }); - service.findUnique = jest.fn().mockResolvedValue(null); - - await service.find({ - ...baselineData, - projectId: TEST_PROJECT.id, - branchName: 'feature', - sourceBranch: 'release', - }); - - expect(projectFindUniqueMock).toHaveBeenCalledWith({ where: { id: TEST_PROJECT.id } }); - expect(service.findUnique).toHaveBeenNthCalledWith(1, expect.objectContaining({ branchName: 'release' })); - expect(service.findUnique).toHaveBeenNthCalledWith(2, expect.objectContaining({ branchName: 'feature' })); + it('create stores missing optional fields as empty strings', async () => { + const { service, m } = await build(); + + await service.create({ + createTestRequestDto: { name: 'only name', projectId: 'projectId', branchName: 'develop' }, }); - it('keeps the current branch variation when it is as new as the main branch one', async () => { - const updatedAt = new Date('2024-01-01'); - const main = generateTestVariation({ id: 'main', updatedAt }); - const current = generateTestVariation({ id: 'current', updatedAt }); - const projectFindUniqueMock = jest.fn().mockResolvedValueOnce(TEST_PROJECT); - service = await initModule({ projectFindUniqueMock }); - service.findUnique = jest.fn().mockResolvedValueOnce(main).mockResolvedValueOnce(current); - - await expect( - service.find({ ...baselineData, projectId: TEST_PROJECT.id, branchName: 'develop' }) - ).resolves.toBe(current); + expect(m.prisma.testVariation.create.mock.calls[0][0].data).toMatchObject({ + name: 'only name', + os: '', + device: '', + browser: '', + viewport: '', + customTags: '', }); }); + }); + + describe('update', () => { + it('update', async () => { + const data = { baselineName: 'baselineName', comment: 'random comment', ignoreAreas: '[]' }; + const { service, m } = await build(); + + await service.update('some id', data); + + expect(m.prisma.testVariation.update).toHaveBeenCalledWith({ where: { id: 'some id' }, data }); + }); it('update connects the test run when one is given', async () => { - const variationUpdateMock = jest.fn().mockResolvedValueOnce('updated'); - service = await initModule({ variationUpdateMock }); + const { service, m } = await build(); + m.prisma.testVariation.update.mockResolvedValueOnce('updated'); const result = await service.update( 'variationId', @@ -734,7 +328,7 @@ describe('TestVariationsService', () => { ); expect(result).toBe('updated'); - expect(variationUpdateMock).toHaveBeenCalledWith({ + expect(m.prisma.testVariation.update).toHaveBeenCalledWith({ where: { id: 'variationId' }, data: { baselineName: 'baseline.png', @@ -744,55 +338,15 @@ describe('TestVariationsService', () => { }, }); }); + }); - it('create leaves testRuns undefined without a test run id', async () => { - const variationCreateMock = jest.fn(); - service = await initModule({ variationCreateMock }); - - await service.create({ - createTestRequestDto: { - ...baselineData, - projectId: 'projectId', - branchName: 'develop', - } as CreateTestRequestDto, - }); - - expect(variationCreateMock.mock.calls[0][0].data).toStrictEqual({ - project: { connect: { id: 'projectId' } }, - testRuns: undefined, - branchName: 'develop', - ...baselineData, - }); - }); - - it('create stores missing optional fields as empty strings', async () => { - const variationCreateMock = jest.fn(); - service = await initModule({ variationCreateMock }); - - await service.create({ - createTestRequestDto: { - name: 'only name', - projectId: 'projectId', - branchName: 'develop', - } as CreateTestRequestDto, - }); - - expect(variationCreateMock.mock.calls[0][0].data).toMatchObject({ - name: 'only name', - os: '', - device: '', - browser: '', - viewport: '', - customTags: '', - }); - }); - + describe('addBaseline', () => { it.each([ ['a user', 'userId', { connect: { id: 'userId' } }], ['no user', undefined, undefined], - ])('addBaseline records a baseline for the test run with %s', async (_, userId, user) => { - const variationUpdateMock = jest.fn().mockResolvedValueOnce('updated'); - service = await initModule({ variationUpdateMock }); + ])('records a baseline for the test run with %s', async (_, userId, user) => { + const { service, m } = await build(); + m.prisma.testVariation.update.mockResolvedValueOnce('updated'); const result = await service.addBaseline({ id: 'variationId', @@ -802,7 +356,7 @@ describe('TestVariationsService', () => { }); expect(result).toBe('updated'); - expect(variationUpdateMock.mock.calls[0][0]).toStrictEqual({ + expect(m.prisma.testVariation.update.mock.calls[0][0]).toStrictEqual({ where: { id: 'variationId' }, data: { baselineName: 'baseline.png', @@ -816,39 +370,93 @@ describe('TestVariationsService', () => { }, }); }); + }); + + describe('merge', () => { + const project = TEST_PROJECT; + const fromBranch = 'develop'; + const targetBranch = 'test'; + const source = (overrides: object = {}) => + generateTestVariation({ id: '123', projectId: project.id, branchName: fromBranch, ...overrides }, []); + + it('merge', async () => { + const mergeBuild = { id: 'a9385fc1-884d-4f9f-915e-40da0e7773d5' }; + const testVariation = source(); + const testVariationSecond = source({ name: 'Test name second' }); + const testVariationNoBaseline = source({ baselineName: null }); + const testVariationTargetBranch = { ...source(), branchName: targetBranch }; + const imageBuffer = PNG.sync.write(new PNG({ width: 10, height: 10 })); + const { service, m } = await build(); + m.builds.findOrCreate.mockResolvedValueOnce(mergeBuild); + m.prisma.testVariation.findMany.mockResolvedValueOnce([ + testVariation, + testVariationSecond, + testVariationNoBaseline, + ]); + m.static.getImageBuffer + .mockReturnValueOnce(imageBuffer) + .mockReturnValueOnce(imageBuffer) + .mockReturnValueOnce(null); + m.prisma.project.findUnique.mockResolvedValue(project); + service.findUnique = jest.fn().mockResolvedValueOnce(null).mockResolvedValueOnce(null); + service.create = jest + .fn() + .mockResolvedValueOnce(testVariationTargetBranch) + .mockResolvedValueOnce(testVariationTargetBranch); + + await service.merge(project.id, fromBranch, targetBranch); + + expect(m.builds.findOrCreate).toHaveBeenCalledWith({ branchName: targetBranch, projectId: project.id }); + expect(service.findUnique).toHaveBeenNthCalledWith(1, keyOf(testVariation, targetBranch)); + expect(service.findUnique).toHaveBeenNthCalledWith(2, keyOf(testVariationSecond, targetBranch)); + [testVariation, testVariationSecond].forEach((variation, index) => + expect(m.testRuns.create).toHaveBeenNthCalledWith(index + 1, { + testVariation: testVariationTargetBranch, + createTestRequestDto: { + ...variation, + branchName: targetBranch, + buildId: mergeBuild.id, + diffTollerancePercent: 0, + merge: true, + ignoreAreas: JSON.parse(variation.ignoreAreas), + }, + imageBuffer, + }) + ); + expect(m.testRuns.create).toHaveBeenCalledTimes(2); + expect(m.prisma.project.findUnique).toHaveBeenCalledTimes(1); + expect(m.testRuns.calculateDiff).toHaveBeenCalledTimes(2); + expect(m.testRuns.calculateDiff).toHaveBeenCalledWith(project.id, undefined, imageBuffer, project); + expect(m.builds.update).toHaveBeenCalledWith(mergeBuild.id, { isRunning: false }); + }); it('merge reuses a variation that already lives on the target branch', async () => { - const source = generateTestVariation({ + const mergeSource = generateTestVariation({ branchName: 'develop', ignoreAreas: '[{"x":1,"y":2,"width":3,"height":4}]', }); const target = generateTestVariation({ id: 'target', branchName: 'master' }); - const build = { id: 'buildId' }; const imageBuffer = Buffer.from('png'); const testRun = { id: 'testRunId' }; - const buildFindOrCreateMock = jest.fn().mockResolvedValueOnce(build); - const variationFindManyMock = jest.fn().mockResolvedValueOnce([source]); - const getImageBufferMock = jest.fn().mockResolvedValueOnce(imageBuffer); - const testRunCreateMock = jest.fn().mockResolvedValueOnce(testRun); - const testRuncalCulateDiffMock = jest.fn(); - const buildUpdateMock = jest.fn().mockResolvedValueOnce('stopped build'); - service = await initModule({ - buildFindOrCreateMock, - variationFindManyMock, - getImageBufferMock, - testRunCreateMock, - testRuncalCulateDiffMock, - buildUpdateMock, - }); + const mergeProject = { ...TEST_PROJECT, id: 'projectId' }; + const { service, m } = await build(); + m.builds.findOrCreate.mockResolvedValueOnce({ id: 'buildId' }); + m.prisma.testVariation.findMany.mockResolvedValueOnce([mergeSource]); + m.static.getImageBuffer.mockResolvedValueOnce(imageBuffer); + m.testRuns.create.mockResolvedValueOnce(testRun); + m.builds.update.mockResolvedValueOnce('stopped build'); + m.prisma.project.findUnique.mockResolvedValue(mergeProject); service.findUnique = jest.fn().mockResolvedValueOnce(target); service.create = jest.fn(); const result = await service.merge('projectId', 'develop', 'master'); - expect(variationFindManyMock).toHaveBeenCalledWith({ where: { projectId: 'projectId', branchName: 'develop' } }); - expect(getImageBufferMock).toHaveBeenCalledWith(source.baselineName); + expect(m.prisma.testVariation.findMany).toHaveBeenCalledWith({ + where: { projectId: 'projectId', branchName: 'develop' }, + }); + expect(m.static.getImageBuffer).toHaveBeenCalledWith(mergeSource.baselineName); expect(service.create).not.toHaveBeenCalled(); - expect(testRunCreateMock).toHaveBeenCalledWith({ + expect(m.testRuns.create).toHaveBeenCalledWith({ testVariation: target, createTestRequestDto: expect.objectContaining({ branchName: 'master', @@ -859,46 +467,202 @@ describe('TestVariationsService', () => { }), imageBuffer, }); - expect(testRuncalCulateDiffMock).toHaveBeenCalledWith('projectId', testRun); - expect(buildUpdateMock).toHaveBeenCalledWith('buildId', { isRunning: false }); + expect(m.testRuns.calculateDiff).toHaveBeenCalledWith('projectId', testRun, imageBuffer, mergeProject); + expect(m.builds.update).toHaveBeenCalledWith('buildId', { isRunning: false }); expect(result).toBe('stopped build'); }); + it('merge into a deleted project answers 404 and stops its build', async () => { + const { service, m } = await build(); + m.builds.findOrCreate.mockResolvedValueOnce({ id: 'buildId' }); + m.builds.update.mockResolvedValueOnce('stopped build'); + m.prisma.project.findUnique.mockResolvedValue(null); + + await expect(service.merge('gone', 'develop', 'master')).rejects.toThrow( + new NotFoundException('Project not found: gone') + ); + + expect(m.builds.update).toHaveBeenCalledWith('buildId', { isRunning: false }); + expect(m.prisma.testVariation.findMany).not.toHaveBeenCalled(); + }); + it('merge stops its build when a step fails and passes the failure on', async () => { - const source = generateTestVariation({ branchName: 'develop' }); - const buildFindOrCreateMock = jest.fn().mockResolvedValueOnce({ id: 'buildId' }); - const variationFindManyMock = jest.fn().mockResolvedValueOnce([source]); - const getImageBufferMock = jest.fn().mockResolvedValueOnce(Buffer.from('png')); - const testRunCreateMock = jest.fn().mockRejectedValueOnce(new Error('storage down')); - const buildUpdateMock = jest.fn().mockResolvedValueOnce('stopped build'); - service = await initModule({ - buildFindOrCreateMock, - variationFindManyMock, - getImageBufferMock, - testRunCreateMock, - buildUpdateMock, - }); + const { service, m } = await build(); + m.builds.findOrCreate.mockResolvedValueOnce({ id: 'buildId' }); + m.prisma.testVariation.findMany.mockResolvedValueOnce([generateTestVariation({ branchName: 'develop' })]); + m.static.getImageBuffer.mockResolvedValueOnce(Buffer.from('png')); + m.testRuns.create.mockRejectedValueOnce(new Error('storage down')); + m.builds.update.mockResolvedValueOnce('stopped build'); + m.prisma.project.findUnique.mockResolvedValue({ ...TEST_PROJECT, id: 'projectId' }); service.findUnique = jest.fn().mockResolvedValueOnce(generateTestVariation({ branchName: 'master' })); await expect(service.merge('projectId', 'develop', 'master')).rejects.toThrow('storage down'); - expect(buildUpdateMock).toHaveBeenCalledWith('buildId', { isRunning: false }); + expect(m.builds.update).toHaveBeenCalledWith('buildId', { isRunning: false }); + }); + }); + + describe('delete', () => { + const twoBaselines = [{ baselineName: 'image 1' }, { baselineName: 'image 2' }]; + + it('removes the rows in one transaction before the files', async () => { + const order: string[] = []; + const { service, m } = await build(); + m.prisma.$transaction.mockImplementation(async (operations: Promise[]) => { + order.push('rows'); + return Promise.all(operations); + }); + m.static.deleteImage.mockImplementation(async () => { + order.push('file'); + }); + m.prisma.baseline.findMany.mockResolvedValueOnce(twoBaselines); + + await service.delete('variationId'); + + expect(m.prisma.baseline.findMany).toHaveBeenCalledWith({ + where: { testVariationId: 'variationId' }, + select: { baselineName: true }, + }); + expect(m.prisma.baseline.deleteMany).toHaveBeenCalledWith({ where: { testVariationId: 'variationId' } }); + expect(m.prisma.testVariation.delete).toHaveBeenCalledWith({ where: { id: 'variationId' } }); + expect(m.prisma.$transaction).toHaveBeenCalledTimes(1); + expect(m.static.deleteImage.mock.calls).toEqual([['image 1'], ['image 2']]); + expect(order).toEqual(['rows', 'file', 'file']); + }); + + it('keeps the files when the rows cannot be deleted', async () => { + const { service, m } = await build(); + m.prisma.$transaction.mockRejectedValueOnce(new Error('db down')); + m.prisma.baseline.findMany.mockResolvedValueOnce(twoBaselines); + + await expect(service.delete('variationId')).rejects.toThrow('db down'); + + expect(m.static.deleteImage).not.toHaveBeenCalled(); + }); + + it('returns the deleted variation', async () => { + const deleted = generateTestVariation({ id: 'variationId' }); + const { service, m } = await build(); + m.prisma.$transaction.mockResolvedValueOnce([{ count: 2 }, deleted]); + m.prisma.baseline.findMany.mockResolvedValueOnce(twoBaselines); + + expect(await service.delete('variationId')).toBe(deleted); + }); + + it('delete runs no raw SQL', async () => { + const { service, m } = await build(); + + await service.delete('id'); + + expect(m.prisma.$executeRaw).not.toHaveBeenCalled(); + }); + }); + + describe('createOrTakeWinner', () => { + const request = { + projectId: 'projectId', + branchName: 'develop', + name: 'Test name', + os: 'OS', + browser: 'browser', + viewport: 'viewport', + device: 'device', + customTags: '', + }; + + it('creates the variation without looking it up first', async () => { + const created = generateTestVariation({ id: 'created' }); + const { service, m } = await build(); + m.prisma.testVariation.create.mockResolvedValueOnce(created); + + expect(await service.createOrTakeWinner(request)).toBe(created); + expect(m.prisma.testVariation.create).toHaveBeenCalledTimes(1); + expect(m.prisma.testVariation.findUnique).not.toHaveBeenCalled(); + }); + + it('returns the variation a concurrent request created', async () => { + const winner = generateTestVariation({ id: 'winner' }); + const { service, m } = await build(); + m.prisma.testVariation.findUnique.mockResolvedValueOnce(winner); + m.prisma.testVariation.create.mockRejectedValueOnce(prismaError('P2002')); + + expect(await service.createOrTakeWinner(request)).toBe(winner); + expect(m.prisma.testVariation.findUnique).toHaveBeenCalledTimes(1); + }); + + it('rethrows a non-P2002 error', async () => { + const { service, m } = await build(); + m.prisma.testVariation.create.mockRejectedValueOnce(new Error('db down')); + + await expect(service.createOrTakeWinner(request)).rejects.toThrow('db down'); + expect(m.prisma.testVariation.findUnique).not.toHaveBeenCalled(); + }); + + it('rethrows P2002 when the winner cannot be found', async () => { + const error = prismaError('P2002'); + const { service, m } = await build(); + m.prisma.testVariation.findUnique.mockResolvedValue(null); + m.prisma.testVariation.create.mockRejectedValueOnce(error); + + await expect(service.createOrTakeWinner(request)).rejects.toBe(error); + }); + }); + + describe('transaction client', () => { + const txWith = (testVariation: Record) => + ({ testVariation }) as unknown as Prisma.TransactionClient; + const idKey = { + projectId: 'p', + branchName: 'b', + name: 'n', + os: '', + device: '', + browser: '', + viewport: '', + customTags: '', + }; + + it('update uses the transaction client it is given', async () => { + const { service, m } = await build(); + const txUpdate = jest.fn(); + + await service.update('id', { comment: 'c' }, 'runId', txWith({ update: txUpdate })); + + expect(txUpdate).toHaveBeenCalledWith(expect.objectContaining({ where: { id: 'id' } })); + expect(m.prisma.testVariation.update).not.toHaveBeenCalled(); }); - it('delete detaches the test runs with a raw update before deleting the variation', async () => { - const $executeRawMock = jest.fn(); - const variationDeleteMock = jest.fn().mockResolvedValueOnce('deleted'); - service = await initModule({ $executeRawMock, variationDeleteMock }); - service.getDetails = jest.fn().mockResolvedValueOnce(generateTestVariation({ id: 'variationId' }, [])); + it('addBaseline uses the transaction client it is given', async () => { + const { service, m } = await build(); + const txUpdate = jest.fn(); - const result = await service.delete('variationId'); + await service.addBaseline({ id: 'id', testRunId: 'runId', baselineName: 'b.png' }, txWith({ update: txUpdate })); - const [strings, ...values] = $executeRawMock.mock.calls[0]; - expect(strings.join('?')).toBe( - 'UPDATE "public"."TestRun" SET "testVariationId" = NULL::text WHERE "testVariationId" = ?' + expect(txUpdate).toHaveBeenCalledWith(expect.objectContaining({ where: { id: 'id' } })); + expect(m.prisma.testVariation.update).not.toHaveBeenCalled(); + }); + + it('findUnique uses the transaction client it is given', async () => { + const { service, m } = await build(); + const txFindUnique = jest.fn(); + + await service.findUnique(idKey, txWith({ findUnique: txFindUnique })); + + expect(txFindUnique).toHaveBeenCalledTimes(1); + expect(m.prisma.testVariation.findUnique).not.toHaveBeenCalled(); + }); + + it('create uses the transaction client it is given', async () => { + const { service, m } = await build(); + const txCreate = jest.fn(); + + await service.create( + { createTestRequestDto: { projectId: 'p', branchName: 'b', name: 'n' } }, + txWith({ create: txCreate }) ); - expect(values).toEqual(['variationId']); - expect(result).toBe('deleted'); + + expect(txCreate).toHaveBeenCalledTimes(1); + expect(m.prisma.testVariation.create).not.toHaveBeenCalled(); }); }); }); diff --git a/src/test-variations/test-variations.service.ts b/src/test-variations/test-variations.service.ts index 69224317..5152dd06 100644 --- a/src/test-variations/test-variations.service.ts +++ b/src/test-variations/test-variations.service.ts @@ -1,16 +1,19 @@ import { Injectable, Inject, forwardRef, Logger, NotFoundException } from '@nestjs/common'; import { PrismaService } from '../prisma/prisma.service'; -import { TestVariation, Baseline, Build, TestRun, User } from '@prisma/client'; +import { isPrismaError } from '../prisma/prisma-errors'; +import { TestVariation, Baseline, Build, Project, TestRun, User, Prisma } from '@prisma/client'; import { StaticService } from '../static/static.service'; import { BuildsService } from '../builds/builds.service'; import { TestRunsService } from '../test-runs/test-runs.service'; import { CreateTestRequestDto } from '../test-runs/dto/create-test-request.dto'; import { BuildDto } from '../builds/dto/build.dto'; -import { getTestVariationUniqueData } from '../utils'; +import { getTestVariationUniqueData, parseIgnoreAreas, variationKey } from '../utils'; import { TestVariationUpdateDto } from './dto/test-variation-update.dto'; import { BaselineDataDto } from '../shared/dto/baseline-data.dto'; import type { WrapperType } from '../shared/wrapper-type'; +export type BaselineRef = Pick; + @Injectable() export class TestVariationsService { private readonly logger = new Logger(TestVariationsService.name); @@ -25,10 +28,9 @@ export class TestVariationsService { ) {} async getDetails(id: string): Promise< - | (TestVariation & { - baselines: (Baseline & { testRun: TestRun | null; user: Omit | null })[]; - }) - | null + TestVariation & { + baselines: (Baseline & { testRun: TestRun | null; user: Omit | null })[]; + } > { const testVariation = await this.prismaService.testVariation.findUnique({ where: { id }, @@ -36,8 +38,6 @@ export class TestVariationsService { baselines: { include: { testRun: true, - // who approved the baseline, for the history list — never their - // credentials: any signed-in user, guests included, can read this user: { omit: { password: true, apiKey: true } }, }, orderBy: { @@ -52,10 +52,37 @@ export class TestVariationsService { return testVariation; } + async findBaselineHistory(testVariationId: string): Promise { + return this.prismaService.baseline.findMany({ + where: { testVariationId }, + orderBy: { createdAt: 'desc' }, + select: { id: true, baselineName: true }, + }); + } + + async findLatestBaselines(testVariationIds: string[]): Promise { + if (testVariationIds.length === 0) { + return []; + } + const baselines = await this.prismaService.baseline.findMany({ + where: { testVariationId: { in: testVariationIds } }, + orderBy: { createdAt: 'desc' }, + select: { id: true, baselineName: true, testVariationId: true }, + }); + const latest = new Map(); + for (const { testVariationId, ...baseline } of baselines) { + if (!latest.has(testVariationId)) { + latest.set(testVariationId, baseline); + } + } + return testVariationIds.flatMap((id) => latest.get(id) ?? []); + } + async findUnique( - data: ReturnType & { projectId: string; branchName: string } + data: ReturnType & { projectId: string; branchName: string }, + tx: Prisma.TransactionClient = this.prismaService ): Promise { - return this.prismaService.testVariation.findUnique({ + return tx.testVariation.findUnique({ where: { projectId_name_browser_device_os_viewport_customTags_branchName: { projectId: data.projectId, @@ -71,8 +98,13 @@ export class TestVariationsService { }); } - async update(id: string, data: TestVariationUpdateDto, testRunId?: string): Promise { - return this.prismaService.testVariation.update({ + async update( + id: string, + data: TestVariationUpdateDto, + testRunId?: string, + tx: Prisma.TransactionClient = this.prismaService + ): Promise { + return tx.testVariation.update({ where: { id }, data: { baselineName: data.baselineName, @@ -83,79 +115,79 @@ export class TestVariationsService { }); } - /** - * Tries to get test variation for the same branch - * Falls back to main branch if not found - * @param projectId - * @param baselineData - * @returns - */ async find( - createTestRequestDto: BaselineDataDto & { projectId: string; sourceBranch?: string } - ): Promise { - const project = (await this.prismaService.project.findUnique({ where: { id: createTestRequestDto.projectId } }))!; - const mainBranch = createTestRequestDto.sourceBranch ?? project.mainBranchName; + createTestRequestDto: BaselineDataDto & { projectId: string; sourceBranch?: string }, + mainBranchName?: string + ): Promise { + const mainBranch = + createTestRequestDto.sourceBranch ?? + mainBranchName ?? + (await this.prismaService.project.findUnique({ where: { id: createTestRequestDto.projectId } }))!.mainBranchName; const [mainBranchTestVariation, currentBranchTestVariation] = await Promise.all([ - // search main branch variation - this.findUnique({ - projectId: createTestRequestDto.projectId, - branchName: mainBranch, - ...getTestVariationUniqueData(createTestRequestDto), - }), - // search current branch variation + this.findUnique(variationKey(createTestRequestDto.projectId, mainBranch, createTestRequestDto)), createTestRequestDto.branchName !== mainBranch && - this.findUnique({ - projectId: createTestRequestDto.projectId, - branchName: createTestRequestDto.branchName, - ...getTestVariationUniqueData(createTestRequestDto), - }), + this.findUnique( + variationKey(createTestRequestDto.projectId, createTestRequestDto.branchName, createTestRequestDto) + ), ]); - if (!!currentBranchTestVariation) { + if (currentBranchTestVariation) { if (mainBranchTestVariation && mainBranchTestVariation.updatedAt > currentBranchTestVariation.updatedAt) { return mainBranchTestVariation; } return currentBranchTestVariation; } - if (!!mainBranchTestVariation) { - return mainBranchTestVariation; - } + return mainBranchTestVariation ?? undefined; } - /** - * Creates empty test variation (no baseline) - */ - create({ - testRunId, - createTestRequestDto, - }: { - testRunId?: string; - createTestRequestDto: Omit; - }): Promise { - return this.prismaService.testVariation.create({ + create( + { createTestRequestDto }: { createTestRequestDto: Omit }, + tx: Prisma.TransactionClient = this.prismaService + ): Promise { + return tx.testVariation.create({ data: { project: { connect: { id: createTestRequestDto.projectId } }, - testRuns: testRunId ? { connect: { id: testRunId } } : undefined, branchName: createTestRequestDto.branchName, ...getTestVariationUniqueData(createTestRequestDto), }, }); } - async addBaseline({ - id, - userId, - testRunId, - baselineName, - }: { - id: string; - userId?: string; - testRunId: string; - baselineName: string; - }): Promise { - return this.prismaService.testVariation.update({ + async createOrTakeWinner( + createTestRequestDto: Omit + ): Promise { + try { + return await this.create({ createTestRequestDto }); + } catch (e) { + if (isPrismaError(e, 'P2002')) { + const winner = await this.findUnique( + variationKey(createTestRequestDto.projectId, createTestRequestDto.branchName, createTestRequestDto) + ); + if (winner) { + return winner; + } + } + throw e; + } + } + + async addBaseline( + { + id, + userId, + testRunId, + baselineName, + }: { + id: string; + userId?: string; + testRunId: string; + baselineName: string; + }, + tx: Prisma.TransactionClient = this.prismaService + ): Promise { + return tx.testVariation.update({ where: { id }, data: { baselineName, @@ -181,95 +213,72 @@ export class TestVariationsService { } async merge(projectId: string, fromBranch: string, toBranch: string): Promise { - // create build const build: Build = await this.buildsService.findOrCreate({ branchName: toBranch, projectId, }); try { - await this.mergeInto(build, projectId, fromBranch, toBranch); + const project = await this.prismaService.project.findUnique({ where: { id: projectId } }); + if (!project) { + throw new NotFoundException(`Project not found: ${projectId}`); + } + await this.mergeInto(build, project, fromBranch, toBranch); } catch (e) { - // the build is stopped either way, so a failed merge does not stay - // "running" in the list forever; the cause still reaches the client await this.buildsService.update(build.id, { isRunning: false }).catch((): void => undefined); throw e; } - // stop build return this.buildsService.update(build.id, { isRunning: false }); } - private async mergeInto(build: Build, projectId: string, fromBranch: string, toBranch: string): Promise { - // find source branch variations + private async mergeInto(build: Build, project: Project, fromBranch: string, toBranch: string): Promise { const testVariations: TestVariation[] = await this.prismaService.testVariation.findMany({ - where: { projectId, branchName: fromBranch }, + where: { projectId: project.id, branchName: fromBranch }, }); - // compare source to destination branch variations for (const sourceBranchTestVariation of testVariations) { const baselineBuffer = await this.staticService.getImageBuffer(sourceBranchTestVariation.baselineName); if (baselineBuffer) { - // get destination branch request const createTestRequestDto: CreateTestRequestDto = { ...sourceBranchTestVariation, branchName: toBranch, buildId: build.id, diffTollerancePercent: 0, merge: true, - ignoreAreas: JSON.parse(sourceBranchTestVariation.ignoreAreas), + ignoreAreas: parseIgnoreAreas(sourceBranchTestVariation.ignoreAreas), }; - // get destination branch variation — on that branch only: find() would - // hand back the main branch's when it is the newer one, and the merge - // would then try to create a second variation on toBranch - let destintionBranchTestVariation = await this.findUnique({ - projectId, - branchName: toBranch, - ...getTestVariationUniqueData(sourceBranchTestVariation), - }); - - if (!destintionBranchTestVariation) { - destintionBranchTestVariation = await this.create({ createTestRequestDto }); - } + // not find(): it may return main's variation and the merge would create a second one on toBranch + const destinationBranchTestVariation = + (await this.findUnique(variationKey(project.id, toBranch, createTestRequestDto))) ?? + (await this.createOrTakeWinner(createTestRequestDto)); const testRun = await this.testRunsService.create({ - testVariation: destintionBranchTestVariation, + testVariation: destinationBranchTestVariation, createTestRequestDto, imageBuffer: baselineBuffer, }); - await this.testRunsService.calculateDiff(projectId, testRun); + await this.testRunsService.calculateDiff(project.id, testRun, baselineBuffer, project); } } } async delete(id: string): Promise { this.logger.debug(`Going to remove TestVariation ${id}`); - const testVariation = (await this.getDetails(id))!; - - // delete Baselines - for (const baseline of testVariation.baselines) { - await this.deleteBaseline(baseline); - } - - // disconnect TestRuns - // workaround due to https://github.com/prisma/prisma/issues/2810 - await this.prismaService - .$executeRaw`UPDATE "public"."TestRun" SET "testVariationId" = NULL::text WHERE "testVariationId" = ${id}`; - - // delete TestVariation - return this.prismaService.testVariation.delete({ - where: { id }, - }); - } - - async deleteBaseline(baseline: Baseline): Promise { - this.logger.debug(`Going to remove Baseline ${baseline.id}`); - - this.staticService.deleteImage(baseline.baselineName); - return this.prismaService.baseline.delete({ - where: { id: baseline.id }, + const baselines = await this.prismaService.baseline.findMany({ + where: { testVariationId: id }, + select: { baselineName: true }, }); + const [, testVariation] = await this.prismaService.$transaction([ + this.prismaService.baseline.deleteMany({ where: { testVariationId: id } }), + this.prismaService.testVariation.delete({ where: { id } }), + ]); + await this.staticService.deleteImages( + baselines.map(({ baselineName }) => baselineName), + 8 + ); + return testVariation; } } diff --git a/src/users/db/dbusers.service.spec.ts b/src/users/db/dbusers.service.spec.ts index cab1d56c..75a3341c 100644 --- a/src/users/db/dbusers.service.spec.ts +++ b/src/users/db/dbusers.service.spec.ts @@ -1,4 +1,4 @@ -import { HttpException, HttpStatus } from '@nestjs/common'; +import { HttpException, HttpStatus, NotFoundException } from '@nestjs/common'; import { Role, User } from '@prisma/client'; import { DbUsersService } from './dbusers.service'; import { PrismaService } from '../../prisma/prisma.service'; @@ -61,18 +61,41 @@ describe('DbUsersService', () => { }); describe('update', () => { - it('updates the profile fields as given and returns a fresh token', async () => { + const current = { ...user, email: 'Test@Example.com' }; + + it('updates the profile fields and returns a fresh token', async () => { + prisma.user.findUnique.mockResolvedValue(current); + prisma.user.update.mockResolvedValue(user); + + const result = await service.update('1', { email: 'test@example.com', firstName: 'J', lastName: 'D' }); + + expect(prisma.user.findUnique).toHaveBeenCalledWith({ where: { id: '1' } }); + expect(auth.signToken).toHaveBeenCalledWith(user); + expect(result).toEqual(new UserLoginResponseDto(user, 'token')); + }); + + it.each([ + ['stores a changed email in lower case', ' New@Example.com ', 'new@example.com'], + ['keeps the stored spelling when only the case differs', 'test@EXAMPLE.com', 'Test@Example.com'], + ])('%s', async (_name, sent, stored) => { + prisma.user.findUnique.mockResolvedValue(current); prisma.user.update.mockResolvedValue(user); - const result = await service.update('1', { email: ' Mixed@Case.com', firstName: 'J', lastName: 'D' }); + await service.update('1', { email: sent, firstName: 'J', lastName: 'D' }); - // NOTE: current behavior — unlike create, the email is stored as given (not trimmed or lower-cased) expect(prisma.user.update).toHaveBeenCalledWith({ where: { id: '1' }, - data: { email: ' Mixed@Case.com', firstName: 'J', lastName: 'D' }, + data: { email: stored, firstName: 'J', lastName: 'D' }, }); - expect(auth.signToken).toHaveBeenCalledWith(user); - expect(result).toEqual(new UserLoginResponseDto(user, 'token')); + }); + + it('answers 404 when the user no longer exists', async () => { + prisma.user.findUnique.mockResolvedValue(null); + + await expect(service.update('1', { email: 'a@b.com', firstName: 'J', lastName: 'D' })).rejects.toBeInstanceOf( + NotFoundException + ); + expect(prisma.user.update).not.toHaveBeenCalled(); }); }); @@ -88,6 +111,8 @@ describe('DbUsersService', () => { }); describe('login', () => { + const invalid = new HttpException('Invalid email or password.', HttpStatus.BAD_REQUEST); + it('returns the user with a token when the password matches', async () => { prisma.user.findUnique.mockResolvedValue(user); auth.compare.mockResolvedValue(true); @@ -102,9 +127,7 @@ describe('DbUsersService', () => { it('rejects an unknown email with the same message as a wrong password', async () => { prisma.user.findUnique.mockResolvedValue(null); - await expect(service.login({ email: 'nobody@example.com', password: 'secret' })).rejects.toThrow( - new HttpException('Invalid email or password.', HttpStatus.BAD_REQUEST) - ); + await expect(service.login({ email: 'nobody@example.com', password: 'secret' })).rejects.toThrow(invalid); expect(auth.compare).not.toHaveBeenCalled(); }); @@ -112,20 +135,37 @@ describe('DbUsersService', () => { prisma.user.findUnique.mockResolvedValue(user); auth.compare.mockResolvedValue(false); - await expect(service.login({ email: 'test@example.com', password: 'wrong' })).rejects.toThrow( - new HttpException('Invalid email or password.', HttpStatus.BAD_REQUEST) - ); + await expect(service.login({ email: 'test@example.com', password: 'wrong' })).rejects.toThrow(invalid); expect(auth.signToken).not.toHaveBeenCalled(); }); - it('looks the email up exactly as typed', async () => { + it('signs in with a differently cased email', async () => { + prisma.user.findUnique.mockResolvedValueOnce(null).mockResolvedValueOnce(user); + auth.compare.mockResolvedValue(true); + + const result = await service.login({ email: ' John@Example.com ', password: 'secret' }); + + expect(prisma.user.findUnique).toHaveBeenNthCalledWith(1, { where: { email: 'John@Example.com' } }); + expect(prisma.user.findUnique).toHaveBeenNthCalledWith(2, { where: { email: 'john@example.com' } }); + expect(result).toEqual(new UserLoginResponseDto(user, 'token')); + }); + + it('keeps signing in a mixed-case row with its exact spelling', async () => { + prisma.user.findUnique.mockResolvedValueOnce({ ...user, email: 'John@Example.com' }); + auth.compare.mockResolvedValue(true); + + await service.login({ email: 'John@Example.com', password: 'secret' }); + + expect(prisma.user.findUnique).toHaveBeenCalledTimes(1); + expect(prisma.user.findUnique).toHaveBeenCalledWith({ where: { email: 'John@Example.com' } }); + }); + + it('looks a lower-case email up once', async () => { prisma.user.findUnique.mockResolvedValue(null); - await expect(service.login({ email: ' Test@Example.com', password: 'secret' })).rejects.toThrow(HttpException); + await expect(service.login({ email: 'nobody@example.com', password: 'secret' })).rejects.toThrow(HttpException); - // NOTE: current behavior — create() lower-cases and trims, login() does not, - // so a user registered as "Test@Example.com" must log in as "test@example.com" - expect(prisma.user.findUnique).toHaveBeenCalledWith({ where: { email: ' Test@Example.com' } }); + expect(prisma.user.findUnique).toHaveBeenCalledTimes(1); }); }); }); diff --git a/src/users/db/dbusers.service.ts b/src/users/db/dbusers.service.ts index da8d4954..e2630dee 100644 --- a/src/users/db/dbusers.service.ts +++ b/src/users/db/dbusers.service.ts @@ -1,4 +1,4 @@ -import { HttpException, HttpStatus, Logger } from '@nestjs/common'; +import { HttpException, HttpStatus, NotFoundException } from '@nestjs/common'; import { CreateUserDto } from '../dto/user-create.dto'; import { UserLoginResponseDto } from '../dto/user-login-response.dto'; import { PrismaService } from '../../prisma/prisma.service'; @@ -9,7 +9,7 @@ import { UserLoginRequestDto } from '../dto/user-login-request.dto'; import { Users } from '../users.interface'; export class DbUsersService implements Users { - private readonly logger: Logger = new Logger(DbUsersService.name); + readonly allowsRegistration = true; constructor( private readonly prismaService: PrismaService, @@ -33,10 +33,17 @@ export class DbUsersService implements Users { } async update(id: string, userDto: UpdateUserDto): Promise { + const current = await this.prismaService.user.findUnique({ where: { id } }); + if (!current) { + throw new NotFoundException(`User not found: ${id}`); + } + const requested = userDto.email.trim(); + // a case-only difference keeps the stored spelling: lower-casing it could collide with another row + const email = requested.toLowerCase() === current.email.toLowerCase() ? current.email : requested.toLowerCase(); const user = await this.prismaService.user.update({ where: { id }, data: { - email: userDto.email, + email, firstName: userDto.firstName, lastName: userDto.lastName, }, @@ -56,9 +63,13 @@ export class DbUsersService implements Users { } async login(userLoginRequestDto: UserLoginRequestDto) { - const user = await this.prismaService.user.findUnique({ - where: { email: userLoginRequestDto.email }, - }); + const email = userLoginRequestDto.email.trim(); + // rows saved before emails were lower-cased keep their spelling, so the exact match goes first + const user = + (await this.prismaService.user.findUnique({ where: { email } })) ?? + (email === email.toLowerCase() + ? null + : await this.prismaService.user.findUnique({ where: { email: email.toLowerCase() } })); if (!user) { throw new HttpException('Invalid email or password.', HttpStatus.BAD_REQUEST); } diff --git a/src/users/dto/change-password.dto.spec.ts b/src/users/dto/change-password.dto.spec.ts new file mode 100644 index 00000000..99c6be2d --- /dev/null +++ b/src/users/dto/change-password.dto.spec.ts @@ -0,0 +1,20 @@ +import { plainToInstance } from 'class-transformer'; +import { validate } from 'class-validator'; +import { ChangePasswordDto } from './change-password.dto'; + +const failedProperties = async (body: object): Promise => + (await validate(plainToInstance(ChangePasswordDto, body))).map((error) => error.property); + +describe('ChangePasswordDto', () => { + it.each([ + ['a missing password', {}], + ['an empty password', { password: '' }], + ['a password that is not a string', { password: 1 }], + ])('rejects %s', async (_name, body) => { + expect(await failedProperties(body)).toEqual(['password']); + }); + + it('accepts a non-empty password', async () => { + expect(await failedProperties({ password: 'x' })).toEqual([]); + }); +}); diff --git a/src/users/dto/change-password.dto.ts b/src/users/dto/change-password.dto.ts new file mode 100644 index 00000000..8ccb3a60 --- /dev/null +++ b/src/users/dto/change-password.dto.ts @@ -0,0 +1,9 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { IsNotEmpty, IsString } from 'class-validator'; + +export class ChangePasswordDto { + @ApiProperty() + @IsString() + @IsNotEmpty() + readonly password!: string; +} diff --git a/src/users/dto/user.dto.ts b/src/users/dto/user.dto.ts index 5f97aa21..6f9af9e7 100644 --- a/src/users/dto/user.dto.ts +++ b/src/users/dto/user.dto.ts @@ -1,4 +1,4 @@ -import { ApiProperty } from '@nestjs/swagger'; +import { ApiProperty, OmitType } from '@nestjs/swagger'; import { Role, User } from '@prisma/client'; export class UserDto { @@ -29,3 +29,13 @@ export class UserDto { this.role = user.role; } } + +export class UserSummaryDto extends OmitType(UserDto, ['apiKey'] as const) {} + +export const toUserSummary = ({ id, email, firstName, lastName, role }: User): UserSummaryDto => ({ + id, + email, + firstName, + lastName, + role, +}); diff --git a/src/users/ldap/ldapusers.concurrency.spec.ts b/src/users/ldap/ldapusers.concurrency.spec.ts new file mode 100644 index 00000000..c7091042 --- /dev/null +++ b/src/users/ldap/ldapusers.concurrency.spec.ts @@ -0,0 +1,100 @@ +import { ConfigService } from '@nestjs/config'; +import { Role, User } from '@prisma/client'; +import { Client as LdapClient } from 'ldapts'; +import { AuthService } from '../../auth/auth.service'; +import { PrismaService } from '../../prisma/prisma.service'; +import { LdapUsersService } from './ldapusers.service'; + +const mockClients: { generation: number; unbind: jest.Mock }[] = []; + +jest.mock('ldapts', () => ({ + Client: jest.fn(() => { + const client = { + generation: 0, + bind: jest.fn(async () => { + const started = client.generation; + await new Promise(setImmediate); + if (client.generation !== started) { + throw new Error('socket closed'); + } + }), + search: jest.fn(async () => { + const started = client.generation; + await new Promise(setImmediate); + if (client.generation !== started) { + throw new Error('socket closed'); + } + return { + searchEntries: [ + { + dn: 'uid=jane,ou=people', + LDAP_ATTRIBUTE_MAIL: 'jane@example.com', + LDAP_ATTRIBUTE_FIRST_NAME: 'Jane', + LDAP_ATTRIBUTE_LAST_NAME: 'Roe', + }, + ], + searchReferences: [], + }; + }), + unbind: jest.fn(async () => { + client.generation++; + }), + }; + mockClients.push(client); + return client; + }), +})); + +const user: User = { + id: '1', + email: 'jane@example.com', + firstName: 'Jane', + lastName: 'Roe', + apiKey: 'key', + password: 'hash', + isActive: true, + role: Role.editor, + updatedAt: new Date(), + createdAt: new Date(), +}; + +describe('LdapUsersService concurrent logins', () => { + let service: LdapUsersService; + + beforeEach(() => { + mockClients.length = 0; + (LdapClient as unknown as jest.Mock).mockClear(); + const configService = { + get: jest.fn((_key: string, defaultValue?: string) => defaultValue), + getOrThrow: jest.fn((key: string) => (key === 'LDAP_USERS_SEARCH_FILTER' ? '(mail={{email}})' : key)), + }; + const prisma = { user: { findUnique: jest.fn().mockResolvedValue(user) } }; + const auth = { signToken: jest.fn().mockReturnValue('token') }; + service = new LdapUsersService( + configService as unknown as ConfigService, + prisma as unknown as PrismaService, + auth as unknown as AuthService + ); + }); + + it("serves two concurrent logins without one closing the other's connection", async () => { + await expect( + Promise.all([ + service.login({ email: 'jane@example.com', password: 'a' }), + service.login({ email: 'jane@example.com', password: 'b' }), + ]) + ).resolves.toHaveLength(2); + }); + + it('opens a connection for the search and one for the user bind, and closes each', async () => { + await Promise.all([ + service.login({ email: 'jane@example.com', password: 'a' }), + service.login({ email: 'jane@example.com', password: 'b' }), + ]); + + expect(LdapClient).toHaveBeenCalledTimes(4); + for (const client of mockClients) { + expect(client.unbind).toHaveBeenCalledTimes(1); + } + }); +}); diff --git a/src/users/ldap/ldapusers.service.spec.ts b/src/users/ldap/ldapusers.service.spec.ts index 7f0b44d6..b1f162ff 100644 --- a/src/users/ldap/ldapusers.service.spec.ts +++ b/src/users/ldap/ldapusers.service.spec.ts @@ -1,41 +1,15 @@ -import { Test, TestingModule } from '@nestjs/testing'; import { LdapUsersService } from './ldapusers.service'; import { AuthService } from '../../auth/auth.service'; import { PrismaService } from '../../prisma/prisma.service'; import { ConfigService } from '@nestjs/config'; import { Role, User } from '@prisma/client'; -import { HttpException, HttpStatus } from '@nestjs/common'; +import { HttpException, HttpStatus, NotFoundException, ServiceUnavailableException } from '@nestjs/common'; import { UserLoginResponseDto } from '../dto/user-login-response.dto'; -// eslint-disable-next-line @typescript-eslint/no-unused-vars -import { Entry as LdapEntry, Client as LdapClient, SearchOptions, SearchResult } from 'ldapts'; +import { Client as LdapClient } from 'ldapts'; jest.mock('ldapts', () => { - const mockLdapClient = { - // eslint-disable-next-line @typescript-eslint/no-unused-vars - search: jest.fn((_searchDN, searchOptions: SearchOptions): Promise => { - if (searchOptions.filter!.toString().includes('exists@example.com')) { - return Promise.resolve({ - searchEntries: [ - { - dn: 'dn', - LDAP_ATTRIBUTE_MAIL: 'exists@example.com', - LDAP_ATTRIBUTE_FIRST_NAME: 'first', - LDAP_ATTRIBUTE_LAST_NAME: 'last', - }, - ], - searchReferences: [], - }); - } else { - return Promise.resolve({ searchEntries: [], searchReferences: [] }); - } - }), - bind: jest.fn(), - unbind: jest.fn(), - }; - - return { - Client: jest.fn(() => mockLdapClient), - }; + const mockLdapClient = { search: jest.fn(), bind: jest.fn(), unbind: jest.fn().mockResolvedValue(undefined) }; + return { Client: jest.fn(() => mockLdapClient) }; }); const user: User = { @@ -51,133 +25,7 @@ const user: User = { createdAt: new Date(), }; -describe('LdapUsersService match from ldap', () => { - let service: LdapUsersService; - let prismaService: PrismaService; - let configService: ConfigService; - let authService: AuthService; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [ - { - provide: ConfigService, - useValue: { - get: jest.fn().mockReturnValue('true'), - getOrThrow: jest.fn((string) => (string === 'LDAP_USERS_SEARCH_FILTER' ? '(&(mail={{email}}))' : string)), - }, - }, - { - provide: PrismaService, - useValue: { - user: { - findMany: jest.fn().mockResolvedValueOnce([]), - findUnique: jest.fn(), - create: jest.fn(), - update: jest.fn(), - }, - }, - }, - { - provide: AuthService, - useValue: { - generateApiKey: jest.fn(() => 'generatedApiKey'), - encryptPassword: jest.fn(() => 'encryptedPassword'), - signToken: jest.fn(() => 'token'), - }, - }, - ], - }).compile(); - configService = module.get(ConfigService); - prismaService = module.get(PrismaService); - authService = module.get(AuthService); - service = new LdapUsersService(configService, prismaService, authService); - }); - - it('should be defined', () => { - expect(service).toBeDefined(); - }); - - it('should create new user on login', async () => { - jest.spyOn(prismaService.user, 'findUnique').mockResolvedValue(null); - const prismaUserCreateMock = jest.spyOn(prismaService.user, 'create').mockResolvedValue(user); - await service.login({ email: 'exists@example.com', password: 'password' }); - expect(prismaUserCreateMock).toHaveBeenCalled(); - }); - - it('should login with user already in db', async () => { - jest.spyOn(prismaService.user, 'findUnique').mockResolvedValue(user); - const prismaUserCreateMock = jest.spyOn(prismaService.user, 'create'); - await service.login({ email: 'exists@example.com', password: 'password' }); - expect(prismaUserCreateMock).not.toHaveBeenCalled(); - }); -}); - -describe('LdapUsersService with no results from ldap', () => { - let service: LdapUsersService; - let prismaService: PrismaService; - let configService: ConfigService; - let authService: AuthService; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [ - { - provide: ConfigService, - useValue: { - get: jest.fn().mockReturnValue('true'), - getOrThrow: jest.fn().mockReturnValue('string'), - }, - }, - { - provide: PrismaService, - useValue: { - user: { - findMany: jest.fn().mockResolvedValueOnce([]), - create: jest.fn(), - update: jest.fn(), - }, - }, - }, - { - provide: AuthService, - useValue: { - generateApiKey: jest.fn(() => 'generatedApiKey'), - encryptPassword: jest.fn(() => 'encryptedPassword'), - }, - }, - ], - }).compile(); - configService = module.get(ConfigService); - prismaService = module.get(PrismaService); - authService = module.get(AuthService); - service = new LdapUsersService(configService, prismaService, authService); - }); - - it('should be defined', () => { - expect(service).toBeDefined(); - }); - - it('should never change a password', async () => { - const prismaUserCreateMock = jest.spyOn(prismaService.user, 'create'); - const prismaUserUpdateMock = jest.spyOn(prismaService.user, 'update'); - const result = await service.changePassword(user, 'newPassword'); - expect(prismaUserCreateMock).not.toHaveBeenCalled(); - expect(prismaUserUpdateMock).not.toHaveBeenCalled(); - expect(result).toBe(true); - }); - - it('should not login when ldap search results are empty', async () => { - expect.assertions(1); - try { - await service.login({ email: 'testÜ*()\\\0@example.com', password: 'password' }); - } catch (e) { - expect((e as Error).message).toBe('Invalid email or password.'); - } - }); -}); - -describe('LdapUsersService characterization', () => { +describe('LdapUsersService', () => { let service: LdapUsersService; let ldapClient: { search: jest.Mock; bind: jest.Mock; unbind: jest.Mock }; let prisma: { user: { findUnique: jest.Mock; create: jest.Mock; update: jest.Mock } }; @@ -201,9 +49,11 @@ describe('LdapUsersService characterization', () => { lastAttr: 'Roe', }; - const createService = (tlsNoVerify = 'false') => { + const createService = (tlsNoVerify = 'false', overrides: Record = {}) => { const configService = { - get: jest.fn((key: string, defaultValue: string) => (key === 'LDAP_TLS_NO_VERIFY' ? tlsNoVerify : defaultValue)), + get: jest.fn((key: string, defaultValue: string) => + key === 'LDAP_TLS_NO_VERIFY' ? tlsNoVerify : (overrides[key] ?? defaultValue) + ), getOrThrow: jest.fn((key: string) => config[key]), }; return new LdapUsersService( @@ -213,6 +63,13 @@ describe('LdapUsersService characterization', () => { ); }; + const found = () => { + ldapClient.search.mockResolvedValueOnce({ searchEntries: [entry], searchReferences: [] }); + prisma.user.findUnique.mockResolvedValueOnce(user); + }; + const login = (svc = service, password = 'pw') => svc.login({ email: 'jane', password }); + const invalidLogin = new HttpException('Invalid email or password.', HttpStatus.BAD_REQUEST); + beforeEach(() => { jest.clearAllMocks(); prisma = { user: { findUnique: jest.fn(), create: jest.fn(), update: jest.fn() } }; @@ -222,18 +79,68 @@ describe('LdapUsersService characterization', () => { signToken: jest.fn().mockReturnValue('token'), }; service = createService(); - ldapClient = (service as any).ldapClient; + ldapClient = new (LdapClient as unknown as jest.Mock)(); + (LdapClient as unknown as jest.Mock).mockClear(); }); it.each([ ['false', true], ['TRUE', false], - ])('creates the client with LDAP_TLS_NO_VERIFY=%s -> rejectUnauthorized %s', (tlsNoVerify, rejectUnauthorized) => { - (LdapClient as unknown as jest.Mock).mockClear(); + ])( + 'creates the client with LDAP_TLS_NO_VERIFY=%s -> rejectUnauthorized %s', + async (tlsNoVerify, rejectUnauthorized) => { + found(); + + await login(createService(tlsNoVerify)); + + expect(LdapClient).toHaveBeenCalledWith(expect.objectContaining({ tlsOptions: { rejectUnauthorized } })); + } + ); + + it('passes timeouts to every client', async () => { + found(); + + await login(); + + expect(LdapClient).toHaveBeenCalledTimes(2); + expect(LdapClient).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ url: 'ldaps://ldap.example.com', timeout: 10000, connectTimeout: 5000 }) + ); + expect(LdapClient).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ url: 'ldaps://ldap.example.com', timeout: 10000, connectTimeout: 5000 }) + ); + }); + + it('takes the timeouts from LDAP_TIMEOUT_MS and LDAP_CONNECT_TIMEOUT_MS', async () => { + found(); - createService(tlsNoVerify); + await login(createService('false', { LDAP_TIMEOUT_MS: '2500', LDAP_CONNECT_TIMEOUT_MS: '700' })); - expect(LdapClient).toHaveBeenCalledWith({ url: 'ldaps://ldap.example.com', tlsOptions: { rejectUnauthorized } }); + expect(LdapClient).toHaveBeenCalledWith(expect.objectContaining({ timeout: 2500, connectTimeout: 700 })); + }); + + it.each(['0', '-5', 'abc', '1.5', ''])('falls back to the default timeouts for %j', async (value) => { + found(); + + await login(createService('false', { LDAP_TIMEOUT_MS: value, LDAP_CONNECT_TIMEOUT_MS: value })); + + expect(LdapClient).toHaveBeenCalledWith(expect.objectContaining({ timeout: 10000, connectTimeout: 5000 })); + }); + + it('never changes a password', async () => { + await expect(service.changePassword(user, 'newPassword')).rejects.toThrow( + new HttpException('Password changes are disabled. Change your password in the directory.', HttpStatus.BAD_REQUEST) + ); + expect(prisma.user.create).not.toHaveBeenCalled(); + expect(prisma.user.update).not.toHaveBeenCalled(); + }); + + it('rejects an email the directory does not know', async () => { + ldapClient.search.mockResolvedValueOnce({ searchEntries: [], searchReferences: [] }); + + await expect(service.login({ email: 'testÜ*()\\\0@example.com', password: 'pw' })).rejects.toThrow(invalidLogin); }); it('refuses to create users', async () => { @@ -245,9 +152,17 @@ describe('LdapUsersService characterization', () => { }); describe('login', () => { + it.each([ + ['an empty string', ''], + ['a missing value', undefined as unknown as string], + ])('rejects %s as the password without contacting LDAP', async (_name, password) => { + await expect(service.login({ email: 'exists@example.com', password })).rejects.toThrow(invalidLogin); + expect(ldapClient.bind).not.toHaveBeenCalled(); + expect(ldapClient.search).not.toHaveBeenCalled(); + }); + it('searches as the bind user with the escaped, trimmed email in every placeholder', async () => { - ldapClient.search.mockResolvedValueOnce({ searchEntries: [entry], searchReferences: [] }); - prisma.user.findUnique.mockResolvedValueOnce(user); + found(); await service.login({ email: ' j*ne(x)\\@example.com ', password: 'pw' }); @@ -260,10 +175,9 @@ describe('LdapUsersService characterization', () => { }); it('binds as the found user and signs in the existing VRT user by the lower-cased LDAP email', async () => { - ldapClient.search.mockResolvedValueOnce({ searchEntries: [entry], searchReferences: [] }); - prisma.user.findUnique.mockResolvedValueOnce(user); + found(); - const result = await service.login({ email: 'jane', password: 'pw' }); + const result = await login(); expect(ldapClient.bind).toHaveBeenNthCalledWith(2, 'uid=jane,ou=people', 'pw'); expect(ldapClient.unbind).toHaveBeenCalledTimes(2); @@ -278,7 +192,7 @@ describe('LdapUsersService characterization', () => { prisma.user.findUnique.mockResolvedValueOnce(null); prisma.user.create.mockResolvedValueOnce(user); - await service.login({ email: 'jane', password: 'pw' }); + await login(); expect(auth.encryptPassword).toHaveBeenCalledWith(expect.any(String)); expect(auth.encryptPassword.mock.calls[0][0]).toHaveLength(12); @@ -298,9 +212,7 @@ describe('LdapUsersService characterization', () => { ldapClient.search.mockResolvedValueOnce({ searchEntries: [entry], searchReferences: [] }); ldapClient.bind.mockResolvedValueOnce(undefined).mockRejectedValueOnce(new Error('invalid credentials')); - await expect(service.login({ email: 'jane', password: 'wrong' })).rejects.toThrow( - new HttpException('Invalid email or password.', HttpStatus.BAD_REQUEST) - ); + await expect(login(service, 'wrong')).rejects.toThrow(invalidLogin); expect(ldapClient.unbind).toHaveBeenCalledTimes(2); expect(prisma.user.findUnique).not.toHaveBeenCalled(); }); @@ -311,46 +223,90 @@ describe('LdapUsersService characterization', () => { const incomplete: Partial = { ...entry, [attribute]: undefined }; ldapClient.search.mockResolvedValueOnce({ searchEntries: [incomplete], searchReferences: [] }); - await expect(service.login({ email: 'jane', password: 'pw' })).rejects.toThrow( - new HttpException('Invalid email or password.', HttpStatus.BAD_REQUEST) - ); + await expect(login()).rejects.toThrow(invalidLogin); expect(ldapClient.bind).toHaveBeenCalledTimes(1); expect(ldapClient.unbind).toHaveBeenCalledTimes(1); } ); - it('passes a failing service bind through and still unbinds', async () => { + it('answers 503 when the service bind fails and still unbinds', async () => { ldapClient.bind.mockRejectedValueOnce(new Error('ldap down')); - await expect(service.login({ email: 'jane', password: 'pw' })).rejects.toThrow('ldap down'); + await expect(login()).rejects.toBeInstanceOf(ServiceUnavailableException); expect(ldapClient.search).not.toHaveBeenCalled(); expect(ldapClient.unbind).toHaveBeenCalledTimes(1); }); + + it('answers 503 when the search fails and still unbinds', async () => { + ldapClient.search.mockRejectedValueOnce(new Error('timeout')); + + await expect(login()).rejects.toBeInstanceOf(ServiceUnavailableException); + expect(ldapClient.unbind).toHaveBeenCalledTimes(1); + }); + + it('does not fail the login when unbind throws', async () => { + found(); + ldapClient.unbind.mockRejectedValue(new Error('socket already closed')); + + await expect(login()).resolves.toEqual(new UserLoginResponseDto(user, 'token')); + ldapClient.unbind.mockResolvedValue(undefined); + }); }); describe('update', () => { - it('overwrites the profile with the LDAP entry found for the new email', async () => { + const own = { ...user, email: 'exists@example.com' }; + + it("refreshes the caller's own LDAP entry and ignores the email sent by the client", async () => { + prisma.user.findUnique.mockResolvedValueOnce(own); + ldapClient.search.mockResolvedValueOnce({ searchEntries: [entry], searchReferences: [] }); + prisma.user.update.mockResolvedValueOnce(user); + + const result = await service.update('1', { email: 'cfo@corp.com', firstName: 'x', lastName: 'y' }); + + expect(prisma.user.findUnique).toHaveBeenCalledWith({ where: { id: '1' } }); + const filter = ldapClient.search.mock.calls[0][1].filter as string; + expect(filter).toContain('exists@example.com'); + expect(filter).not.toContain('cfo@corp.com'); + expect(prisma.user.update).toHaveBeenCalledWith({ + where: { id: '1' }, + data: { email: 'jane@example.com', firstName: 'Jane', lastName: 'Roe' }, + }); + expect(result).toEqual(new UserLoginResponseDto(user, 'token')); + }); + + it('reads the configured attribute names', async () => { + prisma.user.findUnique.mockResolvedValueOnce(own); ldapClient.search.mockResolvedValueOnce({ - searchEntries: [{ ...entry, mail: ' Jane.Roe@Example.com', givenName: 'J.', sn: 'R.' }], + searchEntries: [{ ...entry, mail: 'wrong@example.com', givenName: 'wrong', sn: 'wrong' }], searchReferences: [], }); prisma.user.update.mockResolvedValueOnce(user); - const result = await service.update('1', { email: 'jane', firstName: 'ignored', lastName: 'ignored' }); + await service.update('1', { email: 'jane', firstName: 'a', lastName: 'b' }); - // NOTE: current behavior — update reads the hard-coded `mail`, `givenName` - // and `sn` attributes, not the configured LDAP_ATTRIBUTE_* ones login uses expect(prisma.user.update).toHaveBeenCalledWith({ where: { id: '1' }, - data: { email: 'jane.roe@example.com', firstName: 'J.', lastName: 'R.' }, + data: { email: 'jane@example.com', firstName: 'Jane', lastName: 'Roe' }, }); - expect(result).toEqual(new UserLoginResponseDto(user, 'token')); }); - it('fails with a TypeError when the entry has no `mail` attribute', async () => { - ldapClient.search.mockResolvedValueOnce({ searchEntries: [entry], searchReferences: [] }); + it('answers 404 when the caller no longer exists', async () => { + prisma.user.findUnique.mockResolvedValueOnce(null); - await expect(service.update('1', { email: 'jane', firstName: 'a', lastName: 'b' })).rejects.toThrow(TypeError); + await expect(service.update('1', { email: 'jane', firstName: 'a', lastName: 'b' })).rejects.toBeInstanceOf( + NotFoundException + ); + expect(ldapClient.search).not.toHaveBeenCalled(); + expect(prisma.user.update).not.toHaveBeenCalled(); + }); + + it('answers 503 when the directory is not reachable', async () => { + prisma.user.findUnique.mockResolvedValueOnce(own); + ldapClient.bind.mockRejectedValueOnce(new Error('ldap down')); + + await expect(service.update('1', { email: 'jane', firstName: 'a', lastName: 'b' })).rejects.toBeInstanceOf( + ServiceUnavailableException + ); expect(prisma.user.update).not.toHaveBeenCalled(); }); }); diff --git a/src/users/ldap/ldapusers.service.ts b/src/users/ldap/ldapusers.service.ts index 39aaa490..e5c557bb 100644 --- a/src/users/ldap/ldapusers.service.ts +++ b/src/users/ldap/ldapusers.service.ts @@ -1,4 +1,4 @@ -import { HttpException, HttpStatus, Logger } from '@nestjs/common'; +import { HttpException, HttpStatus, Logger, NotFoundException, ServiceUnavailableException } from '@nestjs/common'; import { CreateUserDto } from '../dto/user-create.dto'; import { UserLoginResponseDto } from '../dto/user-login-response.dto'; import { PrismaService } from '../../prisma/prisma.service'; @@ -21,9 +21,19 @@ type LDAPConfig = { attributeFirstName: string; attributeLastName: string; tlsNoVerify: boolean; + timeoutMs: number; + connectTimeoutMs: number; }; + +const invalidCredentials = () => new HttpException('Invalid email or password.', HttpStatus.BAD_REQUEST); + +const positiveInt = (value: unknown, fallback: number): number => { + const parsed = Number(value); + return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback; +}; + export class LdapUsersService implements Users { - private readonly ldapClient: LdapClient; + readonly allowsRegistration = false; private readonly logger: Logger = new Logger(LdapUsersService.name); private readonly ldapConfig: LDAPConfig; @@ -42,21 +52,27 @@ export class LdapUsersService implements Users { attributeFirstName: this.configService.getOrThrow('LDAP_ATTRIBUTE_FIRST_NAME'), attributeLastName: this.configService.getOrThrow('LDAP_ATTRIBUTE_LAST_NAME'), tlsNoVerify: this.configService.get('LDAP_TLS_NO_VERIFY', 'false').toLowerCase() === 'true', + timeoutMs: positiveInt(this.configService.get('LDAP_TIMEOUT_MS'), 10_000), + connectTimeoutMs: positiveInt(this.configService.get('LDAP_CONNECT_TIMEOUT_MS'), 5_000), }; - this.ldapClient = new LdapClient({ + } + + // one connection per operation: ldapts' unbind() destroys the socket a concurrent login may still be using + private createClient(): LdapClient { + return new LdapClient({ url: this.ldapConfig.url, - tlsOptions: { - rejectUnauthorized: !this.ldapConfig.tlsNoVerify, - }, + timeout: this.ldapConfig.timeoutMs, + connectTimeout: this.ldapConfig.connectTimeoutMs, + tlsOptions: { rejectUnauthorized: !this.ldapConfig.tlsNoVerify }, }); } private async findUserInLdap(email: string): Promise { - //escape the email address for LDAP search email = this.escapeLdap(email.trim()); this.logger.verbose(`search '${email}' in LDAP`); + const client = this.createClient(); try { - await this.ldapClient.bind(this.ldapConfig.bindUser, this.ldapConfig.bindPassword); + await client.bind(this.ldapConfig.bindUser, this.ldapConfig.bindPassword); const attributes = [ 'dn', this.ldapConfig.attributeMail, @@ -64,90 +80,95 @@ export class LdapUsersService implements Users { this.ldapConfig.attributeLastName, ]; - const { searchEntries } = await this.ldapClient.search(this.ldapConfig.searchDN, { + const { searchEntries } = await client.search(this.ldapConfig.searchDN, { filter: this.ldapConfig.usersSearchFilter.replaceAll('{{email}}', email), sizeLimit: 1, attributes: attributes, }); if (searchEntries.length === 0) { this.logger.log('User not found in LDAP.'); - throw new HttpException('Invalid email or password.', HttpStatus.BAD_REQUEST); + throw invalidCredentials(); } for (const attribute of attributes) { if (searchEntries[0][attribute] == null) { this.logger.warn(`Attribute '${attribute}' not found in LDAP entry found for '${email}'`); - throw new HttpException('Invalid email or password.', HttpStatus.BAD_REQUEST); + throw invalidCredentials(); } } return searchEntries[0]; + } catch (e) { + if (e instanceof HttpException) { + throw e; + } + this.logger.error(`LDAP search for '${email}' failed: ${e}`); + throw new ServiceUnavailableException('Directory service is not available.'); } finally { - await this.ldapClient.unbind(); + await client.unbind().catch((): undefined => undefined); } } - private async createUserFromLdapEntry(ldapEntry: LdapEntry): Promise { - const userForVRTDb = { - email: ldapEntry[this.ldapConfig.attributeMail].toString().trim().toLowerCase(), - firstName: ldapEntry[this.ldapConfig.attributeFirstName].toString(), - lastName: ldapEntry[this.ldapConfig.attributeLastName].toString(), - apiKey: this.authService.generateApiKey(), - password: await this.authService.encryptPassword(genSaltSync(1).slice(-12)), - role: Role.editor, + private profileOf(entry: LdapEntry) { + return { + email: entry[this.ldapConfig.attributeMail].toString().trim().toLowerCase(), + firstName: entry[this.ldapConfig.attributeFirstName].toString(), + lastName: entry[this.ldapConfig.attributeLastName].toString(), }; + } + private async createUserFromLdapEntry(ldapEntry: LdapEntry): Promise { return this.prismaService.user.create({ - data: userForVRTDb, + data: { + ...this.profileOf(ldapEntry), + apiKey: this.authService.generateApiKey(), + password: await this.authService.encryptPassword(genSaltSync(1).slice(-12)), + role: Role.editor, + }, }); } - // eslint-disable-next-line @typescript-eslint/no-unused-vars - async create(createUserDto: CreateUserDto): Promise { + async create(_createUserDto: CreateUserDto): Promise { throw new HttpException('User creation is disabled. Use your LDAP-Credentials to login.', HttpStatus.BAD_REQUEST); } - async update(id: string, userDto: UpdateUserDto): Promise { - const userFromLdap = await this.findUserInLdap(userDto.email); - const user = await this.prismaService.user.update({ - where: { id }, - data: { - email: userFromLdap.mail.toString().trim().toLowerCase(), - firstName: userFromLdap.givenName.toString(), - lastName: userFromLdap.sn.toString(), - }, - }); - const token = this.authService.signToken(user); - return new UserLoginResponseDto(user, token); + async update(id: string, _userDto: UpdateUserDto): Promise { + // the directory owns the profile: refresh it from the caller's own entry, never from an email the client sent + const current = await this.prismaService.user.findUnique({ where: { id } }); + if (!current) { + throw new NotFoundException(`User not found: ${id}`); + } + const entry = await this.findUserInLdap(current.email); + const user = await this.prismaService.user.update({ where: { id }, data: this.profileOf(entry) }); + return new UserLoginResponseDto(user, this.authService.signToken(user)); } - // eslint-disable-next-line @typescript-eslint/no-unused-vars - async changePassword(user: User, newPassword: string): Promise { - this.logger.warn(`${user.email} tied to change password - this is not supported for LDAP users`); - return true; + changePassword(user: User, _newPassword: string): Promise { + this.logger.warn(`${user.email} tried to change password - this is not supported for LDAP users`); + return Promise.reject( + new HttpException('Password changes are disabled. Change your password in the directory.', HttpStatus.BAD_REQUEST) + ); } async login(userLoginRequestDto: UserLoginRequestDto) { - const userFromLdap = await this.findUserInLdap(userLoginRequestDto.email); - if (!userFromLdap) { - throw new HttpException('Invalid email or password.', HttpStatus.BAD_REQUEST); + // RFC 4513 5.1.2: a DN with an empty password is an unauthenticated bind, which AD answers with success + if (!userLoginRequestDto.password) { + throw invalidCredentials(); } + const userFromLdap = await this.findUserInLdap(userLoginRequestDto.email); - // check if user password is correct using ldap + const client = this.createClient(); try { - await this.ldapClient.bind(userFromLdap.dn, userLoginRequestDto.password); - } catch (e) { - throw new HttpException('Invalid email or password.', HttpStatus.BAD_REQUEST); + await client.bind(userFromLdap.dn, userLoginRequestDto.password); + } catch { + throw invalidCredentials(); } finally { - await this.ldapClient.unbind(); + await client.unbind().catch((): undefined => undefined); } - const userEmailFromLdap = userFromLdap[this.ldapConfig.attributeMail].toString().trim().toLowerCase(); + const userEmailFromLdap = this.profileOf(userFromLdap).email; - let user = await this.prismaService.user.findUnique({ - where: { email: userEmailFromLdap }, - }); + let user = await this.prismaService.user.findUnique({ where: { email: userEmailFromLdap } }); if (!user) { - // create user if not found in VRT database this.logger.log( `'${userLoginRequestDto.email}' (found in ldap as '${userEmailFromLdap}') successfully ` + 'authenticated via LDAP, but not found in VRT database. Creating user.' @@ -159,13 +180,6 @@ export class LdapUsersService implements Users { return new UserLoginResponseDto(user, token); } - /** - * RFC 2254 Escaping of filter strings - * Raw Escaped - * (o=Parens (R Us)) (o=Parens \28R Us\29) - * (cn=star*) (cn=star\2A) - * (filename=C:\MyFile) (filename=C:\5cMyFile) - */ private escapeLdap(input: string): string { let escapedResult = ''; for (const inputChar of input) { diff --git a/src/users/oidc/oidcusers.service.spec.ts b/src/users/oidc/oidcusers.service.spec.ts index 649fa609..3d06f5a5 100644 --- a/src/users/oidc/oidcusers.service.spec.ts +++ b/src/users/oidc/oidcusers.service.spec.ts @@ -73,6 +73,35 @@ const prismaService = { user: { findUnique, create, update }, } as unknown as PrismaService; +const configWith = (overrides: Record = {}): ConfigService => + ({ + get: jest.fn((key: string, defaultValue?: string) => + key in overrides ? overrides[key] : configService.get(key, defaultValue) + ), + getOrThrow: configService.getOrThrow, + }) as unknown as ConfigService; + +const serviceWith = (overrides: Record = {}) => + new OidcUsersService(configWith(overrides), prismaService, authService); + +const idpReturns = (claims: Record, extra: Record = {}) => + mockCallback.mockResolvedValue({ claims: () => claims, ...extra }); + +const noUser = () => { + findUnique.mockResolvedValue(null); + create.mockImplementation(({ data }) => ({ ...user, ...data })); +}; + +const checks = { state: 'mock-state', codeVerifier: 'mock-verifier' }; +const login = (svc: OidcUsersService, params: Record = { code: 'code', state: 'mock-state' }) => + svc.handleCallback(params, checks); +const expectLoginFailed = (svc: OidcUsersService, params?: Record) => + expect(login(svc, params)).rejects.toThrow('OIDC login failed.'); +const expectCreatedWith = (data: object) => + expect(create).toHaveBeenCalledWith(expect.objectContaining({ data: expect.objectContaining(data) })); +const expectRoleUpdate = (role: Role) => + expect(update).toHaveBeenCalledWith({ where: { id: user.id }, data: { role } }); + describe('OidcUsersService', () => { let service: OidcUsersService; @@ -86,7 +115,7 @@ describe('OidcUsersService', () => { }, }); mockAuthorizationUrl.mockReturnValue('https://idp.example.com/authorize'); - service = new OidcUsersService(configService, prismaService, authService); + service = serviceWith(); }); describe('getAuthorizationUrl', () => { @@ -124,15 +153,7 @@ describe('OidcUsersService', () => { }); it('applies the insecure TLS agent only to the issuer origin', async () => { - const tlsConfigService = { - get: jest.fn((key: string, defaultValue?: string) => - key === 'OIDC_TLS_NO_VERIFY' ? 'true' : configService.get(key, defaultValue) - ), - getOrThrow: configService.getOrThrow, - } as unknown as ConfigService; - const tlsService = new OidcUsersService(tlsConfigService, prismaService, authService); - - await tlsService.getAuthorizationUrl(); + await serviceWith({ OIDC_TLS_NO_VERIFY: 'true' }).getAuthorizationUrl(); const provider = (Issuer as unknown as Record Record>)[ custom.http_options as unknown as string @@ -148,15 +169,8 @@ describe('OidcUsersService', () => { }, }; mockDiscover.mockResolvedValue(issuer); - const tlsConfigService = { - get: jest.fn((key: string, defaultValue?: string) => - key === 'OIDC_TLS_NO_VERIFY' ? 'true' : configService.get(key, defaultValue) - ), - getOrThrow: configService.getOrThrow, - } as unknown as ConfigService; - const tlsService = new OidcUsersService(tlsConfigService, prismaService, authService); - await tlsService.getAuthorizationUrl(); + await serviceWith({ OIDC_TLS_NO_VERIFY: 'true' }).getAuthorizationUrl(); const provider = issuer[custom.http_options as unknown as string] as ( url: URL, @@ -179,160 +193,121 @@ describe('OidcUsersService', () => { }); describe('handleCallback', () => { - const checks = { state: 'mock-state', codeVerifier: 'mock-verifier' }; - it('creates a new user and derives the admin role from the claims', async () => { - mockCallback.mockResolvedValue({ - access_token: 'access-token', - claims: () => ({ + idpReturns( + { email: 'new.user@example.com', email_verified: true, given_name: 'New', family_name: 'User', groups: ['vrt-admin'], - }), - }); - findUnique.mockResolvedValue(null); - create.mockImplementation(({ data }) => ({ ...user, ...data })); + }, + { access_token: 'access-token' } + ); + noUser(); - const result = await service.handleCallback({ code: 'code', state: 'mock-state' }, checks); + const result = await login(service); - expect(create).toHaveBeenCalledWith( - expect.objectContaining({ - data: expect.objectContaining({ - email: 'new.user@example.com', - firstName: 'New', - lastName: 'User', - role: Role.admin, - }), - }) - ); + expectCreatedWith({ email: 'new.user@example.com', firstName: 'New', lastName: 'User', role: Role.admin }); expect(result.token).toBe('vrt-jwt'); }); it('falls back to the editor role when no admin claim matches', async () => { - mockCallback.mockResolvedValue({ - claims: () => ({ - email: 'plain@example.com', - email_verified: true, - given_name: 'Plain', - family_name: 'User', - groups: ['other'], - }), + idpReturns({ + email: 'plain@example.com', + email_verified: true, + given_name: 'Plain', + family_name: 'User', + groups: ['other'], }); - findUnique.mockResolvedValue(null); - create.mockImplementation(({ data }) => ({ ...user, ...data })); + noUser(); - await service.handleCallback({ code: 'code', state: 'mock-state' }, checks); + await login(service); - expect(create).toHaveBeenCalledWith( - expect.objectContaining({ data: expect.objectContaining({ role: Role.editor }) }) - ); + expectCreatedWith({ role: Role.editor }); }); it('updates the role of an existing user from the claims', async () => { - mockCallback.mockResolvedValue({ - claims: () => ({ email: 'test@example.com', email_verified: true, groups: ['admin'] }), - }); + idpReturns({ email: 'test@example.com', email_verified: true, groups: ['admin'] }); findUnique.mockResolvedValue(user); update.mockResolvedValue({ ...user, role: Role.admin }); - const result = await service.handleCallback({ code: 'code', state: 'mock-state' }, checks); + const result = await login(service); - expect(update).toHaveBeenCalledWith({ where: { id: user.id }, data: { role: Role.admin } }); + expectRoleUpdate(Role.admin); expect(result.token).toBe('vrt-jwt'); }); it('throws when the email claim is missing', async () => { - mockCallback.mockResolvedValue({ - claims: () => ({ given_name: 'No', family_name: 'Mail' }), - }); + idpReturns({ given_name: 'No', family_name: 'Mail' }); - await expect(service.handleCallback({ code: 'code', state: 'mock-state' }, checks)).rejects.toThrow( - 'OIDC login failed.' - ); + await expectLoginFailed(service); }); it('throws when the callback validation fails', async () => { mockCallback.mockRejectedValue(new Error('state mismatch')); - await expect(service.handleCallback({ code: 'code', state: 'wrong' }, checks)).rejects.toThrow( - 'OIDC login failed.' - ); + await expectLoginFailed(service, { code: 'code', state: 'wrong' }); }); it('falls back to the userinfo endpoint when the email claim is missing', async () => { - mockCallback.mockResolvedValue({ access_token: 'access-token', claims: () => ({ sub: 'user-1' }) }); + idpReturns({ sub: 'user-1' }, { access_token: 'access-token' }); mockUserinfo.mockResolvedValue({ email: 'from.userinfo@example.com', email_verified: true, given_name: 'Info', family_name: 'User', }); - findUnique.mockResolvedValue(null); - create.mockImplementation(({ data }) => ({ ...user, ...data })); + noUser(); - await service.handleCallback({ code: 'code', state: 'mock-state' }, checks); + await login(service); expect(mockUserinfo).toHaveBeenCalledWith(expect.objectContaining({ access_token: 'access-token' })); - expect(create).toHaveBeenCalledWith( - expect.objectContaining({ data: expect.objectContaining({ email: 'from.userinfo@example.com' }) }) - ); + expectCreatedWith({ email: 'from.userinfo@example.com' }); }); it('still fails when the userinfo request fails and no email is present', async () => { - mockCallback.mockResolvedValue({ access_token: 'access-token', claims: () => ({ sub: 'user-1' }) }); + idpReturns({ sub: 'user-1' }, { access_token: 'access-token' }); mockUserinfo.mockRejectedValue(new Error('userinfo down')); - await expect(service.handleCallback({ code: 'code', state: 'mock-state' }, checks)).rejects.toThrow( - 'OIDC login failed.' - ); + await expectLoginFailed(service); }); it('rejects an email the provider marks as unverified', async () => { - mockCallback.mockResolvedValue({ - claims: () => ({ email: 'unverified@example.com', email_verified: false }), - }); + idpReturns({ email: 'unverified@example.com', email_verified: false }); - await expect(service.handleCallback({ code: 'code', state: 'mock-state' }, checks)).rejects.toThrow( - 'OIDC login failed.' - ); + await expectLoginFailed(service); expect(findUnique).not.toHaveBeenCalled(); }); it('accepts an unverified email when the check is disabled', async () => { - const relaxedConfigService = { - get: jest.fn((key: string, defaultValue?: string) => - key === 'OIDC_REQUIRE_VERIFIED_EMAIL' ? 'false' : configService.get(key, defaultValue) - ), - getOrThrow: configService.getOrThrow, - } as unknown as ConfigService; - const relaxedService = new OidcUsersService(relaxedConfigService, prismaService, authService); - mockCallback.mockResolvedValue({ - claims: () => ({ email: 'unverified@example.com', email_verified: false }), - }); + idpReturns({ email: 'unverified@example.com', email_verified: false }); findUnique.mockResolvedValue(user); - const result = await relaxedService.handleCallback({ code: 'code', state: 'mock-state' }, checks); + const result = await login(serviceWith({ OIDC_REQUIRE_VERIFIED_EMAIL: 'false' })); expect(result.token).toBe('vrt-jwt'); }); + it('rejects a missing email even when unverified emails are allowed', async () => { + idpReturns({ sub: 'user-1' }); + + await expectLoginFailed(serviceWith({ OIDC_REQUIRE_VERIFIED_EMAIL: 'false' })); + + expect(findUnique).not.toHaveBeenCalled(); + }); + it('rejects an empty email claim array', async () => { - mockCallback.mockResolvedValue({ claims: () => ({ email: [] as string[] }) }); + idpReturns({ email: [] as string[] }); - await expect(service.handleCallback({ code: 'code', state: 'mock-state' }, checks)).rejects.toThrow( - 'OIDC login failed.' - ); + await expectLoginFailed(service); }); it('accepts an email_verified claim serialized as a string', async () => { - mockCallback.mockResolvedValue({ - claims: () => ({ email: 'test@example.com', email_verified: 'true', groups: ['editor'] }), - }); + idpReturns({ email: 'test@example.com', email_verified: 'true', groups: ['editor'] }); findUnique.mockResolvedValue(user); - const result = await service.handleCallback({ code: 'code', state: 'mock-state' }, checks); + const result = await login(service); expect(result.token).toBe('vrt-jwt'); }); @@ -345,183 +320,61 @@ describe('OidcUsersService', () => { { 'https://vrt.example.com/roles': ['vrt-admin'] }, ], ])('reads the role claim via %s', async (_label, roleClaim, roleClaims) => { - const claimConfigService = { - get: jest.fn((key: string, defaultValue?: string) => - key === 'OIDC_ROLE_CLAIM' ? roleClaim : configService.get(key, defaultValue) - ), - getOrThrow: configService.getOrThrow, - } as unknown as ConfigService; - const claimService = new OidcUsersService(claimConfigService, prismaService, authService); - mockCallback.mockResolvedValue({ - claims: () => ({ email: 'test@example.com', email_verified: true, ...roleClaims }), - }); + idpReturns({ email: 'test@example.com', email_verified: true, ...roleClaims }); findUnique.mockResolvedValue(user); update.mockResolvedValue({ ...user, role: Role.admin }); - await claimService.handleCallback({ code: 'code', state: 'mock-state' }, checks); + await login(serviceWith({ OIDC_ROLE_CLAIM: roleClaim })); - expect(update).toHaveBeenCalledWith({ where: { id: user.id }, data: { role: Role.admin } }); + expectRoleUpdate(Role.admin); }); it('uses the user a concurrent first login created', async () => { - mockCallback.mockResolvedValue({ - claims: () => ({ email: 'race@example.com', email_verified: true }), - }); + idpReturns({ email: 'race@example.com', email_verified: true }); findUnique.mockResolvedValueOnce(null).mockResolvedValueOnce({ ...user, email: 'race@example.com' }); create.mockRejectedValue(new PrismaClientKnownRequestError('unique', { code: 'P2002', clientVersion: '5' })); - const result = await service.handleCallback({ code: 'code', state: 'mock-state' }, checks); + const result = await login(service); expect(findUnique).toHaveBeenCalledTimes(2); expect(result.token).toBe('vrt-jwt'); }); it('rethrows a unique violation that is not a concurrent login', async () => { - mockCallback.mockResolvedValue({ - claims: () => ({ email: 'race@example.com', email_verified: true }), - }); + idpReturns({ email: 'race@example.com', email_verified: true }); findUnique.mockResolvedValue(null); const error = new PrismaClientKnownRequestError('unique', { code: 'P2002', clientVersion: '5' }); create.mockRejectedValue(error); - await expect(service.handleCallback({ code: 'code', state: 'mock-state' }, checks)).rejects.toBe(error); + await expect(login(service)).rejects.toBe(error); }); it('keeps the role of an existing user when the role claim is missing', async () => { - mockCallback.mockResolvedValue({ - claims: () => ({ email: 'test@example.com', email_verified: true }), - }); + idpReturns({ email: 'test@example.com', email_verified: true }); findUnique.mockResolvedValue({ ...user, role: Role.admin }); - const result = await service.handleCallback({ code: 'code', state: 'mock-state' }, checks); + const result = await login(service); expect(update).not.toHaveBeenCalled(); expect(result.token).toBe('vrt-jwt'); }); it('leaves an existing user unchanged when the role already matches', async () => { - mockCallback.mockResolvedValue({ - claims: () => ({ email: 'test@example.com', email_verified: true, groups: ['editor'] }), - }); + idpReturns({ email: 'test@example.com', email_verified: true, groups: ['editor'] }); findUnique.mockResolvedValue(user); - const result = await service.handleCallback({ code: 'code', state: 'mock-state' }, checks); + const result = await login(service); expect(update).not.toHaveBeenCalled(); expect(result.token).toBe('vrt-jwt'); }); - }); - - describe('configuration', () => { - const configWith = (overrides: Record): ConfigService => - ({ - get: jest.fn((key: string, defaultValue?: string) => - key in overrides ? overrides[key] : configService.get(key, defaultValue) - ), - getOrThrow: configService.getOrThrow, - }) as unknown as ConfigService; - const checks = { state: 'mock-state', codeVerifier: 'mock-verifier' }; - - it.each([ - ['a single admin value', { groups: 'admin' }, Role.admin], - ['values separated by spaces', { groups: 'viewer vrt-admin' }, Role.admin], - ['values separated by commas', { groups: 'viewer,admin' }, Role.admin], - ['admin values in another case', { groups: ['VRT-Admin'] }, Role.admin], - ['values that are only separators', { groups: ' , ' }, Role.editor], - ['object values, which are ignored', { groups: [{ name: 'admin' }] }, Role.editor], - ])('derives the role from %s', async (_case, roleClaims, expected) => { - mockCallback.mockResolvedValue({ - claims: () => ({ email: 'new@example.com', email_verified: true, ...roleClaims }), - }); - findUnique.mockResolvedValue(null); - create.mockImplementation(({ data }) => ({ ...user, ...data })); - - await service.handleCallback({ code: 'code' }, checks); - - expect(create).toHaveBeenCalledWith( - expect.objectContaining({ data: expect.objectContaining({ role: expected }) }) - ); - }); - - it('trims and lower-cases the configured admin values', async () => { - const custom = new OidcUsersService( - configWith({ OIDC_ADMIN_VALUES: ' Ops-Team , ,42' }), - prismaService, - authService - ); - mockCallback.mockResolvedValue({ - claims: () => ({ email: 'new@example.com', email_verified: true, groups: ['ops-team'] }), - }); - findUnique.mockResolvedValue(null); - create.mockImplementation(({ data }) => ({ ...user, ...data })); - - await custom.handleCallback({ code: 'code' }, checks); - - expect(create).toHaveBeenCalledWith( - expect.objectContaining({ data: expect.objectContaining({ role: Role.admin }) }) - ); - }); - - it.each([ - ['a number', 42], - ['a boolean', true], - ])('compares a role claim holding %s as a string', async (_case, value) => { - const custom = new OidcUsersService(configWith({ OIDC_ADMIN_VALUES: String(value) }), prismaService, authService); - mockCallback.mockResolvedValue({ - claims: () => ({ email: 'new@example.com', email_verified: true, groups: [value] }), - }); - findUnique.mockResolvedValue(null); - create.mockImplementation(({ data }) => ({ ...user, ...data })); - - await custom.handleCallback({ code: 'code' }, checks); - - expect(create).toHaveBeenCalledWith( - expect.objectContaining({ data: expect.objectContaining({ role: Role.admin }) }) - ); - }); - - it('gives new users without an admin claim the configured default role', async () => { - const custom = new OidcUsersService(configWith({ OIDC_DEFAULT_ROLE: 'Admin' }), prismaService, authService); - mockCallback.mockResolvedValue({ claims: () => ({ email: 'new@example.com', email_verified: true }) }); - findUnique.mockResolvedValue(null); - create.mockImplementation(({ data }) => ({ ...user, ...data })); - - await custom.handleCallback({ code: 'code' }, checks); - - expect(create).toHaveBeenCalledWith( - expect.objectContaining({ data: expect.objectContaining({ role: Role.admin }) }) - ); - }); - }); - - describe('handleCallback claim sources', () => { - const checks = { state: 'mock-state', codeVerifier: 'mock-verifier' }; - const relaxed = () => - new OidcUsersService( - { - get: jest.fn((key: string, defaultValue?: string) => - key === 'OIDC_REQUIRE_VERIFIED_EMAIL' ? 'false' : configService.get(key, defaultValue) - ), - getOrThrow: configService.getOrThrow, - } as unknown as ConfigService, - prismaService, - authService - ); - - it('rejects a missing email even when unverified emails are allowed', async () => { - mockCallback.mockResolvedValue({ claims: () => ({ sub: 'user-1' }) }); - - await expect(relaxed().handleCallback({ code: 'code' }, checks)).rejects.toThrow('OIDC login failed.'); - - expect(findUnique).not.toHaveBeenCalled(); - }); it('warns and keeps the role of an existing user when the role claim is missing', async () => { - mockCallback.mockResolvedValue({ claims: () => ({ email: 'test@example.com', email_verified: true }) }); + idpReturns({ email: 'test@example.com', email_verified: true }); findUnique.mockResolvedValue({ ...user, role: Role.admin }); const warn = jest.spyOn(service['logger'], 'warn').mockImplementation(() => undefined); - await service.handleCallback({ code: 'code' }, checks); + await login(service); expect(warn).toHaveBeenCalledWith("Claim 'groups' not found for 'test@example.com'. Keeping role 'admin'."); expect(update).not.toHaveBeenCalled(); @@ -532,22 +385,10 @@ describe('OidcUsersService', () => { ['a null parent claim', { realm_access: null }], ['a parent claim that is not an object', { realm_access: 'admin' }], ])('treats a dotted role claim with %s as missing', async (_case, parent) => { - const nested = new OidcUsersService( - { - get: jest.fn((key: string, defaultValue?: string) => - key === 'OIDC_ROLE_CLAIM' ? 'realm_access.length' : configService.get(key, defaultValue) - ), - getOrThrow: configService.getOrThrow, - } as unknown as ConfigService, - prismaService, - authService - ); - mockCallback.mockResolvedValue({ - claims: () => ({ email: 'test@example.com', email_verified: true, ...parent }), - }); + idpReturns({ email: 'test@example.com', email_verified: true, ...parent }); findUnique.mockResolvedValue({ ...user, role: Role.admin }); - await nested.handleCallback({ code: 'code' }, checks); + await login(serviceWith({ OIDC_ROLE_CLAIM: 'realm_access.length' })); expect(update).not.toHaveBeenCalled(); }); @@ -556,19 +397,60 @@ describe('OidcUsersService', () => { ['another prisma error', new PrismaClientKnownRequestError('fk', { code: 'P2003', clientVersion: '5' })], ['a non-prisma error', new Error('db down')], ])('rethrows %s from creating the user without looking it up again', async (_case, error) => { - mockCallback.mockResolvedValue({ claims: () => ({ email: 'race@example.com', email_verified: true }) }); - // a lookup after the failed insert would find the user + idpReturns({ email: 'race@example.com', email_verified: true }); findUnique .mockReset() .mockResolvedValueOnce(null) .mockResolvedValue({ ...user, email: 'race@example.com' }); create.mockRejectedValue(error); - await expect(service.handleCallback({ code: 'code' }, checks)).rejects.toBe(error); + await expect(login(service)).rejects.toBe(error); expect(findUnique).toHaveBeenCalledTimes(1); }); }); + describe('configuration', () => { + const newUserWithClaims = (svc: OidcUsersService, roleClaims: Record) => { + idpReturns({ email: 'new@example.com', email_verified: true, ...roleClaims }); + noUser(); + return login(svc); + }; + + it.each([ + ['a single admin value', { groups: 'admin' }, Role.admin], + ['values separated by spaces', { groups: 'viewer vrt-admin' }, Role.admin], + ['values separated by commas', { groups: 'viewer,admin' }, Role.admin], + ['admin values in another case', { groups: ['VRT-Admin'] }, Role.admin], + ['values that are only separators', { groups: ' , ' }, Role.editor], + ['object values, which are ignored', { groups: [{ name: 'admin' }] }, Role.editor], + ])('derives the role from %s', async (_case, roleClaims, expected) => { + await newUserWithClaims(service, roleClaims); + + expectCreatedWith({ role: expected }); + }); + + it('trims and lower-cases the configured admin values', async () => { + await newUserWithClaims(serviceWith({ OIDC_ADMIN_VALUES: ' Ops-Team , ,42' }), { groups: ['ops-team'] }); + + expectCreatedWith({ role: Role.admin }); + }); + + it.each([ + ['a number', 42], + ['a boolean', true], + ])('compares a role claim holding %s as a string', async (_case, value) => { + await newUserWithClaims(serviceWith({ OIDC_ADMIN_VALUES: String(value) }), { groups: [value] }); + + expectCreatedWith({ role: Role.admin }); + }); + + it('gives new users without an admin claim the configured default role', async () => { + await newUserWithClaims(serviceWith({ OIDC_DEFAULT_ROLE: 'Admin' }), {}); + + expectCreatedWith({ role: Role.admin }); + }); + }); + describe('non-SSO operations', () => { it('updates the name but never the email of an OIDC user', async () => { update.mockResolvedValue(user); @@ -588,6 +470,7 @@ describe('OidcUsersService', () => { 'Password changes are disabled. Use SSO to change your password.' ); }); + it('rejects password login', async () => { await expect(service.login({ email: 'a@b.com', password: 'x' })).rejects.toThrow('Password login is disabled.'); }); diff --git a/src/users/oidc/oidcusers.service.ts b/src/users/oidc/oidcusers.service.ts index b63f94cf..9f68e3e0 100644 --- a/src/users/oidc/oidcusers.service.ts +++ b/src/users/oidc/oidcusers.service.ts @@ -3,7 +3,8 @@ import { Client as OidcClient, Issuer, generators, custom, TokenSet } from 'open import { CreateUserDto } from '../dto/user-create.dto'; import { UserLoginResponseDto } from '../dto/user-login-response.dto'; import { PrismaService } from '../../prisma/prisma.service'; -import { Prisma, Role, User } from '@prisma/client'; +import { isPrismaError } from '../../prisma/prisma-errors'; +import { Role, User } from '@prisma/client'; import { UpdateUserDto } from '../dto/user-update.dto'; import { AuthService } from '../../auth/auth.service'; import { UserLoginRequestDto } from '../dto/user-login-request.dto'; @@ -29,6 +30,7 @@ type OIDCConfig = { }; export class OidcUsersService implements Users { + readonly allowsRegistration = false; private readonly logger: Logger = new Logger(OidcUsersService.name); private readonly oidcConfig: OIDCConfig; private readonly appFrontendUrl: string; @@ -73,10 +75,6 @@ export class OidcUsersService implements Users { } } - /** - * Restricts the insecure HTTPS agent to the configured issuer, so a TLS - * bypass never applies to unrelated openid-client requests. - */ private getHttpOptions(): ((url: URL, options: Record) => Record) | null { if (!this.oidcConfig.tlsNoVerify) { return null; @@ -89,16 +87,13 @@ export class OidcUsersService implements Users { private async discoverClient(): Promise { const httpOptions = this.getHttpOptions(); if (httpOptions) { - // Static discovery cannot be scoped per instance; the provider itself - // only bypasses verification for the configured issuer origin. - Issuer[custom.http_options] = httpOptions as never; + // Discovery only takes a static option; httpOptions still checks the origin + Issuer[custom.http_options] = httpOptions; } const issuer = await Issuer.discover(this.oidcConfig.issuerUrl); if (httpOptions) { - // The JWKS used to validate the ID token is fetched through the issuer - // instance, which does not inherit the static option set above. - issuer[custom.http_options] = httpOptions as never; + issuer[custom.http_options] = httpOptions; } const client = new issuer.Client({ client_id: this.oidcConfig.clientId, @@ -108,7 +103,7 @@ export class OidcUsersService implements Users { }); if (httpOptions) { - client[custom.http_options] = httpOptions as never; + client[custom.http_options] = httpOptions; } return client; } @@ -116,7 +111,6 @@ export class OidcUsersService implements Users { private getClient(): Promise { if (!this.clientPromise) { this.clientPromise = this.discoverClient().catch((error) => { - // Allow a later request to retry discovery after a transient failure. this.clientPromise = null; throw error; }); @@ -124,10 +118,6 @@ export class OidcUsersService implements Users { return this.clientPromise; } - /** - * Builds the IdP authorization URL together with the PKCE/state values that - * must be kept (e.g. in a short-lived cookie) until the callback returns. - */ async getAuthorizationUrl(): Promise<{ url: string; state: string; codeVerifier: string }> { const client = await this.getClient(); const state = generators.state(); @@ -141,10 +131,6 @@ export class OidcUsersService implements Users { return { url, state, codeVerifier }; } - /** - * URL the browser is redirected to after OIDC finished (success or error). - * The token travels in the URL fragment so it never reaches server logs. - */ buildFrontendRedirect(token?: string | null, error?: string): string { const base = `${this.appFrontendUrl}/login`; if (token) { @@ -173,8 +159,6 @@ export class OidcUsersService implements Users { const claims: Record = { ...tokenSet.claims() }; if (tokenSet.access_token && !this.claimToString(claims, this.oidcConfig.attributeMail)) { try { - // Passing the whole token set makes openid-client verify that the - // userinfo `sub` matches the ID token before the claims are merged. Object.assign(claims, await client.userinfo(tokenSet)); } catch (e) { this.logger.warn(`OIDC userinfo request failed: ${e}`); @@ -187,10 +171,7 @@ export class OidcUsersService implements Users { throw new HttpException('OIDC login failed.', HttpStatus.BAD_REQUEST); } - // Never trust an email address the identity provider marks as unverified: - // it could otherwise be used to take over an existing VRT account. Can be - // disabled for providers that do not emit a reliable email_verified claim. - // Some providers serialize the claim as the string "true". + // An unverified email could take over an existing VRT account const emailVerified = claims.email_verified === true || claims.email_verified === 'true'; if (this.oidcConfig.requireVerifiedEmail && !emailVerified) { this.logger.warn(`OIDC login rejected: email '${email}' is not verified by the identity provider`); @@ -198,9 +179,7 @@ export class OidcUsersService implements Users { } const role = this.resolveRole(claims); - // Without the role claim (e.g. a missing scope or mapper) the provider says - // nothing about the role, so keep existing users' roles instead of - // demoting every admin to the default role. + // Without the claim the provider says nothing about roles: keep existing ones rather than demote every admin const hasRoleClaim = this.readRoleClaim(claims) != null; let user = await this.prismaService.user.findUnique({ where: { email } }); @@ -219,12 +198,6 @@ export class OidcUsersService implements Users { return new UserLoginResponseDto(user, token); } - /** - * Two first logins for the same email (e.g. two tabs) can both miss the - * lookup; the losing insert hits the unique email constraint, so use the - * user the other request created. Other unique violations (apiKey) still - * surface because the lookup then finds nothing. - */ private async createUserFromClaimsOrFindConcurrent( claims: Record, email: string, @@ -233,7 +206,7 @@ export class OidcUsersService implements Users { try { return await this.createUserFromClaims(claims, email, role); } catch (e) { - if (!(e instanceof Prisma.PrismaClientKnownRequestError) || e.code !== 'P2002') { + if (!isPrismaError(e, 'P2002')) { throw e; } const user = await this.prismaService.user.findUnique({ where: { email } }); @@ -268,7 +241,6 @@ export class OidcUsersService implements Users { return value == null ? [] : [value]; } - // Objects are ignored rather than stringified to '[object Object]'. private scalarToString(value: unknown): string | null { if (typeof value === 'string') { return value; @@ -279,11 +251,6 @@ export class OidcUsersService implements Users { return null; } - /** - * Reads the role claim either by its exact name (which may itself contain - * dots, e.g. namespaced claims like 'https://example.com/roles') or as a - * dotted path into nested claims (e.g. Keycloak's 'realm_access.roles'). - */ private readRoleClaim(claims: Record): unknown { const name = this.oidcConfig.roleClaim; if (claims[name] != null) { @@ -308,13 +275,11 @@ export class OidcUsersService implements Users { return isAdmin ? Role.admin : this.oidcConfig.defaultRole; } - create(createUserDto: CreateUserDto): Promise { + create(_createUserDto: CreateUserDto): Promise { return Promise.reject(new HttpException('User creation is disabled. Use SSO to login.', HttpStatus.BAD_REQUEST)); } async update(id: string, userDto: UpdateUserDto): Promise { - // The email is the account key for OIDC users, so it is owned by the - // identity provider and must not be changed here. const user = await this.prismaService.user.update({ where: { id }, data: { @@ -326,14 +291,14 @@ export class OidcUsersService implements Users { return new UserLoginResponseDto(user, token); } - changePassword(user: User, newPassword: string): Promise { + changePassword(user: User, _newPassword: string): Promise { this.logger.warn(`${user.email} tried to change password - this is not supported for OIDC users`); return Promise.reject( new HttpException('Password changes are disabled. Use SSO to change your password.', HttpStatus.BAD_REQUEST) ); } - login(userLoginRequestDto: UserLoginRequestDto): Promise { + login(_userLoginRequestDto: UserLoginRequestDto): Promise { return Promise.reject(new HttpException('Password login is disabled. Use SSO to login.', HttpStatus.BAD_REQUEST)); } } diff --git a/src/users/users.controller.spec.ts b/src/users/users.controller.spec.ts index 1976bf40..b29ef06e 100644 --- a/src/users/users.controller.spec.ts +++ b/src/users/users.controller.spec.ts @@ -4,12 +4,15 @@ import { UsersController } from './users.controller'; import { UsersService } from './users.service'; import { ConfigService } from '@nestjs/config'; import { Role, User } from '@prisma/client'; -import { UserDto } from './dto/user.dto'; +import { UserDto, toUserSummary } from './dto/user.dto'; +import { ForbiddenException } from '@nestjs/common'; describe('Users Controller', () => { let controller: UsersController; let usersService: { isOidcEnabled: jest.Mock; + isRegistrationAllowedByConfig: jest.Mock; + isRegistrationEnabled: jest.Mock; oidcGetAuthorizationUrl: jest.Mock; oidcBuildFrontendRedirect: jest.Mock; oidcHandleCallback: jest.Mock; @@ -40,6 +43,8 @@ describe('Users Controller', () => { beforeEach(async () => { usersService = { isOidcEnabled: jest.fn(), + isRegistrationAllowedByConfig: jest.fn().mockReturnValue(true), + isRegistrationEnabled: jest.fn(), oidcGetAuthorizationUrl: jest.fn(), oidcBuildFrontendRedirect: jest.fn(), oidcHandleCallback: jest.fn(), @@ -66,10 +71,6 @@ describe('Users Controller', () => { controller = module.get(UsersController); }); - it('should be defined', () => { - expect(controller).toBeDefined(); - }); - it('returns the current user', () => { const result = controller.me(user); expect(result).toBeInstanceOf(UserDto); @@ -81,165 +82,127 @@ describe('Users Controller', () => { expect(controller.oidcEnabled()).toBe(true); }); - it('redirects to the frontend with an error when the state cookie is missing', async () => { - usersService.isOidcEnabled.mockReturnValue(true); - usersService.oidcBuildFrontendRedirect.mockReturnValue('http://localhost:8080/login#error=oidc_login_failed'); - const res = { clearCookie: jest.fn(), redirect: jest.fn(), status: jest.fn().mockReturnThis(), send: jest.fn() }; - - await controller.oidcCallback({ headers: {}, query: {} } as never, res as never); - - expect(usersService.oidcHandleCallback).not.toHaveBeenCalled(); - expect(res.redirect).toHaveBeenCalledWith('http://localhost:8080/login#error=oidc_login_failed'); - }); - - it('redirects to the frontend with an error when the state cookie is malformed', async () => { - usersService.isOidcEnabled.mockReturnValue(true); - usersService.oidcBuildFrontendRedirect.mockReturnValue('http://localhost:8080/login#error=oidc_login_failed'); - const res = { clearCookie: jest.fn(), redirect: jest.fn(), status: jest.fn().mockReturnThis(), send: jest.fn() }; - - await controller.oidcCallback({ headers: { cookie: 'oidc_state=%E0%A4%A' }, query: {} } as never, res as never); - - expect(usersService.oidcHandleCallback).not.toHaveBeenCalled(); - expect(res.redirect).toHaveBeenCalledWith('http://localhost:8080/login#error=oidc_login_failed'); - }); - - it('redirects to the frontend with an error when the identity provider is unreachable', async () => { - usersService.isOidcEnabled.mockReturnValue(true); - usersService.oidcGetAuthorizationUrl.mockRejectedValue(new Error('discovery down')); - usersService.oidcBuildFrontendRedirect.mockReturnValue('http://localhost:8080/login#error=oidc_login_failed'); - const res = { cookie: jest.fn(), redirect: jest.fn() }; - - await controller.oidcLogin(res as never); - - expect(res.cookie).not.toHaveBeenCalled(); - expect(res.redirect).toHaveBeenCalledWith('http://localhost:8080/login#error=oidc_login_failed'); - }); + describe('OIDC', () => { + const mockRes = () => ({ + cookie: jest.fn(), + clearCookie: jest.fn(), + redirect: jest.fn(), + status: jest.fn().mockReturnThis(), + send: jest.fn(), + }); + const callback = (res: ReturnType, headers: object = {}, query: object = {}) => + controller.oidcCallback({ headers, query } as never, res as never); + const stateCookie = { cookie: 'oidc_state=state.verifier' }; - it('returns 404 on login when OIDC is disabled', async () => { - usersService.isOidcEnabled.mockReturnValue(false); - const res = { cookie: jest.fn(), redirect: jest.fn(), status: jest.fn().mockReturnThis(), send: jest.fn() }; + beforeEach(() => { + usersService.isOidcEnabled.mockReturnValue(true); + usersService.oidcBuildFrontendRedirect.mockImplementation((token?: string, error?: string) => + token ? `front#token=${token}` : `front#error=${error}` + ); + }); - await controller.oidcLogin(res as never); + it.each([ + ['is missing', {}], + ['is missing among other cookies', { cookie: 'a=1; b=2' }], + ['is malformed', { cookie: 'oidc_state=%E0%A4%A' }], + ])('redirects to the frontend with an error when the state cookie %s', async (_case, headers) => { + const res = mockRes(); - expect(res.status).toHaveBeenCalledWith(404); - expect(usersService.oidcGetAuthorizationUrl).not.toHaveBeenCalled(); - }); + await callback(res, headers); - it('redirects to the identity provider and stores the PKCE state cookie', async () => { - usersService.isOidcEnabled.mockReturnValue(true); - usersService.oidcGetAuthorizationUrl.mockResolvedValue({ - url: 'https://idp.example.com/authorize', - state: 'state', - codeVerifier: 'verifier', + expect(usersService.oidcHandleCallback).not.toHaveBeenCalled(); + expect(usersService.oidcBuildFrontendRedirect).toHaveBeenCalledWith(undefined, 'oidc_login_failed'); + expect(res.clearCookie).toHaveBeenCalledWith('oidc_state', { path: '/' }); + expect(res.redirect).toHaveBeenCalledWith('front#error=oidc_login_failed'); }); - const res = { cookie: jest.fn(), redirect: jest.fn() }; - - await controller.oidcLogin(res as never); - - expect(res.cookie).toHaveBeenCalledWith( - 'oidc_state', - 'state.verifier', - expect.objectContaining({ httpOnly: true, sameSite: 'lax' }) - ); - expect(res.redirect).toHaveBeenCalledWith('https://idp.example.com/authorize'); - }); - it('redirects to the frontend with the token after a successful callback', async () => { - usersService.isOidcEnabled.mockReturnValue(true); - usersService.oidcHandleCallback.mockResolvedValue({ token: 'jwt' }); - usersService.oidcBuildFrontendRedirect.mockReturnValue('http://localhost:8080/login#token=jwt'); - const res = { clearCookie: jest.fn(), redirect: jest.fn(), status: jest.fn().mockReturnThis(), send: jest.fn() }; - const req = { - headers: { cookie: 'oidc_state=state.verifier' }, - query: { code: 'code', state: 'state', repeated: ['a', 'b'] }, - }; + it.each([new Error('discovery down'), 'plain failure'])( + 'redirects to the frontend with an error when the authorization url rejects with %j', + async (failure) => { + usersService.oidcGetAuthorizationUrl.mockRejectedValue(failure); + const res = mockRes(); - await controller.oidcCallback(req as never, res as never); + await controller.oidcLogin(res as never); - expect(usersService.oidcHandleCallback).toHaveBeenCalledWith( - { code: 'code', state: 'state' }, - { state: 'state', codeVerifier: 'verifier' } + expect(res.cookie).not.toHaveBeenCalled(); + expect(res.redirect).toHaveBeenCalledWith('front#error=oidc_login_failed'); + } ); - expect(res.redirect).toHaveBeenCalledWith('http://localhost:8080/login#token=jwt'); - }); - it('returns 404 on the callback when OIDC is disabled', async () => { - usersService.isOidcEnabled.mockReturnValue(false); - const res = { clearCookie: jest.fn(), redirect: jest.fn(), status: jest.fn().mockReturnThis(), send: jest.fn() }; + it('returns 404 on login when OIDC is disabled', async () => { + usersService.isOidcEnabled.mockReturnValue(false); + const res = mockRes(); - await controller.oidcCallback({ headers: {}, query: {} } as never, res as never); - - expect(res.status).toHaveBeenCalledWith(404); - expect(res.send).toHaveBeenCalled(); - }); + await controller.oidcLogin(res as never); - describe('OIDC cookie handling', () => { - const okResponse = () => ({ - clearCookie: jest.fn(), - redirect: jest.fn(), - status: jest.fn().mockReturnThis(), - send: jest.fn(), + expect(res.status).toHaveBeenCalledWith(404); + expect(usersService.oidcGetAuthorizationUrl).not.toHaveBeenCalled(); }); - beforeEach(() => { - usersService.isOidcEnabled.mockReturnValue(true); - usersService.oidcBuildFrontendRedirect.mockImplementation((token?: string, error?: string) => - token ? `front#token=${token}` : `front#error=${error}` + it('redirects to the identity provider and stores the PKCE state cookie', async () => { + usersService.oidcGetAuthorizationUrl.mockResolvedValue({ + url: 'https://idp.example.com/authorize', + state: 'state', + codeVerifier: 'verifier', + }); + const res = mockRes(); + + await controller.oidcLogin(res as never); + + expect(res.cookie).toHaveBeenCalledWith( + 'oidc_state', + 'state.verifier', + expect.objectContaining({ httpOnly: true, sameSite: 'lax' }) ); + expect(res.redirect).toHaveBeenCalledWith('https://idp.example.com/authorize'); }); - it('finds the state cookie among other cookies and decodes it', async () => { + it('redirects to the frontend with the token after a successful callback', async () => { usersService.oidcHandleCallback.mockResolvedValue({ token: 'jwt' }); - const res = okResponse(); + const res = mockRes(); - await controller.oidcCallback( - { headers: { cookie: 'a=1; oidc_state=st%2Ea.ver=ifier; b=2' }, query: {} } as never, - res as never - ); + await callback(res, stateCookie, { code: 'code', state: 'state', repeated: ['a', 'b'] }); - // "=" inside the value is kept; after decoding it is split on every "." and - // only the first two parts are used. - // NOTE: current behavior — a verifier containing "." (allowed by RFC 7636) would be truncated - expect(usersService.oidcHandleCallback).toHaveBeenCalledWith({}, { state: 'st', codeVerifier: 'a' }); - expect(res.clearCookie).toHaveBeenCalledWith('oidc_state', { path: '/' }); + expect(usersService.oidcHandleCallback).toHaveBeenCalledWith( + { code: 'code', state: 'state' }, + { state: 'state', codeVerifier: 'verifier' } + ); expect(res.redirect).toHaveBeenCalledWith('front#token=jwt'); }); - it('fails the callback when other cookies are present but not the state cookie', async () => { - const res = okResponse(); + it('returns 404 on the callback when OIDC is disabled', async () => { + usersService.isOidcEnabled.mockReturnValue(false); + const res = mockRes(); - await controller.oidcCallback({ headers: { cookie: 'a=1; b=2' }, query: {} } as never, res as never); + await callback(res); - expect(usersService.oidcHandleCallback).not.toHaveBeenCalled(); - expect(usersService.oidcBuildFrontendRedirect).toHaveBeenCalledWith(undefined, 'oidc_login_failed'); - expect(res.clearCookie).toHaveBeenCalledWith('oidc_state', { path: '/' }); - expect(res.redirect).toHaveBeenCalledWith('front#error=oidc_login_failed'); + expect(res.status).toHaveBeenCalledWith(404); + expect(res.send).toHaveBeenCalled(); }); - it('fails the callback when the identity provider rejects the code', async () => { - usersService.oidcHandleCallback.mockRejectedValue(new Error('invalid_grant')); - const res = okResponse(); + it('finds the state cookie among other cookies and splits it at the first dot (generated verifiers are base64url, without dots)', async () => { + usersService.oidcHandleCallback.mockResolvedValue({ token: 'jwt' }); + const res = mockRes(); - await controller.oidcCallback( - { headers: { cookie: 'oidc_state=state.verifier' }, query: { code: 'c' } } as never, - res as never - ); + await callback(res, { cookie: 'a=1; oidc_state=st%2Ea.ver=ifier; b=2' }); + expect(usersService.oidcHandleCallback).toHaveBeenCalledWith({}, { state: 'st', codeVerifier: 'a' }); expect(res.clearCookie).toHaveBeenCalledWith('oidc_state', { path: '/' }); - expect(res.redirect).toHaveBeenCalledWith('front#error=oidc_login_failed'); + expect(res.redirect).toHaveBeenCalledWith('front#token=jwt'); }); - it('fails the callback when the identity provider rejects with a non-Error', async () => { - usersService.oidcHandleCallback.mockRejectedValue('plain failure'); - const res = okResponse(); + it.each([new Error('invalid_grant'), 'plain failure'])( + 'fails the callback when the identity provider rejects with %j', + async (failure) => { + usersService.oidcHandleCallback.mockRejectedValue(failure); + const res = mockRes(); - await controller.oidcCallback( - { headers: { cookie: 'oidc_state=state.verifier' }, query: {} } as never, - res as never - ); + await callback(res, stateCookie, { code: 'c' }); - expect(res.redirect).toHaveBeenCalledWith('front#error=oidc_login_failed'); - }); + expect(res.clearCookie).toHaveBeenCalledWith('oidc_state', { path: '/' }); + expect(res.redirect).toHaveBeenCalledWith('front#error=oidc_login_failed'); + } + ); it.each([ ['https://vrt.example.com/callback', true], @@ -249,7 +212,7 @@ describe('Users Controller', () => { key === 'OIDC_REDIRECT_URI' ? redirectUri : defaultValue ); usersService.oidcGetAuthorizationUrl.mockResolvedValue({ url: 'idp', state: 's', codeVerifier: 'v' }); - const res = { cookie: jest.fn(), redirect: jest.fn() }; + const res = mockRes(); await controller.oidcLogin(res as never); @@ -262,15 +225,6 @@ describe('Users Controller', () => { path: '/', }); }); - - it('redirects to the login page when the authorization url rejects with a non-Error', async () => { - usersService.oidcGetAuthorizationUrl.mockRejectedValue('plain failure'); - const res = { cookie: jest.fn(), redirect: jest.fn() }; - - await controller.oidcLogin(res as never); - - expect(res.redirect).toHaveBeenCalledWith('front#error=oidc_login_failed'); - }); }); describe('account handlers', () => { @@ -282,6 +236,21 @@ describe('Users Controller', () => { expect(usersService.create).toHaveBeenCalledWith(dto); }); + it('refuses to register a user when registration is switched off', () => { + usersService.isRegistrationAllowedByConfig.mockReturnValue(false); + + expect(() => + controller.register({ email: 'new@example.com', firstName: 'N', lastName: 'U', password: 'secret' }) + ).toThrow(ForbiddenException); + expect(usersService.create).not.toHaveBeenCalled(); + }); + + it('reports whether registration is enabled', () => { + usersService.isRegistrationEnabled.mockReturnValue(false); + + expect(controller.registrationEnabled()).toBe(false); + }); + it('logs a user in', async () => { const dto = { email: 'test@example.com', password: 'secret' }; usersService.login.mockResolvedValue('logged in'); @@ -300,7 +269,7 @@ describe('Users Controller', () => { it('changes the password of the current user', async () => { usersService.changePassword.mockResolvedValue(true); - await expect(controller.changePassword(user, 'new password')).resolves.toBe(true); + await expect(controller.changePassword(user, { password: 'new password' })).resolves.toBe(true); expect(usersService.changePassword).toHaveBeenCalledWith(user, 'new password'); }); @@ -314,14 +283,18 @@ describe('Users Controller', () => { }); describe('admin handlers', () => { - it('lists every user as a UserDto without the password', async () => { - prismaService.user.findMany.mockResolvedValue([user, { ...user, id: '2', email: 'other@example.com' }]); + it('lists every user as a summary without the password and the api key', async () => { + const other = { ...user, id: '2', email: 'other@example.com' }; + prismaService.user.findMany.mockResolvedValue([user, other]); const result = await controller.userList(); expect(prismaService.user.findMany).toHaveBeenCalledWith(); - expect(result).toEqual([new UserDto(user), new UserDto({ ...user, id: '2', email: 'other@example.com' })]); - expect(result[0]).not.toHaveProperty('password'); + expect(result).toEqual([toUserSummary(user), toUserSummary(other)]); + for (const summary of result) { + expect(summary).not.toHaveProperty('password'); + expect(summary).not.toHaveProperty('apiKey'); + } }); it('deletes the users in one go', async () => { diff --git a/src/users/users.controller.ts b/src/users/users.controller.ts index 9d89ea37..40998683 100644 --- a/src/users/users.controller.ts +++ b/src/users/users.controller.ts @@ -11,6 +11,8 @@ import { Res, HttpStatus, Header, + Logger, + ForbiddenException, } from '@nestjs/common'; import { Request, Response } from 'express'; import { UsersService } from './users.service'; @@ -18,7 +20,7 @@ import { ApiOkResponse, ApiBearerAuth, ApiTags } from '@nestjs/swagger'; import { UserLoginResponseDto } from './dto/user-login-response.dto'; import { CreateUserDto } from './dto/user-create.dto'; import { JwtAuthGuard } from '../auth/guards/auth.guard'; -import { UserDto } from './dto/user.dto'; +import { UserDto, UserSummaryDto, toUserSummary } from './dto/user.dto'; import { UpdateUserDto } from './dto/user-update.dto'; import { UserLoginRequestDto } from './dto/user-login-request.dto'; import { CurrentUser } from '../shared/current-user.decorator'; @@ -27,7 +29,7 @@ import { RoleGuard } from '../auth/guards/role.guard'; import { Roles } from '../shared/roles.decorator'; import { PrismaService } from '../prisma/prisma.service'; import { AssignRoleDto } from './dto/assign-role.dto'; -import { Logger } from '@nestjs/common'; +import { ChangePasswordDto } from './dto/change-password.dto'; import { ConfigService } from '@nestjs/config'; const OIDC_STATE_COOKIE = 'oidc_state'; @@ -60,9 +62,19 @@ export class UsersController { @Post('register') @ApiOkResponse({ type: UserLoginResponseDto }) register(@Body() createUserDto: CreateUserDto): Promise { + if (!this.usersService.isRegistrationAllowedByConfig()) { + throw new ForbiddenException('User registration is disabled.'); + } return this.usersService.create(createUserDto); } + @Get('registration/enabled') + @ApiOkResponse({ type: Boolean }) + @Header('Content-Type', 'application/json') + registrationEnabled(): boolean { + return this.usersService.isRegistrationEnabled(); + } + @Post('login') @ApiOkResponse({ type: UserLoginResponseDto }) async login(@Body() userLoginRequestDto: UserLoginRequestDto): Promise { @@ -88,8 +100,6 @@ export class UsersController { try { authorization = await this.usersService.oidcGetAuthorizationUrl(); } catch (e) { - // The browser navigated here directly, so send it back to the login page - // instead of showing a raw JSON error (e.g. when discovery fails). this.logger.warn(`OIDC login failed: ${(e as Error)?.message ?? e}`); res.redirect(this.usersService.oidcBuildFrontendRedirect(undefined, 'oidc_login_failed')); return; @@ -114,13 +124,10 @@ export class UsersController { let target = this.usersService.oidcBuildFrontendRedirect(undefined, 'oidc_login_failed'); try { - // Inside the try: decodeURIComponent throws on a malformed cookie value. const [state, codeVerifier] = (readCookie(req, OIDC_STATE_COOKIE) ?? '').split('.'); if (!state || !codeVerifier) { throw new Error('Missing or malformed OIDC state cookie.'); } - // Repeated query keys produce arrays; keep only scalar strings so the - // callback sees a plain parameter map. const params: Record = {}; for (const [key, value] of Object.entries(req.query)) { if (typeof value === 'string') { @@ -160,8 +167,8 @@ export class UsersController { @ApiOkResponse({ type: Boolean }) @ApiBearerAuth() @UseGuards(JwtAuthGuard) - changePassword(@CurrentUser() user: User, @Body('password') password: string): Promise { - return this.usersService.changePassword(user, password); + changePassword(@CurrentUser() user: User, @Body() body: ChangePasswordDto): Promise { + return this.usersService.changePassword(user, body.password); } @Put() @@ -173,13 +180,13 @@ export class UsersController { } @Get('all') - @ApiOkResponse({ type: UserDto, isArray: true }) + @ApiOkResponse({ type: UserSummaryDto, isArray: true }) @ApiBearerAuth() @UseGuards(JwtAuthGuard, RoleGuard) @Roles(Role.admin) - async userList(): Promise { + async userList(): Promise { const users = await this.prismaService.user.findMany(); - return users.map((user) => new UserDto(user)); + return users.map(toUserSummary); } @Delete() @@ -193,11 +200,11 @@ export class UsersController { } @Patch('assignRole') - @ApiOkResponse({ type: UserDto }) + @ApiOkResponse({ type: UserSummaryDto }) @ApiBearerAuth() @UseGuards(JwtAuthGuard, RoleGuard) @Roles(Role.admin) - async assignRole(@Body() data: AssignRoleDto): Promise { + async assignRole(@Body() data: AssignRoleDto): Promise { this.logger.debug(`Going to assign role ${data.role} to User: ${data.id}`); return this.usersService.assignRole(data); } diff --git a/src/users/users.factory.spec.ts b/src/users/users.factory.spec.ts index 40a9ffce..65cda525 100644 --- a/src/users/users.factory.spec.ts +++ b/src/users/users.factory.spec.ts @@ -48,31 +48,14 @@ describe('UsersFactoryService', () => { configService = module.get(ConfigService); }); - it('should be defined', () => { - expect(service).toBeDefined(); - }); - - it('should return DbUsersService when neither OIDC_ENABLED nor LDAP_ENABLED is true', () => { - withFlags({}); - const result = service.getUsersService(); - expect(result).toBeInstanceOf(DbUsersService); - }); - - it('should return LdapUsersService when LDAP_ENABLED is true', () => { - withFlags({ ldap: true }); - const result = service.getUsersService(); - expect(result).toBeInstanceOf(LdapUsersService); - }); - - it('should return OidcUsersService when OIDC_ENABLED is true', () => { - withFlags({ oidc: true }); - const result = service.getUsersService(); - expect(result).toBeInstanceOf(OidcUsersService); - }); - - it('should prefer OidcUsersService when both OIDC_ENABLED and LDAP_ENABLED are true', () => { - withFlags({ oidc: true, ldap: true }); - const result = service.getUsersService(); - expect(result).toBeInstanceOf(OidcUsersService); + it.each([ + ['neither OIDC_ENABLED nor LDAP_ENABLED is true', {}, DbUsersService], + ['LDAP_ENABLED is true', { ldap: true }, LdapUsersService], + ['OIDC_ENABLED is true', { oidc: true }, OidcUsersService], + ['both OIDC_ENABLED and LDAP_ENABLED are true, preferring OIDC', { oidc: true, ldap: true }, OidcUsersService], + ])('returns the matching implementation when %s', (_case, flags, implementation) => { + withFlags(flags); + + expect(service.getUsersService()).toBeInstanceOf(implementation); }); }); diff --git a/src/users/users.factory.ts b/src/users/users.factory.ts index 5ec1cf2a..4c05fa4a 100644 --- a/src/users/users.factory.ts +++ b/src/users/users.factory.ts @@ -30,14 +30,12 @@ export class UsersFactoryService { return new OidcUsersService(this.configService, this.prismaService, this.authService); } - switch (ldapEnabled) { - case true: - this.logger.debug('users service type: LDAP'); - return new LdapUsersService(this.configService, this.prismaService, this.authService); - case false: - default: - this.logger.debug('users service type: DB'); - return new DbUsersService(this.prismaService, this.authService); + if (ldapEnabled) { + this.logger.debug('users service type: LDAP'); + return new LdapUsersService(this.configService, this.prismaService, this.authService); } + + this.logger.debug('users service type: DB'); + return new DbUsersService(this.prismaService, this.authService); } } diff --git a/src/users/users.interface.ts b/src/users/users.interface.ts index 4ac84267..0d9362f9 100644 --- a/src/users/users.interface.ts +++ b/src/users/users.interface.ts @@ -5,6 +5,7 @@ import { UpdateUserDto } from './dto/user-update.dto'; import { UserLoginRequestDto } from './dto/user-login-request.dto'; export interface Users { + readonly allowsRegistration: boolean; create(createUserDto: CreateUserDto): Promise; update(id: string, userDto: UpdateUserDto): Promise; changePassword(user: User, newPassword: string): Promise; diff --git a/src/users/users.service.spec.ts b/src/users/users.service.spec.ts index 3ebc4228..b7574a48 100644 --- a/src/users/users.service.spec.ts +++ b/src/users/users.service.spec.ts @@ -9,9 +9,10 @@ import { Role, User } from '@prisma/client'; import { UserLoginResponseDto } from './dto/user-login-response.dto'; import { AssignRoleDto } from './dto/assign-role.dto'; import { randomUUID } from 'crypto'; -import { UserDto } from './dto/user.dto'; +import { toUserSummary } from './dto/user.dto'; import { UpdateUserDto } from './dto/user-update.dto'; import { OidcUsersService } from './oidc/oidcusers.service'; +import { LdapUsersService } from './ldap/ldapusers.service'; const user: User = { id: '1', @@ -29,17 +30,17 @@ const user: User = { describe('UsersService with DbUserService implementation', () => { let service: UsersService; let prismaService: PrismaService; + let configService: { get: jest.Mock }; beforeEach(async () => { + configService = { get: jest.fn() }; const module: TestingModule = await Test.createTestingModule({ providers: [ UsersService, UsersFactoryService, { provide: ConfigService, - useValue: { - get: jest.fn(), - }, + useValue: configService, }, { provide: PrismaService, @@ -69,25 +70,17 @@ describe('UsersService with DbUserService implementation', () => { prismaService = module.get(PrismaService); }); - it('should be defined', () => { - expect(service).toBeDefined(); - }); - it('should generate new api key', async () => { - // Arrange const prismaUserUpdateMock = jest.spyOn(prismaService.user, 'update'); prismaUserUpdateMock.mockResolvedValueOnce(user); - // Act const result = await service.generateNewApiKey(user); - // Assert expect(prismaUserUpdateMock).toHaveBeenCalled(); expect(typeof result).toBe('string'); }); it('should create a new user', async () => { - // Arrange const createUserDto = { email: 'test@example.com', firstName: 'John', @@ -98,10 +91,8 @@ describe('UsersService with DbUserService implementation', () => { const prismaUserCreateMock = jest.spyOn(prismaService.user, 'create'); prismaUserCreateMock.mockResolvedValueOnce(user); - // Act const result = await service.create(createUserDto); - // Assert expect(prismaUserCreateMock).toHaveBeenCalledWith({ data: { email: 'test@example.com', @@ -115,7 +106,6 @@ describe('UsersService with DbUserService implementation', () => { }); it('should update a user', async () => { - // Arrange const updateUserDto: UpdateUserDto = { email: 'test@example.com', firstName: 'John', @@ -125,43 +115,33 @@ describe('UsersService with DbUserService implementation', () => { const prismaUserUpdateMock = jest.spyOn(prismaService.user, 'update'); prismaUserUpdateMock.mockResolvedValueOnce(user); - // Act const result = await service.update(user.id, updateUserDto); - // Assert expect(prismaUserUpdateMock).toHaveBeenCalled(); expect(result).toEqual(expect.any(UserLoginResponseDto)); }); it('should change password', async () => { - // Arrange const prismaUserUpdateMock = jest.spyOn(prismaService.user, 'update'); prismaUserUpdateMock.mockResolvedValueOnce(user); - // Act const result = await service.changePassword(user, 'newPassword'); - // Assert expect(result).toEqual(true); }); it('should login existing user', async () => { - // Act const result = await service.login({ email: 'test@example.com', password: 'doesntmatter' }); - // Assert expect(result).toEqual(expect.any(UserLoginResponseDto)); }); it('should delete existing user', async () => { - // Arrange const prismaUserDeleteMock = jest.spyOn(prismaService.user, 'delete'); prismaUserDeleteMock.mockResolvedValueOnce(user); - // Act await service.delete(user.id); - // Assert expect(prismaUserDeleteMock).toHaveBeenCalledWith({ where: { id: user.id } }); }); @@ -190,15 +170,6 @@ describe('UsersService with DbUserService implementation', () => { }); }); - it('should get existing user', async () => { - // Act - const result = await service.get(user.id); - - // Assert - expect(result).toBeInstanceOf(UserDto); - expect(prismaService.user.findUnique).toHaveBeenCalledWith({ where: { id: user.id } }); - }); - it('looks a user up by id', async () => { const found = await service.findOne(user.id); @@ -207,7 +178,6 @@ describe('UsersService with DbUserService implementation', () => { }); it('should assign role to user', async () => { - // Arrange const assignRoleDto: AssignRoleDto = { id: randomUUID(), role: Role.editor, @@ -216,12 +186,28 @@ describe('UsersService with DbUserService implementation', () => { const prismaUserCreateMock = jest.spyOn(prismaService.user, 'update'); prismaUserCreateMock.mockResolvedValueOnce(user); - // Act const result = await service.assignRole(assignRoleDto); - // Assert expect(prismaUserCreateMock).toHaveBeenCalled(); - expect(result).toBeInstanceOf(UserDto); + expect(result).toEqual(toUserSummary(user)); + expect(result).not.toHaveProperty('apiKey'); + }); + + it.each([ + [undefined, true], + ['true', true], + ['TRUE', true], + ['yes', true], + ['', true], + ['false', false], + ['0', false], + [' OFF ', false], + ['No', false], + ])('with REGISTRATION_ENABLED %j registration is allowed: %s', (value, allowed) => { + configService.get.mockImplementation((key: string) => (key === 'REGISTRATION_ENABLED' ? value : undefined)); + + expect(service.isRegistrationAllowedByConfig()).toBe(allowed); + expect(service.isRegistrationEnabled()).toBe(allowed); }); it('reports OIDC as disabled for the DB implementation', () => { @@ -233,34 +219,44 @@ describe('UsersService with DbUserService implementation', () => { }); }); -describe('UsersService with OidcUsersService implementation', () => { - let service: UsersService; - let oidcUsersService: OidcUsersService; +const serviceWith = async (implementation: object, get = jest.fn()) => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ + UsersService, + { provide: UsersFactoryService, useValue: { getUsersService: () => implementation } }, + { provide: PrismaService, useValue: {} }, + { provide: AuthService, useValue: {} }, + { provide: ConfigService, useValue: { get } }, + ], + }).compile(); + return module.get(UsersService); +}; - beforeEach(async () => { - oidcUsersService = Object.assign(Object.create(OidcUsersService.prototype), { +describe('UsersService with an identity-provider implementation', () => { + const withoutRegistration = (prototype: object, members: object = {}) => + Object.assign(Object.create(prototype), { allowsRegistration: false, ...members }); + + it.each([ + ['OIDC', OidcUsersService.prototype, true], + ['LDAP', LdapUsersService.prototype, false], + ])('%s: registration is disabled whatever the flag says', async (_name, prototype, oidc) => { + const service = await serviceWith( + withoutRegistration(prototype), + jest.fn(() => 'true') + ); + + expect(service.isRegistrationAllowedByConfig()).toBe(true); + expect(service.isRegistrationEnabled()).toBe(false); + expect(service.isOidcEnabled()).toBe(oidc); + }); + + it('delegates authorization url, callback and redirect building to OIDC', async () => { + const oidcUsersService = withoutRegistration(OidcUsersService.prototype, { getAuthorizationUrl: jest.fn(), handleCallback: jest.fn(), buildFrontendRedirect: jest.fn(), }) as OidcUsersService; - - const module: TestingModule = await Test.createTestingModule({ - providers: [ - UsersService, - { provide: UsersFactoryService, useValue: { getUsersService: () => oidcUsersService } }, - { provide: PrismaService, useValue: {} }, - { provide: AuthService, useValue: {} }, - ], - }).compile(); - - service = module.get(UsersService); - }); - - it('reports OIDC as enabled', () => { - expect(service.isOidcEnabled()).toBe(true); - }); - - it('delegates authorization url, callback and redirect building', async () => { + const service = await serviceWith(oidcUsersService); const authorization = { url: 'https://idp/authorize', state: 'state', codeVerifier: 'verifier' }; (oidcUsersService.getAuthorizationUrl as jest.Mock).mockResolvedValue(authorization); (oidcUsersService.handleCallback as jest.Mock).mockResolvedValue({ token: 'jwt' }); diff --git a/src/users/users.service.ts b/src/users/users.service.ts index 63cd9f38..2d6c6cc5 100644 --- a/src/users/users.service.ts +++ b/src/users/users.service.ts @@ -1,17 +1,17 @@ -import { Injectable } from '@nestjs/common'; import { CreateUserDto } from './dto/user-create.dto'; import { UserLoginResponseDto } from './dto/user-login-response.dto'; import { PrismaService } from '../prisma/prisma.service'; import { User } from '@prisma/client'; -import { UserDto } from './dto/user.dto'; +import { UserSummaryDto, toUserSummary } from './dto/user.dto'; import { UpdateUserDto } from './dto/user-update.dto'; import { AuthService } from '../auth/auth.service'; import { UserLoginRequestDto } from './dto/user-login-request.dto'; import { AssignRoleDto } from './dto/assign-role.dto'; -import { HttpException, HttpStatus, Logger, NotFoundException } from '@nestjs/common'; +import { HttpException, HttpStatus, Injectable, Logger, NotFoundException } from '@nestjs/common'; import { UsersFactoryService } from './users.factory'; import { Users } from './users.interface'; import { OidcUsersService } from './oidc/oidcusers.service'; +import { ConfigService } from '@nestjs/config'; @Injectable() export class UsersService { @@ -21,7 +21,8 @@ export class UsersService { constructor( private readonly usersFactoryService: UsersFactoryService, private prismaService: PrismaService, - private authService: AuthService + private authService: AuthService, + private readonly configService: ConfigService ) { this.usersService = this.usersFactoryService.getUsersService(); } @@ -42,6 +43,18 @@ export class UsersService { return this.usersService.login(userLoginRequestDto); } + // default true: switching registration off is the operator's choice, not an upgrade side effect + isRegistrationAllowedByConfig(): boolean { + const value = String(this.configService.get('REGISTRATION_ENABLED') ?? 'true') + .trim() + .toLowerCase(); + return !['false', '0', 'no', 'off'].includes(value); + } + + isRegistrationEnabled(): boolean { + return this.usersService.allowsRegistration && this.isRegistrationAllowedByConfig(); + } + isOidcEnabled(): boolean { return this.usersService instanceof OidcUsersService; } @@ -77,10 +90,6 @@ export class UsersService { return this.prismaService.user.delete({ where: { id } }); } - /** - * All or nothing: an unknown id fails the request before anyone is removed, - * instead of after the users listed ahead of it are already gone. - */ async deleteMany(ids: string[]): Promise { this.logger.debug(`Removing Users: ${ids}`); await this.prismaService.$transaction(async (tx) => { @@ -93,12 +102,7 @@ export class UsersService { }); } - async get(id: string): Promise { - const user = await this.findOne(id); - return new UserDto(user!); - } - - async assignRole(data: AssignRoleDto): Promise { + async assignRole(data: AssignRoleDto): Promise { const { id, role } = data; this.logger.debug(`Assigning role ${role} to User: ${id}`); @@ -106,7 +110,7 @@ export class UsersService { where: { id }, data: { role }, }); - return new UserDto(user); + return toUserSummary(user); } async generateNewApiKey(user: User): Promise { diff --git a/src/utils/bounded-cache.ts b/src/utils/bounded-cache.ts new file mode 100644 index 00000000..dfe8fc6b --- /dev/null +++ b/src/utils/bounded-cache.ts @@ -0,0 +1,26 @@ +export class BoundedCache { + private readonly entries = new Map(); + + constructor(private readonly limit: number) {} + + get(key: string): T | undefined { + const value = this.entries.get(key); + if (value !== undefined) { + this.entries.delete(key); + this.entries.set(key, value); + } + return value; + } + + set(key: string, value: T): void { + this.entries.delete(key); + this.entries.set(key, value); + while (this.entries.size > this.limit) { + this.entries.delete(this.entries.keys().next().value!); + } + } + + delete(key: string): void { + this.entries.delete(key); + } +} diff --git a/src/utils/env.spec.ts b/src/utils/env.spec.ts new file mode 100644 index 00000000..5048560a --- /dev/null +++ b/src/utils/env.spec.ts @@ -0,0 +1,44 @@ +import { nonNegativeNumber, positiveNumber } from './env'; + +describe('nonNegativeNumber', () => { + it.each([ + ['unset', undefined], + ['empty', ''], + ['blank', ' '], + ['not a number', 'abc'], + ['negative', '-1'], + ['infinite', 'Infinity'], + ])('falls back when the value is %s', (_title, raw) => { + expect(nonNegativeNumber(raw, 7)).toBe(7); + }); + + it.each([ + ['0', 0], + ['12', 12], + [' 3.5 ', 3.5], + ])('accepts %j', (raw, expected) => { + expect(nonNegativeNumber(raw, 7)).toBe(expected); + }); +}); + +describe('positiveNumber', () => { + it.each([ + ['unset', undefined], + ['empty', ''], + ['blank', ' '], + ['zero', '0'], + ['negative', '-5'], + ['not a number', 'soon'], + ['infinite', 'Infinity'], + ])('falls back when the value is %s', (_title, raw) => { + expect(positiveNumber(raw, 9)).toBe(9); + }); + + it.each([ + ['1', 1], + ['2500', 2500], + ['0.5', 0.5], + ])('accepts %j', (raw, expected) => { + expect(positiveNumber(raw, 9)).toBe(expected); + }); +}); diff --git a/src/utils/env.ts b/src/utils/env.ts new file mode 100644 index 00000000..519a11ef --- /dev/null +++ b/src/utils/env.ts @@ -0,0 +1,10 @@ +export function nonNegativeNumber(raw: string | undefined, fallback: number): number { + if (!raw?.trim()) return fallback; + const value = Number(raw); + return Number.isFinite(value) && value >= 0 ? value : fallback; +} + +export function positiveNumber(raw: string | undefined, fallback: number): number { + const value = Number(raw); + return Number.isFinite(value) && value > 0 ? value : fallback; +} diff --git a/src/utils/index.spec.ts b/src/utils/index.spec.ts index 003f7971..41d4086f 100644 --- a/src/utils/index.spec.ts +++ b/src/utils/index.spec.ts @@ -1,5 +1,5 @@ import { generateTestRun, generateTestVariation } from '../_data_'; -import { getTestVariationUniqueData } from '.'; +import { getTestVariationUniqueData, parseIgnoreAreas, testRunImageNames, variationKey } from '.'; describe('getTestVariationUniqueData', () => { it('picks the fields that identify a test variation', () => { @@ -53,3 +53,70 @@ describe('getTestVariationUniqueData', () => { }); }); }); + +describe('parseIgnoreAreas', () => { + it('returns the stored areas', () => { + const areas = [{ x: 1, y: 2, width: 3, height: 4 }]; + + expect(parseIgnoreAreas(JSON.stringify(areas))).toEqual(areas); + }); + + it.each([ + ['null', null], + ['undefined', undefined], + ['an empty string', ''], + ['JSON null', 'null'], + ['an object', '{}'], + ['a string', '"str"'], + ['text that is not JSON', 'not json'], + ])('treats %s as no areas', (_case, stored) => { + expect(parseIgnoreAreas(stored)).toEqual([]); + }); + + it('drops entries that are not objects', () => { + expect(parseIgnoreAreas('[1,null,{"x":1}]')).toEqual([{ x: 1 }]); + }); +}); + +describe('testRunImageNames', () => { + it('lists every stored file of a run', () => { + expect( + testRunImageNames({ + imageName: 'i.png', + diffName: 'd.png', + imageThumbnailName: 'it.png', + diffThumbnailName: 'dt.png', + }).sort() + ).toEqual(['d.png', 'dt.png', 'i.png', 'it.png']); + }); + + it('drops missing and empty names', () => { + expect( + testRunImageNames({ imageName: 'i.png', diffName: null, imageThumbnailName: '', diffThumbnailName: null }) + ).toEqual(['i.png']); + }); +}); + +describe('variationKey', () => { + it('names the project and branch next to the screen the source belongs to', () => { + const source = { + name: 'screen', + os: 'linux', + browser: null, + device: undefined, + viewport: '800x600', + customTags: '', + }; + + expect(variationKey('projectId', 'feature', source as never)).toStrictEqual({ + projectId: 'projectId', + branchName: 'feature', + name: 'screen', + os: 'linux', + browser: '', + device: '', + viewport: '800x600', + customTags: '', + }); + }); +}); diff --git a/src/utils/index.ts b/src/utils/index.ts index e3d40068..bc948df0 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -1,5 +1,6 @@ import { TestRun, TestVariation } from '@prisma/client'; import { BaselineDataDto } from '../shared/dto/baseline-data.dto'; +import type { IgnoreAreaDto } from '../test-runs/dto/ignore-area.dto'; export const getTestVariationUniqueData = ( object: TestRun | TestVariation | BaselineDataDto @@ -20,3 +21,32 @@ export const getTestVariationUniqueData = ( customTags: object.customTags ?? '', }; }; + +export const variationKey = ( + projectId: string, + branchName: string, + source: TestRun | TestVariation | BaselineDataDto +): ReturnType & { projectId: string; branchName: string } => ({ + projectId, + branchName, + ...getTestVariationUniqueData(source), +}); + +export const testRunImageNames = ( + run: Pick +): string[] => + [run.imageName, run.diffName, run.imageThumbnailName, run.diffThumbnailName].filter((name): name is string => !!name); + +export const parseIgnoreAreas = (json: string | null | undefined): IgnoreAreaDto[] => { + if (!json) { + return []; + } + try { + const parsed: unknown = JSON.parse(json); + return Array.isArray(parsed) + ? parsed.filter((area): area is IgnoreAreaDto => typeof area === 'object' && area !== null) + : []; + } catch { + return []; + } +}; diff --git a/src/utils/map-with-concurrency.ts b/src/utils/map-with-concurrency.ts new file mode 100644 index 00000000..f688caef --- /dev/null +++ b/src/utils/map-with-concurrency.ts @@ -0,0 +1,12 @@ +export async function mapWithConcurrency(items: T[], limit: number, task: (item: T) => Promise): Promise { + const results = new Array(items.length); + let next = 0; + const worker = async (): Promise => { + while (next < items.length) { + const index = next++; + results[index] = await task(items[index]); + } + }; + await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker)); + return results; +} diff --git a/src/utils/random-key.ts b/src/utils/random-key.ts index d4f9c91d..fb914169 100644 --- a/src/utils/random-key.ts +++ b/src/utils/random-key.ts @@ -1,32 +1,17 @@ import { randomBytes } from 'crypto'; -// Crockford's base32 digits: no I, L, O or U, so a key read aloud or retyped -// is not mistaken. The alphabet of the keys issued before (uuid-apikey). const CROCKFORD_DIGITS = '0123456789ABCDEFGHJKMNPQRSTVWXYZ'; const KEY_LENGTH = 28; -/** - * A random 28-character key in the format API keys and image names have - * always had — Crockford base32, upper case, no dashes — so keys issued - * before keep working and clients see no change. Each character carries 5 - * random bits (140 in all, more than the 122 a v4 UUID gave). - */ export function generateRandomKey(): string { const bytes = randomBytes(KEY_LENGTH); let key = ''; for (const byte of bytes) { - // 256 is a multiple of 32, so taking the low 5 bits keeps every digit equally likely key += CROCKFORD_DIGITS[byte & 31]; } return key; } -/** - * Whether a project reference is an id rather than a name: dashes where a - * UUID has its first three, and 32 characters without them. The check - * uuid-apikey made, so references resolve as they did; an empty string is - * simply not a UUID instead of an exception. - */ export function isUuidShaped(value: string): boolean { return ( !!value && diff --git a/test/api-helpers.ts b/test/api-helpers.ts index b82a6b93..92afe1b3 100644 --- a/test/api-helpers.ts +++ b/test/api-helpers.ts @@ -1,23 +1,98 @@ -import { INestApplication, ValidationPipe } from '@nestjs/common'; +import { INestApplication, Type } from '@nestjs/common'; +import { NestExpressApplication } from '@nestjs/platform-express'; import { Test, TestingModule } from '@nestjs/testing'; +import { Build } from '@prisma/client'; +import { randomUUID } from 'crypto'; import { existsSync } from 'fs'; import path from 'path'; import { PNG } from 'pngjs'; +import { TEST_PROJECT } from '../src/_data_'; import { AppModule } from '../src/app.module'; +import { ProjectsService } from '../src/projects/projects.service'; +import { GlobalValidationPipe } from '../src/shared/validation'; import { HDD_IMAGE_PATH } from '../src/static/hdd/constants'; +import { TestRunsService } from '../src/test-runs/test-runs.service'; +import { LoggedUser, requestWithApiKey, requestWithAuth } from './preconditions'; -/** Boots the whole app the same way the other e2e specs do. */ -export const createE2eApp = async (): Promise<{ app: INestApplication; moduleFixture: TestingModule }> => { - const moduleFixture: TestingModule = await Test.createTestingModule({ - imports: [AppModule], - }).compile(); +interface E2eAppOptions { + overrides?: Array<[Type, unknown]>; + validation?: boolean; + configure?: (app: NestExpressApplication) => void; +} - const app = moduleFixture.createNestApplication(); - app.useGlobalPipes(new ValidationPipe()); +export const createE2eApp = async ({ overrides = [], validation = true, configure }: E2eAppOptions = {}) => { + let builder = Test.createTestingModule({ imports: [AppModule] }); + for (const [token, value] of overrides) { + builder = builder.overrideProvider(token).useValue(value); + } + const moduleFixture: TestingModule = await builder.compile(); + const app = moduleFixture.createNestApplication(); + if (validation) { + app.useGlobalPipes(new GlobalValidationPipe()); + } + configure?.(app); await app.init(); - return { app, moduleFixture }; + return { app, moduleFixture, get: (token: Type): T => moduleFixture.get(token) }; }; +export const haveProject = ( + projectsService: ProjectsService, + label: string, + overrides: Partial[0]> = {} +) => projectsService.create({ ...TEST_PROJECT, ...overrides, name: `${label} ${randomUUID()}` }); + +export const postRunDirect = (testRunsService: TestRunsService, build: Build, image: Buffer, name = 'Screen') => + testRunsService.postTestRun({ + createTestRequestDto: { projectId: build.projectId, buildId: build.id, branchName: build.branchName!, name }, + imageBuffer: image, + }); + +export const postRunBase64 = async ( + app: INestApplication, + user: LoggedUser, + build: Build, + image: Buffer, + extra: Record = {} +): Promise> => + ( + await requestWithApiKey(app, 'post', '/test-runs', user.apiKey) + .send({ + projectId: build.projectId, + buildId: build.id, + branchName: build.branchName, + name: 'Screen', + imageBase64: image.toString('base64'), + ...extra, + }) + .expect(201) + ).body; + +export const setEnv = (vars: Record): (() => void) => { + const previous = Object.fromEntries(Object.keys(vars).map((key) => [key, process.env[key]])); + Object.assign(process.env, vars); + return () => { + for (const [key, value] of Object.entries(previous)) { + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } + } + }; +}; + +export const withEnv = async (vars: Record, fn: () => PromiseLike): Promise => { + const restore = setEnv(vars); + try { + return await fn(); + } finally { + restore(); + } +}; + +export const approveRuns = (app: INestApplication, user: LoggedUser, ids: string[], merge = false) => + requestWithAuth(app, 'post', `/test-runs/approve?merge=${merge}`, user.token).send(ids).expect(201); + export interface Rect { x: number; y: number; @@ -26,10 +101,6 @@ export interface Rect { color: [number, number, number]; } -/** - * A tiny white PNG with solid rectangles painted on it. Synthetic images keep - * the diffs fast and make the change (its colour and its area) exact. - */ export const pngBuffer = (rects: Rect[] = [], width = 60, height = 60): Buffer => { const png = new PNG({ width, height }); png.data.fill(255); @@ -50,12 +121,14 @@ export const pngBuffer = (rects: Rect[] = [], width = 60, height = 60): Buffer = export const RED: [number, number, number] = [255, 0, 0]; export const BLUE: [number, number, number] = [0, 0, 255]; -/** Where the HDD storage keeps a stored image (the default STATIC_SERVICE). */ +export const WHITE = pngBuffer(); +export const redAt = (x: number, y: number, size = 10): Rect => ({ x, y, w: size, h: size, color: RED }); +export const CHANGED = pngBuffer([redAt(5, 5)]); + export const storedImagePath = (imageName: string): string => path.resolve(HDD_IMAGE_PATH, imageName); export const imageExists = (imageName: string | null): boolean => !!imageName && existsSync(storedImagePath(imageName)); -/** Polls until the predicate holds — for image deletions the app fires and does not await. */ export const waitFor = async (predicate: () => boolean | Promise, timeoutMs = 5000): Promise => { const deadline = Date.now() + timeoutMs; while (!(await predicate())) { @@ -66,7 +139,6 @@ export const waitFor = async (predicate: () => boolean | Promise, timeo } }; -/** Keys of TestRunDto — what the API exposes for a test run. */ export const TEST_RUN_DTO_KEYS = [ 'baselineBranchName', 'baselineName', diff --git a/test/body-limit.e2e-spec.ts b/test/body-limit.e2e-spec.ts new file mode 100644 index 00000000..a15475f7 --- /dev/null +++ b/test/body-limit.e2e-spec.ts @@ -0,0 +1,36 @@ +import { NestExpressApplication } from '@nestjs/platform-express'; +import request from 'supertest'; +import { createE2eApp } from './api-helpers'; + +describe('JSON body limit (e2e)', () => { + let app: NestExpressApplication; + + beforeAll(async () => { + ({ app } = await createE2eApp({ configure: (created) => created.useBodyParser('json', { limit: '1kb' }) })); + }); + + afterAll(async () => { + await app.close(); + }); + + it('answers 413 with the usual error body for a JSON body over the limit', async () => { + const res = await request(app.getHttpServer()) + .post('/users/login') + .send({ email: 'someone@example.com', password: 'x'.repeat(2048) }) + .expect(413); + + expect(res.body).toEqual({ + statusCode: 413, + path: '/users/login', + name: 'PayloadTooLargeError', + message: 'request entity too large', + }); + }); + + it('still answers a small body normally', async () => { + await request(app.getHttpServer()) + .post('/users/login') + .send({ email: 'nobody@example.com', password: 'x' }) + .expect((res) => expect(res.status).not.toBe(413)); + }); +}); diff --git a/test/builds-api.e2e-spec.ts b/test/builds-api.e2e-spec.ts index 9c66e54a..7aac7b80 100644 --- a/test/builds-api.e2e-spec.ts +++ b/test/builds-api.e2e-spec.ts @@ -1,15 +1,14 @@ import { INestApplication } from '@nestjs/common'; import { Build, Project, TestRun, TestStatus } from '@prisma/client'; import { randomUUID } from 'crypto'; -import { TEST_PROJECT } from '../src/_data_'; import { BuildDto } from '../src/builds/dto/build.dto'; import { BuildsService } from '../src/builds/builds.service'; import { PrismaService } from '../src/prisma/prisma.service'; import { ProjectsService } from '../src/projects/projects.service'; import { TestRunsService } from '../src/test-runs/test-runs.service'; import { UsersService } from '../src/users/users.service'; -import { createE2eApp, imageExists, pngBuffer, RED } from './api-helpers'; -import { haveUserLogged, requestWithApiKey, requestWithAuth, LoggedUser } from './preconditions'; +import { CHANGED, createE2eApp, haveProject, imageExists, postRunDirect, WHITE } from './api-helpers'; +import { haveUserLogged, requestWithApiKey, requestWithAuth, LoggedUser, withUser } from './preconditions'; describe('Builds API (e2e)', () => { let app: INestApplication; @@ -21,30 +20,23 @@ describe('Builds API (e2e)', () => { let user: LoggedUser; let project: Project; - const WHITE = pngBuffer(); - const CHANGED = pngBuffer([{ x: 5, y: 5, w: 10, h: 10, color: RED }]); - beforeAll(async () => { const created = await createE2eApp(); app = created.app; - usersService = created.moduleFixture.get(UsersService); - projectsService = created.moduleFixture.get(ProjectsService); - buildsService = created.moduleFixture.get(BuildsService); - testRunsService = created.moduleFixture.get(TestRunsService); - prisma = created.moduleFixture.get(PrismaService); + usersService = created.get(UsersService); + projectsService = created.get(ProjectsService); + buildsService = created.get(BuildsService); + testRunsService = created.get(TestRunsService); + prisma = created.get(PrismaService); }); beforeEach(async () => { user = await haveUserLogged(usersService); - project = await projectsService.create({ - ...TEST_PROJECT, - autoApproveFeature: false, - maxBuildAllowed: 100, - name: `Builds API E2E ${randomUUID()}`, - }); + project = await haveProject(projectsService, 'Builds API E2E', { autoApproveFeature: false, maxBuildAllowed: 100 }); }); afterEach(async () => { + await buildsService.whenRetentionIdle(); await projectsService.remove(project.id); await usersService.delete(user.id); }); @@ -56,13 +48,16 @@ describe('Builds API (e2e)', () => { const newBuild = (ciBuildId?: string, projectId = project.id): Promise => buildsService.findOrCreate({ projectId, branchName: 'master', ciBuildId }); - const postRun = (build: Build, image: Buffer, name = 'Screen') => - testRunsService.postTestRun({ - createTestRequestDto: { projectId: build.projectId, buildId: build.id, branchName: build.branchName!, name }, - imageBuffer: image, - }); + const postRun = (build: Build, image: Buffer, name?: string) => postRunDirect(testRunsService, build, image, name); + + const createBuild = async (ciBuildId?: string) => { + const res = await requestWithApiKey(app, 'post', '/builds', user.apiKey) + .send({ project: project.id, ciBuildId }) + .expect(201); + await buildsService.whenRetentionIdle(); + return res; + }; - /** The project's builds in the order the list endpoint promises: newest first, id breaking ties. */ const listOrder = async (): Promise => ( await prisma.build.findMany({ @@ -106,7 +101,7 @@ describe('Builds API (e2e)', () => { }); it('aroundBuildId that holds no page falls back to skip', async () => { - const otherProject = await projectsService.create({ ...TEST_PROJECT, name: `Other ${randomUUID()}` }); + const otherProject = await haveProject(projectsService, 'Other'); try { const foreign = await newBuild(undefined, otherProject.id); @@ -143,7 +138,6 @@ describe('Builds API (e2e)', () => { expect(res.body).toMatchObject({ total: 2, take: 1, skip: 1 }); expect(res.body.data.map((build: BuildDto) => build.id)).toEqual([oldestNightly]); - // a build the filter leaves out holds no page const excluded = await list(`take=1&skip=0&ciBuildId=nightly&aroundBuildId=${ordered[4]}`).expect(200); expect(excluded.body.skip).toBe(0); }); @@ -165,6 +159,7 @@ describe('Builds API (e2e)', () => { await list('take=2').expect(400); await list('take=two&skip=0').expect(400); await list('take=2&skip=0&aroundBuildId=nope').expect(400); + await list('take=10&skip=-1').expect(400); }); }); @@ -211,7 +206,6 @@ describe('Builds API (e2e)', () => { const res = await requestWithAuth(app, 'delete', `/builds/${build.id}`, user.token).expect(200); - // the raw build row with the runs it had expect(res.body.id).toBe(build.id); expect(res.body.testRuns.map((testRun: TestRun) => testRun.id)).toEqual([run.id]); expect(await prisma.build.findUnique({ where: { id: build.id } })).toBeNull(); @@ -239,12 +233,9 @@ describe('Builds API (e2e)', () => { it('403 for a guest', async () => { const build = await newBuild(); - const guest = await haveUserLogged(usersService, 'guest'); - try { - await requestWithAuth(app, 'delete', `/builds/${build.id}`, guest.token).expect(403); - } finally { - await usersService.delete(guest.id); - } + await withUser(usersService, 'guest', (guest) => + requestWithAuth(app, 'delete', `/builds/${build.id}`, guest.token).expect(403) + ); expect(await prisma.build.findUnique({ where: { id: build.id } })).not.toBeNull(); }); }); @@ -258,7 +249,7 @@ describe('Builds API (e2e)', () => { const unresolved = await postRun(build, CHANGED, 'Unresolved'); const fresh = await postRun(build, WHITE, 'New'); const failed = await postRun(build, WHITE, 'Failed'); - await testRunsService.setStatus(failed.id, TestStatus.failed); + await prisma.testRun.update({ where: { id: failed.id }, data: { status: TestStatus.failed } }); await requestWithAuth(app, 'patch', `/builds/${build.id}/approve?merge=false`, user.token).expect(200); @@ -270,7 +261,6 @@ describe('Builds API (e2e)', () => { [fresh.id]: TestStatus.approved, [failed.id]: TestStatus.failed, }); - // like POST /test-runs/approve, each new baseline records who approved it const baselines = await prisma.baseline.findMany({ where: { testRunId: { in: [unresolved.id, fresh.id] } } }); expect(baselines.map((baseline) => baseline.userId)).toEqual([user.id, user.id]); }); @@ -287,12 +277,9 @@ describe('Builds API (e2e)', () => { it('403 for a guest', async () => { const build = await newBuild(); - const guest = await haveUserLogged(usersService, 'guest'); - try { - await requestWithAuth(app, 'patch', `/builds/${build.id}/approve?merge=false`, guest.token).expect(403); - } finally { - await usersService.delete(guest.id); - } + await withUser(usersService, 'guest', (guest) => + requestWithAuth(app, 'patch', `/builds/${build.id}/approve?merge=false`, guest.token).expect(403) + ); }); }); @@ -347,8 +334,13 @@ describe('Builds API (e2e)', () => { }); describe('POST /builds retention (maxBuildAllowed)', () => { - const createBuild = () => - requestWithApiKey(app, 'post', '/builds', user.apiKey).send({ project: project.id }).expect(201); + beforeAll(() => { + process.env.BUILD_RETENTION_GRACE_MINUTES = '0'; + }); + + afterAll(() => { + delete process.env.BUILD_RETENTION_GRACE_MINUTES; + }); it('keeps at most maxBuildAllowed builds and removes the files of the dropped ones', async () => { await prisma.project.update({ where: { id: project.id }, data: { maxBuildAllowed: 2 } }); @@ -377,18 +369,39 @@ describe('Builds API (e2e)', () => { it('never prunes the build a CI build id reuses, and counts it as one of those kept', async () => { await prisma.project.update({ where: { id: project.id }, data: { maxBuildAllowed: 2 } }); - const joinShared = () => - requestWithApiKey(app, 'post', '/builds', user.apiKey) - .send({ project: project.id, ciBuildId: 'shared' }) - .expect(201); + const joinShared = () => createBuild('shared'); const shared = (await joinShared()).body; const newer = (await createBuild()).body; - // a parallel CI job joins the shared build, which is now the older of the two const joined = (await joinShared()).body; expect(joined.id).toBe(shared.id); expect((await listOrder()).sort()).toEqual([shared.id, newer.id].sort()); }); }); + + describe('POST /builds retention of builds in use', () => { + it('spares a build that is still receiving test runs', async () => { + await prisma.project.update({ where: { id: project.id }, data: { maxBuildAllowed: 1 } }); + const live = (await createBuild()).body; + await postRun({ ...live, projectId: project.id } as Build, WHITE); + + const newest = (await createBuild()).body; + + expect((await listOrder()).sort()).toEqual([live.id, newest.id].sort()); + }); + + it('removes a build idle for longer than the grace period', async () => { + await prisma.project.update({ where: { id: project.id }, data: { maxBuildAllowed: 1 } }); + const idle = (await createBuild()).body; + await postRun({ ...idle, projectId: project.id } as Build, WHITE); + const then = new Date(Date.now() - 2 * 60 * 60 * 1000); + await prisma.$executeRaw`UPDATE "Build" SET "updatedAt" = ${then} WHERE "id" = ${idle.id}`; + await prisma.testRun.updateMany({ where: { buildId: idle.id }, data: { createdAt: then } }); + + const newest = (await createBuild()).body; + + expect(await listOrder()).toEqual([newest.id]); + }); + }); }); diff --git a/test/builds.e2e-spec.ts b/test/builds.e2e-spec.ts index d4883764..6d279819 100644 --- a/test/builds.e2e-spec.ts +++ b/test/builds.e2e-spec.ts @@ -1,53 +1,43 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { INestApplication, ValidationPipe } from '@nestjs/common'; -import { AppModule } from '../src/app.module'; -import { UsersService } from '../src/users/users.service'; -import { haveTestRunCreated, haveUserLogged, requestWithApiKey, requestWithAuth, LoggedUser } from './preconditions'; -import { BuildsService } from '../src/builds/builds.service'; -import { CreateBuildDto } from '../src/builds/dto/build-create.dto'; +import { INestApplication } from '@nestjs/common'; import { Project } from '@prisma/client'; +import { BuildsController } from '../src/builds/builds.controller'; +import { CreateBuildDto } from '../src/builds/dto/build-create.dto'; +import { BuildsService } from '../src/builds/builds.service'; import { ProjectsService } from '../src/projects/projects.service'; import { TestRunsService } from '../src/test-runs/test-runs.service'; -import { BuildsController } from '../src/builds/builds.controller'; +import { UsersService } from '../src/users/users.service'; import { TEST_PROJECT } from '../src/_data_'; -import { randomUUID } from 'crypto'; +import { createE2eApp, haveProject } from './api-helpers'; +import { haveTestRunCreated, haveUserLogged, requestWithApiKey, requestWithAuth, LoggedUser } from './preconditions'; describe('Builds (e2e)', () => { let app: INestApplication; let buildsService: BuildsService; let buildsController: BuildsController; - let projecstService: ProjectsService; + let projectsService: ProjectsService; let usersService: UsersService; let testRunsService: TestRunsService; let user: LoggedUser; let project: Project; beforeAll(async () => { - const moduleFixture: TestingModule = await Test.createTestingModule({ - imports: [AppModule], - }).compile(); - - app = moduleFixture.createNestApplication(); - app.useGlobalPipes(new ValidationPipe()); - buildsService = moduleFixture.get(BuildsService); - buildsController = moduleFixture.get(BuildsController); - usersService = moduleFixture.get(UsersService); - projecstService = moduleFixture.get(ProjectsService); - testRunsService = moduleFixture.get(TestRunsService); - - await app.init(); + const created = await createE2eApp(); + app = created.app; + buildsService = created.get(BuildsService); + buildsController = created.get(BuildsController); + usersService = created.get(UsersService); + projectsService = created.get(ProjectsService); + testRunsService = created.get(TestRunsService); }); beforeEach(async () => { user = await haveUserLogged(usersService); - project = await projecstService.create({ - ...TEST_PROJECT, - name: `Builds E2E test${randomUUID()}`, - }); + project = await haveProject(projectsService, 'Builds E2E test'); }); afterEach(async () => { - await projecstService.remove(project.id); + await buildsService.whenRetentionIdle(); + await projectsService.remove(project.id); await usersService.delete(user.id); }); @@ -55,169 +45,85 @@ describe('Builds (e2e)', () => { await app.close(); }); - describe('POST /', () => { - it('201 by id', () => { - const createBuildDto: CreateBuildDto = { - branchName: 'branchName', - project: project.id, - }; - return requestWithApiKey(app, 'post', '/builds', user.apiKey) - .send(createBuildDto) - .expect(201) - .expect((res) => { - expect(res.body.projectId).toBe(project.id); - expect(res.body.branchName).toBe(createBuildDto.branchName); - expect(res.body.failedCount).toBe(0); - expect(res.body.passedCount).toBe(0); - expect(res.body.unresolvedCount).toBe(0); - expect(res.body.isRunning).toBe(true); - }); - }); + const haveBuild = () => buildsController.create({ project: project.id, branchName: 'develop' }); + const haveRun = (merge: boolean) => + haveTestRunCreated(buildsService, testRunsService, project.id, project.mainBranchName, './test/image.png', merge); - it('201 by null branchname', () => { - // what an older SDK sends; the DTO type does not allow it - const createBuildDto = { - branchName: null, - project: project.id, - }; - return requestWithApiKey(app, 'post', '/builds', user.apiKey) - .send(createBuildDto) - .expect(201) - .expect((res) => { - expect(res.body.projectId).toBe(project.id); - expect(res.body.branchName).toBe(TEST_PROJECT.mainBranchName); - expect(res.body.failedCount).toBe(0); - expect(res.body.passedCount).toBe(0); - expect(res.body.unresolvedCount).toBe(0); - expect(res.body.isRunning).toBe(true); - }); - }); - - it('201 with no branchname and no ciBuildId', () => { - const createBuildDto: CreateBuildDto = { - project: project.id, - }; - return requestWithApiKey(app, 'post', '/builds', user.apiKey) - .send(createBuildDto) - .expect(201) - .expect((res) => { - expect(res.body.projectId).toBe(project.id); - expect(res.body.branchName).toBe(TEST_PROJECT.mainBranchName); - expect(res.body.failedCount).toBe(0); - expect(res.body.passedCount).toBe(0); - expect(res.body.unresolvedCount).toBe(0); - expect(res.body.isRunning).toBe(true); - }); - }); - - it('201 by name', () => { - const createBuildDto: CreateBuildDto = { - branchName: 'branchName', - project: project.name, - }; - return requestWithApiKey(app, 'post', '/builds', user.apiKey) + describe('POST /', () => { + it.each([ + ['by id', { branchName: 'branchName' }, 'branchName', false], + ['by null branchname', { branchName: null }, TEST_PROJECT.mainBranchName, false], + ['with no branchname and no ciBuildId', {}, TEST_PROJECT.mainBranchName, false], + ['by name', { branchName: 'branchName' }, 'branchName', true], + ['by ciBuildId', { ciBuildId: 'ciBuildId', branchName: 'branchName' }, 'branchName', true], + ])('201 %s', async (_name, dto, branchName, byName) => { + const createBuildDto = { ...dto, project: byName ? project.name : project.id } as CreateBuildDto; + const existing = createBuildDto.ciBuildId ? await buildsController.create(createBuildDto) : undefined; + + await requestWithApiKey(app, 'post', '/builds', user.apiKey) .send(createBuildDto) .expect(201) .expect((res) => { - expect(res.body.projectId).toBe(project.id); - expect(res.body.branchName).toBe(createBuildDto.branchName); - expect(res.body.failedCount).toBe(0); - expect(res.body.passedCount).toBe(0); - expect(res.body.unresolvedCount).toBe(0); - expect(res.body.isRunning).toBe(true); + expect(res.body).toMatchObject({ + projectId: project.id, + branchName, + failedCount: 0, + passedCount: 0, + unresolvedCount: 0, + isRunning: true, + }); + if (existing) { + expect(res.body.id).toBe(existing.id); + expect(res.body.ciBuildId).toBe(createBuildDto.ciBuildId); + } }); }); - it('201 by ciBuildId', async () => { + it('201 cuncurrent', async () => { const createBuildDto: CreateBuildDto = { ciBuildId: 'ciBuildId', branchName: 'branchName', project: project.name, }; - const build = await buildsController.create(createBuildDto); + await haveRun(false); + await haveRun(false); - return requestWithApiKey(app, 'post', '/builds', user.apiKey) - .send(createBuildDto) - .expect(201) - .expect((res) => { - expect(res.body.id).toBe(build.id); - expect(res.body.ciBuildId).toBe(createBuildDto.ciBuildId); - expect(res.body.projectId).toBe(project.id); - expect(res.body.branchName).toBe(createBuildDto.branchName); - expect(res.body.failedCount).toBe(0); - expect(res.body.passedCount).toBe(0); - expect(res.body.unresolvedCount).toBe(0); - expect(res.body.isRunning).toBe(true); - }); + const builds = await Promise.all(Array.from({ length: 5 }, () => buildsController.create(createBuildDto))); + + expect(new Set(builds.map((build) => build.id)).size).toBe(1); }); - it('201 cuncurrent', async () => { - const createBuildDto: CreateBuildDto = { - ciBuildId: 'ciBuildId', - branchName: 'branchName', - project: project.name, - }; - await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - project.mainBranchName, - './test/image.png', - false - ); - await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - project.mainBranchName, - './test/image.png', - false - ); - - const builds = await Promise.all([ - buildsController.create(createBuildDto), - buildsController.create(createBuildDto), - buildsController.create(createBuildDto), - buildsController.create(createBuildDto), - buildsController.create(createBuildDto), - ]); + it('parallel creates of one new ciBuildId number it once', async () => { + const dto: CreateBuildDto = { ciBuildId: 'fresh', project: project.name }; - expect(new Set(builds.map((build) => build.id)).size).toBe(1); + const builds = await Promise.all(Array.from({ length: 8 }, () => buildsController.create(dto))); + + expect(new Set(builds.map((build) => build.number))).toEqual(new Set([1])); + expect((await projectsService.findOne(project.id)).buildsCounter).toBe(1); }); it('404', () => { - const createBuildDto: CreateBuildDto = { - branchName: 'branchName', - project: 'random', - }; - return requestWithApiKey(app, 'post', '/builds', user.apiKey).send(createBuildDto).expect(404); + return requestWithApiKey(app, 'post', '/builds', user.apiKey) + .send({ branchName: 'branchName', project: 'random' }) + .expect(404); }); it('403', () => { - const createBuildDto: CreateBuildDto = { - branchName: 'branchName', - project: project.id, - }; - return requestWithApiKey(app, 'post', '/builds', '').send(createBuildDto).expect(403); + return requestWithApiKey(app, 'post', '/builds', '') + .send({ branchName: 'branchName', project: project.id }) + .expect(403); }); }); describe('GET /', () => { it('200', async () => { - const build = await buildsController.create({ project: project.id, branchName: 'develop' }); + const build = await haveBuild(); return requestWithAuth(app, 'get', `/builds?projectId=${project.id}&take=${5}&skip=${0}`, user.token) .send() .expect(200) .expect((res) => { - expect(JSON.stringify(res.body)).toEqual( - JSON.stringify({ - data: [build], - total: 1, - take: 5, - skip: 0, - }) - ); + expect(JSON.stringify(res.body)).toEqual(JSON.stringify({ data: [build], total: 1, take: 5, skip: 0 })); }); }); @@ -228,7 +134,7 @@ describe('Builds (e2e)', () => { describe('GET /:id', () => { it('200', async () => { - const build = await buildsController.create({ project: project.id, branchName: 'develop' }); + const build = await haveBuild(); return requestWithAuth(app, 'get', `/builds/${build.id}`, user.token) .send() @@ -240,14 +146,8 @@ describe('Builds (e2e)', () => { }); describe('DELETE /', () => { - it('200', async () => { - const build = await buildsController.create({ project: project.id, branchName: 'develop' }); - - return requestWithAuth(app, 'delete', `/builds/${build.id}`, user.token).send().expect(200); - }); - it('401', async () => { - const build = await buildsController.create({ project: project.id, branchName: 'develop' }); + const build = await haveBuild(); return requestWithAuth(app, 'delete', `/builds/${build.id}`, '').send().expect(401); }); @@ -255,100 +155,49 @@ describe('Builds (e2e)', () => { describe('PATCH /', () => { it('200 jwt', async () => { - const build = await buildsController.create({ project: project.id, branchName: 'develop' }); + const build = await haveBuild(); - return requestWithAuth(app, 'patch', `/builds/${build.id}`, user.token) + const res = await requestWithAuth(app, 'patch', `/builds/${build.id}`, user.token) .send({ isRunning: false }) - .expect(200) - .expect((res) => { - expect(res.body.isRunning).toBe(false); - }); - }); - - it('200 api', async () => { - const build = await buildsController.create({ project: project.id, branchName: 'develop' }); + .expect(200); - return requestWithApiKey(app, 'patch', `/builds/${build.id}`, user.apiKey) - .send({ isRunning: false }) - .expect(200) - .expect((res) => { - expect(res.body.isRunning).toBe(false); - }); + expect(res.body.isRunning).toBe(false); }); - it('200 api without body stops build', async () => { - const build = await buildsController.create({ project: project.id, branchName: 'develop' }); - - return requestWithApiKey(app, 'patch', `/builds/${build.id}`, user.apiKey) - .send() - .expect(200) - .expect((res) => { - expect(res.body.isRunning).toBe(false); - }); - }); + it.each([ + ['without body', undefined], + ['with empty body', {}], + ])('200 api %s stops build', async (_name, body) => { + const build = await haveBuild(); - it('200 api with empty body stops build', async () => { - const build = await buildsController.create({ project: project.id, branchName: 'develop' }); + const res = await requestWithApiKey(app, 'patch', `/builds/${build.id}`, user.apiKey).send(body).expect(200); - return requestWithApiKey(app, 'patch', `/builds/${build.id}`, user.apiKey) - .send({}) - .expect(200) - .expect((res) => { - expect(res.body.isRunning).toBe(false); - }); + expect(res.body.isRunning).toBe(false); }); it('403', async () => { - const build = await buildsController.create({ project: project.id, branchName: 'develop' }); + const build = await haveBuild(); return requestWithAuth(app, 'patch', `/builds/${build.id}`, '').send().expect(403); }); }); describe('PATCH /:id/approve', () => { - it('200 default', async () => { - const { build } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - project.mainBranchName, - './test/image.png', - false - ); - - await buildsService.approve(build.id, false); - - const result = await buildsService.findOne(build.id); - expect(result).toEqual({ - ...build, - status: 'passed', - passedCount: 1, - failedCount: 0, - unresolvedCount: 0, - merge: false, - }); - }); - - it('200 with merge', async () => { - const { build } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - project.mainBranchName, - './test/image.png', - true - ); + it.each([ + ['default', false], + ['with merge', true], + ])('200 %s', async (_name, merge) => { + const { build } = await haveRun(merge); - await buildsService.approve(build.id, true); + await buildsService.approve(build.id, merge); - const result = await buildsService.findOne(build.id); - expect(result).toEqual({ + expect(await buildsService.findOne(build.id)).toEqual({ ...build, status: 'passed', passedCount: 1, failedCount: 0, unresolvedCount: 0, - merge: true, + merge, }); }); }); diff --git a/test/comparators.e2e-spec.ts b/test/comparators.e2e-spec.ts index 05df6494..903250ab 100644 --- a/test/comparators.e2e-spec.ts +++ b/test/comparators.e2e-spec.ts @@ -1,11 +1,9 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { INestApplication, ValidationPipe } from '@nestjs/common'; +import { INestApplication } from '@nestjs/common'; import { ImageComparison, Project, TestStatus } from '@prisma/client'; import { existsSync, readFileSync, promises as fs } from 'fs'; import { createServer } from 'net'; import { PNG } from 'pngjs'; import { randomUUID } from 'crypto'; -import { AppModule } from '../src/app.module'; import { UsersService } from '../src/users/users.service'; import { ProjectsService } from '../src/projects/projects.service'; import { BuildsService } from '../src/builds/builds.service'; @@ -18,20 +16,9 @@ import { DEFAULT_CONFIG as LOOKS_SAME_DEFAULT_CONFIG } from '../src/compare/libs import { DEFAULT_CONFIG as ODIFF_DEFAULT_CONFIG } from '../src/compare/libs/odiff/odiff.service'; import { DEFAULT_CONFIG as VLM_DEFAULT_CONFIG } from '../src/compare/libs/vlm/vlm.service'; import { Response } from 'supertest'; +import { createE2eApp, withEnv } from './api-helpers'; import { haveUserLogged, requestWithApiKey, requestWithAuth, LoggedUser } from './preconditions'; -/** - * Every comparator end to end: a real project with the comparator selected, - * test runs posted over HTTP (multipart, like the SDKs), the real looks-same - * library, the native odiff-bin binary and the HDD storage underneath. - * - * VLM: no Ollama or Gemini is reachable from the test machine. The real - * OllamaService is used for the "provider unreachable" case (it is pointed at - * a closed local port), the real GeminiService for the "no API key" case, and - * the model's verdict is stubbed with jest.spyOn on the OllamaService instance - * the app resolved, so everything around the provider call stays real. - */ - const SIZE = 360; const TOTAL_PIXELS = SIZE * SIZE; const SLOW_TEST = 90000; @@ -40,7 +27,6 @@ const image = readFileSync('./test/image.png'); const imageEdited = readFileSync('./test/image_edited.png'); const imageEdited2 = readFileSync('./test/image_edited_2.png'); -// The same picture with its last 60 columns cut off (300x360): a layout change const cropWidth = (buffer: Buffer, width: number): Buffer => { const source = PNG.sync.read(buffer); const target = new PNG({ width, height: source.height }); @@ -49,6 +35,11 @@ const cropWidth = (buffer: Buffer, width: number): Buffer => { }; const imageNarrow = cropWidth(image, 300); +const slow = (name: string, fn: () => Promise) => it(name, fn, SLOW_TEST); + +const NO_DIFF = { status: TestStatus.ok, diffPercent: 0, pixelMisMatchCount: 0, diffName: null }; +const NO_NUMBERS = { status: TestStatus.unresolved, diffName: null, diffPercent: null, pixelMisMatchCount: null }; + const WHOLE_IMAGE: IgnoreAreaDto[] = [{ x: 0, y: 0, width: SIZE, height: SIZE }]; const closedPort = (): Promise => @@ -74,21 +65,15 @@ describe('Comparators (e2e)', () => { const projectIds: string[] = []; beforeAll(async () => { - const moduleFixture: TestingModule = await Test.createTestingModule({ - imports: [AppModule], - }).compile(); - - app = moduleFixture.createNestApplication(); - app.useGlobalPipes(new ValidationPipe()); - usersService = moduleFixture.get(UsersService); - projectsService = moduleFixture.get(ProjectsService); - buildsService = moduleFixture.get(BuildsService); - testRunsService = moduleFixture.get(TestRunsService); - hddService = moduleFixture.get(HddService); - prismaService = moduleFixture.get(PrismaService); - ollamaService = moduleFixture.get(OllamaService); - - await app.init(); + const created = await createE2eApp(); + app = created.app; + usersService = created.get(UsersService); + projectsService = created.get(ProjectsService); + buildsService = created.get(BuildsService); + testRunsService = created.get(TestRunsService); + hddService = created.get(HddService); + prismaService = created.get(PrismaService); + ollamaService = created.get(OllamaService); user = await haveUserLogged(usersService); }); @@ -116,7 +101,6 @@ describe('Comparators (e2e)', () => { bulkApproveGroupBy: 'customTags', }); - // created through the projects API, the way the UI does it const DEFAULT_CONFIGS: Partial> = { [ImageComparison.lookSame]: LOOKS_SAME_DEFAULT_CONFIG, [ImageComparison.odiff]: ODIFF_DEFAULT_CONFIG, @@ -134,7 +118,6 @@ describe('Comparators (e2e)', () => { return res.body; }; - // resolves with the response: an async helper cannot hand back the chainable supertest request const postTestRun = async ( project: Project, imageBuffer: Buffer, @@ -164,407 +147,180 @@ describe('Comparators (e2e)', () => { const diffExists = (imageName: string) => existsSync(hddService.getImagePath(imageName)); - // posts the first image and approves it, so the next run of the same screen has a baseline - // and resolves with the name of the baseline image file const haveBaseline = async (project: Project, imageBuffer: Buffer = image): Promise => { const res = await postTestRun(project, imageBuffer); expect(res.body.status).toBe(TestStatus.new); const approved = await testRunsService.approve(res.body.id); - // approving gave the run a variation and the variation a baseline const baselineName = (await prismaService.testVariation.findUnique({ where: { id: approved.testVariationId! } }))! .baselineName!; expect(diffExists(baselineName)).toBe(true); return baselineName; }; - describe('lookSame', () => { - it( - 'new without a baseline, ok for the same image', - async () => { - const project = await haveProject(ImageComparison.lookSame); - await haveBaseline(project); - - const res = await postTestRun(project, image); - - expect(res.body).toMatchObject({ - status: TestStatus.ok, - diffPercent: 0, - pixelMisMatchCount: 0, - diffName: null, - }); - }, - SLOW_TEST - ); - - it( - 'unresolved with a diff file for a changed image', - async () => { - const project = await haveProject(ImageComparison.lookSame); - await haveBaseline(project); - - const res = await postTestRun(project, imageEdited); - - expect(res.body).toMatchObject({ - status: TestStatus.unresolved, - pixelMisMatchCount: 31068, - diffPercent: (31068 * 100) / TOTAL_PIXELS, - // thumbnails and change signatures are pixelmatch-only - imageThumbnailName: null, - diffThumbnailName: null, - }); - expect(res.body.diffName).toMatch(/\.diff\.png$/); - expect(diffExists(res.body.diffName)).toBe(true); - // looks-same writes a full-size diff image - const diff = PNG.sync.read(await fs.readFile(hddService.getImagePath(res.body.diffName))); - expect([diff.width, diff.height]).toEqual([SIZE, SIZE]); - }, - SLOW_TEST - ); - - it( - 'ok, without a diff, when the change is within diffTollerancePercent', - async () => { - const project = await haveProject(ImageComparison.lookSame); - await haveBaseline(project); + const compare = async ( + imageComparison: ImageComparison, + imageComparisonConfig: string | undefined, + candidate: Buffer, + extra: Parameters[2] = {} + ) => { + const project = await haveProject(imageComparison, imageComparisonConfig); + await haveBaseline(project); + return { project, res: await postTestRun(project, candidate, extra) }; + }; - const res = await postTestRun(project, imageEdited, { diffTollerancePercent: 100 }); + describe.each([ + { + comparator: ImageComparison.lookSame, + changedCount: 31068, + changedPercent: (31068 * 100) / TOTAL_PIXELS, + resizedConfig: { allowDiffDimensions: true }, + resizedPercent: (60 * SIZE * 100) / TOTAL_PIXELS, + lenientConfig: { tolerance: 100 }, + }, + { + comparator: ImageComparison.odiff, + changedCount: 31145, + changedPercent: 24.03, + resizedConfig: { outputDiffMask: true, failOnLayoutDiff: false, threshold: 0, antialiasing: true }, + resizedPercent: 16.67, + lenientConfig: { threshold: 1 }, + }, + ])('$comparator', ({ comparator, changedCount, changedPercent, resizedConfig, resizedPercent, lenientConfig }) => { + slow('ok for the same image', async () => { + const { res } = await compare(comparator, undefined, image); + + expect(res.body).toMatchObject(NO_DIFF); + }); - expect(res.body).toMatchObject({ diffTollerancePercent: 100, status: TestStatus.ok, diffName: null }); - expect(res.body.pixelMisMatchCount).toBeGreaterThan(0); - }, - SLOW_TEST - ); + slow('unresolved with a diff file for a changed image', async () => { + const { res } = await compare(comparator, undefined, imageEdited); + + expect(res.body).toMatchObject({ + status: TestStatus.unresolved, + pixelMisMatchCount: changedCount, + diffPercent: changedPercent, + imageThumbnailName: null, + diffThumbnailName: null, + }); + expect(res.body.diffName).toMatch(/\.diff\.png$/); + expect(diffExists(res.body.diffName)).toBe(true); + const diff = PNG.sync.read(await fs.readFile(hddService.getImagePath(res.body.diffName))); + expect([diff.width, diff.height]).toEqual([SIZE, SIZE]); + }); - it( - 'unresolved when the change is over diffTollerancePercent', - async () => { - const project = await haveProject(ImageComparison.lookSame); - await haveBaseline(project); + slow('ok, without a diff, when the change is within diffTollerancePercent', async () => { + const { res } = await compare(comparator, undefined, imageEdited, { diffTollerancePercent: 100 }); - const res = await postTestRun(project, imageEdited, { diffTollerancePercent: 0.01 }); + expect(res.body).toMatchObject({ + diffTollerancePercent: 100, + status: TestStatus.ok, + diffName: null, + pixelMisMatchCount: changedCount, + diffPercent: changedPercent, + }); + }); - expect(res.body.status).toBe(TestStatus.unresolved); - expect(diffExists(res.body.diffName)).toBe(true); - }, - SLOW_TEST - ); + slow('unresolved when the change is over diffTollerancePercent', async () => { + const { res } = await compare(comparator, undefined, imageEdited, { diffTollerancePercent: 0.01 }); - it( - 'ok when an ignore area covers the whole change', - async () => { - const project = await haveProject(ImageComparison.lookSame); - await haveBaseline(project); + expect(res.body.status).toBe(TestStatus.unresolved); + expect(diffExists(res.body.diffName)).toBe(true); + }); - const res = await postTestRun(project, imageEdited, { ignoreAreas: WHOLE_IMAGE }); + slow('ok when an ignore area covers the whole change', async () => { + const { res } = await compare(comparator, undefined, imageEdited, { ignoreAreas: WHOLE_IMAGE }); - expect(res.body).toMatchObject({ - status: TestStatus.ok, - diffPercent: 0, - pixelMisMatchCount: 0, - diffName: null, - }); - }, - SLOW_TEST - ); + expect(res.body).toMatchObject(NO_DIFF); + }); - it( - 'unresolved without numbers for a different size', - async () => { - const project = await haveProject(ImageComparison.lookSame); - await haveBaseline(project); + slow('unresolved without numbers for a different size', async () => { + const { res } = await compare(comparator, undefined, imageNarrow); - const res = await postTestRun(project, imageNarrow); + expect(res.body).toMatchObject(NO_NUMBERS); + }); - expect(res.body).toMatchObject({ - status: TestStatus.unresolved, - diffName: null, - diffPercent: null, - pixelMisMatchCount: null, - }); - }, - SLOW_TEST - ); + slow('compares a different size when the project config allows it', async () => { + const { res } = await compare(comparator, JSON.stringify(resizedConfig), imageNarrow); - it( - 'compares a different size when the project config allows it', - async () => { - const project = await haveProject(ImageComparison.lookSame, JSON.stringify({ allowDiffDimensions: true })); - await haveBaseline(project); + expect(res.body).toMatchObject({ + status: TestStatus.unresolved, + pixelMisMatchCount: 60 * SIZE, + diffPercent: resizedPercent, + }); + expect(diffExists(res.body.diffName)).toBe(true); + }); - const res = await postTestRun(project, imageNarrow); + slow('reads the tolerance from the project config', async () => { + const { res } = await compare(comparator, JSON.stringify(lenientConfig), imageEdited); - // the 60 missing columns are the difference, counted against the baseline's area - expect(res.body).toMatchObject({ - status: TestStatus.unresolved, - pixelMisMatchCount: 60 * SIZE, - diffPercent: (60 * SIZE * 100) / TOTAL_PIXELS, - }); - expect(diffExists(res.body.diffName)).toBe(true); - }, - SLOW_TEST - ); + expect(res.body).toMatchObject(NO_DIFF); + }); - it( - 'reads tolerance from the project config', - async () => { - // a partial config replaces the defaults instead of being merged into them - const project = await haveProject(ImageComparison.lookSame, JSON.stringify({ tolerance: 100 })); - await haveBaseline(project); + slow('treats a missing baseline file as no baseline', async () => { + const project = await haveProject(comparator); + await fs.unlink(hddService.getImagePath(await haveBaseline(project))); - const res = await postTestRun(project, imageEdited); + const res = await postTestRun(project, imageEdited); - // a ΔE tolerance of 100 accepts every colour change in this picture - expect(res.body).toMatchObject({ - status: TestStatus.ok, - diffPercent: 0, - pixelMisMatchCount: 0, - diffName: null, - }); - }, - SLOW_TEST - ); + expect(res.body).toMatchObject({ status: TestStatus.new, diffName: null, diffPercent: null }); + }); + }); + describe('lookSame', () => { it('the projects API rejects a config that is not JSON', async () => { await requestWithAuth(app, 'post', '/projects', user.token) .send(projectBody(ImageComparison.lookSame, 'not json')) .expect(400) .expect((res) => { expect(res.body.message).toBe('imageComparisonConfig must be a json string'); - expect(res.body.exception.response.message).toEqual(['imageComparisonConfig must be a json string']); }); }); - it( - 'falls back to the default config when the stored config is not JSON', - async () => { - const created = await haveProject(ImageComparison.lookSame); - // only reachable by writing the column directly: the API validates the JSON - const project = await prismaService.project.update({ - where: { id: created.id }, - data: { imageComparisonConfig: 'not json' }, - }); - await haveBaseline(project); - - const res = await postTestRun(project, imageEdited); - - // same numbers as with DEFAULT_CONFIG - expect(res.body).toMatchObject({ status: TestStatus.unresolved, pixelMisMatchCount: 31068 }); - expect(diffExists(res.body.diffName)).toBe(true); - }, - SLOW_TEST - ); - - it( - 'falls back to the default config when looks-same rejects the config', - async () => { - // looks-same refuses strict together with tolerance - const project = await haveProject(ImageComparison.lookSame, JSON.stringify({ strict: true, tolerance: 5 })); - await haveBaseline(project); + slow('falls back to the default config when the stored config is not JSON', async () => { + const created = await haveProject(ImageComparison.lookSame); + const project = await prismaService.project.update({ + where: { id: created.id }, + data: { imageComparisonConfig: 'not json' }, + }); + await haveBaseline(project); - const res = await postTestRun(project, imageEdited); + const res = await postTestRun(project, imageEdited); - // the same numbers as with the default config (see the config that is not JSON) - expect(res.body).toMatchObject({ status: TestStatus.unresolved, pixelMisMatchCount: 31068 }); - expect(diffExists(res.body.diffName)).toBe(true); - }, - SLOW_TEST - ); - - it( - 'treats a missing baseline file as no baseline', - async () => { - const project = await haveProject(ImageComparison.lookSame); - const baselineName = await haveBaseline(project); - await fs.unlink(hddService.getImagePath(baselineName)); + expect(res.body).toMatchObject({ status: TestStatus.unresolved, pixelMisMatchCount: 31068 }); + expect(diffExists(res.body.diffName)).toBe(true); + }); - const res = await postTestRun(project, imageEdited); + slow('falls back to the default config when looks-same rejects the config', async () => { + const { res } = await compare( + ImageComparison.lookSame, + JSON.stringify({ strict: true, tolerance: 5 }), + imageEdited + ); - expect(res.body).toMatchObject({ status: TestStatus.new, diffName: null, diffPercent: null }); - }, - SLOW_TEST - ); + expect(res.body).toMatchObject({ status: TestStatus.unresolved, pixelMisMatchCount: 31068 }); + expect(diffExists(res.body.diffName)).toBe(true); + }); }); describe('odiff', () => { - it( - 'ok for the same image', - async () => { - const project = await haveProject(ImageComparison.odiff); - await haveBaseline(project); - - const res = await postTestRun(project, image); - - expect(res.body).toMatchObject({ - status: TestStatus.ok, - diffPercent: 0, - pixelMisMatchCount: 0, - diffName: null, - }); - }, - SLOW_TEST - ); - - it( - 'unresolved with a diff file for a changed image', - async () => { - const project = await haveProject(ImageComparison.odiff); - await haveBaseline(project); - - const res = await postTestRun(project, imageEdited); - - // odiff rounds the percentage to two decimals - expect(res.body).toMatchObject({ - status: TestStatus.unresolved, - pixelMisMatchCount: 31145, - diffPercent: 24.03, - imageThumbnailName: null, - diffThumbnailName: null, - }); - expect(res.body.diffName).toMatch(/\.diff\.png$/); - expect(diffExists(res.body.diffName)).toBe(true); - }, - SLOW_TEST - ); - - it( - 'ok within diffTollerancePercent, without keeping a diff', - async () => { - const project = await haveProject(ImageComparison.odiff); - await haveBaseline(project); - - const res = await postTestRun(project, imageEdited, { diffTollerancePercent: 100 }); - - // the numbers are kept, the diff image odiff wrote is deleted - expect(res.body).toMatchObject({ - status: TestStatus.ok, - diffName: null, - pixelMisMatchCount: 31145, - diffPercent: 24.03, - }); - }, - SLOW_TEST - ); - - it( - 'ok when an ignore region covers the whole change', - async () => { - const project = await haveProject(ImageComparison.odiff); - await haveBaseline(project); - - const res = await postTestRun(project, imageEdited, { ignoreAreas: WHOLE_IMAGE }); - - expect(res.body).toMatchObject({ - status: TestStatus.ok, - diffPercent: 0, - pixelMisMatchCount: 0, - diffName: null, - }); - }, - SLOW_TEST - ); - - it( - 'unresolved without numbers for a different size', - async () => { - const project = await haveProject(ImageComparison.odiff); - await haveBaseline(project); - - const res = await postTestRun(project, imageNarrow); - - expect(res.body).toMatchObject({ - status: TestStatus.unresolved, - diffName: null, - diffPercent: null, - pixelMisMatchCount: null, - }); - }, - SLOW_TEST - ); - - it( - 'compares a different size when failOnLayoutDiff is off', - async () => { - const project = await haveProject( - ImageComparison.odiff, - JSON.stringify({ outputDiffMask: true, failOnLayoutDiff: false, threshold: 0, antialiasing: true }) + slow('fails the request when storage is not HDD', async () => { + const project = await haveProject(ImageComparison.odiff); + await haveBaseline(project); + + await withEnv({ STATIC_SERVICE: 's3' }, async () => { + const res = await postTestRun(project, imageEdited, {}, 500); + expect(res.body.message).toBe( + 'Odiff can only be used with HDD static service. Please use another image comparison lib in project settings or switch STATIC_SERVICE envitonmental variable to HDD.' ); - await haveBaseline(project); - - const res = await postTestRun(project, imageNarrow); - - expect(res.body).toMatchObject({ - status: TestStatus.unresolved, - pixelMisMatchCount: 60 * SIZE, - diffPercent: 16.67, - }); - expect(diffExists(res.body.diffName)).toBe(true); - }, - SLOW_TEST - ); - - it( - 'reads threshold from the project config', - async () => { - // a partial config replaces the defaults instead of being merged into them - const project = await haveProject(ImageComparison.odiff, JSON.stringify({ threshold: 1 })); - await haveBaseline(project); - - const res = await postTestRun(project, imageEdited); - - // the maximum colour threshold treats every changed pixel as equal - expect(res.body).toMatchObject({ - status: TestStatus.ok, - diffPercent: 0, - pixelMisMatchCount: 0, - diffName: null, - }); - }, - SLOW_TEST - ); - - it( - 'treats a missing baseline file as no baseline', - async () => { - const project = await haveProject(ImageComparison.odiff); - const baselineName = await haveBaseline(project); - await fs.unlink(hddService.getImagePath(baselineName)); - - const res = await postTestRun(project, imageEdited); + }); - expect(res.body).toMatchObject({ status: TestStatus.new, diffName: null, diffPercent: null }); - }, - SLOW_TEST - ); - - it( - 'fails the request when storage is not HDD', - async () => { - const project = await haveProject(ImageComparison.odiff); - await haveBaseline(project); - const previous = process.env.STATIC_SERVICE; - process.env.STATIC_SERVICE = 's3'; - try { - const res = await postTestRun(project, imageEdited, {}, 500); - expect(res.body.message).toBe( - 'Odiff can only be used with HDD static service. Please use another image comparison lib in project settings or switch STATIC_SERVICE envitonmental variable to HDD.' - ); - } finally { - if (previous === undefined) { - delete process.env.STATIC_SERVICE; - } else { - process.env.STATIC_SERVICE = previous; - } - } - // refused before the run was stored: only the baseline's run is there - const runs = await prismaService.testRun.findMany({ where: { projectId: project.id } }); - expect(runs.map((run) => run.status)).toEqual([TestStatus.approved]); - }, - SLOW_TEST - ); + const runs = await prismaService.testRun.findMany({ where: { projectId: project.id } }); + expect(runs.map((run) => run.status)).toEqual([TestStatus.approved]); + }); }); describe.each([ImageComparison.lookSame, ImageComparison.odiff])('%s on a feature branch', (imageComparison) => { - it( + slow( 'auto-approves a run that matches an older main-branch baseline, and compares without keeping diffs', async () => { const created = await haveProject(imageComparison); @@ -573,41 +329,32 @@ describe('Comparators (e2e)', () => { data: { autoApproveFeature: true }, }); await haveBaseline(project, image); - // a second approval on main: the variation's baseline history is now [edited, image] const res = await postTestRun(project, imageEdited); expect(res.body.status).toBe(TestStatus.unresolved); await testRunsService.approve(res.body.id); const matching = await postTestRun(project, image, { branchName: 'feature' }); - // compared with the older baseline with saveDiffAsFile off, then auto-approved expect(matching.body.status).toBe(TestStatus.autoApproved); const other = await postTestRun(project, imageEdited2, { branchName: 'feature' }); expect(other.body.status).toBe(TestStatus.unresolved); - // the diff kept is the one against the current baseline expect(diffExists(other.body.diffName)).toBe(true); - }, - SLOW_TEST + } ); }); describe('vlm', () => { const vlmConfig = JSON.stringify({ provider: 'ollama', model: 'llava:7b', prompt: 'compare', temperature: 0 }); + const answer = (verdict: object) => ({ content: JSON.stringify(verdict) }); - it( - 'ok for the same image without asking the model', - async () => { - const generate = jest.spyOn(ollamaService, 'generate'); - const project = await haveProject(ImageComparison.vlm, vlmConfig); - await haveBaseline(project); + slow('ok for the same image without asking the model', async () => { + const generate = jest.spyOn(ollamaService, 'generate'); - const res = await postTestRun(project, image); + const { res } = await compare(ImageComparison.vlm, vlmConfig, image); - expect(res.body).toMatchObject({ status: TestStatus.ok, diffName: null, vlmDescription: null }); - expect(generate).not.toHaveBeenCalled(); - }, - SLOW_TEST - ); + expect(res.body).toMatchObject({ status: TestStatus.ok, diffName: null, vlmDescription: null }); + expect(generate).not.toHaveBeenCalled(); + }); it.each([ [true, TestStatus.ok], @@ -617,11 +364,9 @@ describe('Comparators (e2e)', () => { async (identical, status) => { const generate = jest .spyOn(ollamaService, 'generate') - .mockResolvedValue({ content: JSON.stringify({ identical, description: 'Button moved' }) }); - const project = await haveProject(ImageComparison.vlm, vlmConfig); - await haveBaseline(project); + .mockResolvedValue(answer({ identical, description: 'Button moved' })); - const res = await postTestRun(project, imageEdited); + const { res } = await compare(ImageComparison.vlm, vlmConfig, imageEdited); expect(res.body.status).toBe(status); expect(res.body.vlmDescription).toBe('Button moved'); @@ -636,24 +381,19 @@ describe('Comparators (e2e)', () => { SLOW_TEST ); - it( - 'reads the verdict from thinking when useThinking is set', - async () => { - jest - .spyOn(ollamaService, 'generate') - .mockResolvedValue({ content: '', thinking: JSON.stringify({ identical: true, description: 'Same' }) }); - const project = await haveProject( - ImageComparison.vlm, - JSON.stringify({ ...JSON.parse(vlmConfig), useThinking: true }) - ); - await haveBaseline(project); + slow('reads the verdict from thinking when useThinking is set', async () => { + jest + .spyOn(ollamaService, 'generate') + .mockResolvedValue({ content: '', thinking: JSON.stringify({ identical: true, description: 'Same' }) }); - const res = await postTestRun(project, imageEdited); + const { res } = await compare( + ImageComparison.vlm, + JSON.stringify({ ...JSON.parse(vlmConfig), useThinking: true }), + imageEdited + ); - expect(res.body).toMatchObject({ status: TestStatus.ok, vlmDescription: 'Same' }); - }, - SLOW_TEST - ); + expect(res.body).toMatchObject({ status: TestStatus.ok, vlmDescription: 'Same' }); + }); it.each([ [ @@ -663,12 +403,10 @@ describe('Comparators (e2e)', () => { [{ content: '', thinking: '' }, 'VLM analysis failed: Empty response from model'], ])( 'keeps the pixelmatch verdict when the model answers %j', - async (answer, description) => { - jest.spyOn(ollamaService, 'generate').mockResolvedValue(answer); - const project = await haveProject(ImageComparison.vlm, vlmConfig); - await haveBaseline(project); + async (response, description) => { + jest.spyOn(ollamaService, 'generate').mockResolvedValue(response); - const res = await postTestRun(project, imageEdited); + const { res } = await compare(ImageComparison.vlm, vlmConfig, imageEdited); expect(res.body.status).toBe(TestStatus.unresolved); expect(res.body.vlmDescription).toBe(description); @@ -676,74 +414,44 @@ describe('Comparators (e2e)', () => { SLOW_TEST ); - it( - 'keeps the pixelmatch verdict when Ollama is unreachable', - async () => { - // the Ollama client is created lazily from OLLAMA_BASE_URL on first use - const previous = process.env.OLLAMA_BASE_URL; - process.env.OLLAMA_BASE_URL = `http://127.0.0.1:${await closedPort()}`; - try { - const project = await haveProject(ImageComparison.vlm, vlmConfig); - await haveBaseline(project); - - const res = await postTestRun(project, imageEdited); - - expect(res.body.status).toBe(TestStatus.unresolved); - expect(res.body.vlmDescription).toBe('VLM analysis failed: fetch failed'); - expect(diffExists(res.body.diffName)).toBe(true); - } finally { - if (previous === undefined) { - delete process.env.OLLAMA_BASE_URL; - } else { - process.env.OLLAMA_BASE_URL = previous; - } - } - }, - SLOW_TEST - ); - - it( - 'keeps the pixelmatch verdict when the Gemini config has no API key', - async () => { - const project = await haveProject( - ImageComparison.vlm, - JSON.stringify({ provider: 'gemini', model: 'gemini-2.0-flash', prompt: 'compare', temperature: 0 }) - ); - await haveBaseline(project); - - const res = await postTestRun(project, imageEdited); + slow('keeps the pixelmatch verdict when Ollama is unreachable', async () => { + await withEnv({ OLLAMA_BASE_URL: `http://127.0.0.1:${await closedPort()}` }, async () => { + const { res } = await compare(ImageComparison.vlm, vlmConfig, imageEdited); expect(res.body.status).toBe(TestStatus.unresolved); - expect(res.body.vlmDescription).toBe('VLM analysis failed: Gemini API key is required'); - }, - SLOW_TEST - ); + expect(res.body.vlmDescription).toBe('VLM analysis failed: fetch failed'); + expect(diffExists(res.body.diffName)).toBe(true); + }); + }); - it( - 'sends the project config as is when it is not a VLM config', - async () => { - const generate = jest - .spyOn(ollamaService, 'generate') - .mockResolvedValue({ content: JSON.stringify({ identical: false, description: 'Changed' }) }); - // no imageComparisonConfig: the column default is a pixelmatch config - const project = await projectsService.create({ - ...projectBody(ImageComparison.vlm, undefined as unknown as string), - }); - projectIds.push(project.id); - await haveBaseline(project); + slow('keeps the pixelmatch verdict when the Gemini config has no API key', async () => { + const { res } = await compare( + ImageComparison.vlm, + JSON.stringify({ provider: 'gemini', model: 'gemini-2.0-flash', prompt: 'compare', temperature: 0 }), + imageEdited + ); - const res = await postTestRun(project, imageEdited); + expect(res.body.status).toBe(TestStatus.unresolved); + expect(res.body.vlmDescription).toBe('VLM analysis failed: Gemini API key is required'); + }); - expect(res.body).toMatchObject({ status: TestStatus.unresolved, vlmDescription: 'Changed' }); - // the column default is pixelmatch's, laid over the VLM defaults: Ollama still gets a model and a prompt - expect(generate).toHaveBeenCalledTimes(1); - expect(generate.mock.calls[0][0]).toMatchObject({ - model: VLM_DEFAULT_CONFIG.model, - prompt: VLM_DEFAULT_CONFIG.prompt, - temperature: VLM_DEFAULT_CONFIG.temperature, - }); - }, - SLOW_TEST - ); + slow('sends the project config as is when it is not a VLM config', async () => { + const generate = jest + .spyOn(ollamaService, 'generate') + .mockResolvedValue(answer({ identical: false, description: 'Changed' })); + const project = await projectsService.create(projectBody(ImageComparison.vlm, undefined as unknown as string)); + projectIds.push(project.id); + await haveBaseline(project); + + const res = await postTestRun(project, imageEdited); + + expect(res.body).toMatchObject({ status: TestStatus.unresolved, vlmDescription: 'Changed' }); + expect(generate).toHaveBeenCalledTimes(1); + expect(generate.mock.calls[0][0]).toMatchObject({ + model: VLM_DEFAULT_CONFIG.model, + prompt: VLM_DEFAULT_CONFIG.prompt, + temperature: VLM_DEFAULT_CONFIG.temperature, + }); + }); }); }); diff --git a/test/diff-worker-pool.e2e-spec.ts b/test/diff-worker-pool.e2e-spec.ts new file mode 100644 index 00000000..0e3e9c7c --- /dev/null +++ b/test/diff-worker-pool.e2e-spec.ts @@ -0,0 +1,71 @@ +import { execFileSync } from 'child_process'; +import { existsSync, readFileSync } from 'fs'; +import { join } from 'path'; + +const ROOT = join(__dirname, '..'); +const DIST = join(ROOT, 'dist', 'compare'); + +type PoolModule = typeof import('../src/compare/diff-worker-pool'); +type JobModule = typeof import('../src/compare/libs/pixelmatch/worker-job'); + +describe('DiffWorkerPool on real worker threads (compiled build)', () => { + const baseline = readFileSync(join(__dirname, 'image.png')); + const image = readFileSync(join(__dirname, 'image_edited.png')); + const workersCount = process.env.DIFF_WORKERS_COUNT; + let pool: InstanceType; + let runWorkerJob: JobModule['runWorkerJob']; + + beforeAll(() => { + execFileSync('npm', ['run', 'build'], { cwd: ROOT, stdio: 'pipe' }); + const { DiffWorkerPool } = require(join(DIST, 'diff-worker-pool.js')) as PoolModule; + ({ runWorkerJob } = require(join(DIST, 'libs', 'pixelmatch', 'worker-job.js')) as JobModule); + process.env.DIFF_WORKERS_COUNT = '2'; + pool = new DiffWorkerPool(); + }, 120000); + + afterAll(async () => { + await pool?.onApplicationShutdown(); + if (workersCount === undefined) delete process.env.DIFF_WORKERS_COUNT; + else process.env.DIFF_WORKERS_COUNT = workersCount; + }); + + const internals = () => pool as unknown as { inline: boolean; workers: unknown[] }; + + it('finds the worker file where the build puts it', () => { + expect(existsSync(join(DIST, 'libs', 'pixelmatch', 'pixelmatch.worker.js'))).toBe(true); + expect(internals().inline).toBe(false); + }); + + it.each([ + [ + 'diff', + { + kind: 'diff', + ignoreAreas: [], + threshold: 0.1, + includeAA: false, + diffTolerancePercent: 0, + saveDiff: true, + }, + ], + ['signature', { kind: 'signature', ignoreAreas: [], threshold: 0.1, includeAA: false }], + ['blur-amplitude', { kind: 'blur-amplitude', ignoreAreas: [], gateIfNoise: { threshold: 80 } }], + [ + 'looks-same', + { + kind: 'looks-same', + ignoreAreas: [], + config: { strict: false, ignoreAntialiasing: true, ignoreCaret: true, allowDiffDimensions: false }, + saveDiff: true, + }, + ], + ['vlm-inputs', { kind: 'vlm-inputs', ignoreAreas: [] }], + ])('runs a %s job in a worker with the same result as inline', async (_kind, params) => { + const input = { ...params, baseline, image } as Parameters[0]; + + const output = await pool.run(input); + + expect(internals().workers.length).toBeGreaterThan(0); + expect(output).toEqual(await runWorkerJob(input)); + }); +}); diff --git a/test/jest.config.ts b/test/jest.config.ts index 013478ae..86301f1e 100644 --- a/test/jest.config.ts +++ b/test/jest.config.ts @@ -1,4 +1,3 @@ -/** @returns {Promise} */ module.exports = async () => { return { displayName: 'E2E Tests', @@ -6,7 +5,7 @@ module.exports = async () => { testTimeout: 30000, testRegex: '.e2e-spec.ts$', transform: { - '^.+\\.tsx?$': ['ts-jest', { tsconfig: '/../tsconfig.jest.json' }], + '^.+\\.(t|j)s$': '@swc/jest', }, coverageDirectory: '../coverage', testEnvironment: 'node', diff --git a/test/ollama.e2e-spec.ts b/test/ollama.e2e-spec.ts index 32191aba..b4187c90 100644 --- a/test/ollama.e2e-spec.ts +++ b/test/ollama.e2e-spec.ts @@ -1,10 +1,9 @@ -import { Test, TestingModule } from '@nestjs/testing'; import { INestApplication } from '@nestjs/common'; import { Role } from '@prisma/client'; import request from 'supertest'; -import { AppModule } from '../src/app.module'; import { UsersService } from '../src/users/users.service'; import { OllamaService } from '../src/compare/libs/vlm/providers/ollama/ollama.service'; +import { createE2eApp } from './api-helpers'; import { haveUserLogged, requestWithApiKey, requestWithAuth } from './preconditions'; const image = './test/image.png'; @@ -26,17 +25,9 @@ describe('Ollama (e2e)', () => { }; beforeAll(async () => { - const moduleFixture: TestingModule = await Test.createTestingModule({ - imports: [AppModule], - }) - .overrideProvider(OllamaService) - .useValue(ollamaServiceMock) - .compile(); - - app = moduleFixture.createNestApplication(); - usersService = moduleFixture.get(UsersService); - - await app.init(); + const created = await createE2eApp({ validation: false, overrides: [[OllamaService, ollamaServiceMock]] }); + app = created.app; + usersService = created.get(UsersService); }); beforeEach(() => { @@ -79,8 +70,6 @@ describe('Ollama (e2e)', () => { }); describe('POST /ollama/compare', () => { - // Guards reject before the upload is read, so rejected requests are sent without files: - // streaming a multipart body into an already closed connection fails with EPIPE. it('401 without authentication', async () => { await request(app.getHttpServer()).post(compareUrl).expect(401); expect(ollamaServiceMock.generate).not.toHaveBeenCalled(); diff --git a/test/preconditions.ts b/test/preconditions.ts index 26babde6..1a3e0b2d 100644 --- a/test/preconditions.ts +++ b/test/preconditions.ts @@ -10,7 +10,6 @@ import { Build, Role } from '@prisma/client'; import { CreateUserDto } from 'src/users/dto/user-create.dto'; import { UserLoginResponseDto } from 'src/users/dto/user-login-response.dto'; -/** A signed-in user: login always issues a token, only creating a user answers without one. */ export type LoggedUser = UserLoginResponseDto & { token: string }; export const generateUser = (password: string): CreateUserDto => ({ @@ -49,6 +48,19 @@ export const haveUserLogged = async (usersService: UsersService, role: Role = 'a }) as Promise; }; +export const withUser = async ( + usersService: UsersService, + role: Role, + fn: (user: LoggedUser) => PromiseLike +): Promise => { + const user = await haveUserLogged(usersService, role); + try { + return await fn(user); + } finally { + await usersService.delete(user.id); + } +}; + export const haveTestRunCreated = async ( buildsService: BuildsService, testRunsService: TestRunsService, diff --git a/test/project-favorites.e2e-spec.ts b/test/project-favorites.e2e-spec.ts index 2e748249..fb3a32df 100644 --- a/test/project-favorites.e2e-spec.ts +++ b/test/project-favorites.e2e-spec.ts @@ -1,13 +1,11 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { INestApplication, ValidationPipe } from '@nestjs/common'; +import { INestApplication } from '@nestjs/common'; import { Project, Role } from '@prisma/client'; import request from 'supertest'; import { randomUUID } from 'crypto'; -import { AppModule } from '../src/app.module'; import { UsersService } from '../src/users/users.service'; import { PrismaService } from '../src/prisma/prisma.service'; import { ProjectsService } from '../src/projects/projects.service'; -import { TEST_PROJECT } from '../src/_data_'; +import { createE2eApp, haveProject as createProject } from './api-helpers'; import { haveUserLogged, requestWithAuth, LoggedUser } from './preconditions'; describe('Project favourites (e2e)', () => { @@ -25,7 +23,7 @@ describe('Project favourites (e2e)', () => { }; const haveProject = async (): Promise => { - const project = await projectsService.create({ ...TEST_PROJECT, name: `Favourite ${randomUUID()}` }); + const project = await createProject(projectsService, 'Favourite'); projectIds.push(project.id); return project; }; @@ -34,20 +32,14 @@ describe('Project favourites (e2e)', () => { (await requestWithAuth(app, 'get', '/projects/favorites', user.token).expect(200)).body; beforeAll(async () => { - const moduleFixture: TestingModule = await Test.createTestingModule({ - imports: [AppModule], - }).compile(); - - app = moduleFixture.createNestApplication(); - app.useGlobalPipes(new ValidationPipe()); - usersService = moduleFixture.get(UsersService); - projectsService = moduleFixture.get(ProjectsService); - prismaService = moduleFixture.get(PrismaService); - await app.init(); + const created = await createE2eApp(); + app = created.app; + usersService = created.get(UsersService); + projectsService = created.get(ProjectsService); + prismaService = created.get(PrismaService); }); afterEach(async () => { - // some tests delete their project or user already await prismaService.project.deleteMany({ where: { id: { in: projectIds.splice(0) } } }); await prismaService.user.deleteMany({ where: { id: { in: userIds.splice(0) } } }); }); diff --git a/test/projects-remove.e2e-spec.ts b/test/projects-remove.e2e-spec.ts new file mode 100644 index 00000000..48054183 --- /dev/null +++ b/test/projects-remove.e2e-spec.ts @@ -0,0 +1,71 @@ +import { HttpException, HttpStatus, INestApplication } from '@nestjs/common'; +import { Project } from '@prisma/client'; +import { BuildsService } from '../src/builds/builds.service'; +import { PrismaService } from '../src/prisma/prisma.service'; +import { ProjectsService } from '../src/projects/projects.service'; +import { TestRunsService } from '../src/test-runs/test-runs.service'; +import { createE2eApp, haveProject, imageExists } from './api-helpers'; +import { haveTestRunCreated } from './preconditions'; + +describe('Project removal (e2e)', () => { + let app: INestApplication; + let projectsService: ProjectsService; + let buildsService: BuildsService; + let testRunsService: TestRunsService; + let prisma: PrismaService; + let project: Project; + + beforeAll(async () => { + const created = await createE2eApp(); + app = created.app; + projectsService = created.get(ProjectsService); + buildsService = created.get(BuildsService); + testRunsService = created.get(TestRunsService); + prisma = created.get(PrismaService); + }); + + beforeEach(async () => { + project = await haveProject(projectsService, 'Remove E2E'); + }); + + afterEach(async () => { + await prisma.project.deleteMany({ where: { id: project.id } }); + }); + + afterAll(async () => { + await app.close(); + }); + + it('removes builds, runs, variations, baselines and their files in one go', async () => { + const { testRun } = await haveTestRunCreated( + buildsService, + testRunsService, + project.id, + project.mainBranchName, + './test/image.png' + ); + const approved = await testRunsService.approve(testRun.id, false); + const baseline = await prisma.baseline.findFirstOrThrow({ where: { testRunId: approved.id } }); + const { imageName } = await prisma.testRun.findUniqueOrThrow({ where: { id: testRun.id } }); + expect(imageExists(imageName)).toBe(true); + expect(imageExists(baseline.baselineName)).toBe(true); + + await projectsService.remove(project.id); + + expect(await prisma.project.count({ where: { id: project.id } })).toBe(0); + expect(await prisma.build.count({ where: { projectId: project.id } })).toBe(0); + expect(await prisma.testVariation.count({ where: { projectId: project.id } })).toBe(0); + expect(await prisma.testRun.count({ where: { id: testRun.id } })).toBe(0); + expect(await prisma.baseline.count({ where: { id: baseline.id } })).toBe(0); + expect(imageExists(imageName)).toBe(false); + expect(imageExists(baseline.baselineName)).toBe(false); + }); + + it('answers 404 for a project removed already', async () => { + await projectsService.remove(project.id); + + await expect(projectsService.remove(project.id)).rejects.toThrow( + new HttpException('Project not found', HttpStatus.NOT_FOUND) + ); + }); +}); diff --git a/test/projects.e2e-spec.ts b/test/projects.e2e-spec.ts index 24d9762e..3d2a409c 100644 --- a/test/projects.e2e-spec.ts +++ b/test/projects.e2e-spec.ts @@ -1,7 +1,6 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { INestApplication, ValidationPipe } from '@nestjs/common'; -import { AppModule } from '../src/app.module'; +import { INestApplication } from '@nestjs/common'; import { UsersService } from '../src/users/users.service'; +import { createE2eApp } from './api-helpers'; import { requestWithAuth, haveUserLogged, LoggedUser } from './preconditions'; import { ProjectsService } from '../src/projects/projects.service'; import { randomUUID } from 'crypto'; @@ -36,18 +35,9 @@ describe('Projects (e2e)', () => { let loggedUser: LoggedUser; beforeAll(async () => { - const moduleFixture: TestingModule = await Test.createTestingModule({ - imports: [AppModule], - }) - .overrideProvider(ProjectsService) - .useValue(projectServiceMock) - .compile(); - - app = moduleFixture.createNestApplication(); - app.useGlobalPipes(new ValidationPipe()); - usersService = moduleFixture.get(UsersService); - - await app.init(); + const created = await createE2eApp({ overrides: [[ProjectsService, projectServiceMock]] }); + app = created.app; + usersService = created.get(UsersService); }); beforeEach(async () => { @@ -62,72 +52,59 @@ describe('Projects (e2e)', () => { await app.close(); }); - describe('POST /', () => { - it('200', () => { - return requestWithAuth(app, 'post', '/projects', loggedUser.token) - .send(project) - .expect(201) - .expect((res) => { - expect(res.body.name).toBe(project.name); - expect(res.body.mainBranchName).toBe(project.mainBranchName); - expect(res.body.autoApproveFeature).toBe(project.autoApproveFeature); - expect(res.body.imageComparison).toBe(project.imageComparison); - expect(res.body.imageComparisonConfig).toBe(project.imageComparisonConfig); - expect(res.body.maxBuildAllowed).toBe(project.maxBuildAllowed); - expect(res.body.maxBranchLifetime).toBe(project.maxBranchLifetime); + it.each([ + ['post', '/projects', ''], + ['get', '/projects', ''], + ['delete', `/projects/${project.id}`, 'asd'], + ['put', '/projects', 'asd'], + ] as const)('%s %s answers 401 for token "%s"', (method, url, token) => + requestWithAuth(app, method, url, token).send(project).expect(401) + ); + + it('POST / creates a project', () => { + return requestWithAuth(app, 'post', '/projects', loggedUser.token) + .send(project) + .expect(201) + .expect((res) => { + expect(res.body).toMatchObject({ + name: project.name, + mainBranchName: project.mainBranchName, + autoApproveFeature: project.autoApproveFeature, + imageComparison: project.imageComparison, + imageComparisonConfig: project.imageComparisonConfig, + maxBuildAllowed: project.maxBuildAllowed, + maxBranchLifetime: project.maxBranchLifetime, }); - }); - - it('401', () => { - return requestWithAuth(app, 'post', '/projects', '').send(project).expect(401); - }); + }); }); - describe('GET /', () => { - it('200', async () => { - const res = await requestWithAuth(app, 'get', '/projects', loggedUser.token).send().expect(200); + it('GET / lists the projects', async () => { + const res = await requestWithAuth(app, 'get', '/projects', loggedUser.token).send().expect(200); - expect(res.body).toEqual(expect.arrayContaining(projectServiceMock.findAll())); - }); - - it('401', async () => { - await requestWithAuth(app, 'get', '/projects', '').send().expect(401); - }); + expect(res.body).toEqual(expect.arrayContaining(projectServiceMock.findAll())); }); - describe('DELETE /', () => { - it('can delete', async () => { - const res = await requestWithAuth(app, 'delete', `/projects/${project.id}`, loggedUser.token).send().expect(200); + it('DELETE /:id removes the project', async () => { + const res = await requestWithAuth(app, 'delete', `/projects/${project.id}`, loggedUser.token).send().expect(200); - expect(res.body).toStrictEqual({ - ...projectServiceMock.remove(), - createdAt: expect.any(String), - updatedAt: expect.any(String), - }); - }); - - it('not valid UUID', async () => { - await requestWithAuth(app, 'delete', `/projects/123`, loggedUser.token).send().expect(400); - }); - - it('not valid token', async () => { - await requestWithAuth(app, 'delete', `/projects/${project.id}`, 'asd').send().expect(401); + expect(res.body).toStrictEqual({ + ...projectServiceMock.remove(), + createdAt: expect.any(String), + updatedAt: expect.any(String), }); }); - describe('PUT /', () => { - it('can edit', async () => { - const res = await requestWithAuth(app, 'put', `/projects`, loggedUser.token).send(project).expect(200); + it('DELETE /:id answers 400 for an id that is not a UUID', async () => { + await requestWithAuth(app, 'delete', `/projects/123`, loggedUser.token).send().expect(400); + }); - expect(res.body).toStrictEqual({ - ...projectServiceMock.update(), - createdAt: expect.any(String), - updatedAt: expect.any(String), - }); - }); + it('PUT / edits the project', async () => { + const res = await requestWithAuth(app, 'put', `/projects`, loggedUser.token).send(project).expect(200); - it('not valid token', async () => { - await requestWithAuth(app, 'put', `/projects`, 'asd').send(project).expect(401); + expect(res.body).toStrictEqual({ + ...projectServiceMock.update(), + createdAt: expect.any(String), + updatedAt: expect.any(String), }); }); }); diff --git a/test/static.e2e-spec.ts b/test/static.e2e-spec.ts index 28b064ef..91e0633a 100644 --- a/test/static.e2e-spec.ts +++ b/test/static.e2e-spec.ts @@ -3,14 +3,12 @@ import { Project, TestStatus } from '@prisma/client'; import { readFileSync } from 'fs'; import { PNG } from 'pngjs'; import request from 'supertest'; -import { randomUUID } from 'crypto'; -import { TEST_PROJECT } from '../src/_data_'; import { BuildsService } from '../src/builds/builds.service'; import { PrismaService } from '../src/prisma/prisma.service'; import { ProjectsService } from '../src/projects/projects.service'; import { StaticService } from '../src/static/static.service'; import { TestRunsService } from '../src/test-runs/test-runs.service'; -import { createE2eApp, pngBuffer, RED, storedImagePath } from './api-helpers'; +import { CHANGED, createE2eApp, haveProject, pngBuffer, postRunDirect, storedImagePath } from './api-helpers'; describe('Images (e2e)', () => { let app: INestApplication; @@ -21,12 +19,9 @@ describe('Images (e2e)', () => { let prisma: PrismaService; let project: Project; - // supertest does not follow redirects, so the redirect itself is what is asserted const get = (path: string) => request(app.getHttpServer()).get(path); - /** Binary response bodies as a Buffer. */ const asBuffer = (res: request.Response, callback: (err: Error | null, body: Buffer) => void) => { - // in Node the parser is handed the raw response stream, which superagent's types do not model const stream = res as unknown as NodeJS.ReadableStream; const chunks: Buffer[] = []; stream.on('data', (chunk: Buffer) => chunks.push(chunk)); @@ -36,19 +31,15 @@ describe('Images (e2e)', () => { beforeAll(async () => { const created = await createE2eApp(); app = created.app; - projectsService = created.moduleFixture.get(ProjectsService); - buildsService = created.moduleFixture.get(BuildsService); - testRunsService = created.moduleFixture.get(TestRunsService); - staticService = created.moduleFixture.get(StaticService); - prisma = created.moduleFixture.get(PrismaService); + projectsService = created.get(ProjectsService); + buildsService = created.get(BuildsService); + testRunsService = created.get(TestRunsService); + staticService = created.get(StaticService); + prisma = created.get(PrismaService); }); beforeEach(async () => { - project = await projectsService.create({ - ...TEST_PROJECT, - autoApproveFeature: false, - name: `Images E2E ${randomUUID()}`, - }); + project = await haveProject(projectsService, 'Images E2E', { autoApproveFeature: false }); }); afterEach(async () => { @@ -60,13 +51,12 @@ describe('Images (e2e)', () => { await app.close(); }); - const postRun = async (image: Buffer) => { - const build = await buildsService.findOrCreate({ projectId: project.id, branchName: 'master' }); - return testRunsService.postTestRun({ - createTestRequestDto: { projectId: project.id, buildId: build.id, branchName: 'master', name: 'Screen' }, - imageBuffer: image, - }); - }; + const postRun = async (image: Buffer) => + postRunDirect( + testRunsService, + await buildsService.findOrCreate({ projectId: project.id, branchName: 'master' }), + image + ); describe('GET /images/:fileName', () => { it('redirects to the stored file, cacheable privately, without authentication', async () => { @@ -78,16 +68,13 @@ describe('Images (e2e)', () => { expect(res.headers['cache-control']).toBe('private, max-age=3000'); }); - it('a name that is not stored is still redirected', async () => { - // NOTE: current behavior — the HDD backend does not check the file, the static handler 404s after the redirect + it('redirects a well-formed name without checking that it is stored', async () => { const res = await get('/images/missing.screenshot.png').expect(302); expect(res.headers.location).toBe('/missing.screenshot.png'); }); - // a leading slash used to come back as a protocol-relative Location - // ('//example.com'): an open redirect to another host - it.each(['%2Fexample.com', '%2F%2Fexample.com', 'nested%2Fimage.png', '..'])( + it.each(['%2Fexample.com', '%2F%2Fexample.com', 'nested%2Fimage.png', '..', '%5Cevil.com'])( 'refuses %s as an image name instead of redirecting', async (name) => { const res = await get(`/images/${name}`).expect(400); @@ -100,7 +87,7 @@ describe('Images (e2e)', () => { it('500 when storage cannot produce a URL', async () => { jest.spyOn(staticService, 'getImageUrl').mockRejectedValueOnce(new Error('storage down')); - const res = await get('/images/any.png').expect(500); + const res = await get('/images/any.screenshot.png').expect(500); expect(res.text).toBe('Error occurred while getting the file.'); expect(res.headers['cache-control']).toBeUndefined(); @@ -115,13 +102,14 @@ describe('Images (e2e)', () => { expect(res.headers['content-type']).toBe('image/png'); expect(res.headers['content-disposition']).toBe(`attachment; filename="${run.imageName}"`); + expect(res.headers['cache-control']).toBe('private, max-age=31536000, immutable'); expect(Buffer.compare(res.body, readFileSync(storedImagePath(run.imageName)))).toBe(0); }); it('serves the diff and thumbnails of an unresolved run', async () => { const first = await postRun(pngBuffer()); await testRunsService.approve(first.id); - const run = await postRun(pngBuffer([{ x: 5, y: 5, w: 10, h: 10, color: RED }])); + const run = await postRun(CHANGED); expect(run.status).toBe(TestStatus.unresolved); const stored = (await prisma.testRun.findUnique({ where: { id: run.id } }))!; @@ -138,8 +126,8 @@ describe('Images (e2e)', () => { expect(res.body.message).toBe('Image not found: missing.screenshot.png'); }); - it('a name that merely starts with dots stays inside the image directory', async () => { - await get('/images/..missing.png/download').expect(404); + it('400 for a name that merely starts with dots, it is not a generated name', async () => { + await get('/images/..missing.png/download').expect(400); }); it.each([ diff --git a/test/tasks.e2e-spec.ts b/test/tasks.e2e-spec.ts index 7d2778f8..77d556ea 100644 --- a/test/tasks.e2e-spec.ts +++ b/test/tasks.e2e-spec.ts @@ -1,9 +1,6 @@ -import { Test, TestingModule } from '@nestjs/testing'; import { INestApplication } from '@nestjs/common'; import { Project, TestStatus } from '@prisma/client'; import { existsSync } from 'fs'; -import { randomUUID } from 'crypto'; -import { AppModule } from '../src/app.module'; import { CLEANUP_LOCK_KEY, TasksService } from '../src/shared/tasks/tasks.service'; import { PrismaService } from '../src/prisma/prisma.service'; import { ProjectsService } from '../src/projects/projects.service'; @@ -11,18 +8,12 @@ import { BuildsService } from '../src/builds/builds.service'; import { TestRunsService } from '../src/test-runs/test-runs.service'; import { HddService } from '../src/static/hdd/hdd.service'; import { TestVariationsService } from '../src/test-variations/test-variations.service'; -import { TEST_PROJECT } from '../src/_data_'; +import { createE2eApp, haveProject as createProject, waitFor } from './api-helpers'; import { haveTestRunCreated } from './preconditions'; const DAY = 24 * 60 * 60 * 1000; const image = './test/image.png'; -/** - * The nightly cleanup (@Cron at midnight) called directly on the TasksService - * instance of the app. It walks every project in the database, so the other - * e2e files running in parallel are affected only if they keep non-main - * variations untouched for longer than their project's maxBranchLifetime. - */ describe('Tasks: cleanOldTestVariations (e2e)', () => { let app: INestApplication; let tasksService: TasksService; @@ -35,19 +26,15 @@ describe('Tasks: cleanOldTestVariations (e2e)', () => { const projectIds: string[] = []; beforeAll(async () => { - const moduleFixture: TestingModule = await Test.createTestingModule({ - imports: [AppModule], - }).compile(); - - app = moduleFixture.createNestApplication(); - tasksService = moduleFixture.get(TasksService); - prismaService = moduleFixture.get(PrismaService); - projectsService = moduleFixture.get(ProjectsService); - buildsService = moduleFixture.get(BuildsService); - testRunsService = moduleFixture.get(TestRunsService); - hddService = moduleFixture.get(HddService); - testVariationsService = moduleFixture.get(TestVariationsService); - await app.init(); + const created = await createE2eApp({ validation: false }); + app = created.app; + tasksService = created.get(TasksService); + prismaService = created.get(PrismaService); + projectsService = created.get(ProjectsService); + buildsService = created.get(BuildsService); + testRunsService = created.get(TestRunsService); + hddService = created.get(HddService); + testVariationsService = created.get(TestVariationsService); }); afterEach(async () => { @@ -62,9 +49,7 @@ describe('Tasks: cleanOldTestVariations (e2e)', () => { }); const haveProject = async (maxBranchLifetime: number): Promise => { - const project = await projectsService.create({ - ...TEST_PROJECT, - name: `Tasks E2E ${randomUUID()}`, + const project = await createProject(projectsService, 'Tasks E2E', { maxBuildAllowed: 100, maxBranchLifetime, autoApproveFeature: false, @@ -73,10 +58,8 @@ describe('Tasks: cleanOldTestVariations (e2e)', () => { return project; }; - // an approved run on the branch: a variation with a baseline, a test run and their images const haveApprovedVariation = async (project: Project, branchName: string) => { const { testRun } = await haveTestRunCreated(buildsService, testRunsService, project.id, branchName, image); - // merge keeps the variation on this branch instead of the main one const approved = await testRunsService.approve(testRun.id, true); const variation = (await prismaService.testVariation.findUnique({ where: { id: approved.testVariationId! }, @@ -87,24 +70,12 @@ describe('Tasks: cleanOldTestVariations (e2e)', () => { return { variation, testRun: approved }; }; - // the variation and the runs compared against it, as if last used days ago const ageBy = async (variationId: string, days: number) => { const then = new Date(Date.now() - days * DAY); - // @updatedAt would be overwritten by an update through Prisma await prismaService.$executeRaw`UPDATE "TestVariation" SET "updatedAt" = ${then} WHERE "id" = ${variationId}`; await prismaService.testRun.updateMany({ where: { testVariationId: variationId }, data: { createdAt: then } }); }; - const waitUntil = async (condition: () => boolean, timeout = 5000) => { - const start = Date.now(); - while (!condition()) { - if (Date.now() - start > timeout) { - throw new Error('condition not met in time'); - } - await new Promise((resolve) => setTimeout(resolve, 50)); - } - }; - it('removes feature-branch variations not used within maxBranchLifetime, with their baselines', async () => { const project = await haveProject(5); const old = await haveApprovedVariation(project, 'feature/old'); @@ -121,14 +92,11 @@ describe('Tasks: cleanOldTestVariations (e2e)', () => { const variations = await prismaService.testVariation.findMany({ where: { projectId: project.id } }); expect(variations.map((variation) => variation.id).sort()).toEqual([fresh.variation.id, main.variation.id].sort()); expect(await prismaService.baseline.findMany({ where: { testVariationId: old.variation.id } })).toEqual([]); - // the baseline image is deleted without being awaited - await waitUntil(() => !existsSync(oldBaselineFile)); + await waitFor(() => !existsSync(oldBaselineFile)); - // the test runs stay in their builds, detached from the removed variation const oldRun = (await prismaService.testRun.findUnique({ where: { id: old.testRun.id } }))!; expect(oldRun).toMatchObject({ testVariationId: null, status: TestStatus.approved, branchName: 'feature/old' }); expect(existsSync(hddService.getImagePath(oldRun.imageName))).toBe(true); - // the kept variations keep their baselines expect(await prismaService.baseline.count({ where: { testVariationId: fresh.variation.id } })).toBe(1); expect(await prismaService.baseline.count({ where: { testVariationId: main.variation.id } })).toBe(1); }); @@ -184,7 +152,6 @@ describe('Tasks: cleanOldTestVariations (e2e)', () => { await ageBy(feature.variation.id, 10); const { testRun } = await haveTestRunCreated(buildsService, testRunsService, project.id, 'feature/active', image); expect(testRun.status).toBe(TestStatus.ok); - // a run that matches the baseline leaves the variation as it was const { updatedAt } = (await prismaService.testVariation.findUnique({ where: { id: feature.variation.id } }))!; expect(updatedAt.getTime()).toBeLessThan(Date.now() - 5 * DAY); @@ -206,7 +173,6 @@ describe('Tasks: cleanOldTestVariations (e2e)', () => { } const deleteSpy = jest.spyOn(testVariationsService, 'delete'); - // each run opens its own lock connection, as on two instances await Promise.all([tasksService.cleanOldTestVariations(), tasksService.cleanOldTestVariations()]); const deleted = deleteSpy.mock.calls.map(([id]) => id).filter((id) => stale.includes(id)); @@ -232,7 +198,6 @@ describe('Tasks: cleanOldTestVariations (e2e)', () => { await otherInstance.end(); } - // released with the other instance's connection await tasksService.cleanOldTestVariations(); expect(await prismaService.testVariation.findUnique({ where: { id: variation.id } })).toBeNull(); diff --git a/test/test-runs-review.e2e-spec.ts b/test/test-runs-review.e2e-spec.ts index 81cbc18b..8ef5030c 100644 --- a/test/test-runs-review.e2e-spec.ts +++ b/test/test-runs-review.e2e-spec.ts @@ -1,14 +1,25 @@ import { INestApplication } from '@nestjs/common'; import { Build, Project, TestStatus } from '@prisma/client'; import { randomUUID } from 'crypto'; -import { TEST_PROJECT } from '../src/_data_'; import { BuildsService } from '../src/builds/builds.service'; import { PrismaService } from '../src/prisma/prisma.service'; import { ProjectsService } from '../src/projects/projects.service'; import { TestRunDto } from '../src/test-runs/dto/testRun.dto'; import { UsersService } from '../src/users/users.service'; -import { BLUE, createE2eApp, imageExists, pngBuffer, RED, Rect, TEST_RUN_DTO_KEYS, waitFor } from './api-helpers'; -import { haveUserLogged, requestWithApiKey, requestWithAuth, LoggedUser } from './preconditions'; +import { + approveRuns, + BLUE, + createE2eApp, + haveProject, + imageExists, + pngBuffer, + postRunBase64, + redAt, + TEST_RUN_DTO_KEYS, + waitFor, + WHITE, +} from './api-helpers'; +import { haveUserLogged, requestWithApiKey, requestWithAuth, LoggedUser, withUser } from './preconditions'; describe('TestRuns review API (e2e)', () => { let app: INestApplication; @@ -19,25 +30,20 @@ describe('TestRuns review API (e2e)', () => { let user: LoggedUser; let project: Project; - const WHITE = pngBuffer(); - const redAt = (x: number, y: number, size = 10): Rect => ({ x, y, w: size, h: size, color: RED }); - beforeAll(async () => { const created = await createE2eApp(); app = created.app; - usersService = created.moduleFixture.get(UsersService); - projectsService = created.moduleFixture.get(ProjectsService); - buildsService = created.moduleFixture.get(BuildsService); - prisma = created.moduleFixture.get(PrismaService); + usersService = created.get(UsersService); + projectsService = created.get(ProjectsService); + buildsService = created.get(BuildsService); + prisma = created.get(PrismaService); }); beforeEach(async () => { user = await haveUserLogged(usersService); - project = await projectsService.create({ - ...TEST_PROJECT, + project = await haveProject(projectsService, 'TestRuns review E2E', { autoApproveFeature: false, maxBuildAllowed: 100, - name: `TestRuns review E2E ${randomUUID()}`, }); }); @@ -53,29 +59,14 @@ describe('TestRuns review API (e2e)', () => { const newBuild = (branchName = project.mainBranchName): Promise => buildsService.findOrCreate({ projectId: project.id, branchName }); - /** Posts a screenshot through the public base64 endpoint and returns the response body. */ - const postRun = async ( - build: Build, - image: Buffer, - extra: Record = {} - ): Promise> => { - const res = await requestWithApiKey(app, 'post', '/test-runs', user.apiKey) - .send({ - projectId: project.id, - buildId: build.id, - branchName: build.branchName, - name: 'Screen', - imageBase64: image.toString('base64'), - ...extra, - }) - .expect(201); - return res.body; - }; + const postRun = (build: Build, image: Buffer, extra: Record = {}) => + postRunBase64(app, user, build, image, extra); - const approve = (ids: string[], merge = false) => - requestWithAuth(app, 'post', `/test-runs/approve?merge=${merge}`, user.token).send(ids).expect(201); + const variationOf = async (id: string) => (await prisma.testVariation.findUnique({ where: { id } }))!; + const runOf = async (id: string) => (await prisma.testRun.findUnique({ where: { id } }))!; + + const approve = (ids: string[], merge = false) => approveRuns(app, user, ids, merge); - /** Approves `baseline` as the master baseline of the variation, then posts `image` against it in `build`. */ const runAgainstBaseline = async ( build: Build, baseline: Buffer, @@ -88,6 +79,22 @@ describe('TestRuns review API (e2e)', () => { return postRun(build, image, extra); }; + it.each([ + ['approve', 'post', '/test-runs/approve?merge=false', (id: string) => [id]], + ['reject', 'post', '/test-runs/reject', (id: string) => [id]], + ['revert', 'post', '/test-runs/revert', (id: string) => [id]], + ['delete', 'post', '/test-runs/delete', (id: string) => [id]], + ['update', 'patch', '/test-runs/update/:id', () => ({ comment: 'x' })], + ] as const)('403 for a guest on %s', async (_name, method, url, body) => { + const run = await postRun(await newBuild(), WHITE); + + await withUser(usersService, 'guest', (guest) => + requestWithAuth(app, method, url.replace(':id', run.id), guest.token).send(body(run.id)).expect(403) + ); + + expect((await runOf(run.id)).status).toBe(TestStatus.new); + }); + describe('POST /test-runs (base64)', () => { it('creates a new run and stores the image on disk', async () => { const build = await newBuild(); @@ -118,6 +125,14 @@ describe('TestRuns review API (e2e)', () => { it('403 without an API key', async () => { await requestWithApiKey(app, 'post', '/test-runs', '').send({}).expect(403); }); + + it('parallel builds uploading the same brand-new screen all succeed', async () => { + const builds = await Promise.all(Array.from({ length: 8 }, () => newBuild())); + + await Promise.all(builds.map((build) => postRun(build, WHITE, { name: 'Brand new screen' }))); + + expect(await prisma.testVariation.count({ where: { projectId: project.id, name: 'Brand new screen' } })).toBe(1); + }); }); describe('GET /test-runs?buildId=', () => { @@ -131,7 +146,6 @@ describe('TestRuns review API (e2e)', () => { const res = await requestWithAuth(app, 'get', `/test-runs?buildId=${build.id}`, user.token).expect(200); expect(res.body.map((run: TestRunDto) => run.id).sort()).toEqual([a.id, b.id].sort()); - // internal columns (pixelMisMatchCount, changeSignature, timestamps, projectId) are not exposed expect(Object.keys(res.body[0]).sort()).toEqual(TEST_RUN_DTO_KEYS); }); @@ -163,7 +177,6 @@ describe('TestRuns review API (e2e)', () => { expect(res.body.browser).toBe('firefox'); expect(res.body.testVariation.id).toBe(run.testVariationId); expect(res.body.testVariation.browser).toBe('firefox'); - // unlike the list, the details route returns the raw row expect(res.body).toHaveProperty('pixelMisMatchCount'); expect(res.body).toHaveProperty('changeSignature'); expect(res.body.projectId).toBe(project.id); @@ -189,7 +202,7 @@ describe('TestRuns review API (e2e)', () => { await approve([run.id]); - const stored = (await prisma.testRun.findUnique({ where: { id: run.id } }))!; + const stored = await runOf(run.id); expect(stored.status).toBe(TestStatus.approved); const variation = (await prisma.testVariation.findUnique({ where: { id: run.testVariationId }, @@ -213,7 +226,7 @@ describe('TestRuns review API (e2e)', () => { await approve([featureRun.id]); - const stored = (await prisma.testRun.findUnique({ where: { id: featureRun.id } }))!; + const stored = await runOf(featureRun.id); expect(stored.status).toBe(TestStatus.approved); expect(stored.testVariationId).not.toBe(masterRun.testVariationId); const featureVariation = (await prisma.testVariation.findUnique({ @@ -223,7 +236,6 @@ describe('TestRuns review API (e2e)', () => { expect(featureVariation.branchName).toBe('feature'); expect(featureVariation.comment).toBe('feature comment'); expect(featureVariation.baselines).toHaveLength(1); - // master keeps its single baseline expect(await prisma.baseline.count({ where: { testVariationId: masterRun.testVariationId } })).toBe(1); }); @@ -236,7 +248,7 @@ describe('TestRuns review API (e2e)', () => { await approve([featureRun.id], true); - const stored = (await prisma.testRun.findUnique({ where: { id: featureRun.id } }))!; + const stored = await runOf(featureRun.id); expect(stored.status).toBe(TestStatus.approved); expect(stored.merge).toBe(true); expect(stored.testVariationId).toBe(masterRun.testVariationId); @@ -249,30 +261,44 @@ describe('TestRuns review API (e2e)', () => { await approve([featureBaseline.id]); const masterBaseline = await postRun(await newBuild(), WHITE); await approve([masterBaseline.id]); - // master was updated last, so the new feature run is compared against master const featureRun = await postRun(await newBuild('feature'), pngBuffer([redAt(0, 0)])); expect(featureRun.testVariationId).toBe(masterBaseline.testVariationId); expect(featureRun.baselineBranchName).toBe(project.mainBranchName); await approve([featureRun.id]); - const stored = (await prisma.testRun.findUnique({ where: { id: featureRun.id } }))!; + const stored = await runOf(featureRun.id); expect(stored.testVariationId).toBe(featureBaseline.testVariationId); expect(await prisma.baseline.count({ where: { testVariationId: featureBaseline.testVariationId } })).toBe(2); expect(await prisma.baseline.count({ where: { testVariationId: masterBaseline.testVariationId } })).toBe(1); }); + it('approving a feature-branch run twice keeps one baseline', async () => { + const masterRun = await postRun(await newBuild(), WHITE); + await approve([masterRun.id]); + const featureRun = await postRun(await newBuild('feature'), pngBuffer([redAt(0, 0)])); + await approve([featureRun.id]); + const { testVariationId } = await runOf(featureRun.id); + const { baselineName } = await variationOf(testVariationId!); + + await approve([featureRun.id]); + + const baselines = await prisma.baseline.findMany({ where: { testVariationId: testVariationId! } }); + expect(baselines).toHaveLength(1); + expect(baselines[0].testRunId).toBe(featureRun.id); + expect((await variationOf(testVariationId!)).baselineName).toBe(baselineName); + }); + it('a run whose variation was deleted', async () => { const run = await postRun(await newBuild(), WHITE); await requestWithAuth(app, 'delete', `/test-variations/${run.testVariationId}`, user.token).expect(200); - // NOTE: current behavior — a 500 ("No test variation found. Re-create test run"), and the run stays new const res = await requestWithAuth(app, 'post', `/test-runs/approve?merge=false`, user.token) .send([run.id]) - .expect(500); + .expect(409); expect(res.body.message).toBe('No test variation found. Re-create test run'); - expect((await prisma.testRun.findUnique({ where: { id: run.id } }))!.status).toBe(TestStatus.new); + expect((await runOf(run.id)).status).toBe(TestStatus.new); }); it('400 without the merge query parameter', async () => { @@ -281,19 +307,7 @@ describe('TestRuns review API (e2e)', () => { await requestWithAuth(app, 'post', `/test-runs/approve`, user.token).send([run.id]).expect(400); - expect((await prisma.testRun.findUnique({ where: { id: run.id } }))!.status).toBe(TestStatus.new); - }); - - it('403 for a guest', async () => { - const guest = await haveUserLogged(usersService, 'guest'); - try { - const build = await newBuild(); - const run = await postRun(build, WHITE); - - await requestWithAuth(app, 'post', `/test-runs/approve?merge=false`, guest.token).send([run.id]).expect(403); - } finally { - await usersService.delete(guest.id); - } + expect((await runOf(run.id)).status).toBe(TestStatus.new); }); it('404 for an unknown id', async () => { @@ -311,17 +325,115 @@ describe('TestRuns review API (e2e)', () => { const runs = await prisma.testRun.findMany({ where: { buildId: build.id } }); expect(runs.map((run) => run.status)).toEqual([TestStatus.failed, TestStatus.failed]); - // no baseline is created expect(await prisma.baseline.count({ where: { testVariation: { projectId: project.id } } })).toBe(0); }); + }); - it('403 for a guest', async () => { - const guest = await haveUserLogged(usersService, 'guest'); - try { - await requestWithAuth(app, 'post', `/test-runs/reject`, guest.token).send([]).expect(403); - } finally { - await usersService.delete(guest.id); - } + describe('POST /test-runs/revert', () => { + const revert = (ids: string[], token = user.token) => + requestWithAuth(app, 'post', '/test-runs/revert', token).send(ids); + + it('approve, then revert: the variation gets its previous baseline back and the run its status', async () => { + const run = await runAgainstBaseline(await newBuild(), WHITE, pngBuffer([redAt(0, 0)])); + expect(run.status).toBe(TestStatus.unresolved); + const before = await variationOf(run.testVariationId); + await approve([run.id]); + const approvedName = (await variationOf(run.testVariationId)).baselineName!; + expect(approvedName).not.toBe(before.baselineName); + expect(imageExists(approvedName)).toBe(true); + + await revert([run.id]).expect(204); + + expect((await variationOf(run.testVariationId)).baselineName).toBe(before.baselineName); + const stored = await runOf(run.id); + expect(stored.status).toBe(TestStatus.unresolved); + expect(stored.statusBeforeDecision).toBeNull(); + expect(await prisma.baseline.count({ where: { testRunId: run.id } })).toBe(0); + expect(await prisma.baseline.count({ where: { testVariationId: run.testVariationId } })).toBe(1); + expect(imageExists(approvedName)).toBe(false); + expect(imageExists(before.baselineName)).toBe(true); + }); + + it('reject, then revert: the status is restored', async () => { + const run = await runAgainstBaseline(await newBuild(), WHITE, pngBuffer([redAt(0, 0)])); + await requestWithAuth(app, 'post', '/test-runs/reject', user.token).send([run.id]).expect(201); + expect((await runOf(run.id)).status).toBe(TestStatus.failed); + + await revert([run.id]).expect(204); + + expect((await runOf(run.id)).status).toBe(TestStatus.unresolved); + }); + + it('two approvals of one screen: reverting the first is a 409 and nothing changes', async () => { + const first = await postRun(await newBuild(), WHITE); + await approve([first.id]); + const second = await postRun(await newBuild(), pngBuffer([redAt(0, 0)])); + await approve([second.id]); + const variationBefore = await variationOf(first.testVariationId); + + const res = await revert([first.id]).expect(409); + + expect(res.body.message).toContain(first.id); + expect(await variationOf(first.testVariationId)).toEqual(variationBefore); + expect((await runOf(first.id)).status).toBe(TestStatus.approved); + expect((await runOf(second.id)).status).toBe(TestStatus.approved); + expect(await prisma.baseline.count({ where: { testVariationId: first.testVariationId } })).toBe(2); + expect(imageExists(variationBefore.baselineName)).toBe(true); + }); + + it('two approvals of one screen are reverted in one request, latest first', async () => { + const first = await postRun(await newBuild(), WHITE); + await approve([first.id]); + const second = await postRun(await newBuild(), pngBuffer([redAt(0, 0)])); + await approve([second.id]); + + await revert([first.id, second.id]).expect(204); + + expect((await variationOf(first.testVariationId)).baselineName).toBeNull(); + expect(await prisma.baseline.count({ where: { testVariationId: first.testVariationId } })).toBe(0); + expect((await runOf(first.id)).status).toBe(TestStatus.new); + expect((await runOf(second.id)).status).toBe(TestStatus.unresolved); + }); + + it('a feature-branch approval: the feature variation it created is deleted and the run is back on main', async () => { + const masterRun = await postRun(await newBuild(), WHITE); + await approve([masterRun.id]); + const featureRun = await postRun(await newBuild('feature'), pngBuffer([redAt(0, 0)])); + await approve([featureRun.id]); + const featureVariationId = (await runOf(featureRun.id)).testVariationId!; + expect(featureVariationId).not.toBe(masterRun.testVariationId); + const featureBaselineName = (await variationOf(featureVariationId)).baselineName!; + + await revert([featureRun.id]).expect(204); + + const stored = await runOf(featureRun.id); + expect(stored.testVariationId).toBe(masterRun.testVariationId); + expect(stored.status).toBe(TestStatus.unresolved); + expect(await prisma.testVariation.count({ where: { projectId: project.id, branchName: 'feature' } })).toBe(0); + expect(imageExists(featureBaselineName)).toBe(false); + expect(await prisma.baseline.count({ where: { testVariationId: masterRun.testVariationId } })).toBe(1); + }); + + it('409 for a run that was not decided', async () => { + const run = await postRun(await newBuild(), WHITE); + + const res = await revert([run.id]).expect(409); + + expect(res.body.message).toContain(run.id); + }); + + it('409 for an empty list', async () => { + await revert([]).expect(409); + }); + + it('404 for an unknown id', async () => { + await revert([randomUUID()]).expect(404); + }); + + it('400 for an id that is not a uuid', async () => { + const run = await postRun(await newBuild(), WHITE); + + await revert([run.id, 'not-a-uuid']).expect(400); }); }); @@ -330,7 +442,7 @@ describe('TestRuns review API (e2e)', () => { const build = await newBuild(); const run = await runAgainstBaseline(build, WHITE, pngBuffer([redAt(5, 5)])); expect(run.status).toBe(TestStatus.unresolved); - const stored = (await prisma.testRun.findUnique({ where: { id: run.id } }))!; + const stored = await runOf(run.id); const files = [stored.imageName, stored.diffName, stored.imageThumbnailName, stored.diffThumbnailName]; expect(files.every((name) => imageExists(name))).toBe(true); const baselineName = stored.baselineName; @@ -339,42 +451,31 @@ describe('TestRuns review API (e2e)', () => { expect(await prisma.testRun.findUnique({ where: { id: run.id } })).toBeNull(); expect(files.some((name) => imageExists(name))).toBe(false); - // the baseline belongs to the variation and stays expect(imageExists(baselineName)).toBe(true); }); it('ignores ids that do not exist', async () => { await requestWithAuth(app, 'post', `/test-runs/delete`, user.token).send([randomUUID()]).expect(201); }); - - it('403 for a guest', async () => { - const guest = await haveUserLogged(usersService, 'guest'); - try { - await requestWithAuth(app, 'post', `/test-runs/delete`, guest.token).send([]).expect(403); - } finally { - await usersService.delete(guest.id); - } - }); }); describe('POST /test-runs/ignoreAreas/*', () => { const area = { x: 0, y: 0, width: 30, height: 30 }; const otherArea = { x: 30, y: 30, width: 30, height: 30 }; + const setAreas = (action: 'update' | 'add', ids: string[], ignoreAreas: unknown[]) => + requestWithAuth(app, 'post', `/test-runs/ignoreAreas/${action}`, user.token).send({ ids, ignoreAreas }); it('update: stores the areas on run and variation and recomputes the diff', async () => { const build = await newBuild(); const run = await runAgainstBaseline(build, WHITE, pngBuffer([redAt(5, 5)])); expect(run.status).toBe(TestStatus.unresolved); - const before = (await prisma.testRun.findUnique({ where: { id: run.id } }))!; + const before = await runOf(run.id); - await requestWithAuth(app, 'post', `/test-runs/ignoreAreas/update`, user.token) - .send({ ids: [run.id], ignoreAreas: [area] }) - .expect(201); + await setAreas('update', [run.id], [area]).expect(201); const after = (await prisma.testRun.findUnique({ where: { id: run.id }, include: { testVariation: true } }))!; expect(JSON.parse(after.ignoreAreas)).toEqual([area]); expect(JSON.parse(after.testVariation!.ignoreAreas)).toEqual([area]); - // the change is now masked: the run passes and its diff pictures are gone expect(after.status).toBe(TestStatus.ok); expect(after.diffName).toBeNull(); expect(after.imageThumbnailName).toBeNull(); @@ -389,24 +490,17 @@ describe('TestRuns review API (e2e)', () => { const build = await newBuild(); const run = await postRun(build, WHITE); - await requestWithAuth(app, 'post', `/test-runs/ignoreAreas/update`, user.token) - .send({ ids: [run.id], ignoreAreas: [area] }) - .expect(201); - await requestWithAuth(app, 'post', `/test-runs/ignoreAreas/add`, user.token) - .send({ ids: [run.id], ignoreAreas: [otherArea] }) - .expect(201); + await setAreas('update', [run.id], [area]).expect(201); + await setAreas('add', [run.id], [otherArea]).expect(201); const stored = (await prisma.testRun.findUnique({ where: { id: run.id }, include: { testVariation: true } }))!; expect(JSON.parse(stored.ignoreAreas)).toEqual([area, otherArea]); expect(JSON.parse(stored.testVariation!.ignoreAreas)).toEqual([area, otherArea]); - // no baseline: still new after the recompute expect(stored.status).toBe(TestStatus.new); - await requestWithAuth(app, 'post', `/test-runs/ignoreAreas/update`, user.token) - .send({ ids: [run.id], ignoreAreas: [] }) - .expect(201); + await setAreas('update', [run.id], []).expect(201); - const cleared = (await prisma.testRun.findUnique({ where: { id: run.id } }))!; + const cleared = await runOf(run.id); expect(JSON.parse(cleared.ignoreAreas)).toEqual([]); }); @@ -422,7 +516,6 @@ describe('TestRuns review API (e2e)', () => { expect(JSON.parse(stored.tempIgnoreAreas)).toEqual([area]); expect(JSON.parse(stored.ignoreAreas)).toEqual([]); expect(JSON.parse(stored.testVariation!.ignoreAreas)).toEqual([]); - // the previous unresolved run of the same variation in the build was replaced expect(await prisma.testRun.findUnique({ where: { id: run.id } })).toBeNull(); }); @@ -430,15 +523,11 @@ describe('TestRuns review API (e2e)', () => { const build = await newBuild(); const run = await postRun(build, WHITE); - await requestWithAuth(app, 'post', `/test-runs/ignoreAreas/update`, user.token) - .send({ ids: [run.id], ignoreAreas: [{ ...area, x: 'left' }] }) - .expect(400); + await setAreas('update', [run.id], [{ ...area, x: 'left' }]).expect(400); }); it('404 for an unknown id', async () => { - await requestWithAuth(app, 'post', `/test-runs/ignoreAreas/update`, user.token) - .send({ ids: [randomUUID()], ignoreAreas: [area] }) - .expect(404); + await setAreas('update', [randomUUID()], [area]).expect(404); }); }); @@ -452,10 +541,25 @@ describe('TestRuns review API (e2e)', () => { .expect(200); expect(res.body.comment).toBe('looks fine'); - const variation = (await prisma.testVariation.findUnique({ where: { id: run.testVariationId } }))!; + const variation = await variationOf(run.testVariationId); expect(variation.comment).toBe('looks fine'); }); + it('only the comment reaches the variation', async () => { + const run = await postRun(await newBuild(), WHITE); + await approve([run.id]); + const before = await variationOf(run.testVariationId); + + await requestWithAuth(app, 'patch', `/test-runs/update/${run.id}`, user.token) + .send({ comment: 'x', baselineName: 'evil.png', ignoreAreas: 'not json' }) + .expect(200); + + const after = await variationOf(run.testVariationId); + expect(after.comment).toBe('x'); + expect(after.baselineName).toBe(before.baselineName); + expect(after.ignoreAreas).toBe(before.ignoreAreas); + }); + it('the variation comment is the default for the next run', async () => { const build = await newBuild(); const run = await postRun(build, WHITE); @@ -474,20 +578,6 @@ describe('TestRuns review API (e2e)', () => { await requestWithAuth(app, 'patch', `/test-runs/update/${run.id}`, user.token).send({}).expect(400); }); - - it('403 for a guest', async () => { - const guest = await haveUserLogged(usersService, 'guest'); - try { - const build = await newBuild(); - const run = await postRun(build, WHITE); - - await requestWithAuth(app, 'patch', `/test-runs/update/${run.id}`, guest.token) - .send({ comment: 'x' }) - .expect(403); - } finally { - await usersService.delete(guest.id); - } - }); }); describe('GET /test-runs/matchingSiblings/:id', () => { @@ -499,17 +589,10 @@ describe('TestRuns review API (e2e)', () => { }); const byTag = (list: Array<{ customTags: string; reason?: string }>) => [...list].sort((a, b) => a.customTags.localeCompare(b.customTags)); + const idsOf = (res: { body: { variations: TestRunDto[] } }) => res.body.variations.map((run) => run.id); + const skippedOf = (res: { body: { skipped: Array<{ customTags: string; reason?: string }> } }) => + byTag(res.body.skipped).map((run) => [run.customTags, run.reason]); - /** - * One screen in several locales of the same build, each compared against a - * white baseline: - * - en (reviewed) and de: a 10x10 red square, at different positions - * - fr: a 10x10 blue square (same size, other colour) - * - es: a 30x30 red square (9x the change area) - * - it: unchanged (ok — not a review candidate) - * - pt: no baseline yet (new — nothing to compare) - * plus runs that differ on another axis or by name, which are never siblings. - */ const haveLocalesBuild = async () => { const build = await newBuild(); const en = await runAgainstBaseline(build, WHITE, pngBuffer([redAt(5, 5)]), { customTags: 'en' }); @@ -542,7 +625,7 @@ describe('TestRuns review API (e2e)', () => { const res = await matching(en.id).expect(200); - expect(res.body.variations.map((run: TestRunDto) => run.id)).toEqual([en.id]); + expect(idsOf(res)).toEqual([en.id]); expect(res.body.skipped).toEqual([]); expect(Object.keys(res.body.variations[0]).sort()).toEqual(TEST_RUN_DTO_KEYS); }); @@ -553,14 +636,13 @@ describe('TestRuns review API (e2e)', () => { const res = await matching(runs.en.id).expect(200); - expect(res.body.variations.map((run: TestRunDto) => run.id)).toEqual([runs.en.id, runs.de.id]); - expect(byTag(res.body.skipped).map((run) => [run.customTags, run.reason])).toEqual([ + expect(idsOf(res)).toEqual([runs.en.id, runs.de.id]); + expect(skippedOf(res)).toEqual([ ['es', 'different change size'], ['fr', 'different change pattern'], ['pt', 'no diff to match'], ]); expect(Object.keys(res.body.skipped[0]).sort()).toEqual([...TEST_RUN_DTO_KEYS, 'reason'].sort()); - // read-only: nothing changed status const statuses = await prisma.testRun.findMany({ where: { buildId: runs.build.id }, select: { status: true } }); expect(statuses.filter((run) => run.status === TestStatus.approved)).toHaveLength(0); }, 60000); @@ -579,12 +661,11 @@ describe('TestRuns review API (e2e)', () => { await prisma.testRun.update({ where: { id: runs.fr.id }, data: { changeSignature: stampedWithOtherThreshold } }); const first = await matching(runs.en.id).expect(200); - // second request is served from the in-memory signature memo const second = await matching(runs.en.id).expect(200); for (const res of [first, second]) { - expect(res.body.variations.map((run: TestRunDto) => run.id)).toEqual([runs.en.id, runs.de.id]); - expect(byTag(res.body.skipped).map((run) => [run.customTags, run.reason])).toEqual([ + expect(idsOf(res)).toEqual([runs.en.id, runs.de.id]); + expect(skippedOf(res)).toEqual([ ['es', 'different change size'], ['fr', 'different change pattern'], ['pt', 'no diff to match'], @@ -602,8 +683,8 @@ describe('TestRuns review API (e2e)', () => { const res = await matching(runs.en.id).expect(200); - expect(res.body.variations.map((run: TestRunDto) => run.id)).toEqual([runs.en.id]); - expect(byTag(res.body.skipped).map((run) => [run.customTags, run.reason])).toEqual([ + expect(idsOf(res)).toEqual([runs.en.id]); + expect(skippedOf(res)).toEqual([ ['de', 'no diff to match'], ['es', 'different change size'], ['fr', 'different change pattern'], @@ -617,8 +698,8 @@ describe('TestRuns review API (e2e)', () => { const res = await matching(runs.pt.id).expect(200); - expect(res.body.variations.map((run: TestRunDto) => run.id)).toEqual([runs.pt.id]); - expect(byTag(res.body.skipped).map((run) => [run.customTags, run.reason])).toEqual([ + expect(idsOf(res)).toEqual([runs.pt.id]); + expect(skippedOf(res)).toEqual([ ['de', 'no reference diff'], ['en', 'no reference diff'], ['es', 'no reference diff'], @@ -630,13 +711,12 @@ describe('TestRuns review API (e2e)', () => { const build = await newBuild(); const mac = await runAgainstBaseline(build, WHITE, pngBuffer([redAt(5, 5)]), { os: 'mac' }); const win = await runAgainstBaseline(build, WHITE, pngBuffer([redAt(30, 30)]), { os: 'win' }); - // same os as the reviewed run but another locale: not a sibling under groupBy=os await runAgainstBaseline(build, WHITE, pngBuffer([redAt(5, 5)]), { os: 'mac', customTags: 'de' }); await enableBulkApprove('os'); const res = await matching(mac.id).expect(200); - expect(res.body.variations.map((run: TestRunDto) => run.id)).toEqual([mac.id, win.id]); + expect(idsOf(res)).toEqual([mac.id, win.id]); expect(res.body.skipped).toEqual([]); }, 60000); @@ -649,7 +729,7 @@ describe('TestRuns review API (e2e)', () => { const res = await matching(en.id).expect(200); - expect(res.body.variations.map((run: TestRunDto) => run.id)).toEqual([en.id, de.id]); + expect(idsOf(res)).toEqual([en.id, de.id]); }, 60000); it('404 for an unknown id', async () => { diff --git a/test/test-runs-validation.e2e-spec.ts b/test/test-runs-validation.e2e-spec.ts new file mode 100644 index 00000000..c4325083 --- /dev/null +++ b/test/test-runs-validation.e2e-spec.ts @@ -0,0 +1,124 @@ +import { INestApplication } from '@nestjs/common'; +import { randomUUID } from 'crypto'; +import { PNG } from 'pngjs'; +import { Build, Project } from '@prisma/client'; +import { UsersService } from '../src/users/users.service'; +import { ProjectsService } from '../src/projects/projects.service'; +import { BuildsService } from '../src/builds/builds.service'; +import { TestRunsService } from '../src/test-runs/test-runs.service'; +import { createE2eApp, haveProject, withEnv } from './api-helpers'; +import { haveUserLogged, LoggedUser, requestWithApiKey, requestWithAuth } from './preconditions'; + +describe('Test runs input validation (e2e)', () => { + let app: INestApplication; + let usersService: UsersService; + let projectsService: ProjectsService; + let buildsService: BuildsService; + let testRunsService: TestRunsService; + let user: LoggedUser; + let project: Project; + let build: Build; + + const image = './test/image.png'; + + beforeAll(async () => { + const created = await createE2eApp(); + app = created.app; + usersService = created.get(UsersService); + projectsService = created.get(ProjectsService); + buildsService = created.get(BuildsService); + testRunsService = created.get(TestRunsService); + }); + + beforeEach(async () => { + user = await haveUserLogged(usersService); + project = await haveProject(projectsService, 'Validation E2E'); + build = await buildsService.findOrCreate({ projectId: project.id, branchName: project.mainBranchName }); + }); + + afterEach(async () => { + await projectsService.remove(project.id); + await usersService.delete(user.id); + }); + + afterAll(async () => { + await app.close(); + }); + + const multipart = () => + requestWithApiKey(app, 'post', '/test-runs/multipart', user.apiKey) + .set('Content-type', 'multipart/form-data') + .field('name', 'Multipart image') + .field('branchName', project.mainBranchName) + .field('buildId', build.id) + .field('projectId', project.id); + + const postBase64 = (imageBase64: string) => + requestWithApiKey(app, 'post', '/test-runs', user.apiKey).send({ + name: 'Base64 image', + branchName: project.mainBranchName, + buildId: build.id, + projectId: project.id, + imageBase64, + }); + + it('keeps every multipart field, the area id included', async () => { + await multipart() + .field('diffTollerancePercent', '0.12') + .field('merge', 'true') + .field('ignoreAreas', '[{"id":"1692476944110","x":182.5,"y":28,"width":38,"height":30}]') + .attach('image', image) + .expect(201); + + const [testRun] = await testRunsService.findMany(build.id); + expect(testRun.merge).toBe(true); + expect(testRun.diffTollerancePercent).toBe(0.12); + expect(JSON.parse(testRun.tempIgnoreAreas)).toEqual([ + { id: '1692476944110', x: 182.5, y: 28, width: 38, height: 30 }, + ]); + }); + + it('400 for a multipart request without the image, storing nothing', async () => { + await multipart().expect(400); + + expect(await testRunsService.findMany(build.id)).toEqual([]); + }); + + it('400 for multipart ignore areas that are not JSON', async () => { + await multipart().field('ignoreAreas', 'not-json').attach('image', image).expect(400); + + expect(await testRunsService.findMany(build.id)).toEqual([]); + }); + + it('413 for a file over the multipart limit', async () => { + await withEnv({ MULTIPART_FILE_SIZE_LIMIT: '1000' }, () => multipart().attach('image', image).expect(413)); + }); + + it('400 for a base64 image that is not a PNG', async () => { + const res = await postBase64(Buffer.from('GIF89a').toString('base64')).expect(400); + + expect(res.body.message).toBe('Image is not a PNG file'); + expect(await testRunsService.findMany(build.id)).toEqual([]); + }); + + it('accepts a base64 image with line breaks', async () => { + const imageBase64 = PNG.sync + .write(new PNG({ width: 40, height: 40 })) + .toString('base64') + .replace(/(.{76})/g, '$1\r\n'); + + await postBase64(imageBase64).expect(201); + }); + + it.each(['approve?merge=false', 'reject', 'delete'])('400 for a %s body that is not an array', async (path) => { + await requestWithAuth(app, 'post', `/test-runs/${path}`, user.token) + .send({ ids: [randomUUID()] }) + .expect(400); + }); + + it('400 for ignore areas that are an object', async () => { + await requestWithAuth(app, 'post', '/test-runs/ignoreAreas/update', user.token) + .send({ ids: [randomUUID()], ignoreAreas: { x: 1, y: 2, width: 3, height: 4 } }) + .expect(400); + }); +}); diff --git a/test/test-runs.e2e-spec.ts b/test/test-runs.e2e-spec.ts index f2125f8e..775cc256 100644 --- a/test/test-runs.e2e-spec.ts +++ b/test/test-runs.e2e-spec.ts @@ -1,22 +1,19 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { INestApplication, ValidationPipe } from '@nestjs/common'; -import { AppModule } from '../src/app.module'; -import { UsersService } from '../src/users/users.service'; -import { haveTestRunCreated, haveUserLogged, requestWithApiKey, requestWithAuth, LoggedUser } from './preconditions'; -import { TestRunsService } from '../src/test-runs/test-runs.service'; -import { ProjectsService } from '../src/projects/projects.service'; +import { INestApplication } from '@nestjs/common'; import { Project, TestStatus } from '@prisma/client'; +import { randomUUID } from 'crypto'; import { BuildsService } from '../src/builds/builds.service'; +import { ProjectsService } from '../src/projects/projects.service'; +import { TestRunsService } from '../src/test-runs/test-runs.service'; import { TestVariationsService } from '../src/test-variations/test-variations.service'; -import { TEST_PROJECT } from '../src/_data_'; -import { randomUUID } from 'crypto'; -import { UpdateIgnoreAreasDto } from 'src/test-runs/dto/update-ignore-area.dto'; +import { UsersService } from '../src/users/users.service'; +import { createE2eApp, haveProject } from './api-helpers'; +import { haveTestRunCreated, haveUserLogged, requestWithApiKey, LoggedUser } from './preconditions'; describe('TestRuns (e2e)', () => { let app: INestApplication; let testRunsService: TestRunsService; let usersService: UsersService; - let projecstService: ProjectsService; + let projectsService: ProjectsService; let buildsService: BuildsService; let testVariationsService: TestVariationsService; let user: LoggedUser; @@ -27,31 +24,22 @@ describe('TestRuns (e2e)', () => { const image_v3 = './test/image_edited_2.png'; beforeAll(async () => { - const moduleFixture: TestingModule = await Test.createTestingModule({ - imports: [AppModule], - }).compile(); - - app = moduleFixture.createNestApplication(); - app.useGlobalPipes(new ValidationPipe()); - testRunsService = moduleFixture.get(TestRunsService); - usersService = moduleFixture.get(UsersService); - projecstService = moduleFixture.get(ProjectsService); - buildsService = moduleFixture.get(BuildsService); - testVariationsService = moduleFixture.get(TestVariationsService); - - await app.init(); + const created = await createE2eApp(); + app = created.app; + testRunsService = created.get(TestRunsService); + usersService = created.get(UsersService); + projectsService = created.get(ProjectsService); + buildsService = created.get(BuildsService); + testVariationsService = created.get(TestVariationsService); }); beforeEach(async () => { user = await haveUserLogged(usersService); - project = await projecstService.create({ - ...TEST_PROJECT, - name: `TestRun E2E test${randomUUID()}`, - }); + project = await haveProject(projectsService, 'TestRun E2E test'); }); afterEach(async () => { - await projecstService.remove(project.id); + await projectsService.remove(project.id); await usersService.delete(user.id); }); @@ -59,194 +47,114 @@ describe('TestRuns (e2e)', () => { await app.close(); }); + const haveRun = async (branchName: string, image: string, merge?: boolean) => + (await haveTestRunCreated(buildsService, testRunsService, project.id, branchName, image, merge)).testRun; + const haveApprovedRun = async (branchName: string, image: string) => { + const testRun = await haveRun(branchName, image); + await testRunsService.approve(testRun.id); + return testRun; + }; + const detailsOf = async (testVariationId: string | null) => testVariationsService.getDetails(testVariationId!); + describe('POST /', () => { it('New if no baseline', async () => { - const { testRun } = await haveTestRunCreated(buildsService, testRunsService, project.id, 'develop', image_v1); + const testRun = await haveRun('develop', image_v1); expect(testRun.status).toBe(TestStatus.new); }); it('Unresolved if diff found', async () => { - const { testRun: testRun1 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - project.mainBranchName, - image_v1 - ); - await testRunsService.approve(testRun1.id); - const { testRun: testRun2 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - project.mainBranchName, - image_v2 - ); - await testRunsService.approve(testRun2.id); - const { testRun: testRun3 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - 'feature', - image_v2 - ); - await testRunsService.approve(testRun3.id); - - const { testRun } = await haveTestRunCreated(buildsService, testRunsService, project.id, 'develop', image_v3); + await haveApprovedRun(project.mainBranchName, image_v1); + await haveApprovedRun(project.mainBranchName, image_v2); + await haveApprovedRun('feature', image_v2); + + const testRun = await haveRun('develop', image_v3); expect(testRun.status).toBe(TestStatus.unresolved); }); it('Ok', async () => { - const { testRun: testRun1 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - project.mainBranchName, - image_v1 - ); - await testRunsService.approve(testRun1.id); + await haveApprovedRun(project.mainBranchName, image_v1); - const { testRun } = await haveTestRunCreated(buildsService, testRunsService, project.id, 'develop', image_v1); + const testRun = await haveRun('develop', image_v1); expect(testRun.status).toBe(TestStatus.ok); }); it('Auto approve not rebased feature branch then Ok after rebase', async () => { - await projecstService.update({ ...project, autoApproveFeature: true }); - const { testRun: testRun1 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - project.mainBranchName, - image_v1 - ); - await testRunsService.approve(testRun1.id); - const { testRun: testRun2 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - project.mainBranchName, - image_v2 - ); - await testRunsService.approve(testRun2.id); - - const { testRun: notRebasedTestRun } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - 'develop', - image_v1 - ); + await projectsService.update({ ...project, autoApproveFeature: true }); + await haveApprovedRun(project.mainBranchName, image_v1); + await haveApprovedRun(project.mainBranchName, image_v2); + + const notRebasedTestRun = await haveRun('develop', image_v1); expect(notRebasedTestRun.status).toBe(TestStatus.autoApproved); - const { testRun: rebasedTestRun } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - 'develop', - image_v2 - ); + const rebasedTestRun = await haveRun('develop', image_v2); expect(rebasedTestRun.status).toBe(TestStatus.ok); }); it('Auto approve merged feature into feature branch', async () => { - await projecstService.update({ ...project, autoApproveFeature: true }); - const { testRun: testRun1 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - 'feature1', - image_v1 - ); - await testRunsService.approve(testRun1.id); - - const { testRun } = await haveTestRunCreated(buildsService, testRunsService, project.id, 'feature2', image_v1); + await projectsService.update({ ...project, autoApproveFeature: true }); + await haveApprovedRun('feature1', image_v1); + + const testRun = await haveRun('feature2', image_v1); expect(testRun.status).toBe(TestStatus.autoApproved); }); it('Auto approve merged feature into main branch', async () => { - await projecstService.update({ ...project, autoApproveFeature: true }); - const { testRun: testRun1 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - project.mainBranchName, - image_v1 - ); - await testRunsService.approve(testRun1.id); - const { testRun: testRun2 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - 'develop', - image_v2 - ); - await testRunsService.approve(testRun2.id); - - const { testRun } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - project.mainBranchName, - image_v2 - ); + await projectsService.update({ ...project, autoApproveFeature: true }); + await haveApprovedRun(project.mainBranchName, image_v1); + await haveApprovedRun('develop', image_v2); + + const testRun = await haveRun(project.mainBranchName, image_v2); expect(testRun.status).toBe(TestStatus.autoApproved); }); }); describe('POST /multipart', () => { - const url = '/test-runs/multipart'; + const post = async (projectId = project.id) => { + const build = await buildsService.findOrCreate({ projectId: project.id, branchName: project.mainBranchName }); + return { + build, + req: requestWithApiKey(app, 'post', '/test-runs/multipart', user.apiKey) + .set('Content-type', 'multipart/form-data') + .field('name', 'Multipart image') + .field('branchName', project.mainBranchName) + .field('buildId', build.id) + .field('projectId', projectId) + .attach('image', image_v1), + }; + }; it('should post multipart all fields', async () => { - const build = await buildsService.findOrCreate({ projectId: project.id, branchName: project.mainBranchName }); + const { req } = await post(); - await requestWithApiKey(app, 'post', url, user.apiKey) - .set('Content-type', 'multipart/form-data') - .field('name', 'Multipart image') + await req .field('os', 'Windows') .field('browser', 'Browser') .field('viewport', '123x456') .field('device', 'Desktop') - .field('branchName', project.mainBranchName) - .field('buildId', build.id) - .field('projectId', project.id) .field('diffTollerancePercent', '0.12') .field('merge', 'true') .field('ignoreAreas', '[{"id":"1692476944110","x":182,"y":28,"width":38,"height":30}]') .field('comment', 'Comment') - .attach('image', image_v1) .expect(201); }); it('should post multipart required fields', async () => { - const build = await buildsService.findOrCreate({ projectId: project.id, branchName: project.mainBranchName }); + const { req } = await post(); - await requestWithApiKey(app, 'post', url, user.apiKey) - .set('Content-type', 'multipart/form-data') - .field('name', 'Multipart image') - .field('branchName', project.mainBranchName) - .field('buildId', build.id) - .field('projectId', project.id) - .attach('image', image_v1) - .expect(201); + await req.expect(201); }); it('404 for a project that does not exist, storing nothing', async () => { - const build = await buildsService.findOrCreate({ projectId: project.id, branchName: project.mainBranchName }); const projectId = randomUUID(); + const { build, req } = await post(projectId); - const res = await requestWithApiKey(app, 'post', url, user.apiKey) - .set('Content-type', 'multipart/form-data') - .field('name', 'Multipart image') - .field('branchName', project.mainBranchName) - .field('buildId', build.id) - .field('projectId', projectId) - .attach('image', image_v1) - .expect(404); + const res = await req.expect(404); expect(res.body.message).toBe(`Project not found: ${projectId}`); expect(await testRunsService.findMany(build.id)).toEqual([]); @@ -255,249 +163,82 @@ describe('TestRuns (e2e)', () => { describe('POST /approve', () => { it('approve changes in new main branch', async () => { - const { testRun: testRun1 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - project.mainBranchName, - image_v1 - ); + const testRun = await haveRun(project.mainBranchName, image_v1); - const result = await testRunsService.approve(testRun1.id); + const result = await testRunsService.approve(testRun.id); expect(result.status).toBe(TestStatus.approved); - const testVariation = (await testVariationsService.getDetails(result.testVariationId!))!; - expect(testVariation.baselines).toHaveLength(1); + expect((await detailsOf(result.testVariationId)).baselines).toHaveLength(1); }); it('approve changes in new feature branch', async () => { - const { testRun: testRun1 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - 'develop', - image_v1 - ); + const testRun = await haveRun('develop', image_v1); - const result = await testRunsService.approve(testRun1.id); + const result = await testRunsService.approve(testRun.id); expect(result.status).toBe(TestStatus.approved); - const testVariation = (await testVariationsService.getDetails(result.testVariationId!))!; + const testVariation = await detailsOf(result.testVariationId); expect(testVariation.baselines).toHaveLength(1); expect(testVariation.branchName).toBe('develop'); }); it('approve changes in main vs feature branch', async () => { - const { testRun: testRun1 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - project.mainBranchName, - image_v1 - ); - await testRunsService.approve(testRun1.id); - const { testRun: testRun2 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - 'develop', - image_v2 - ); - - const featureBranchResult = await testRunsService.approve(testRun2.id); + await haveApprovedRun(project.mainBranchName, image_v1); + const testRun = await haveRun('develop', image_v2); + + const featureBranchResult = await testRunsService.approve(testRun.id); expect(featureBranchResult.status).toBe(TestStatus.approved); - const testVariation = (await testVariationsService.getDetails(featureBranchResult.testVariationId!))!; + const testVariation = await detailsOf(featureBranchResult.testVariationId); expect(testVariation.baselines).toHaveLength(1); expect(testVariation.branchName).toBe('develop'); }); it('approve changes in updated main vs feature branch', async () => { - const { testRun: testRun1 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - 'develop', - image_v1 - ); - await testRunsService.approve(testRun1.id); - const { testRun: testRun2 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - project.mainBranchName, - image_v2 - ); - await testRunsService.approve(testRun2.id); - const { testRun: testRun3 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - 'develop', - image_v1 - ); - - const result = await testRunsService.approve(testRun3.id); + await haveApprovedRun('develop', image_v1); + await haveApprovedRun(project.mainBranchName, image_v2); + const testRun = await haveRun('develop', image_v1); + + const result = await testRunsService.approve(testRun.id); expect(result.status).toBe(TestStatus.approved); expect(result.baselineBranchName).toBe(project.mainBranchName); - const testVariation = (await testVariationsService.getDetails(result.testVariationId!))!; + const testVariation = await detailsOf(result.testVariationId); expect(testVariation.branchName).toBe('develop'); expect(testVariation.baselines).toHaveLength(2); }); it('approve changes with merge', async () => { - const { testRun: testRun1 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - project.mainBranchName, - image_v1 - ); - const mainBranchResult = await testRunsService.approve(testRun1.id); - const { testRun: testRun2 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - 'develop', - image_v2, - true - ); - - const featureBranchResult = await testRunsService.approve(testRun2.id, true); + const mainBranchResult = await testRunsService.approve((await haveRun(project.mainBranchName, image_v1)).id); + const testRun = await haveRun('develop', image_v2, true); + + const featureBranchResult = await testRunsService.approve(testRun.id, true); expect(featureBranchResult.status).toBe(TestStatus.approved); expect(featureBranchResult.merge).toBe(true); - const mainBranchTestVariation = (await testVariationsService.getDetails(mainBranchResult.testVariationId!))!; + const mainBranchTestVariation = await detailsOf(mainBranchResult.testVariationId); expect(mainBranchTestVariation.baselines).toHaveLength(2); expect(mainBranchTestVariation.baselines.shift()!.baselineName).toBe(mainBranchTestVariation.baselineName); }); it('approve feature branch testVariation', async () => { - const { testRun: testRun1 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - project.mainBranchName, - image_v1 - ); - const mainBranchResult = await testRunsService.approve(testRun1.id); - const { testRun: testRun2 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - 'develop', - image_v2 - ); - - const featureBranchResult = await testRunsService.approve(testRun2.id); + const mainBranchResult = await testRunsService.approve((await haveRun(project.mainBranchName, image_v1)).id); + const testRun = await haveRun('develop', image_v2); + + const featureBranchResult = await testRunsService.approve(testRun.id); expect(featureBranchResult.status).toBe(TestStatus.approved); - const mainBranchTestVariation = (await testVariationsService.getDetails(mainBranchResult.testVariationId!))!; - expect(mainBranchTestVariation.baselines).toHaveLength(1); + expect((await detailsOf(mainBranchResult.testVariationId)).baselines).toHaveLength(1); }); it('autoApprove feature branch testVariation', async () => { - const { testRun: testRun1 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - project.mainBranchName, - image_v1 - ); - const mainBranchResult = await testRunsService.approve(testRun1.id); - const { testRun: testRun2 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - 'develop', - image_v2 - ); - - const featureBranchResult = await testRunsService.approve(testRun2.id, false, true); + const mainBranchResult = await testRunsService.approve((await haveRun(project.mainBranchName, image_v1)).id); + const testRun = await haveRun('develop', image_v2); + + const featureBranchResult = await testRunsService.approve(testRun.id, false, true); expect(featureBranchResult.status).toBe(TestStatus.autoApproved); expect(mainBranchResult.testVariationId).toBe(featureBranchResult.testVariationId); }); }); - - describe('POST /delete', () => { - it('Should delete', async () => { - const { testRun: testRun1 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - 'develop', - image_v1 - ); - const { testRun: testRun2 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - 'develop', - image_v1 - ); - - await requestWithAuth(app, 'post', `/test-runs/delete`, user.token).send([testRun1.id, testRun2.id]).expect(201); - }); - }); - - describe('POST /reject', () => { - it('200', async () => { - const { testRun: testRun1 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - 'develop', - image_v1 - ); - const { testRun: testRun2 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - 'develop', - image_v1 - ); - - await requestWithAuth(app, 'post', `/test-runs/reject`, user.token).send([testRun1.id, testRun2.id]).expect(201); - }); - }); - - describe('POST /ignoreAreas/update', () => { - it('200', async () => { - const { testRun } = await haveTestRunCreated(buildsService, testRunsService, project.id, 'develop', image_v1); - - const request: UpdateIgnoreAreasDto = { - ids: [testRun.id], - ignoreAreas: [ - { - x: 182, - y: 28, - width: 38, - height: 30, - }, - ], - }; - await requestWithAuth(app, 'post', `/test-runs/ignoreAreas/update`, user.token).send(request).expect(201); - }); - }); - - describe('POST /ignoreAreas/add', () => { - it('200', async () => { - const { testRun } = await haveTestRunCreated(buildsService, testRunsService, project.id, 'develop', image_v1); - - const request: UpdateIgnoreAreasDto = { - ids: [testRun.id], - ignoreAreas: [ - { - x: 182, - y: 28, - width: 38, - height: 30, - }, - ], - }; - await requestWithAuth(app, 'post', `/test-runs/ignoreAreas/add`, user.token).send(request).expect(201); - }); - }); }); diff --git a/test/test-variations-api.e2e-spec.ts b/test/test-variations-api.e2e-spec.ts index a680b312..69367bed 100644 --- a/test/test-variations-api.e2e-spec.ts +++ b/test/test-variations-api.e2e-spec.ts @@ -64,7 +64,6 @@ describe('TestVariations API (e2e)', () => { }); }; - /** Posts and approves (as the logged-in user), so the variation of that branch gets a baseline. */ const haveBaseline = async (branchName: string, image: Buffer, name = 'Screen') => { const run = await postRun(branchName, image, name); await requestWithAuth(app, 'post', '/test-runs/approve?merge=false', user.token).send([run.id]).expect(201); @@ -131,7 +130,6 @@ describe('TestVariations API (e2e)', () => { expect(res.body.baselineName).toBe(res.body.baselines[0].baselineName); expect(res.body.baselines[0].testRun.id).toBe(second.id); expect(res.body.baselines[0].user.id).toBe(user.id); - // who approved, but never their credentials: any signed-in user can read this expect(res.body.baselines[0].user).toMatchObject({ email: user.email, firstName: user.firstName, @@ -169,7 +167,6 @@ describe('TestVariations API (e2e)', () => { await haveBaseline(project.mainBranchName, WHITE, 'Same'); await haveBaseline('feature', WHITE, 'Same'); await haveBaseline('feature', CHANGED, 'Only on feature'); - // a feature variation that never got a baseline is not merged await postRun('feature', WHITE, 'No baseline'); const res = await merge('feature', project.mainBranchName).expect(200); @@ -189,12 +186,10 @@ describe('TestVariations API (e2e)', () => { expect(byName['Changed']).toMatchObject({ status: TestStatus.unresolved, merge: true, branchName: 'master' }); expect(byName['Same'].status).toBe(TestStatus.ok); expect(byName['Only on feature'].status).toBe(TestStatus.new); - // a master variation was created for the screen that only existed on feature const created = await prisma.testVariation.findUnique({ where: { id: byName['Only on feature'].testVariationId! }, }); expect(created).toMatchObject({ branchName: 'master', baselineName: null }); - // the merged screenshot is a stored copy of the feature baseline expect(imageExists(byName['Changed'].imageName)).toBe(true); }); @@ -212,7 +207,6 @@ describe('TestVariations API (e2e)', () => { const res = await merge('feature', 'release').expect(200); - // compared against release's own baseline, not main's newer one expect(res.body).toMatchObject({ branchName: 'release', isRunning: false, merge: true, unresolvedCount: 1 }); const [release] = await prisma.testVariation.findMany({ where: { projectId: project.id, branchName: 'release' }, @@ -261,7 +255,6 @@ describe('TestVariations API (e2e)', () => { expect(await prisma.testVariation.findUnique({ where: { id: first.testVariationId! } })).toBeNull(); expect(await prisma.baseline.count({ where: { testVariationId: first.testVariationId! } })).toBe(0); await waitFor(() => baselines.every((baseline) => !imageExists(baseline.baselineName))); - // the runs stay, without a variation, and keep their own screenshots const runs = await prisma.testRun.findMany({ where: { id: { in: [first.id, second.id] } } }); expect(runs.map((run) => run.testVariationId)).toEqual([null, null]); expect(runs.every((run) => imageExists(run.imageName))).toBe(true); diff --git a/test/test-variations.e2e-spec.ts b/test/test-variations.e2e-spec.ts deleted file mode 100644 index 9f147aa9..00000000 --- a/test/test-variations.e2e-spec.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { INestApplication, ValidationPipe } from '@nestjs/common'; -import { AppModule } from '../src/app.module'; -import { UsersService } from '../src/users/users.service'; -import { haveTestRunCreated, haveUserLogged, LoggedUser } from './preconditions'; -import { TestRunsService } from '../src/test-runs/test-runs.service'; -import { ProjectsService } from '../src/projects/projects.service'; -import { Project } from '@prisma/client'; -import { BuildsService } from '../src/builds/builds.service'; -import { TestVariationsService } from '../src/test-variations/test-variations.service'; -import { TEST_PROJECT } from '../src/_data_'; -import { randomUUID } from 'crypto'; - -describe('TestVariations (e2e)', () => { - let app: INestApplication; - let testRunsService: TestRunsService; - let usersService: UsersService; - let projecstService: ProjectsService; - let buildsService: BuildsService; - let user: LoggedUser; - let project: Project; - let testVariationsService: TestVariationsService; - - beforeAll(async () => { - const moduleFixture: TestingModule = await Test.createTestingModule({ - imports: [AppModule], - }).compile(); - - app = moduleFixture.createNestApplication(); - app.useGlobalPipes(new ValidationPipe()); - testRunsService = moduleFixture.get(TestRunsService); - usersService = moduleFixture.get(UsersService); - projecstService = moduleFixture.get(ProjectsService); - buildsService = moduleFixture.get(BuildsService); - testVariationsService = moduleFixture.get(TestVariationsService); - - await app.init(); - }); - - beforeEach(async () => { - user = await haveUserLogged(usersService); - project = await projecstService.create({ - ...TEST_PROJECT, - name: `TestVariations E2E test${randomUUID()}`, - }); - }); - - afterEach(async () => { - await projecstService.remove(project.id); - await usersService.delete(user.id); - }); - - afterAll(async () => { - await app.close(); - }); - - describe('DELETE /', () => { - const image_v1 = './test/image.png'; - - it('can delete', async () => { - const { testRun } = await haveTestRunCreated(buildsService, testRunsService, project.id, 'develop', image_v1); - await testRunsService.approve(testRun.id); - - await testVariationsService.delete(testRun.testVariationId!); - - expect((await testRunsService.findOne(testRun.id))!.testVariationId).toBeNull(); - }); - }); -}); diff --git a/test/users-admin.e2e-spec.ts b/test/users-admin.e2e-spec.ts index 51d314bc..3143dff6 100644 --- a/test/users-admin.e2e-spec.ts +++ b/test/users-admin.e2e-spec.ts @@ -1,18 +1,16 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { INestApplication, ValidationPipe } from '@nestjs/common'; +import { INestApplication } from '@nestjs/common'; import { Role, TestStatus } from '@prisma/client'; import request from 'supertest'; import { createHash, generateKeyPairSync, KeyObject, randomUUID, sign } from 'crypto'; import { createServer, IncomingMessage, Server } from 'http'; import { generateRandomKey } from '../src/utils/random-key'; -import { AppModule } from '../src/app.module'; import { UsersService } from '../src/users/users.service'; import { PrismaService } from '../src/prisma/prisma.service'; import { ProjectsService } from '../src/projects/projects.service'; import { BuildsService } from '../src/builds/builds.service'; import { TestRunsService } from '../src/test-runs/test-runs.service'; -import { TEST_PROJECT } from '../src/_data_'; import { UserDto } from '../src/users/dto/user.dto'; +import { approveRuns, createE2eApp, haveProject, setEnv } from './api-helpers'; import { generateUser, haveTestRunCreated, @@ -24,6 +22,13 @@ import { const FORBIDDEN = 'You do not have permission to perform this operation.'; +const expectJsonBoolean = (app: INestApplication, url: string, value: boolean) => + request(app.getHttpServer()) + .get(url) + .expect(200) + .expect('Content-Type', 'application/json; charset=utf-8') + .expect((res) => expect(res.body).toBe(value)); + describe('Users admin and RBAC (e2e)', () => { let app: INestApplication; let usersService: UsersService; @@ -37,19 +42,13 @@ describe('Users admin and RBAC (e2e)', () => { }; beforeAll(async () => { - const moduleFixture: TestingModule = await Test.createTestingModule({ - imports: [AppModule], - }).compile(); - - app = moduleFixture.createNestApplication(); - app.useGlobalPipes(new ValidationPipe()); - usersService = moduleFixture.get(UsersService); - prismaService = moduleFixture.get(PrismaService); - await app.init(); + const created = await createE2eApp(); + app = created.app; + usersService = created.get(UsersService); + prismaService = created.get(PrismaService); }); afterEach(async () => { - // some tests delete their users through the API already await prismaService.user.deleteMany({ where: { id: { in: userIds.splice(0) } } }); }); @@ -57,9 +56,11 @@ describe('Users admin and RBAC (e2e)', () => { await app.close(); }); - describe('GET /users/me', () => { - it('401 without a token', () => request(app.getHttpServer()).get('/users/me').expect(401)); + it.each(['/users/me', '/users/all'])('GET %s answers 401 without a token', (url) => + request(app.getHttpServer()).get(url).expect(401) + ); + describe('GET /users/me', () => { it.each([Role.admin, Role.editor, Role.guest])('returns the current %s without the password', async (role) => { const user = await haveUser(role); @@ -77,8 +78,6 @@ describe('Users admin and RBAC (e2e)', () => { }); describe('GET /users/all', () => { - it('401 without a token', () => request(app.getHttpServer()).get('/users/all').expect(401)); - it('401 with an admin api key only: the list needs a JWT', async () => { const admin = await haveUser(Role.admin); @@ -107,12 +106,12 @@ describe('Users admin and RBAC (e2e)', () => { email: guest.email, firstName: 'fName', lastName: 'lName', - apiKey: guest.apiKey, role: 'guest', }, ]) ); expect(res.body.every((user: UserDto) => !('password' in user))).toBe(true); + expect(res.body.every((user: UserDto) => !('apiKey' in user))).toBe(true); }); }); @@ -138,8 +137,9 @@ describe('Users admin and RBAC (e2e)', () => { .expect(200); expect(res.body).toEqual(expect.objectContaining({ id: target.id, email: target.email, role: Role.admin })); + expect(res.body).not.toHaveProperty('apiKey'); + expect(res.body).not.toHaveProperty('password'); expect((await usersService.findOne(target.id))!.role).toBe(Role.admin); - // the role is read from the database on every request, not from the token await requestWithAuth(app, 'get', '/users/all', target.token).expect(200); }); @@ -222,25 +222,11 @@ describe('Users admin and RBAC (e2e)', () => { expect(await usersService.findOne(last.id)).not.toBeNull(); }); - it('deletes every listed user', async () => { - const admin = await haveUser(Role.admin); - const first = await haveUser(Role.guest); - const last = await haveUser(Role.editor); - - await requestWithAuth(app, 'delete', '/users', admin.token).send([first.id, last.id]).expect(200); - - expect(await usersService.findOne(first.id)).toBeNull(); - expect(await usersService.findOne(last.id)).toBeNull(); - }); - it('keeps the baselines a deleted user approved, without the user', async () => { const admin = await haveUser(Role.admin); const editor = await haveUser(Role.editor); const projectsService = app.get(ProjectsService); - const project = await projectsService.create({ - ...TEST_PROJECT, - name: `Users admin E2E ${randomUUID()}`, - }); + const project = await haveProject(projectsService, 'Users admin E2E'); try { const { testRun } = await haveTestRunCreated( app.get(BuildsService), @@ -249,9 +235,7 @@ describe('Users admin and RBAC (e2e)', () => { project.mainBranchName, './test/image.png' ); - await requestWithAuth(app, 'post', '/test-runs/approve?merge=false', editor.token) - .send([testRun.id]) - .expect(201); + await approveRuns(app, editor, [testRun.id]); const [baseline] = await prismaService.baseline.findMany({ where: { testRunId: testRun.id } }); expect(baseline.userId).toBe(editor.id); @@ -269,12 +253,7 @@ describe('Users admin and RBAC (e2e)', () => { }); describe('OIDC disabled (default DB users)', () => { - it('GET /users/oidc/enabled is false', () => - request(app.getHttpServer()) - .get('/users/oidc/enabled') - .expect(200) - .expect('Content-Type', 'application/json; charset=utf-8') - .expect((res) => expect(res.body).toBe(false))); + it('GET /users/oidc/enabled is false', () => expectJsonBoolean(app, '/users/oidc/enabled', false)); it.each(['/users/oidc/login', '/users/oidc/callback?code=abc&state=xyz'])('GET %s is 404', (url) => request(app.getHttpServer()).get(url).expect(404).expect('OIDC is not enabled.') @@ -282,21 +261,12 @@ describe('Users admin and RBAC (e2e)', () => { }); }); -/** - * OIDC needs an identity provider. Instead of a real one, a minimal in-process - * provider serves discovery, JWKS and the token endpoint on 127.0.0.1, signs ID - * tokens with an RSA key generated for the run and checks PKCE. The app runs - * with the real UsersFactoryService/OidcUsersService and openid-client, - * switched to OIDC through the same environment variables as in production. - * The browser's visit to the provider's authorization page is simulated: the - * test reads state and code_challenge from the redirect and registers a code. - */ describe('Users OIDC (e2e)', () => { const CLIENT_ID = 'vrt-e2e'; const CLIENT_SECRET = 'e2e-secret'; const REDIRECT_URI = 'http://localhost:4200/users/oidc/callback'; const FRONTEND_LOGIN = 'http://localhost:8080/login'; - const OIDC_ENV = ['OIDC_ENABLED', 'OIDC_ISSUER_URL', 'OIDC_CLIENT_ID', 'OIDC_CLIENT_SECRET', 'OIDC_REDIRECT_URI']; + const LOGIN_FAILED = `${FRONTEND_LOGIN}#error=oidc_login_failed`; let app: INestApplication; let prismaService: PrismaService; @@ -307,7 +277,7 @@ describe('Users OIDC (e2e)', () => { const tokenRequests: URLSearchParams[] = []; const codes = new Map }>(); const emails: string[] = []; - const previousEnv: Record = {}; + let restoreEnv: () => void; const base64url = (input: Buffer | string) => Buffer.from(input).toString('base64url'); @@ -381,25 +351,17 @@ describe('Users OIDC (e2e)', () => { await new Promise((resolve) => idp.listen(0, '127.0.0.1', resolve)); issuer = `http://127.0.0.1:${(idp.address() as { port: number }).port}`; - const env: Record = { + restoreEnv = setEnv({ OIDC_ENABLED: 'true', OIDC_ISSUER_URL: issuer, OIDC_CLIENT_ID: CLIENT_ID, OIDC_CLIENT_SECRET: CLIENT_SECRET, OIDC_REDIRECT_URI: REDIRECT_URI, - }; - for (const key of OIDC_ENV) { - previousEnv[key] = process.env[key]; - process.env[key] = env[key]; - } + }); - const moduleFixture: TestingModule = await Test.createTestingModule({ - imports: [AppModule], - }).compile(); - app = moduleFixture.createNestApplication(); - app.useGlobalPipes(new ValidationPipe()); - prismaService = moduleFixture.get(PrismaService); - await app.init(); + const created = await createE2eApp(); + app = created.app; + prismaService = created.get(PrismaService); }); afterEach(async () => { @@ -412,13 +374,7 @@ describe('Users OIDC (e2e)', () => { afterAll(async () => { await app.close(); await new Promise((resolve) => idp.close(resolve)); - for (const key of OIDC_ENV) { - if (previousEnv[key] === undefined) { - delete process.env[key]; - } else { - process.env[key] = previousEnv[key]; - } - } + restoreEnv(); }); const newEmail = () => { @@ -427,7 +383,6 @@ describe('Users OIDC (e2e)', () => { return email; }; - // GET /users/oidc/login, then the "browser" at the provider: returns what the callback needs const startLogin = async (claims: Record) => { const res = await request(app.getHttpServer()).get('/users/oidc/login').expect(302); const location = new URL(res.headers.location); @@ -452,23 +407,19 @@ describe('Users OIDC (e2e)', () => { return decodeURIComponent(location.slice(`${FRONTEND_LOGIN}#token=`.length)); }; - // first on purpose: openid-client's discovery result is cached once it succeeds it('GET /users/oidc/login sends the browser back to the login page when discovery fails', async () => { discoveryDown = true; await request(app.getHttpServer()) .get('/users/oidc/login') .expect(302) - .expect('Location', `${FRONTEND_LOGIN}#error=oidc_login_failed`) + .expect('Location', LOGIN_FAILED) .expect((res) => expect(res.headers['set-cookie']).toBeUndefined()); }); - it('GET /users/oidc/enabled is true', () => - request(app.getHttpServer()) - .get('/users/oidc/enabled') - .expect(200) - .expect('Content-Type', 'application/json; charset=utf-8') - .expect((res) => expect(res.body).toBe(true))); + it('GET /users/registration/enabled is false', () => expectJsonBoolean(app, '/users/registration/enabled', false)); + + it('GET /users/oidc/enabled is true', () => expectJsonBoolean(app, '/users/oidc/enabled', true)); it('GET /users/oidc/login redirects to the provider with PKCE and keeps state in a cookie', async () => { const { state, location, setCookie } = await startLogin({}); @@ -497,7 +448,6 @@ describe('Users OIDC (e2e)', () => { groups: ['staff', 'VRT-Admin'], }); - // other cookies of the site travel along with the state cookie const res = await callback(`code=${code}&state=${state}`, `theme=dark; ${cookie}; lang=en`); const token = tokenFrom(res.headers.location); @@ -524,7 +474,6 @@ describe('Users OIDC (e2e)', () => { const third = await startLogin({ email, email_verified: true, groups: ['developers'] }); tokenFrom((await callback(`code=${third.code}&state=${third.state}`, third.cookie)).headers.location); - // OIDC_DEFAULT_ROLE defaults to editor expect((await prismaService.user.findUnique({ where: { email } }))!.role).toBe(Role.editor); }); @@ -532,15 +481,11 @@ describe('Users OIDC (e2e)', () => { const email = newEmail(); const { state, code, cookie } = await startLogin({ email, email_verified: false, groups: ['admin'] }); - await callback(`code=${code}&state=${state}`, cookie).expect( - 'Location', - `${FRONTEND_LOGIN}#error=oidc_login_failed` - ); + await callback(`code=${code}&state=${state}`, cookie).expect('Location', LOGIN_FAILED); expect(await prismaService.user.findUnique({ where: { email } })).toBeNull(); }); - // cookie: undefined = none, null = the one set by /users/oidc/login it.each<[string, { cookie?: string | null; state?: string }]>([ ['without the state cookie', { cookie: undefined }], ['with only other cookies', { cookie: 'theme=dark; lang=en' }], @@ -554,10 +499,7 @@ describe('Users OIDC (e2e)', () => { const state = arrange.state ?? login.state; const cookie = arrange.cookie === null ? login.cookie : arrange.cookie; - await callback(`code=${code}&state=${state}`, cookie).expect( - 'Location', - `${FRONTEND_LOGIN}#error=oidc_login_failed` - ); + await callback(`code=${code}&state=${state}`, cookie).expect('Location', LOGIN_FAILED); expect(tokenRequests).toHaveLength(0); expect(await prismaService.user.findUnique({ where: { email } })).toBeNull(); @@ -566,10 +508,7 @@ describe('Users OIDC (e2e)', () => { it('fails when the provider refuses the code', async () => { const { state, cookie } = await startLogin({ email: newEmail(), email_verified: true }); - await callback(`code=unknown&state=${state}`, cookie).expect( - 'Location', - `${FRONTEND_LOGIN}#error=oidc_login_failed` - ); + await callback(`code=unknown&state=${state}`, cookie).expect('Location', LOGIN_FAILED); expect(tokenRequests).toHaveLength(1); }); @@ -578,11 +517,7 @@ describe('Users OIDC (e2e)', () => { const email = newEmail(); const { state, code, cookie } = await startLogin({ email, email_verified: true }); - // a repeated code parameter is dropped, so openid-client sees no code at all - await callback(`code=${code}&code=${code}&state=${state}`, cookie).expect( - 'Location', - `${FRONTEND_LOGIN}#error=oidc_login_failed` - ); + await callback(`code=${code}&code=${code}&state=${state}`, cookie).expect('Location', LOGIN_FAILED); expect(tokenRequests).toHaveLength(0); }); diff --git a/test/users.e2e-spec.ts b/test/users.e2e-spec.ts index 92f534a1..0be349a3 100644 --- a/test/users.e2e-spec.ts +++ b/test/users.e2e-spec.ts @@ -1,27 +1,36 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { INestApplication, ValidationPipe } from '@nestjs/common'; +import { INestApplication } from '@nestjs/common'; +import { User } from '@prisma/client'; +import { compareSync } from 'bcryptjs'; import request from 'supertest'; -import { AppModule } from '../src/app.module'; import { UsersService } from '../src/users/users.service'; -import { UserLoginRequestDto } from '../src/users/dto/user-login-request.dto'; -import { compareSync } from 'bcryptjs'; -import { requestWithAuth, generateUser } from './preconditions'; -import { User } from '@prisma/client'; +import { createE2eApp, withEnv } from './api-helpers'; +import { generateUser, requestWithAuth } from './preconditions'; describe('Users (e2e)', () => { let app: INestApplication; let usersService: UsersService; let user: Partial; + const password = '123456'; + const haveUserLoggedIn = async () => { + user = await usersService.create(generateUser(password)); + return usersService.login({ email: user.email!, password }); + }; + const expectProfile = (body: Record, expected: Partial) => { + expect(body).toMatchObject({ + id: expected.id, + email: expected.email, + firstName: expected.firstName, + lastName: expected.lastName, + apiKey: user.apiKey, + }); + expect(body.token).not.toBeNull(); + }; + beforeAll(async () => { - const moduleFixture: TestingModule = await Test.createTestingModule({ - imports: [AppModule], - }).compile(); - - app = moduleFixture.createNestApplication(); - app.useGlobalPipes(new ValidationPipe()); - usersService = moduleFixture.get(UsersService); - await app.init(); + const { app: created, get } = await createE2eApp(); + app = created; + usersService = get(UsersService); }); afterAll(async () => { @@ -33,7 +42,7 @@ describe('Users (e2e)', () => { }); it('POST /register', () => { - user = generateUser('123456'); + user = generateUser(password); return request(app.getHttpServer()) .post('/users/register') .send(user) @@ -47,35 +56,43 @@ describe('Users (e2e)', () => { }); }); + it('GET /registration/enabled is true by default', async () => { + user = await usersService.create(generateUser(password)); + + await request(app.getHttpServer()) + .get('/users/registration/enabled') + .expect(200) + .expect('Content-Type', /application\/json/) + .expect((res) => expect(res.body).toBe(true)); + }); + + it('POST /register answers 403 when REGISTRATION_ENABLED=false', async () => { + user = await usersService.create(generateUser(password)); + await withEnv({ REGISTRATION_ENABLED: 'false' }, async () => { + await request(app.getHttpServer()) + .post('/users/register') + .send(generateUser(password)) + .expect(403) + .expect((res) => expect(res.body.message).toBe('User registration is disabled.')); + await request(app.getHttpServer()) + .get('/users/registration/enabled') + .expect(200) + .expect((res) => expect(res.body).toBe(false)); + }); + }); + it('POST /login', async () => { - const password = '123456'; user = await usersService.create(generateUser(password)); - const loginData: UserLoginRequestDto = { - email: user.email!, - password, - }; - return request(app.getHttpServer()) + await request(app.getHttpServer()) .post('/users/login') - .send(loginData) + .send({ email: user.email!, password }) .expect(201) - .expect((res) => { - expect(res.body.id).toBe(user.id); - expect(res.body.email).toBe(user.email); - expect(res.body.firstName).toBe(user.firstName); - expect(res.body.lastName).toBe(user.lastName); - expect(res.body.apiKey).toBe(user.apiKey); - expect(res.body.token).not.toBeNull(); - }); + .expect((res) => expectProfile(res.body, user)); }); it('GET /newApiKey', async () => { - const password = '123456'; - user = await usersService.create(generateUser(password)); - const loggedUser = await usersService.login({ - email: user.email!, - password, - }); + const loggedUser = await haveUserLoggedIn(); const res = await requestWithAuth(app, 'get', '/users/newApiKey', loggedUser.token!).send().expect(200); @@ -85,41 +102,44 @@ describe('Users (e2e)', () => { }); it('PUT /password', async () => { - const newPassword = 'newPassword'; - const password = '123456'; - user = await usersService.create(generateUser(password)); - const loggedUser = await usersService.login({ - email: user.email!, - password, - }); + const loggedUser = await haveUserLoggedIn(); - await requestWithAuth(app, 'put', '/users/password', loggedUser.token!).send({ password: newPassword }).expect(200); + await requestWithAuth(app, 'put', '/users/password', loggedUser.token!) + .send({ password: 'newPassword' }) + .expect(200); const newUser = (await usersService.findOne(user.id!))!; - expect(compareSync(newPassword, newUser.password)).toBe(true); + expect(compareSync('newPassword', newUser.password)).toBe(true); + }); + + it('PUT /password answers 400 for a body without password', async () => { + const loggedUser = await haveUserLoggedIn(); + + await requestWithAuth(app, 'put', '/users/password', loggedUser.token!).send({}).expect(400); + await requestWithAuth(app, 'put', '/users/password', loggedUser.token!).send({ password: '' }).expect(400); + + const unchanged = (await usersService.findOne(user.id!))!; + expect(compareSync(password, unchanged.password)).toBe(true); }); it('PUT /', async () => { - const password = '123456'; - user = await usersService.create(generateUser(password)); - const editedUser = { - email: 'edited' + user.email, - firstName: 'EDITEDfName', - lastName: 'EDITEDlName', - }; - - const loggedUser = await usersService.login({ - email: user.email!, - password, - }); + const loggedUser = await haveUserLoggedIn(); + const editedUser = { email: 'edited' + user.email, firstName: 'EDITEDfName', lastName: 'EDITEDlName' }; const res = await requestWithAuth(app, 'put', '/users', loggedUser.token!).send(editedUser).expect(200); + expectProfile(res.body, { id: user.id, ...editedUser }); + }); + + it('a token keeps working after the user changes their email', async () => { + const loggedUser = await haveUserLoggedIn(); + + await requestWithAuth(app, 'put', '/users', loggedUser.token!) + .send({ email: 'changed' + user.email, firstName: 'F', lastName: 'L' }) + .expect(200); + + const res = await requestWithAuth(app, 'get', '/users/me', loggedUser.token!).send().expect(200); expect(res.body.id).toBe(user.id); - expect(res.body.email).toBe(editedUser.email); - expect(res.body.firstName).toBe(editedUser.firstName); - expect(res.body.lastName).toBe(editedUser.lastName); - expect(res.body.apiKey).toBe(user.apiKey); - expect(res.body.token).not.toBeNull(); + expect(res.body.email).toBe('changed' + user.email); }); }); diff --git a/test/validation-whitelist.e2e-spec.ts b/test/validation-whitelist.e2e-spec.ts new file mode 100644 index 00000000..b7059de2 --- /dev/null +++ b/test/validation-whitelist.e2e-spec.ts @@ -0,0 +1,117 @@ +import { INestApplication } from '@nestjs/common'; +import { Build, Project, TestStatus } from '@prisma/client'; +import { BuildsService } from '../src/builds/builds.service'; +import { PrismaService } from '../src/prisma/prisma.service'; +import { ProjectsService } from '../src/projects/projects.service'; +import { UsersService } from '../src/users/users.service'; +import { approveRuns, createE2eApp, haveProject, postRunBase64, WHITE } from './api-helpers'; +import { haveUserLogged, LoggedUser, requestWithApiKey, requestWithAuth } from './preconditions'; + +describe('Global whitelist (e2e)', () => { + let app: INestApplication; + let usersService: UsersService; + let projectsService: ProjectsService; + let buildsService: BuildsService; + let prisma: PrismaService; + let user: LoggedUser; + let project: Project; + + beforeAll(async () => { + const created = await createE2eApp(); + app = created.app; + usersService = created.get(UsersService); + projectsService = created.get(ProjectsService); + buildsService = created.get(BuildsService); + prisma = created.get(PrismaService); + }); + + beforeEach(async () => { + user = await haveUserLogged(usersService); + project = await haveProject(projectsService, 'Whitelist E2E'); + }); + + afterEach(async () => { + await projectsService.remove(project.id); + await usersService.delete(user.id); + }); + + afterAll(async () => { + await app.close(); + }); + + const newBuild = (branchName = project.mainBranchName): Promise => + buildsService.findOrCreate({ projectId: project.id, branchName }); + + const postRun = async (build: Build, extra: Record = {}) => + postRunBase64(app, user, build, WHITE, extra); + + it('PATCH /test-runs/update/:id writes the comment and nothing else', async () => { + const run = await postRun(await newBuild()); + await approveRuns(app, user, [run.id]); + const before = (await prisma.testVariation.findUnique({ where: { id: run.testVariationId } }))!; + + await requestWithAuth(app, 'patch', `/test-runs/update/${run.id}`, user.token) + .send({ comment: 'fine', baselineName: 'other.baseline.png' }) + .expect(200); + + const after = (await prisma.testVariation.findUnique({ where: { id: run.testVariationId } }))!; + expect(after.comment).toBe('fine'); + expect(after.baselineName).toBe(before.baselineName); + }); + + it('POST /test-runs/ignoreAreas/update keeps the id of every area', async () => { + const run = await postRun(await newBuild()); + const areas = [ + { id: 'a', x: 1, y: 2, width: 3, height: 4 }, + { id: 'b', x: 5, y: 6, width: 7, height: 8 }, + ]; + + await requestWithAuth(app, 'post', '/test-runs/ignoreAreas/update', user.token) + .send({ ids: [run.id], ignoreAreas: areas }) + .expect(201); + + const stored = (await prisma.testRun.findUnique({ where: { id: run.id }, include: { testVariation: true } }))!; + expect(JSON.parse(stored.ignoreAreas)).toEqual(areas); + expect(JSON.parse(stored.testVariation!.ignoreAreas)).toEqual(areas); + }); + + it('POST /test-runs/multipart keeps the image, the form values and the area ids', async () => { + const build = await newBuild(); + + await requestWithApiKey(app, 'post', '/test-runs/multipart', user.apiKey) + .set('Content-type', 'multipart/form-data') + .field('name', 'Multipart image') + .field('branchName', project.mainBranchName) + .field('buildId', build.id) + .field('projectId', project.id) + .field('merge', 'true') + .field('ignoreAreas', '[{"id":"1692476944110","x":1,"y":2,"width":3,"height":4}]') + .field('unknown', 'dropped') + .attach('image', './test/image.png') + .expect(201); + + const [testRun] = await prisma.testRun.findMany({ where: { buildId: build.id } }); + expect(testRun.merge).toBe(true); + expect(JSON.parse(testRun.tempIgnoreAreas)).toEqual([{ id: '1692476944110', x: 1, y: 2, width: 3, height: 4 }]); + }); + + it('POST /test-runs compares against the sourceBranch variation', async () => { + const release = await postRun(await newBuild('release')); + await approveRuns(app, user, [release.id]); + const feature = await newBuild('feature'); + + const compared = await postRun(feature, { sourceBranch: 'release' }); + + expect(compared.status).toBe(TestStatus.ok); + }); + + it('PATCH /builds/:id still answers 400 for fields it does not declare', async () => { + const build = await newBuild(); + + await requestWithApiKey(app, 'patch', `/builds/${build.id}`, user.apiKey) + .send({ branchName: 'rewritten', number: 4242 }) + .expect(400); + + expect(await prisma.build.findUnique({ where: { id: build.id } })).toMatchObject({ isRunning: true }); + }); +}); diff --git a/test_acceptance/acceptance.spec.ts b/test_acceptance/acceptance.spec.ts index 43b2ab0e..f725923f 100644 --- a/test_acceptance/acceptance.spec.ts +++ b/test_acceptance/acceptance.spec.ts @@ -6,7 +6,7 @@ import { randomUUID } from 'crypto'; axios.defaults.baseURL = 'http://localhost:4200'; -let user: Partial = { +const user: Partial = { email: `${randomUUID()}@example.com`, password: '123456', firstName: 'fName', diff --git a/test_acceptance/jest.config.ts b/test_acceptance/jest.config.ts index c5a70de6..f09f5849 100644 --- a/test_acceptance/jest.config.ts +++ b/test_acceptance/jest.config.ts @@ -1,4 +1,3 @@ -/** @returns {Promise} */ module.exports = async () => { return { displayName: 'Acceptance', @@ -7,7 +6,7 @@ module.exports = async () => { testRegex: '.spec.ts$', moduleFileExtensions: ['js', 'json', 'ts'], transform: { - '^.+\\.(t|j)s$': 'ts-jest', + '^.+\\.(t|j)s$': '@swc/jest', }, testEnvironment: 'node', }; diff --git a/test_ldap/jest.config.ts b/test_ldap/jest.config.ts index a7ec3362..9b9a0900 100644 --- a/test_ldap/jest.config.ts +++ b/test_ldap/jest.config.ts @@ -1,4 +1,3 @@ -/** @returns {Promise} */ module.exports = async () => { return { displayName: 'Ldap', @@ -7,7 +6,7 @@ module.exports = async () => { testRegex: '.spec.ts$', moduleFileExtensions: ['js', 'json', 'ts'], transform: { - '^.+\\.(t|j)s$': 'ts-jest', + '^.+\\.(t|j)s$': '@swc/jest', }, testEnvironment: 'node', }; diff --git a/test_ldap/ldap.spec.ts b/test_ldap/ldap.spec.ts index 475507eb..af115649 100644 --- a/test_ldap/ldap.spec.ts +++ b/test_ldap/ldap.spec.ts @@ -27,4 +27,10 @@ describe('Ldap', () => { expect(error.response.data.message).toContain('Invalid email or password.'); }); }); + + test('Rejects an empty password', async () => { + await expect(axios.post('/users/login', { email: loginData.email, password: '' })).rejects.toMatchObject({ + response: { status: 400, data: { message: expect.stringContaining('Invalid email or password.') } }, + }); + }); }); diff --git a/test_oidc/jest.config.ts b/test_oidc/jest.config.ts index accf320d..841ed8fa 100644 --- a/test_oidc/jest.config.ts +++ b/test_oidc/jest.config.ts @@ -1,4 +1,3 @@ -/** @returns {Promise} */ module.exports = async () => { return { displayName: 'Oidc', @@ -7,7 +6,7 @@ module.exports = async () => { testRegex: '.spec.ts$', moduleFileExtensions: ['js', 'json', 'ts'], transform: { - '^.+\\.(t|j)s$': 'ts-jest', + '^.+\\.(t|j)s$': '@swc/jest', }, testEnvironment: 'node', }; diff --git a/test_oidc/oidc.spec.ts b/test_oidc/oidc.spec.ts index c998d437..aea22e72 100644 --- a/test_oidc/oidc.spec.ts +++ b/test_oidc/oidc.spec.ts @@ -3,9 +3,6 @@ import { UserLoginRequestDto } from 'src/users/dto/user-login-request.dto'; axios.defaults.baseURL = 'http://localhost:4200'; -// These tests run against a running api that was started with OIDC enabled -// (see docker-compose.oidc.yml). The full browser redirect flow cannot be -// automated here, so we verify the pieces the backend owns. describe('Oidc', () => { test('reports OIDC as enabled', async () => { const response = await axios.get('/users/oidc/enabled'); diff --git a/tsconfig.build.json b/tsconfig.build.json index 602ef977..f76cab09 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -4,5 +4,5 @@ "rootDir": "./src" }, "include": ["src"], - "exclude": ["node_modules", "dist", "**/*spec.ts", "src/jest.config.ts"] + "exclude": ["node_modules", "dist", "**/*spec.ts", "**/test-utils.ts", "src/jest.config.ts"] } diff --git a/tsconfig.jest.json b/tsconfig.jest.json deleted file mode 100644 index 48586555..00000000 --- a/tsconfig.jest.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "extends": "./tsconfig.json", - "compilerOptions": { - // transpile each file on its own: ts-jest then skips type-checking, which - // `npm run typecheck` does once for the whole project instead - "isolatedModules": true, - "incremental": false - } -} diff --git a/tsconfig.json b/tsconfig.json index 489d84b4..0c37f237 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -18,6 +18,10 @@ "multer" ], "strict": true, + "noUnusedLocals": true, + "noImplicitReturns": true, + "noImplicitOverride": true, + "noFallthroughCasesInSwitch": true, "paths": { "src/*": [ "./src/*" diff --git a/tslint.json b/tslint.json deleted file mode 100644 index 5601443f..00000000 --- a/tslint.json +++ /dev/null @@ -1,22 +0,0 @@ - -{ - "defaultSeverity": "error", - "extends": ["tslint:recommended"], - "jsRules": { - "no-unused-expression": true - }, - "rules": { - "quotemark": [true, "single"], - "member-access": [false], - "ordered-imports": [false], - "max-line-length": [true, 150], - "member-ordering": [false], - "interface-name": [false], - "arrow-parens": false, - "object-literal-sort-keys": false - }, - "rulesDirectory": [], - "linterOptions": { - "exclude": ["config/**"] - } - } \ No newline at end of file From b47b130a353efe748c4ca0d627f1ec17b127076a Mon Sep 17 00:00:00 2001 From: Pavel Strunkin Date: Sun, 1 Nov 2026 21:25:08 +0200 Subject: [PATCH 2/8] fix(events): also send testRun_updated for a run still queued as created Clients skip testRun_created for runs they already fetched over HTTP, so an approve within the debounce window left the UI on the old status. The queued created entry is still refreshed, and the run now goes to the updated queue too, so both events carry the latest state. --- src/shared/events/events.gateway.spec.ts | 8 +++++--- src/shared/events/events.gateway.ts | 9 +++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/shared/events/events.gateway.spec.ts b/src/shared/events/events.gateway.spec.ts index 8d32d50b..3421e785 100644 --- a/src/shared/events/events.gateway.spec.ts +++ b/src/shared/events/events.gateway.spec.ts @@ -195,17 +195,19 @@ describe('EventsGateway', () => { }); describe('ordering and de-duplication', () => { - it('sends an update of a run still queued as created with the created event, not before it', async () => { + it('sends an update of a run still queued as created with both the created and the updated event', async () => { const { gateway, emitMock } = await initGateway(); const diffed = { ...testRunB, status: TestStatus.unresolved, diffName: 'diff.png' }; + const approved = { ...diffed, status: TestStatus.approved }; gateway.testRunCreated(testRunA); gateway.testRunCreated(testRunB); gateway.testRunUpdated(diffed); + gateway.testRunUpdated(approved); jest.advanceTimersByTime(1500); - expect(emitsOf(emitMock, 'testRun_updated')).toEqual([]); - expect(emitsOf(emitMock, 'testRun_created')).toEqual([[new TestRunDto(testRunA)], [new TestRunDto(diffed)]]); + expect(emitsOf(emitMock, 'testRun_created')).toEqual([[new TestRunDto(testRunA)], [new TestRunDto(approved)]]); + expect(emitsOf(emitMock, 'testRun_updated')).toEqual([[new TestRunDto(diffed)], [new TestRunDto(approved)]]); }); it('sends an update of a run already announced as created as an update', async () => { diff --git a/src/shared/events/events.gateway.ts b/src/shared/events/events.gateway.ts index ae3dc893..94f904a7 100644 --- a/src/shared/events/events.gateway.ts +++ b/src/shared/events/events.gateway.ts @@ -46,10 +46,11 @@ export class EventsGateway implements OnApplicationShutdown { } testRunUpdated(testRun: TestRun): void { - this.enqueueTestRun( - this.testRunQueues.testRun_created.has(testRun.id) ? 'testRun_created' : 'testRun_updated', - testRun - ); + // Clients skip testRun_created for runs they already fetched, so the update must go out as well + if (this.testRunQueues.testRun_created.has(testRun.id)) { + this.testRunQueues.testRun_created.set(testRun.id, new TestRunDto(testRun)); + } + this.enqueueTestRun('testRun_updated', testRun); this.buildUpdated(testRun.buildId); } From 8f83d91f56f816048b06df66836c9b296c1e760a Mon Sep 17 00:00:00 2001 From: Pavel Strunkin Date: Sun, 1 Nov 2026 21:32:48 +0200 Subject: [PATCH 3/8] test(events): assert a deleted run does not go out as an update --- src/shared/events/events.gateway.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/shared/events/events.gateway.spec.ts b/src/shared/events/events.gateway.spec.ts index 3421e785..6240b2ee 100644 --- a/src/shared/events/events.gateway.spec.ts +++ b/src/shared/events/events.gateway.spec.ts @@ -146,6 +146,7 @@ describe('EventsGateway', () => { jest.advanceTimersByTime(1500); expect(emitsOf(emitMock, 'testRun_created')).toEqual([[new TestRunDto(testRunA)], [new TestRunDto(testRunC)]]); + expect(emitsOf(emitMock, 'testRun_updated')).toEqual([[new TestRunDto(testRunA)]]); expect(emitsOf(emitMock, 'testRun_deleted')).toEqual([[new TestRunDto(testRunB)]]); }); From aeb310c0a5b126ead0ab22337d8b323a2802a4e4 Mon Sep 17 00:00:00 2001 From: Pavel Strunkin Date: Sun, 1 Nov 2026 21:37:32 +0200 Subject: [PATCH 4/8] ci: install the prisma package before linting its type-aware files --- .github/workflows/workflow.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 0040edce..1373aa32 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -33,6 +33,10 @@ jobs: - name: Install npm dependencies run: npm ci + - name: Install prisma npm dependencies + working-directory: prisma + run: npm ci + - name: Static analysis run: npm run lint From 0b0f5e532cea126e1fd1806b82aa306beebcec36 Mon Sep 17 00:00:00 2001 From: Pavel Strunkin Date: Sun, 1 Nov 2026 21:39:13 +0200 Subject: [PATCH 5/8] fix(config): warn about a public JWT_SECRET instead of refusing to start Deployments that never changed the sample secret kept starting before this branch; an upgrade must not stop them. --- .env | 1 - README.md | 3 +-- src/config/env.validation.spec.ts | 19 ++----------------- src/config/env.validation.ts | 13 +++++-------- 4 files changed, 8 insertions(+), 28 deletions(-) diff --git a/.env b/.env index 51b88c22..10fd3c95 100644 --- a/.env +++ b/.env @@ -84,7 +84,6 @@ OLLAMA_BASE_URL=http://localhost:11434 #DINOV2_URL=http://localhost:8000 #DINOV2_TIMEOUT_MS=60000 #DATABASE_URL=postgresql://postgres:postgres@localhost:5432/vrt_db_dev # required; docker-compose.yml builds it from POSTGRES_* -#NODE_ENV=production # production refuses to start with the sample JWT_SECRET #VRT_RUN_AS_ROOT=false # docker entrypoint: true keeps the container on root #MAX_IMAGE_PIXELS=100000000 # largest image in pixels (width x height); 0 turns the limit off #MULTIPART_FILE_SIZE_LIMIT=104857600 # largest multipart upload in bytes (100 MiB) diff --git a/README.md b/README.md index c41fa223..abe3e81e 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ lines in `.env` (section `# optional`) show the same variables. | `APP_PORT` | `3000` | Port the API listens on. | | `APP_FRONTEND_URL` | none | Frontend base URL, used to build links in test run results. | | `BODY_PARSER_JSON_LIMIT` | body-parser default | Maximum JSON request body size, e.g. `5mb`. | -| `JWT_SECRET` | none, required | Key that signs login tokens; startup fails when it is unset. The public sample `jwtPrivateKey` is refused with `NODE_ENV=production`, and the sample in `.env` only logs a warning. Generate one with `openssl rand -hex 32`. | +| `JWT_SECRET` | none, required | Key that signs login tokens; startup fails when it is unset. Public sample values (`jwtPrivateKey`, the one in `.env`) log a warning. Generate one with `openssl rand -hex 32`. | | `JWT_LIFE_TIME` | `1d` | Token lifetime as a timespan such as `1d`, `12h` or `3600s`. An invalid value falls back to `1d` with a warning. | | `JWT_LEGACY_EMAIL_TOKENS` | `true` | Accept tokens issued before the `sub` claim existed (they carry only the email). Set `false` once those tokens have expired. | | `SERVER_TIMEOUT` | Node.js default | HTTP `server.timeout`, in milliseconds. | @@ -57,7 +57,6 @@ lines in `.env` (section `# optional`) show the same variables. | `BUILD_RETENTION_GRACE_MINUTES` | `60` | Automatic build pruning keeps builds active within this many minutes. `0` turns that protection off. | | `STRICT_BUILD_PROJECT_CHECK` | `false` | `true` (case-insensitive) makes `POST /test-runs` reject a build that belongs to another project with `400`. By default such a request is accepted with a warning, because older clients may send such pairs. | | `LOG_LEVEL` | all levels | Comma-separated Nest log levels to keep, e.g. `log,warn,error,fatal`. | -| `NODE_ENV` | unset | `production` makes startup refuse the public sample `JWT_SECRET`. | | `HTTPS_KEY_PATH` | `./secrets/ssl.key` | TLS private key. HTTPS is used only when both files exist; otherwise the API serves HTTP. | | `HTTPS_CERT_PATH` | `./secrets/ssl.cert` | TLS certificate. | | `VRT_RUN_AS_ROOT` | `false` | Read by `docker-entrypoint.sh`, not by the app. The container drops to the `node` user when the image volume and TLS files allow it; `true` keeps root. | diff --git a/src/config/env.validation.spec.ts b/src/config/env.validation.spec.ts index bd903121..5040c52e 100644 --- a/src/config/env.validation.spec.ts +++ b/src/config/env.validation.spec.ts @@ -1,10 +1,5 @@ import { Logger } from '@nestjs/common'; -import { - DEFAULT_JWT_LIFE_TIME, - DEVELOPMENT_JWT_SECRET, - REFUSED_IN_PRODUCTION_JWT_SECRETS, - validateEnv, -} from './env.validation'; +import { DEFAULT_JWT_LIFE_TIME, DEVELOPMENT_JWT_SECRET, validateEnv } from './env.validation'; describe('validateEnv', () => { const GOOD = { JWT_SECRET: 'a-random-secret', JWT_LIFE_TIME: '12h', APP_FRONTEND_URL: 'http://localhost:8080' }; @@ -32,13 +27,8 @@ describe('validateEnv', () => { expect(() => validateEnv({ ...GOOD, JWT_SECRET: value }, logger)).toThrow('JWT_SECRET is not set'); }); - it('refuses the public sample value when NODE_ENV=production', () => { - expect(() => validateEnv({ ...GOOD, JWT_SECRET: 'jwtPrivateKey', NODE_ENV: 'production' }, logger)).toThrow( - 'Refusing to start' - ); - }); - it.each([ + ['the public sample value when NODE_ENV=production', 'jwtPrivateKey', 'production'], ['the public sample value when NODE_ENV=development', 'jwtPrivateKey', 'development'], ['the public sample value when NODE_ENV is unset', 'jwtPrivateKey', undefined], ['the committed development secret in production', DEVELOPMENT_JWT_SECRET, 'production'], @@ -46,11 +36,6 @@ describe('validateEnv', () => { expect(() => validateEnv({ ...GOOD, JWT_SECRET: secret, NODE_ENV: nodeEnv }, logger)).not.toThrow(); expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('publicly known')); }); - - it('lists the sample value among the refused ones', () => { - expect(REFUSED_IN_PRODUCTION_JWT_SECRETS).toContain('jwtPrivateKey'); - expect(REFUSED_IN_PRODUCTION_JWT_SECRETS).not.toContain(DEVELOPMENT_JWT_SECRET); - }); }); describe('JWT_LIFE_TIME', () => { diff --git a/src/config/env.validation.ts b/src/config/env.validation.ts index 53211a8d..68cea789 100644 --- a/src/config/env.validation.ts +++ b/src/config/env.validation.ts @@ -1,8 +1,8 @@ import { Logger } from '@nestjs/common'; export const DEFAULT_JWT_LIFE_TIME = '1d'; -export const REFUSED_IN_PRODUCTION_JWT_SECRETS: readonly string[] = ['jwtPrivateKey']; export const DEVELOPMENT_JWT_SECRET = 'vrt-local-development-only-jwt-secret'; +const PUBLIC_JWT_SECRETS: readonly string[] = ['jwtPrivateKey', DEVELOPMENT_JWT_SECRET]; const TIMESPAN = /^(-?(?:\d+)?\.?\d+) *(milliseconds?|msecs?|ms|seconds?|secs?|s|minutes?|mins?|m|hours?|hrs?|h|days?|d|weeks?|w|years?|yrs?|y)?$/i; @@ -21,13 +21,10 @@ export function validateEnv( if (!secret) { throw new Error('JWT_SECRET is not set. Generate one, e.g. `openssl rand -hex 32`.'); } - if (REFUSED_IN_PRODUCTION_JWT_SECRETS.includes(secret) || secret === DEVELOPMENT_JWT_SECRET) { - const problem = - 'JWT_SECRET is a publicly known sample value: anyone can sign tokens for any user. Set a random one, e.g. `openssl rand -hex 32`.'; - if (text(env.NODE_ENV) === 'production' && REFUSED_IN_PRODUCTION_JWT_SECRETS.includes(secret)) { - throw new Error(`${problem} Refusing to start with NODE_ENV=production.`); - } - logger.warn(problem); + if (PUBLIC_JWT_SECRETS.includes(secret)) { + logger.warn( + 'JWT_SECRET is a publicly known sample value: anyone can sign tokens for any user. Set a random one, e.g. `openssl rand -hex 32`.' + ); } const lifeTime = text(env.JWT_LIFE_TIME); From c407cd352e19cac4ae67c2b7bf19c708498ce9e5 Mon Sep 17 00:00:00 2001 From: Pavel Strunkin Date: Sun, 1 Nov 2026 21:45:38 +0200 Subject: [PATCH 6/8] refactor(pixelmatch): drop the byte-equal check the service already makes before a diff job --- .../libs/pixelmatch/pixelmatch.core.spec.ts | 21 ------------------- .../libs/pixelmatch/pixelmatch.core.ts | 5 ----- 2 files changed, 26 deletions(-) diff --git a/src/compare/libs/pixelmatch/pixelmatch.core.spec.ts b/src/compare/libs/pixelmatch/pixelmatch.core.spec.ts index 3e2b94c8..fb76e170 100644 --- a/src/compare/libs/pixelmatch/pixelmatch.core.spec.ts +++ b/src/compare/libs/pixelmatch/pixelmatch.core.spec.ts @@ -96,27 +96,6 @@ describe('computePixelmatchDiff with a signature', () => { }); }); -describe('computePixelmatchDiff with byte-equal input', () => { - afterEach(() => jest.restoreAllMocks()); - - it('answers equal for byte-equal input without decoding it', () => { - const read = jest.spyOn(PNG.sync, 'read'); - const other = new Uint8Array(blank.length + 3); - other.set(blank, 3); - - const result = diffJob(blank, other.subarray(3) as unknown as Buffer); - - expect(result).toStrictEqual({ equal: true }); - expect(read).not.toHaveBeenCalled(); - }); - - it('decodes and rejects a byte-equal but truncated pair as before', () => { - const truncated = blank.subarray(0, blank.length - 20); - - expect(() => diffJob(truncated, Buffer.from(truncated))).toThrow(); - }); -}); - describe('computePixelmatchDiff with thumbnails', () => { it('returns a small picture of both the checkpoint and the diff', () => { const result = diffJob(blank, block(10, 10, RED)); diff --git a/src/compare/libs/pixelmatch/pixelmatch.core.ts b/src/compare/libs/pixelmatch/pixelmatch.core.ts index 05322f27..6d7be66c 100644 --- a/src/compare/libs/pixelmatch/pixelmatch.core.ts +++ b/src/compare/libs/pixelmatch/pixelmatch.core.ts @@ -1,6 +1,5 @@ import { PNG } from 'pngjs'; import Pixelmatch from 'pixelmatch'; -import { isByteEqualWellFormedPng } from '../../../static/utils'; import { IgnoreAreaDto } from '../../../test-runs/dto/ignore-area.dto'; import { applyIgnoreAreas, readPng, scaleImageToSize } from '../../utils'; import { @@ -42,10 +41,6 @@ export interface PixelmatchJobOutput { } export function computePixelmatchDiff(input: PixelmatchJobInput): PixelmatchJobOutput { - if (isByteEqualWellFormedPng(input.baseline, input.image)) { - return { equal: true }; - } - const baseline = readPng(input.baseline); const image = readPng(input.image); From 01de78f4a84463a65c4e8ec1e0bc488909c56a1f Mon Sep 17 00:00:00 2001 From: Pavel Strunkin Date: Sun, 1 Nov 2026 21:45:38 +0200 Subject: [PATCH 7/8] refactor: replace thirteen new environment variables with fixed defaults Timeouts (Ollama, Gemini, S3, LDAP, Redis, diff job), VLM concurrency and the build retention grace become constants. Foreign build ids are always accepted with a warning, email-only tokens are always accepted, and the entrypoint drops root whenever the volume allows it. None of these variables reached master. --- .env | 13 ---- README.md | 15 +---- docker-compose.yml | 1 - docker-entrypoint.sh | 2 +- src/auth/jwt.strategy.spec.ts | 25 -------- src/auth/jwt.strategy.ts | 10 +--- src/builds/builds.service.spec.ts | 29 +-------- src/builds/builds.service.ts | 10 +--- src/compare/diff-worker-pool.ts | 13 ++-- src/compare/diff-worker-pool.workers.spec.ts | 60 ++++--------------- .../providers/gemini/gemini.service.spec.ts | 32 ++-------- .../vlm/providers/gemini/gemini.service.ts | 5 +- .../providers/ollama/ollama.service.spec.ts | 23 +++---- .../vlm/providers/ollama/ollama.service.ts | 8 +-- src/compare/libs/vlm/vlm.service.spec.ts | 28 ++------- src/compare/libs/vlm/vlm.service.ts | 2 +- src/redis-io.adapter.spec.ts | 13 +--- src/redis-io.adapter.ts | 6 +- src/static/aws/s3.service.spec.ts | 23 ------- src/static/aws/s3.service.ts | 7 +-- src/test-runs/test-runs.service.spec.ts | 16 +---- src/test-runs/test-runs.service.ts | 24 ++------ src/users/ldap/ldapusers.service.spec.ts | 22 +------ src/users/ldap/ldapusers.service.ts | 13 +--- test/builds-api.e2e-spec.ts | 15 ++--- 25 files changed, 73 insertions(+), 342 deletions(-) diff --git a/.env b/.env index 10fd3c95..45fe68b5 100644 --- a/.env +++ b/.env @@ -31,21 +31,14 @@ AWS_S3_BUCKET_NAME= # Multi-instance socket.io fan-out (optional). Unset = in-memory adapter. #REDIS_URL=redis://localhost:6379 # How long the first Redis connection may take before the API exits instead of waiting (default 30000). -#REDIS_CONNECT_TIMEOUT_MS=30000 # Log levels, comma separated; unset logs everything. Read from the process environment, not from this file. #LOG_LEVEL=log,warn,error,fatal # S3 request timeouts in milliseconds (STATIC_SERVICE=s3) -#S3_CONNECTION_TIMEOUT_MS=5000 -#S3_REQUEST_TIMEOUT_MS=60000 -#BUILD_RETENTION_GRACE_MINUTES=60 # automatic build pruning spares builds active within this many minutes; 0 disables #REGISTRATION_ENABLED=true -#JWT_LEGACY_EMAIL_TOKENS=true -#LDAP_TIMEOUT_MS=10000 -#LDAP_CONNECT_TIMEOUT_MS=5000 #LDAP_ENABLED=true #LDAP_TLS_NO_VERIFY=false @@ -84,14 +77,8 @@ OLLAMA_BASE_URL=http://localhost:11434 #DINOV2_URL=http://localhost:8000 #DINOV2_TIMEOUT_MS=60000 #DATABASE_URL=postgresql://postgres:postgres@localhost:5432/vrt_db_dev # required; docker-compose.yml builds it from POSTGRES_* -#VRT_RUN_AS_ROOT=false # docker entrypoint: true keeps the container on root #MAX_IMAGE_PIXELS=100000000 # largest image in pixels (width x height); 0 turns the limit off #MULTIPART_FILE_SIZE_LIMIT=104857600 # largest multipart upload in bytes (100 MiB) #DIFF_WORKERS_COUNT= # diff worker threads; unset: CPUs minus 1, between 1 and 8 #DIFF_QUEUE_LIMIT=256 # diff jobs waiting for a worker; when full the API answers 503 -#DIFF_JOB_TIMEOUT_MS=120000 # one diff job, ms; 0 turns the timeout off #DIFF_WORKER_MAX_HEAP_MB=512 # heap limit of each diff worker, MB -#VLM_CONCURRENCY=4 # concurrent VLM requests per API instance -#OLLAMA_TIMEOUT_MS=300000 # Ollama request timeout, ms -#GEMINI_TIMEOUT_MS=300000 # Gemini request timeout, ms -#STRICT_BUILD_PROJECT_CHECK=false # true: POST /test-runs rejects a build of another project with 400 diff --git a/README.md b/README.md index abe3e81e..63a1fc8e 100644 --- a/README.md +++ b/README.md @@ -50,16 +50,12 @@ lines in `.env` (section `# optional`) show the same variables. | `BODY_PARSER_JSON_LIMIT` | body-parser default | Maximum JSON request body size, e.g. `5mb`. | | `JWT_SECRET` | none, required | Key that signs login tokens; startup fails when it is unset. Public sample values (`jwtPrivateKey`, the one in `.env`) log a warning. Generate one with `openssl rand -hex 32`. | | `JWT_LIFE_TIME` | `1d` | Token lifetime as a timespan such as `1d`, `12h` or `3600s`. An invalid value falls back to `1d` with a warning. | -| `JWT_LEGACY_EMAIL_TOKENS` | `true` | Accept tokens issued before the `sub` claim existed (they carry only the email). Set `false` once those tokens have expired. | | `SERVER_TIMEOUT` | Node.js default | HTTP `server.timeout`, in milliseconds. | | `SERVER_HEADERS_TIMEOUT` | Node.js default | HTTP `headersTimeout`, in milliseconds. | | `SERVER_KEEP_ALIVE_TIMEOUT` | Node.js default | HTTP `keepAliveTimeout`, in milliseconds. | -| `BUILD_RETENTION_GRACE_MINUTES` | `60` | Automatic build pruning keeps builds active within this many minutes. `0` turns that protection off. | -| `STRICT_BUILD_PROJECT_CHECK` | `false` | `true` (case-insensitive) makes `POST /test-runs` reject a build that belongs to another project with `400`. By default such a request is accepted with a warning, because older clients may send such pairs. | | `LOG_LEVEL` | all levels | Comma-separated Nest log levels to keep, e.g. `log,warn,error,fatal`. | | `HTTPS_KEY_PATH` | `./secrets/ssl.key` | TLS private key. HTTPS is used only when both files exist; otherwise the API serves HTTP. | | `HTTPS_CERT_PATH` | `./secrets/ssl.cert` | TLS certificate. | -| `VRT_RUN_AS_ROOT` | `false` | Read by `docker-entrypoint.sh`, not by the app. The container drops to the `node` user when the image volume and TLS files allow it; `true` keeps root. | ### Storage @@ -69,8 +65,6 @@ lines in `.env` (section `# optional`) show the same variables. | `AWS_S3_BUCKET_NAME` | none | Bucket for `STATIC_SERVICE=s3`; required then. | | `AWS_REGION` | from the AWS SDK | Read by the AWS SDK v3 default chain, not by VRT code. | | `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` | from the AWS SDK | Same chain. An IAM role or shared AWS config works without them. | -| `S3_CONNECTION_TIMEOUT_MS` | `5000` | S3 connection timeout, in milliseconds. | -| `S3_REQUEST_TIMEOUT_MS` | `60000` | S3 request timeout, in milliseconds. A request that exceeds it fails. | ### Image comparison @@ -80,12 +74,8 @@ lines in `.env` (section `# optional`) show the same variables. | `MULTIPART_FILE_SIZE_LIMIT` | `104857600` (100 MiB) | Largest accepted multipart file upload, in bytes. | | `DIFF_WORKERS_COUNT` | CPUs minus 1, between 1 and 8 | Number of diff worker threads. | | `DIFF_QUEUE_LIMIT` | `256` | Diff jobs allowed to wait for a worker. When the queue is full the API answers `503` with `Retry-After: 5`. | -| `DIFF_JOB_TIMEOUT_MS` | `120000` | Timeout for one diff job, in milliseconds. `0` turns it off. | | `DIFF_WORKER_MAX_HEAP_MB` | `512` | Heap limit of each diff worker, in MB. | -| `VLM_CONCURRENCY` | `4` | Concurrent VLM requests per API instance (minimum 1). | | `OLLAMA_BASE_URL` | none | Ollama server for VLM comparisons; required when the Ollama provider is used. | -| `OLLAMA_TIMEOUT_MS` | `300000` | Timeout for one Ollama request, in milliseconds. | -| `GEMINI_TIMEOUT_MS` | `300000` | Timeout for one Gemini request, in milliseconds. The API key comes from the project's VLM settings. | | `DINOV2_URL` | none | DINOv2 service for the `dinov2Check` noise filter. With the filter on and this unset, runs go to review. | | `DINOV2_TIMEOUT_MS` | `60000` | Timeout for one DINOv2 request, in milliseconds. | @@ -102,8 +92,6 @@ lines in `.env` (section `# optional`) show the same variables. | `LDAP_USERS_SEARCH_FILTER` | none | User search filter; `{{email}}` is replaced by the login email. Required when LDAP is enabled. | | `LDAP_ATTRIBUTE_MAIL`, `LDAP_ATTRIBUTE_FIRST_NAME`, `LDAP_ATTRIBUTE_LAST_NAME` | none | Attributes that hold the email, first name and last name. Required when LDAP is enabled. | | `LDAP_TLS_NO_VERIFY` | `false` | Skip TLS certificate verification for LDAP. Test servers only. | -| `LDAP_TIMEOUT_MS` | `10000` | LDAP operation timeout, in milliseconds. | -| `LDAP_CONNECT_TIMEOUT_MS` | `5000` | LDAP connection timeout, in milliseconds. | | `OIDC_ENABLED` | `false` | Enables OIDC (SSO) login; see the OIDC section. | | `OIDC_ISSUER_URL` | none | Issuer URL of the identity provider. Required when OIDC is enabled. | | `OIDC_CLIENT_ID` | none | OIDC client ID. Required when OIDC is enabled. | @@ -124,7 +112,6 @@ lines in `.env` (section `# optional`) show the same variables. | Variable | Default | What it does | |---|---|---| | `REDIS_URL` | unset, in-memory adapter | Redis for socket.io fan-out between instances (see Scaling to multiple instances). | -| `REDIS_CONNECT_TIMEOUT_MS` | `30000` | How long the first Redis connection may take before the API exits instead of waiting. | ### Seed @@ -141,7 +128,7 @@ Read by `prisma db seed`. - **Redis** runs several API instances behind one socket.io fan-out. Set `REDIS_URL`; see [Scaling to multiple instances](#scaling-to-multiple-instances-real-time-updates). - **DINOv2** serves the `dinov2Check` rendering-noise filter (see `dinov2/README.md`). Start it with `docker compose --profile dinov2 up -d`. The service listens on port 8000 inside the compose network only, so the API container uses `DINOV2_URL=http://dinov2:8000`. - **Ollama** runs the VLM comparison locally. Set `OLLAMA_BASE_URL`; from the API container, which is started by compose, the host is `http://host.docker.internal:11434`. -- **Gemini** runs the VLM comparison in the cloud. Its API key is set in the project's VLM settings, not in the environment; `GEMINI_TIMEOUT_MS` sets the request timeout. +- **Gemini** runs the VLM comparison in the cloud. Its API key is set in the project's VLM settings, not in the environment. ## Scaling to multiple instances (real-time updates) diff --git a/docker-compose.yml b/docker-compose.yml index a1811b5c..7c9f7959 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,7 +19,6 @@ services: OLLAMA_BASE_URL: ${OLLAMA_BASE_URL} DINOV2_URL: ${DINOV2_URL} REDIS_URL: ${REDIS_URL:-redis://redis:6379} - BUILD_RETENTION_GRACE_MINUTES: ${BUILD_RETENTION_GRACE_MINUTES} ports: - "${APP_PORT}:3000" expose: diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index ca3ed1d7..d5588965 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -14,7 +14,7 @@ can_drop_root() { done } -if [ "$(id -u)" = "0" ] && [ "${VRT_RUN_AS_ROOT:-false}" != "true" ]; then +if [ "$(id -u)" = "0" ]; then if can_drop_root; then exec setpriv --reuid=node --regid=node --init-groups "$@" fi diff --git a/src/auth/jwt.strategy.spec.ts b/src/auth/jwt.strategy.spec.ts index a7a9b388..52f22f11 100644 --- a/src/auth/jwt.strategy.spec.ts +++ b/src/auth/jwt.strategy.spec.ts @@ -16,10 +16,6 @@ describe('JwtStrategy', () => { const createStrategy = () => new JwtStrategy(configService as unknown as ConfigService, prismaService as unknown as PrismaService); - const withLegacyTokens = (value: string) => { - configService.get.mockImplementation((key: string) => (key === 'JWT_LEGACY_EMAIL_TOKENS' ? value : SECRET)); - strategy = createStrategy(); - }; beforeEach(() => { prismaService = { user: { findUnique: jest.fn() } }; @@ -114,27 +110,6 @@ describe('JwtStrategy', () => { expect(outcome.success).toHaveBeenCalledWith(user, undefined); }); - it.each(['false', ' FALSE '])('rejects an email-only token when JWT_LEGACY_EMAIL_TOKENS=%j', async (value) => { - withLegacyTokens(value); - - const outcome = authenticate(sign({ email: user.email }, SECRET)); - await settle(); - - expect(prismaService.user.findUnique).not.toHaveBeenCalled(); - expect(outcome.error).toHaveBeenCalledWith(expect.any(UnauthorizedException)); - expect(outcome.success).not.toHaveBeenCalled(); - }); - - it('keeps accepting tokens with sub when JWT_LEGACY_EMAIL_TOKENS=false', async () => { - withLegacyTokens('false'); - prismaService.user.findUnique.mockResolvedValue(user); - - const outcome = authenticate(sign({ sub: 'user-id', email: user.email }, SECRET)); - await settle(); - - expect(outcome.success).toHaveBeenCalledWith(user, undefined); - }); - it('errors with a 401 when the token belongs to an unknown user', async () => { prismaService.user.findUnique.mockResolvedValue(null); diff --git a/src/auth/jwt.strategy.ts b/src/auth/jwt.strategy.ts index f4bf2b95..2b5c8fd6 100644 --- a/src/auth/jwt.strategy.ts +++ b/src/auth/jwt.strategy.ts @@ -8,8 +8,6 @@ import { PrismaService } from '../prisma/prisma.service'; @Injectable() export class JwtStrategy extends PassportStrategy(Strategy) { - private readonly acceptEmailOnlyTokens: boolean; - constructor( configService: ConfigService, private prismaService: PrismaService @@ -19,11 +17,6 @@ export class JwtStrategy extends PassportStrategy(Strategy) { ignoreExpiration: false, secretOrKey: configService.get('JWT_SECRET'), }); - // tokens issued before `sub` existed carry only the email; drop them once they have all expired - this.acceptEmailOnlyTokens = - String(configService.get('JWT_LEGACY_EMAIL_TOKENS') ?? 'true') - .trim() - .toLowerCase() !== 'false'; } // @nestjs/passport calls `done` with whatever this returns, so it must not call `done` itself @@ -39,7 +32,8 @@ export class JwtStrategy extends PassportStrategy(Strategy) { if (typeof payload.sub === 'string' && payload.sub) { return this.prismaService.user.findUnique({ where: { id: payload.sub } }); } - if (this.acceptEmailOnlyTokens && typeof payload.email === 'string' && payload.email) { + // tokens issued before `sub` existed carry only the email + if (typeof payload.email === 'string' && payload.email) { return this.prismaService.user.findUnique({ where: { email: payload.email } }); } return Promise.resolve(null); diff --git a/src/builds/builds.service.spec.ts b/src/builds/builds.service.spec.ts index 5e8d0a6e..261fdbc4 100644 --- a/src/builds/builds.service.spec.ts +++ b/src/builds/builds.service.spec.ts @@ -1,5 +1,5 @@ import { Test } from '@nestjs/testing'; -import { BuildsService, retentionGraceMinutes } from './builds.service'; +import { BuildsService } from './builds.service'; import { PrismaService } from '../prisma/prisma.service'; import { TestRunsService } from '../test-runs/test-runs.service'; import { EventsGateway } from '../shared/events/events.gateway'; @@ -605,7 +605,6 @@ describe('BuildsService', () => { }; afterEach(() => { - delete process.env.BUILD_RETENTION_GRACE_MINUTES; jest.useRealTimers(); }); @@ -633,23 +632,6 @@ describe('BuildsService', () => { expect(prisma.build.findMany).toHaveBeenCalledWith(expect.objectContaining({ skip: 0 })); }); - it.each([ - [undefined, 60], - ['', 60], - ['15', 15], - ['0', 0], - ['-5', 60], - ['abc', 60], - ])('retentionGraceMinutes with %p is %p', (raw, expected) => { - if (raw === undefined) { - delete process.env.BUILD_RETENTION_GRACE_MINUTES; - } else { - process.env.BUILD_RETENTION_GRACE_MINUTES = raw; - } - - expect(retentionGraceMinutes()).toBe(expected); - }); - it('passes the grace cutoff to remove', async () => { jest.useFakeTimers({ now: new Date('2026-05-01T12:00:00Z'), doNotFake: ['setImmediate', 'nextTick'] }); const { service } = await init(ids('build1')); @@ -659,15 +641,6 @@ describe('BuildsService', () => { expect(service.remove).toHaveBeenCalledWith('build1', { onlyIfIdleSince: new Date('2026-05-01T11:00:00Z') }); }); - it('with grace 0 removes without the idle check', async () => { - process.env.BUILD_RETENTION_GRACE_MINUTES = '0'; - const { service } = await init(ids('build1')); - - await service.deleteOldBuilds(projectId, 1, 'b-new'); - - expect(service.remove).toHaveBeenCalledWith('build1', { onlyIfIdleSince: undefined }); - }); - it('runs a separate pruning for another kept build of the same project', async () => { const { service, prisma } = await setup(); prisma.build.findMany.mockResolvedValue([]); diff --git a/src/builds/builds.service.ts b/src/builds/builds.service.ts index ec51995a..1ce092d1 100644 --- a/src/builds/builds.service.ts +++ b/src/builds/builds.service.ts @@ -15,7 +15,6 @@ import { EventsGateway } from '../shared/events/events.gateway'; import { BuildDto } from './dto/build.dto'; import { getBuildsStats } from './build-stats'; import { testRunImageNames } from '../utils'; -import { nonNegativeNumber } from '../utils/env'; import { PaginatedBuildDto } from './dto/build-paginated.dto'; import { ModifyBuildDto } from './dto/build-modify.dto'; import type { WrapperType } from '../shared/wrapper-type'; @@ -25,11 +24,7 @@ const TEST_RUN_CHUNK = 500; const MAX_REMOVE_ATTEMPTS = 3; const BUILD_ORDER: Prisma.BuildOrderByWithRelationInput[] = [{ createdAt: 'desc' }, { id: 'desc' }]; -const DEFAULT_RETENTION_GRACE_MINUTES = 60; - -export function retentionGraceMinutes(): number { - return nonNegativeNumber(process.env.BUILD_RETENTION_GRACE_MINUTES, DEFAULT_RETENTION_GRACE_MINUTES); -} +const RETENTION_GRACE_MS = 60 * 60_000; class BuildAlreadyNumbered extends Error {} @@ -218,8 +213,7 @@ export class BuildsService implements BeforeApplicationShutdown { const deletionPromise = (async () => { this.logger.debug(`Starting to delete old builds for project ${projectId}, keeping ${keepBuilds} builds.`); - const graceMinutes = retentionGraceMinutes(); - const onlyIfIdleSince = graceMinutes > 0 ? new Date(Date.now() - graceMinutes * 60_000) : undefined; + const onlyIfIdleSince = new Date(Date.now() - RETENTION_GRACE_MS); const oldBuilds = await this.prismaService.build.findMany({ where: { projectId, id: { not: keepBuildId } }, orderBy: BUILD_ORDER, diff --git a/src/compare/diff-worker-pool.ts b/src/compare/diff-worker-pool.ts index 6a6cfa5f..0454d4a5 100644 --- a/src/compare/diff-worker-pool.ts +++ b/src/compare/diff-worker-pool.ts @@ -39,7 +39,7 @@ export class ImageDiffJobTimeoutError extends Error { } } -const DEFAULT_JOB_TIMEOUT_MS = 120_000; +const JOB_TIMEOUT_MS = 120_000; const DEFAULT_WORKER_MAX_HEAP_MB = 512; interface Job { @@ -58,7 +58,6 @@ export class DiffWorkerPool implements OnApplicationShutdown { Math.min(Number(process.env.DIFF_WORKERS_COUNT) || availableParallelism() - 1, 8) ); private readonly queueLimit = Number(process.env.DIFF_QUEUE_LIMIT) || 256; - private readonly jobTimeoutMs = nonNegativeNumber(process.env.DIFF_JOB_TIMEOUT_MS, DEFAULT_JOB_TIMEOUT_MS); private readonly maxHeapMb = nonNegativeNumber(process.env.DIFF_WORKER_MAX_HEAP_MB, DEFAULT_WORKER_MAX_HEAP_MB); private inline = !existsSync(WORKER_FILE); private workers: Worker[] = []; @@ -205,18 +204,16 @@ export class DiffWorkerPool implements OnApplicationShutdown { const job = this.queue.shift()!; this.inFlight.set(worker, job); worker.postMessage(job.input); - if (this.jobTimeoutMs > 0) { - job.timer = setTimeout(() => this.timeOut(worker, job), this.jobTimeoutMs); - job.timer.unref(); - } + job.timer = setTimeout(() => this.timeOut(worker, job), JOB_TIMEOUT_MS); + job.timer.unref(); } } private timeOut(worker: Worker, job: Job): void { if (this.inFlight.get(worker) !== job) return; this.forget(worker); - this.logger.error(`Image diff job ran over ${this.jobTimeoutMs} ms, replacing its worker`); - job.reject(new ImageDiffJobTimeoutError(this.jobTimeoutMs)); + this.logger.error(`Image diff job ran over ${JOB_TIMEOUT_MS} ms, replacing its worker`); + job.reject(new ImageDiffJobTimeoutError(JOB_TIMEOUT_MS)); worker.terminate().catch(() => undefined); this.spawn(); this.dispatch(); diff --git a/src/compare/diff-worker-pool.workers.spec.ts b/src/compare/diff-worker-pool.workers.spec.ts index 37bf6dcf..cb1f0fda 100644 --- a/src/compare/diff-worker-pool.workers.spec.ts +++ b/src/compare/diff-worker-pool.workers.spec.ts @@ -88,13 +88,11 @@ describe('DiffWorkerPool with worker threads', () => { const createPool = ({ workers = '1', queueLimit, - jobTimeout, maxHeap, - }: { workers?: string; queueLimit?: string; jobTimeout?: string; maxHeap?: string } = {}) => { + }: { workers?: string; queueLimit?: string; maxHeap?: string } = {}) => { const env = { DIFF_WORKER_MAX_HEAP_MB: maxHeap, DIFF_WORKERS_COUNT: workers, - DIFF_JOB_TIMEOUT_MS: jobTimeout, DIFF_QUEUE_LIMIT: queueLimit, }; for (const [name, value] of Object.entries(env)) { @@ -567,29 +565,29 @@ describe('DiffWorkerPool with worker threads', () => { jest.useRealTimers(); }); - it('rejects a job that runs longer than DIFF_JOB_TIMEOUT_MS and replaces its worker', async () => { - createPool({ jobTimeout: '1000' }); + it('rejects a job that runs longer than 120 s and replaces its worker', async () => { + createPool(); const hung = settle(pool.run(job(1))); ready(mockWorkers[0]); - jest.advanceTimersByTime(1000); + jest.advanceTimersByTime(120_000); expect((await hung).error).toBeInstanceOf(ImageDiffJobTimeoutError); - expect((await hung).error?.message).toBe('Image diff job timed out after 1000 ms'); + expect((await hung).error?.message).toBe('Image diff job timed out after 120000 ms'); expect(mockWorkers[0].terminate).toHaveBeenCalledTimes(1); expect(mockWorkers).toHaveLength(2); - expect(loggerErrorSpy).toHaveBeenCalledWith('Image diff job ran over 1000 ms, replacing its worker'); + expect(loggerErrorSpy).toHaveBeenCalledWith('Image diff job ran over 120000 ms, replacing its worker'); void settle(pool.run(job(2))); expect(mockWorkers[1].postMessage).toHaveBeenCalledWith(job(2)); }); it('does not time out a job answered in time', async () => { - createPool({ jobTimeout: '1000' }); + createPool(); const result = pool.run(job(1)); ready(mockWorkers[0]); - jest.advanceTimersByTime(999); + jest.advanceTimersByTime(119_999); mockWorkers[0].emit('message', output(1)); jest.advanceTimersByTime(5000); @@ -598,43 +596,11 @@ describe('DiffWorkerPool with worker threads', () => { expect(mockWorkers).toHaveLength(1); }); - it('defaults to 120 s', async () => { - createPool(); - - const result = settle(pool.run(job(1))); - jest.advanceTimersByTime(119_999); - expect(mockWorkers[0].terminate).not.toHaveBeenCalled(); - jest.advanceTimersByTime(1); - - expect((await result).error).toBeInstanceOf(ImageDiffJobTimeoutError); - }); - - it.each([ - ['turns off with 0', '0'], - ['falls back to the default for garbage', 'abc'], - ['falls back to the default for a negative number', '-5'], - ['falls back to the default for an empty value', ''], - ])('%s', async (title, jobTimeout) => { - createPool({ jobTimeout }); - - const result = settle(pool.run(job(1))); - jest.advanceTimersByTime(600_000); - - if (jobTimeout === '0') { - expect(mockWorkers[0].terminate).not.toHaveBeenCalled(); - mockWorkers[0].emit('message', output(1)); - expect((await result).value).toEqual(output(1)); - } else { - expect((await result).error).toBeInstanceOf(ImageDiffJobTimeoutError); - expect((await result).error?.message).toBe('Image diff job timed out after 120000 ms'); - } - }); - it('ignores the exit of the worker it terminated', async () => { - createPool({ jobTimeout: '1000' }); + createPool(); const hung = settle(pool.run(job(1))); - jest.advanceTimersByTime(1000); + jest.advanceTimersByTime(120_000); await hung; mockWorkers[0].emit('exit', 1); mockWorkers[0].emit('error', new Error('terminated')); @@ -643,11 +609,11 @@ describe('DiffWorkerPool with worker threads', () => { }); it('does not retry the timed-out job and does not count it as a load failure', async () => { - createPool({ jobTimeout: '1000' }); + createPool(); for (const id of [1, 2, 3]) { const hung = settle(pool.run(job(id))); - jest.advanceTimersByTime(1000); + jest.advanceTimersByTime(120_000); expect((await hung).error).toBeInstanceOf(ImageDiffJobTimeoutError); } @@ -657,7 +623,7 @@ describe('DiffWorkerPool with worker threads', () => { }); it('does not spawn a replacement for a pool that is shut down while the job runs', async () => { - createPool({ jobTimeout: '1000' }); + createPool(); const pending = settle(pool.run(job(1))); await pool.onApplicationShutdown(); diff --git a/src/compare/libs/vlm/providers/gemini/gemini.service.spec.ts b/src/compare/libs/vlm/providers/gemini/gemini.service.spec.ts index 66641335..ef7abe64 100644 --- a/src/compare/libs/vlm/providers/gemini/gemini.service.spec.ts +++ b/src/compare/libs/vlm/providers/gemini/gemini.service.spec.ts @@ -109,36 +109,12 @@ describe('GeminiService', () => { }); }); - describe('request timeout', () => { - afterEach(() => { - delete process.env.GEMINI_TIMEOUT_MS; - }); - - it('defaults to five minutes', async () => { - mockGenerateContent.mockResolvedValue(createMockResponse('{}')); - - await service.generate(createConfig(), []); - - expect(GoogleGenAI).toHaveBeenCalledWith({ apiKey: 'test-api-key', httpOptions: { timeout: 300000 } }); - }); - - it('falls back to five minutes for a negative GEMINI_TIMEOUT_MS', async () => { - process.env.GEMINI_TIMEOUT_MS = '-1'; - mockGenerateContent.mockResolvedValue(createMockResponse('{}')); + it('gives the SDK a five-minute request timeout', async () => { + mockGenerateContent.mockResolvedValue(createMockResponse('{}')); - await service.generate(createConfig(), []); + await service.generate(createConfig(), []); - expect(GoogleGenAI).toHaveBeenCalledWith({ apiKey: 'test-api-key', httpOptions: { timeout: 300000 } }); - }); - - it('is set by GEMINI_TIMEOUT_MS', async () => { - process.env.GEMINI_TIMEOUT_MS = '5000'; - mockGenerateContent.mockResolvedValue(createMockResponse('{}')); - - await service.generate(createConfig(), []); - - expect(GoogleGenAI).toHaveBeenCalledWith({ apiKey: 'test-api-key', httpOptions: { timeout: 5000 } }); - }); + expect(GoogleGenAI).toHaveBeenCalledWith({ apiKey: 'test-api-key', httpOptions: { timeout: 300000 } }); }); it.each([ diff --git a/src/compare/libs/vlm/providers/gemini/gemini.service.ts b/src/compare/libs/vlm/providers/gemini/gemini.service.ts index ffb751a6..c96919cc 100644 --- a/src/compare/libs/vlm/providers/gemini/gemini.service.ts +++ b/src/compare/libs/vlm/providers/gemini/gemini.service.ts @@ -4,9 +4,8 @@ import { zodToJsonSchema } from 'zod-to-json-schema'; import { VlmProvider, VlmProviderResponse } from '../../vlm-provider.interface'; import { VlmConfig } from '../../vlm.types'; import { VlmComparisonResultSchema } from '../../vlm-result.schema'; -import { positiveNumber } from '../../../../../utils/env'; -const DEFAULT_TIMEOUT_MS = 300_000; +const TIMEOUT_MS = 300_000; @Injectable() export class GeminiService implements VlmProvider { @@ -18,7 +17,7 @@ export class GeminiService implements VlmProvider { } return new GoogleGenAI({ apiKey, - httpOptions: { timeout: positiveNumber(process.env.GEMINI_TIMEOUT_MS, DEFAULT_TIMEOUT_MS) }, + httpOptions: { timeout: TIMEOUT_MS }, }); } diff --git a/src/compare/libs/vlm/providers/ollama/ollama.service.spec.ts b/src/compare/libs/vlm/providers/ollama/ollama.service.spec.ts index 75528c84..da0bd7ce 100644 --- a/src/compare/libs/vlm/providers/ollama/ollama.service.spec.ts +++ b/src/compare/libs/vlm/providers/ollama/ollama.service.spec.ts @@ -162,8 +162,7 @@ describe('OllamaService', () => { global.fetch = realFetch; }); - const sdkFetch = async (timeoutMs?: string): Promise => { - mockConfigService.get.mockReturnValue(timeoutMs); + const sdkFetch = async (): Promise => { mockChat.mockResolvedValue(createMockResponse('{}')); const fresh = new OllamaService(mockConfigService as any); await fresh.generate(createConfig(), []); @@ -194,11 +193,14 @@ describe('OllamaService', () => { expect(init.signal.aborted).toBe(false); }); - it('aborts a request that outlasts OLLAMA_TIMEOUT_MS', async () => { + it('aborts a request that outlasts its timeout', async () => { + const realTimeout = AbortSignal.timeout.bind(AbortSignal); + const timeout = jest.spyOn(AbortSignal, 'timeout').mockImplementation(() => realTimeout(1)); global.fetch = stalledFetch(); - const fetchForSdk = await sdkFetch('1'); + const fetchForSdk = await sdkFetch(); await expect(fetchForSdk('http://localhost:11434/api/chat')).rejects.toMatchObject({ name: 'TimeoutError' }); + timeout.mockRestore(); }); it('defaults to five minutes', async () => { @@ -212,20 +214,9 @@ describe('OllamaService', () => { timeout.mockRestore(); }); - it('falls back to five minutes for a negative OLLAMA_TIMEOUT_MS', async () => { - const timeout = jest.spyOn(AbortSignal, 'timeout'); - global.fetch = jest.fn().mockResolvedValue({}); - const fetchForSdk = await sdkFetch('-1'); - - await fetchForSdk('http://localhost:11434/api/chat'); - - expect(timeout).toHaveBeenCalledWith(300_000); - timeout.mockRestore(); - }); - it('still honours the signal of the caller', async () => { global.fetch = stalledFetch(); - const fetchForSdk = await sdkFetch('60000'); + const fetchForSdk = await sdkFetch(); const controller = new AbortController(); const request = fetchForSdk('http://localhost:11434/api/chat', { signal: controller.signal }); diff --git a/src/compare/libs/vlm/providers/ollama/ollama.service.ts b/src/compare/libs/vlm/providers/ollama/ollama.service.ts index 2b2d2cd3..180b3458 100644 --- a/src/compare/libs/vlm/providers/ollama/ollama.service.ts +++ b/src/compare/libs/vlm/providers/ollama/ollama.service.ts @@ -5,9 +5,8 @@ import { zodToJsonSchema } from 'zod-to-json-schema'; import { VlmProvider, VlmProviderResponse } from '../../vlm-provider.interface'; import { VlmConfig } from '../../vlm.types'; import { VlmComparisonResultSchema } from '../../vlm-result.schema'; -import { positiveNumber } from '../../../../../utils/env'; -const DEFAULT_TIMEOUT_MS = 300_000; +const TIMEOUT_MS = 300_000; @Injectable() export class OllamaService implements VlmProvider { @@ -19,7 +18,6 @@ export class OllamaService implements VlmProvider { private getOllamaClient(): Ollama { if (!this.ollamaClient) { const baseUrl = this.configService.getOrThrow('OLLAMA_BASE_URL'); - const timeoutMs = positiveNumber(this.configService.get('OLLAMA_TIMEOUT_MS'), DEFAULT_TIMEOUT_MS); this.ollamaClient = new Ollama({ host: baseUrl, // non-streaming chat sends headers only when done: without this a stalled model holds the upload for good @@ -27,8 +25,8 @@ export class OllamaService implements VlmProvider { fetch(url, { ...init, signal: init?.signal - ? AbortSignal.any([init.signal, AbortSignal.timeout(timeoutMs)]) - : AbortSignal.timeout(timeoutMs), + ? AbortSignal.any([init.signal, AbortSignal.timeout(TIMEOUT_MS)]) + : AbortSignal.timeout(TIMEOUT_MS), }), }); } diff --git a/src/compare/libs/vlm/vlm.service.spec.ts b/src/compare/libs/vlm/vlm.service.spec.ts index bcffda3f..3faa481d 100644 --- a/src/compare/libs/vlm/vlm.service.spec.ts +++ b/src/compare/libs/vlm/vlm.service.spec.ts @@ -427,12 +427,7 @@ describe('VlmService', () => { }); describe('model concurrency', () => { - afterEach(() => { - delete process.env.VLM_CONCURRENCY; - }); - - it('asks the model at most VLM_CONCURRENCY times at once', async () => { - process.env.VLM_CONCURRENCY = '1'; + it('asks the model at most four times at once', async () => { const gates: { resolve: (value: { content: string }) => void }[] = []; const ollama = jest.fn().mockImplementation( () => @@ -446,27 +441,16 @@ describe('VlmService', () => { }); const answer = reply(true, 'ok'); - const first = service.getDiff(input, DEFAULT_CONFIG); - const second = service.getDiff(input, DEFAULT_CONFIG); + const diffs = [1, 2, 3, 4, 5].map(() => service.getDiff(input, DEFAULT_CONFIG)); await new Promise((resolve) => setTimeout(resolve, 50)); - expect(ollama).toHaveBeenCalledTimes(1); + expect(ollama).toHaveBeenCalledTimes(4); gates[0].resolve(answer); - await first; await new Promise((resolve) => setTimeout(resolve, 50)); - expect(ollama).toHaveBeenCalledTimes(2); - - gates[1].resolve(answer); - await second; - }); + expect(ollama).toHaveBeenCalledTimes(5); - it('falls back to the default when VLM_CONCURRENCY is zero', async () => { - process.env.VLM_CONCURRENCY = '0'; - const { service } = await setup({ ollama: modelSaying(true) }); - - const result = await service.getDiff(input, DEFAULT_CONFIG); - - expect(result.status).toBe(TestStatus.ok); + gates.slice(1).forEach((gate) => gate.resolve(answer)); + await Promise.all(diffs); }); }); diff --git a/src/compare/libs/vlm/vlm.service.ts b/src/compare/libs/vlm/vlm.service.ts index 13983fc2..67a4db3a 100644 --- a/src/compare/libs/vlm/vlm.service.ts +++ b/src/compare/libs/vlm/vlm.service.ts @@ -35,7 +35,7 @@ export const DEFAULT_CONFIG: OllamaVlmConfig = { export class VlmService implements ImageComparator { private readonly logger: Logger = new Logger(VlmService.name); readonly appliesBlurAmplitudeFilter = true; - private readonly limiter = new ConcurrencyLimiter(Math.max(1, Number(process.env.VLM_CONCURRENCY) || 4)); + private readonly limiter = new ConcurrencyLimiter(4); constructor( private readonly staticService: StaticService, diff --git a/src/redis-io.adapter.spec.ts b/src/redis-io.adapter.spec.ts index 1f14d1cc..838b5530 100644 --- a/src/redis-io.adapter.spec.ts +++ b/src/redis-io.adapter.spec.ts @@ -144,19 +144,12 @@ describe('RedisIoAdapter', () => { await expect(makeAdapter().connectToRedis('redis://localhost:6379')).rejects.toThrow('redis down'); }); - it.each([ - ['2000', 2_000], - ['abc', 30_000], - ['', 30_000], - ['0', 30_000], - ['-1', 30_000], - ])('gives up after the REDIS_CONNECT_TIMEOUT_MS=%p it reads (%p ms)', async (value, ms) => { - process.env.REDIS_CONNECT_TIMEOUT_MS = value; + it('gives up after 30 seconds by default', async () => { const { strategy, finish } = connectPending('redis://localhost:6379'); - now.mockReturnValue(1_000_000 + ms - 1); + now.mockReturnValue(1_000_000 + 30_000 - 1); expect(typeof strategy(1, new Error('x'))).toBe('number'); - now.mockReturnValue(1_000_000 + ms); + now.mockReturnValue(1_000_000 + 30_000); expect(strategy(2, new Error('x'))).toBeInstanceOf(Error); await finish(); }); diff --git a/src/redis-io.adapter.ts b/src/redis-io.adapter.ts index 82d59367..cf0ed7d4 100644 --- a/src/redis-io.adapter.ts +++ b/src/redis-io.adapter.ts @@ -3,14 +3,10 @@ import { IoAdapter } from '@nestjs/platform-socket.io'; import { createAdapter } from '@socket.io/redis-adapter'; import { createClient } from 'redis'; import { ServerOptions } from 'socket.io'; -import { positiveNumber } from './utils/env'; const DEFAULT_CONNECT_TIMEOUT_MS = 30_000; const MAX_RECONNECT_DELAY_MS = 5_000; -const connectTimeoutFromEnv = (): number => - positiveNumber(process.env.REDIS_CONNECT_TIMEOUT_MS, DEFAULT_CONNECT_TIMEOUT_MS); - const describeUrl = (url: string): string => { try { return new URL(url).host; @@ -23,7 +19,7 @@ export class RedisIoAdapter extends IoAdapter { private static readonly logger = new Logger(RedisIoAdapter.name); private adapterConstructor!: ReturnType; - async connectToRedis(url: string, connectTimeoutMs = connectTimeoutFromEnv()): Promise { + async connectToRedis(url: string, connectTimeoutMs = DEFAULT_CONNECT_TIMEOUT_MS): Promise { const startedAt = Date.now(); let connected = false; const reconnectStrategy = (retries: number, cause: Error): number | Error => diff --git a/src/static/aws/s3.service.spec.ts b/src/static/aws/s3.service.spec.ts index 35d4f5cd..eb304a6b 100644 --- a/src/static/aws/s3.service.spec.ts +++ b/src/static/aws/s3.service.spec.ts @@ -188,35 +188,12 @@ describe('AWSS3Service', () => { describe('constructor', () => { it('creates one S3 client with the default credential chain and request timeouts', () => { - delete process.env.S3_CONNECTION_TIMEOUT_MS; - delete process.env.S3_REQUEST_TIMEOUT_MS; - new AWSS3Service(); expect(S3Client).toHaveBeenCalledTimes(1); expect(S3Client).toHaveBeenCalledWith(defaultTimeouts); }); - it('reads the timeouts from S3_CONNECTION_TIMEOUT_MS and S3_REQUEST_TIMEOUT_MS', () => { - process.env.S3_CONNECTION_TIMEOUT_MS = '2000'; - process.env.S3_REQUEST_TIMEOUT_MS = '1000'; - - new AWSS3Service(); - - expect(S3Client).toHaveBeenCalledWith({ - requestHandler: { connectionTimeout: 2000, requestTimeout: 1000, throwOnRequestTimeout: true }, - }); - }); - - it.each(['abc', '', '0', '-5'])('falls back to the default timeout for %p', (value) => { - process.env.S3_CONNECTION_TIMEOUT_MS = value; - process.env.S3_REQUEST_TIMEOUT_MS = value; - - new AWSS3Service(); - - expect(S3Client).toHaveBeenCalledWith(defaultTimeouts); - }); - it('reads the bucket name when the service is constructed, not on each call', async () => { const service = new AWSS3Service(); process.env.AWS_S3_BUCKET_NAME = 'changed-later'; diff --git a/src/static/aws/s3.service.ts b/src/static/aws/s3.service.ts index 2859910a..594d13ae 100644 --- a/src/static/aws/s3.service.ts +++ b/src/static/aws/s3.service.ts @@ -10,12 +10,9 @@ import { import { Readable } from 'stream'; import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; import { generateNewImageName } from '../utils'; -import { positiveNumber } from '../../utils/env'; export const URL_WINDOW_SECONDS = 3600; -const msFromEnv = (name: string, fallback: number): number => positiveNumber(process.env[name], fallback); - export class AWSS3Service implements Static { private readonly logger: Logger = new Logger(AWSS3Service.name); private readonly AWS_S3_BUCKET_NAME = process.env.AWS_S3_BUCKET_NAME; @@ -25,8 +22,8 @@ export class AWSS3Service implements Static { constructor() { this.s3Client = new S3Client({ requestHandler: { - connectionTimeout: msFromEnv('S3_CONNECTION_TIMEOUT_MS', 5_000), - requestTimeout: msFromEnv('S3_REQUEST_TIMEOUT_MS', 60_000), + connectionTimeout: 5_000, + requestTimeout: 60_000, throwOnRequestTimeout: true, }, }); diff --git a/src/test-runs/test-runs.service.spec.ts b/src/test-runs/test-runs.service.spec.ts index 8da59340..7c30bf57 100644 --- a/src/test-runs/test-runs.service.spec.ts +++ b/src/test-runs/test-runs.service.spec.ts @@ -1,5 +1,4 @@ -import { BadRequestException, ConflictException, Logger, NotFoundException } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; +import { ConflictException, Logger, NotFoundException } from '@nestjs/common'; import { mocked } from 'jest-mock'; import { Test } from '@nestjs/testing'; import { SIGNATURE_CONCURRENCY, TestRunsService } from './test-runs.service'; @@ -35,7 +34,6 @@ const createMocks = () => { baseline: { count: jest.fn().mockResolvedValue(0) }, project: { findUnique: jest.fn() }, build: { findUnique: jest.fn() }, - config: { get: jest.fn((_key: string, fallback?: string) => fallback) }, static: { getImageBuffer: jest.fn(), copyImage: jest.fn(), saveImage: jest.fn(), deleteImage: jest.fn() }, events: { testRunUpdated: jest.fn(), testRunCreated: jest.fn(), testRunDeleted: jest.fn() }, tv: { @@ -74,7 +72,6 @@ const build = async () => { build: m.build, }, }, - { provide: ConfigService, useValue: m.config }, { provide: StaticService, useValue: { @@ -941,7 +938,7 @@ describe('TestRunsService', () => { describe('a build of another project', () => { const message = `Build ${createTestRequestDto.buildId} does not belong to project ${createTestRequestDto.projectId}`; - it('is accepted with a warning by default', async () => { + it('is accepted with a warning', async () => { const warn = jest.spyOn(Logger.prototype, 'warn').mockImplementation(); const { service, diffed, createdVariation } = await initPostService({ buildRow: { projectId: 'other' } }); @@ -951,15 +948,6 @@ describe('TestRunsService', () => { expect(mocked(TestRunResultDto)).toHaveBeenCalledWith(diffed, createdVariation); warn.mockRestore(); }); - - it('is rejected when STRICT_BUILD_PROJECT_CHECK is true', async () => { - const { service, m } = await initPostService({ buildRow: { projectId: 'other' } }); - m.config.get.mockReturnValue('TRUE'); - - await expect(post(service)).rejects.toThrow(new BadRequestException(message)); - expect(m.config.get).toHaveBeenCalledWith('STRICT_BUILD_PROJECT_CHECK', 'false'); - expect(m.static.saveImage).not.toHaveBeenCalled(); - }); }); it("fails before storing anything when the project's comparator cannot run here", async () => { diff --git a/src/test-runs/test-runs.service.ts b/src/test-runs/test-runs.service.ts index 4fd904cd..41d27fe8 100644 --- a/src/test-runs/test-runs.service.ts +++ b/src/test-runs/test-runs.service.ts @@ -1,13 +1,4 @@ -import { - BadRequestException, - ConflictException, - forwardRef, - Inject, - Injectable, - Logger, - NotFoundException, -} from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; +import { ConflictException, forwardRef, Inject, Injectable, Logger, NotFoundException } from '@nestjs/common'; import { CreateTestRequestDto } from './dto/create-test-request.dto'; import { IgnoreAreaDto } from './dto/ignore-area.dto'; import { StaticService } from '../static/static.service'; @@ -41,8 +32,7 @@ export class TestRunsService { private prismaService: PrismaService, private staticService: StaticService, private compareService: CompareService, - private eventsGateway: EventsGateway, - private configService: ConfigService + private eventsGateway: EventsGateway ) {} async findMany(buildId: string): Promise { @@ -87,7 +77,7 @@ export class TestRunsService { throw new NotFoundException(`Build not found: ${createTestRequestDto.buildId}`); } if (build.projectId !== createTestRequestDto.projectId) { - this.rejectForeignBuild(createTestRequestDto); + this.warnForeignBuild(createTestRequestDto); } this.compareService.getComparator(project.imageComparison); @@ -157,12 +147,8 @@ export class TestRunsService { ); } - private rejectForeignBuild({ buildId, projectId }: CreateTestRequestDto): void { - const message = `Build ${buildId} does not belong to project ${projectId}`; - if (this.configService.get('STRICT_BUILD_PROJECT_CHECK', 'false').toLowerCase() === 'true') { - throw new BadRequestException(message); - } - this.logger.warn(message); + private warnForeignBuild({ buildId, projectId }: CreateTestRequestDto): void { + this.logger.warn(`Build ${buildId} does not belong to project ${projectId}`); } async approve(id: string, merge = false, autoApprove = false, userId?: string): Promise { diff --git a/src/users/ldap/ldapusers.service.spec.ts b/src/users/ldap/ldapusers.service.spec.ts index b1f162ff..48fb11fd 100644 --- a/src/users/ldap/ldapusers.service.spec.ts +++ b/src/users/ldap/ldapusers.service.spec.ts @@ -49,11 +49,9 @@ describe('LdapUsersService', () => { lastAttr: 'Roe', }; - const createService = (tlsNoVerify = 'false', overrides: Record = {}) => { + const createService = (tlsNoVerify = 'false') => { const configService = { - get: jest.fn((key: string, defaultValue: string) => - key === 'LDAP_TLS_NO_VERIFY' ? tlsNoVerify : (overrides[key] ?? defaultValue) - ), + get: jest.fn((key: string, defaultValue: string) => (key === 'LDAP_TLS_NO_VERIFY' ? tlsNoVerify : defaultValue)), getOrThrow: jest.fn((key: string) => config[key]), }; return new LdapUsersService( @@ -113,22 +111,6 @@ describe('LdapUsersService', () => { ); }); - it('takes the timeouts from LDAP_TIMEOUT_MS and LDAP_CONNECT_TIMEOUT_MS', async () => { - found(); - - await login(createService('false', { LDAP_TIMEOUT_MS: '2500', LDAP_CONNECT_TIMEOUT_MS: '700' })); - - expect(LdapClient).toHaveBeenCalledWith(expect.objectContaining({ timeout: 2500, connectTimeout: 700 })); - }); - - it.each(['0', '-5', 'abc', '1.5', ''])('falls back to the default timeouts for %j', async (value) => { - found(); - - await login(createService('false', { LDAP_TIMEOUT_MS: value, LDAP_CONNECT_TIMEOUT_MS: value })); - - expect(LdapClient).toHaveBeenCalledWith(expect.objectContaining({ timeout: 10000, connectTimeout: 5000 })); - }); - it('never changes a password', async () => { await expect(service.changePassword(user, 'newPassword')).rejects.toThrow( new HttpException('Password changes are disabled. Change your password in the directory.', HttpStatus.BAD_REQUEST) diff --git a/src/users/ldap/ldapusers.service.ts b/src/users/ldap/ldapusers.service.ts index e5c557bb..b44f8991 100644 --- a/src/users/ldap/ldapusers.service.ts +++ b/src/users/ldap/ldapusers.service.ts @@ -21,17 +21,10 @@ type LDAPConfig = { attributeFirstName: string; attributeLastName: string; tlsNoVerify: boolean; - timeoutMs: number; - connectTimeoutMs: number; }; const invalidCredentials = () => new HttpException('Invalid email or password.', HttpStatus.BAD_REQUEST); -const positiveInt = (value: unknown, fallback: number): number => { - const parsed = Number(value); - return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback; -}; - export class LdapUsersService implements Users { readonly allowsRegistration = false; private readonly logger: Logger = new Logger(LdapUsersService.name); @@ -52,8 +45,6 @@ export class LdapUsersService implements Users { attributeFirstName: this.configService.getOrThrow('LDAP_ATTRIBUTE_FIRST_NAME'), attributeLastName: this.configService.getOrThrow('LDAP_ATTRIBUTE_LAST_NAME'), tlsNoVerify: this.configService.get('LDAP_TLS_NO_VERIFY', 'false').toLowerCase() === 'true', - timeoutMs: positiveInt(this.configService.get('LDAP_TIMEOUT_MS'), 10_000), - connectTimeoutMs: positiveInt(this.configService.get('LDAP_CONNECT_TIMEOUT_MS'), 5_000), }; } @@ -61,8 +52,8 @@ export class LdapUsersService implements Users { private createClient(): LdapClient { return new LdapClient({ url: this.ldapConfig.url, - timeout: this.ldapConfig.timeoutMs, - connectTimeout: this.ldapConfig.connectTimeoutMs, + timeout: 10_000, + connectTimeout: 5_000, tlsOptions: { rejectUnauthorized: !this.ldapConfig.tlsNoVerify }, }); } diff --git a/test/builds-api.e2e-spec.ts b/test/builds-api.e2e-spec.ts index 7aac7b80..f3f5ff79 100644 --- a/test/builds-api.e2e-spec.ts +++ b/test/builds-api.e2e-spec.ts @@ -334,19 +334,18 @@ describe('Builds API (e2e)', () => { }); describe('POST /builds retention (maxBuildAllowed)', () => { - beforeAll(() => { - process.env.BUILD_RETENTION_GRACE_MINUTES = '0'; - }); - - afterAll(() => { - delete process.env.BUILD_RETENTION_GRACE_MINUTES; - }); + const idleForTwoHours = async () => { + const then = new Date(Date.now() - 2 * 60 * 60 * 1000); + await prisma.$executeRaw`UPDATE "Build" SET "updatedAt" = ${then} WHERE "projectId" = ${project.id}`; + await prisma.testRun.updateMany({ where: { build: { projectId: project.id } }, data: { createdAt: then } }); + }; it('keeps at most maxBuildAllowed builds and removes the files of the dropped ones', async () => { await prisma.project.update({ where: { id: project.id }, data: { maxBuildAllowed: 2 } }); const oldest = (await createBuild()).body; const run = await postRun({ ...oldest, projectId: project.id } as Build, WHITE); await createBuild(); + await idleForTwoHours(); const newest = (await createBuild()).body; @@ -362,6 +361,7 @@ describe('Builds API (e2e)', () => { await createBuild(); await createBuild(); + await idleForTwoHours(); const newest = (await createBuild()).body; expect(await listOrder()).toEqual([newest.id]); @@ -372,6 +372,7 @@ describe('Builds API (e2e)', () => { const joinShared = () => createBuild('shared'); const shared = (await joinShared()).body; const newer = (await createBuild()).body; + await idleForTwoHours(); const joined = (await joinShared()).body; From ec452243e6b119b0bf63c66b406489eac255fb0a Mon Sep 17 00:00:00 2001 From: Pavel Strunkin Date: Sun, 1 Nov 2026 21:58:28 +0200 Subject: [PATCH 8/8] ci: publish Docker images only from release tags; CI already builds and runs api and migration on pull requests --- .github/workflows/docker.yml | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 9e6d0e0c..710ba862 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -6,14 +6,6 @@ on: - 'api-v*' - 'migration-v*' - 'dinov2-v*' - pull_request: - branches: [master] - paths: - - Dockerfile - - prisma/Dockerfile - - prisma/schema.prisma - - dinov2/** - - .github/workflows/docker.yml workflow_dispatch: inputs: tag: @@ -44,13 +36,12 @@ jobs: TAG: ${{ inputs.tag || github.ref_name }} PREFIX: ${{ matrix.image }}-v run: | - if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then - echo "build=true" >> "$GITHUB_OUTPUT" - echo "push=false" >> "$GITHUB_OUTPUT" - elif [ "${TAG#"$PREFIX"}" != "$TAG" ]; then - echo "build=true" >> "$GITHUB_OUTPUT" - echo "push=true" >> "$GITHUB_OUTPUT" - echo "version=${TAG#"$PREFIX"}" >> "$GITHUB_OUTPUT" + if [ "${TAG#"$PREFIX"}" != "$TAG" ]; then + { + echo "build=true" + echo "push=true" + echo "version=${TAG#"$PREFIX"}" + } >> "$GITHUB_OUTPUT" else echo "Tag $TAG is not a ${{ matrix.image }} release, skipping" echo "build=false" >> "$GITHUB_OUTPUT" @@ -85,6 +76,6 @@ jobs: context: ${{ matrix.context }} platforms: linux/amd64,linux/arm64 push: ${{ steps.version.outputs.push == 'true' }} - tags: visualregressiontracker/${{ matrix.image }}:${{ steps.version.outputs.version || 'ci' }} + tags: visualregressiontracker/${{ matrix.image }}:${{ steps.version.outputs.version }} cache-from: type=gha,scope=${{ matrix.image }} cache-to: type=gha,mode=max,scope=${{ matrix.image }}