diff --git a/.claude/skills/pre-pr-review/SKILL.md b/.claude/skills/pre-pr-review/SKILL.md new file mode 100644 index 00000000..1b66066c --- /dev/null +++ b/.claude/skills/pre-pr-review/SKILL.md @@ -0,0 +1,41 @@ +--- +name: pre-pr-review +description: Review a branch before its pull request is opened or updated. Use whenever a PR or merge request is about to be created, pushed or marked ready, or the user asks "can we open the PR", "make a PR", "push this", even when they do not mention a review. +--- + +# Before a pull request + +No pull request is opened, and no branch is pushed for one, until these steps +pass. A step that fails is fixed on the branch, not noted for later. + +1. **Checks.** `npm run typecheck`, `npm run lint`, `npm run format:check`, + `npm run test:cov`, `npm run build`, and `npm run test:e2e` against a + database of its own (`docker run -d --name vrt-e2e-pr -e POSTGRES_PASSWORD=pg + -p 55499:5432 postgres:15.3-alpine3.18`, then with + `DATABASE_URL=postgresql://postgres:pg@localhost:55499/postgres`: + `npx prisma migrate deploy`, `npx prisma migrate diff --from-config-datasource + --to-schema prisma/schema.prisma --exit-code`, `npm run test:e2e`; remove it + with `docker rm -fv vrt-e2e-pr`). If the branch touches the Dockerfiles, + compose files, the seed or auth, also bring the stack up with + `docker compose up --build`, check `/health`, run `npm run test:acceptance` + (and `test:ldap` / `test:oidc` with their compose overlays when auth + changed), then `docker compose down --volumes`. If `dinov2/` changed, run + `python -m unittest test_patches` there. +2. **Code review.** Run the `code-review` skill on the branch against its base + (`master` unless the PR targets another branch), at `high` effort or above + for a branch of more than a few commits. Fix every confirmed finding on the + branch; for a finding left as is, say why in the PR description. +3. **Comments.** Go through the diff's comments with the rule in `AGENTS.md` + (Code conventions, "Comments"): delete any that restate the code, narrate + history, point at review notes or ids that are not in the repo, or would + not change what a reader does. Keep only non-obvious whys, one line each. + Test names follow the same rule. +4. **Plan status.** If the branch works through a plan in `docs/`, tick the + done tasks there and record follow-ups. +5. **Commits.** Author is the repo's configured git identity; no + `Co-Authored-By` or "Generated with" lines. + +Then open the PR. Its description lists what changed by area, how it was +verified (the commands above and their results), what needs attention when +deploying (migrations, new or changed environment variables, behaviour +changes), and what is deliberately left for later. diff --git a/.env b/.env index ef80ee54..45fe68b5 100644 --- a/.env +++ b/.env @@ -1,4 +1,4 @@ -JWT_SECRET='jwtPrivateKey' +JWT_SECRET='vrt-local-development-only-jwt-secret' JWT_LIFE_TIME=1d BODY_PARSER_JSON_LIMIT="5mb" @@ -30,6 +30,15 @@ AWS_S3_BUCKET_NAME= # Multi-instance socket.io fan-out (optional). Unset = in-memory adapter. #REDIS_URL=redis://localhost:6379 +# How long the first Redis connection may take before the API exits instead of waiting (default 30000). + +# Log levels, comma separated; unset logs everything. Read from the process environment, not from this file. +#LOG_LEVEL=log,warn,error,fatal + +# S3 request timeouts in milliseconds (STATIC_SERVICE=s3) + + +#REGISTRATION_ENABLED=true #LDAP_ENABLED=true #LDAP_TLS_NO_VERIFY=false @@ -67,3 +76,9 @@ OLLAMA_BASE_URL=http://localhost:11434 # DINOv2 service for the rendering-noise filter's "dinov2Check" (see dinov2/README.md) #DINOV2_URL=http://localhost:8000 #DINOV2_TIMEOUT_MS=60000 +#DATABASE_URL=postgresql://postgres:postgres@localhost:5432/vrt_db_dev # required; docker-compose.yml builds it from POSTGRES_* +#MAX_IMAGE_PIXELS=100000000 # largest image in pixels (width x height); 0 turns the limit off +#MULTIPART_FILE_SIZE_LIMIT=104857600 # largest multipart upload in bytes (100 MiB) +#DIFF_WORKERS_COUNT= # diff worker threads; unset: CPUs minus 1, between 1 and 8 +#DIFF_QUEUE_LIMIT=256 # diff jobs waiting for a worker; when full the API answers 503 +#DIFF_WORKER_MAX_HEAP_MB=512 # heap limit of each diff worker, MB diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 6aa54ce8..0b352283 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -46,10 +46,11 @@ updates: - "*" - package-ecosystem: docker - # base images of the api (Dockerfile) and the migration image (prisma/Dockerfile) + # base images of the api (Dockerfile), the migration image (prisma/Dockerfile) and dinov2/Dockerfile directories: - "/" - "/prisma" + - "/dinov2" schedule: interval: weekly groups: @@ -57,6 +58,16 @@ updates: patterns: - "*" + - package-ecosystem: pip + # the DINOv2 service image installs dinov2/requirements.txt + directory: "/dinov2" + schedule: + interval: weekly + groups: + pip: + patterns: + - "*" + - package-ecosystem: github-actions directory: "/" schedule: diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 09837f5b..710ba862 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,41 +1,23 @@ name: Docker images -# Builds visualregressiontracker/api, visualregressiontracker/migration and -# visualregressiontracker/dinov2 for linux/amd64 and linux/arm64 (Apple -# Silicon, Graviton), so docker compose runs them natively instead of under -# emulation. -# - pushing an api-vX.Y.Z, migration-vX.Y.Z or dinov2-vX.Y.Z tag publishes that image as X.Y.Z -# - "Run workflow" with an existing tag (re)publishes it, e.g. to add arm64 to a released version -# - pull requests that touch the images only build them, without pushing -# Publishing needs the DOCKERHUB_USERNAME and DOCKERHUB_TOKEN repository secrets. - on: push: tags: - 'api-v*' - 'migration-v*' - 'dinov2-v*' - pull_request: - branches: [master] - paths: - - Dockerfile - - prisma/Dockerfile - - prisma/schema.prisma - - dinov2/** - - .github/workflows/docker.yml workflow_dispatch: inputs: tag: description: Existing tag to publish, e.g. api-v5.7.0, migration-v5.5.0 or dinov2-v1.0.0 required: true -# GITHUB_TOKEN is only used to check out the code; image pushes use the Docker Hub secrets permissions: contents: read jobs: docker: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: @@ -54,24 +36,22 @@ jobs: TAG: ${{ inputs.tag || github.ref_name }} PREFIX: ${{ matrix.image }}-v run: | - if [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then - echo "build=true" >> "$GITHUB_OUTPUT" - echo "push=false" >> "$GITHUB_OUTPUT" - elif [ "${TAG#"$PREFIX"}" != "$TAG" ]; then - echo "build=true" >> "$GITHUB_OUTPUT" - echo "push=true" >> "$GITHUB_OUTPUT" - echo "version=${TAG#"$PREFIX"}" >> "$GITHUB_OUTPUT" + if [ "${TAG#"$PREFIX"}" != "$TAG" ]; then + { + echo "build=true" + echo "push=true" + echo "version=${TAG#"$PREFIX"}" + } >> "$GITHUB_OUTPUT" else echo "Tag $TAG is not a ${{ matrix.image }} release, skipping" echo "build=false" >> "$GITHUB_OUTPUT" fi - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 if: steps.version.outputs.build == 'true' with: # a manual run checks out the tag itself, never a branch with the same name ref: ${{ inputs.tag && format('refs/tags/{0}', inputs.tag) || github.ref }} - # keep the token out of .git/config, which is part of the build context persist-credentials: false - name: Set up QEMU @@ -96,6 +76,6 @@ jobs: context: ${{ matrix.context }} platforms: linux/amd64,linux/arm64 push: ${{ steps.version.outputs.push == 'true' }} - tags: visualregressiontracker/${{ matrix.image }}:${{ steps.version.outputs.version || 'ci' }} + tags: visualregressiontracker/${{ matrix.image }}:${{ steps.version.outputs.version }} cache-from: type=gha,scope=${{ matrix.image }} cache-to: type=gha,mode=max,scope=${{ matrix.image }} diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 185b7757..1373aa32 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -6,7 +6,6 @@ on: pull_request: branches: [master] -# a new push to a pull request supersedes its running checks; master runs to completion concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} @@ -16,15 +15,14 @@ permissions: jobs: checks: - # https://github.com/actions/runner-images#available-images runs-on: ubuntu-24.04 timeout-minutes: 15 steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Setup Node.js environment - uses: actions/setup-node@v7 + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: '24' cache: npm @@ -35,6 +33,10 @@ jobs: - name: Install npm dependencies run: npm ci + - name: Install prisma npm dependencies + working-directory: prisma + run: npm ci + - name: Static analysis run: npm run lint @@ -55,7 +57,6 @@ jobs: fail-fast: false matrix: include: - # acceptance tests call the api container, e2e tests run the app in-process against its database - suite: e2e compose-file: docker-compose.yml test: npm run test:acceptance && npm run test:e2e @@ -66,14 +67,14 @@ jobs: compose-file: docker-compose.yml:docker-compose.oidc.yml test: npm run test:oidc env: - # read by every docker compose command below COMPOSE_FILE: ${{ matrix.compose-file }} + COMPOSE_PROJECT_NAME: vrt steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Setup Node.js environment - uses: actions/setup-node@v7 + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: '24' cache: npm @@ -84,12 +85,32 @@ jobs: - name: Install npm dependencies run: npm ci - - name: Build and run containers - run: docker compose up --build -d + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + + - name: Build api image + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: . + load: true + tags: vrt-api:latest + cache-from: type=gha,scope=ci-api + cache-to: type=gha,mode=max,scope=ci-api + + - name: Build migration image + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: prisma + load: true + tags: vrt-migration:latest + cache-from: type=gha,scope=ci-migration + cache-to: type=gha,mode=max,scope=ci-migration + + - name: Run containers + run: docker compose up --no-build -d - name: Wait for migrations and the api run: | - # the migration container applies the migrations, seeds the default user and exits status=$(docker wait vrt_migration) if [ "$status" != 0 ]; then echo "::error::migration container exited with $status" @@ -113,12 +134,28 @@ jobs: # See: https://github.com/Visual-Regression-Tracker/backend/issues/273 runs-on: ubuntu-24.04 timeout-minutes: 10 + services: + postgres: + image: postgres:15.3-alpine3.18 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: vrt_migrations + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres -d vrt_migrations" + --health-interval 5s + --health-timeout 5s + --health-retries 20 + env: + DATABASE_URL: postgresql://postgres:postgres@localhost:5432/vrt_migrations steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Setup Node.js environment - uses: actions/setup-node@v7 + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: '24' cache: npm @@ -130,19 +167,23 @@ jobs: working-directory: prisma run: npm ci - - name: Static analysis + - name: Unit tests working-directory: prisma run: npm run test + - name: Migrations match the schema + working-directory: prisma + run: | + npx prisma migrate deploy + npx prisma migrate diff --from-config-datasource --to-schema schema.prisma --exit-code + test-dinov2: runs-on: ubuntu-24.04 timeout-minutes: 10 steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - # the logic is tested with a stand-in encoder: numpy only, no model; - # the runner's own Python 3.12, in a venv (system pip is locked down) - name: Unit tests working-directory: dinov2 run: | @@ -150,8 +191,7 @@ jobs: .venv/bin/pip install numpy==2.5.4 .venv/bin/python -m unittest test_patches - # Branch protection requires a check named "build": it passes only when every job above did, - # so the required check keeps covering the whole pipeline after the split. + # branch protection requires a check named "build" build: if: ${{ !cancelled() }} needs: [checks, integration, test-migrations, test-dinov2] diff --git a/.gitignore b/.gitignore index 519d9953..912982c9 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,7 @@ lerna-debug.log* # TypeScript incremental build info *.tsbuildinfo + +# Claude Code +.claude/settings.local.json +.claude/worktrees/ diff --git a/AGENTS.md b/AGENTS.md index 966b429e..9163b84d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -55,7 +55,15 @@ Run typecheck, lint, format check and the unit tests before calling work done. Fields older clients do not send must be `@IsOptional()`, or saving breaks for them. - Format only the files you changed (`npx prettier --write `), not whole directories: prettier also rewrites untouched files, markdown included. -- Comments explain *why*, in the surrounding style; match existing naming. +- Comments: by default, none. Write one only for a non-obvious _why_ that a + reader would otherwise get wrong, and keep it to one line. Delete on sight: + comments that restate the code or its names, narrate past bugs or earlier + versions (that goes in the commit message), explain what a well-named + function already says, point at review notes or ticket ids that are not in + the repo, or run for several lines without changing what a reader would do. + No TODOs or "known issue" notes (open an issue). Tests follow the same rule: + the test name says what is checked. Match existing naming. Before a pull + request, see the `pre-pr-review` skill. ## Services beside the API diff --git a/Dockerfile b/Dockerfile index 093de53c..044c017b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,38 +1,30 @@ FROM node:24-bookworm-slim AS builder -# Create app directory WORKDIR /app COPY prisma.config.ts ./ COPY ./prisma/schema.prisma ./prisma/ -# A wildcard is used to ensure both package.json AND package-lock.json are copied COPY package*.json ./ -# Install app dependencies -ENV NODE_TLS_REJECT_UNAUTHORIZED='0' RUN npm ci COPY tsconfig*.json nest-cli.json .swcrc ./ COPY src ./src -# SWC compiles without checking types, so the image is gated on the type -# check here, as tsc's build used to gate it. Dev dependencies are only needed -# to build; the generated Prisma client under node_modules/.prisma survives -# the prune RUN npm run typecheck && npm run build && npm prune --omit=dev FROM node:24-bookworm-slim ENV NODE_ENV=production -# tini as PID 1: the kernel ignores a signal PID 1 has no handler for, so the -# SIGTERM Nest re-raises after its shutdown hooks would never end a node PID 1. -# It also reaps the processes odiff leaves behind. +# tini as PID 1: the SIGTERM Nest re-raises would never end a node PID 1 RUN apt-get update && apt-get install -y --no-install-recommends tini && rm -rf /var/lib/apt/lists/* -# No WORKDIR: the app resolves imageUploads/ against /, where deployments -# mount the image volume +# No WORKDIR: imageUploads/ resolves against /, where the image volume is mounted +RUN mkdir -p /imageUploads && chown node:node /imageUploads COPY --from=builder /app/node_modules ./node_modules COPY --from=builder /app/package*.json ./ COPY --from=builder /app/dist ./dist +COPY docker-entrypoint.sh /usr/local/bin/vrt-entrypoint.sh +RUN chmod 755 /usr/local/bin/vrt-entrypoint.sh EXPOSE 3000 # node without npm in between, so SIGTERM reaches the app's shutdown hooks -ENTRYPOINT [ "/usr/bin/tini", "--" ] +ENTRYPOINT [ "/usr/bin/tini", "--", "/usr/local/bin/vrt-entrypoint.sh" ] CMD [ "node", "dist/main" ] diff --git a/README.md b/README.md index f0e78732..63a1fc8e 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,122 @@ - Seed initial data `npx prisma db seed` - `npm run start:debug` +## Running the test suites + +| Suite | Command | Needs | Covers | +|---|---|---|---| +| unit | `npm test` | nothing | services with mocked dependencies (`src/**/*.spec.ts`) | +| e2e | `npm run test:e2e` | Postgres with the migrations applied (`DATABASE_URL`); run `npm run build` first | the app in-process against that database | +| acceptance | `npm run test:acceptance` | `docker compose up -d`; the API on `localhost:4200` | the running API over HTTP and socket.io | +| ldap | `npm run test:ldap` | `COMPOSE_FILE=docker-compose.yml:docker-compose.ldap.yml docker compose up -d` | LDAP login against the LDAP mock | +| oidc | `npm run test:oidc` | `COMPOSE_FILE=docker-compose.yml:docker-compose.oidc.yml docker compose up -d` | the OIDC endpoints against the OIDC mock | +| prisma package | `cd prisma && npm test` | `npm ci` in `prisma/` | the `prisma/` package's own tests | + +The e2e suite loads the diff worker threads from `dist/`. Without a build they run +inline, so `npm run build` makes the e2e run match production. + +Schema drift check (AGENTS.md: every schema change ships with its migration). Run it +against a database where the migrations are applied; exit code 2 means the schema has +changes without a migration: +``` +cd prisma && npx prisma migrate deploy && npx prisma migrate diff --from-config-datasource --to-schema schema.prisma --exit-code +``` + +## Environment variables + +Defaults are what the code uses when the variable is unset or empty. The commented +lines in `.env` (section `# optional`) show the same variables. + +### Core + +| Variable | Default | What it does | +|---|---|---| +| `DATABASE_URL` | none, required | Postgres connection string. `connection_limit` sets the pool size per instance (see Database connections). | +| `APP_PORT` | `3000` | Port the API listens on. | +| `APP_FRONTEND_URL` | none | Frontend base URL, used to build links in test run results. | +| `BODY_PARSER_JSON_LIMIT` | body-parser default | Maximum JSON request body size, e.g. `5mb`. | +| `JWT_SECRET` | none, required | Key that signs login tokens; startup fails when it is unset. Public sample values (`jwtPrivateKey`, the one in `.env`) log a warning. Generate one with `openssl rand -hex 32`. | +| `JWT_LIFE_TIME` | `1d` | Token lifetime as a timespan such as `1d`, `12h` or `3600s`. An invalid value falls back to `1d` with a warning. | +| `SERVER_TIMEOUT` | Node.js default | HTTP `server.timeout`, in milliseconds. | +| `SERVER_HEADERS_TIMEOUT` | Node.js default | HTTP `headersTimeout`, in milliseconds. | +| `SERVER_KEEP_ALIVE_TIMEOUT` | Node.js default | HTTP `keepAliveTimeout`, in milliseconds. | +| `LOG_LEVEL` | all levels | Comma-separated Nest log levels to keep, e.g. `log,warn,error,fatal`. | +| `HTTPS_KEY_PATH` | `./secrets/ssl.key` | TLS private key. HTTPS is used only when both files exist; otherwise the API serves HTTP. | +| `HTTPS_CERT_PATH` | `./secrets/ssl.cert` | TLS certificate. | + +### Storage + +| Variable | Default | What it does | +|---|---|---| +| `STATIC_SERVICE` | `hdd` | Where images are stored: `hdd` (local disk) or `s3`. Odiff works only with `hdd`. | +| `AWS_S3_BUCKET_NAME` | none | Bucket for `STATIC_SERVICE=s3`; required then. | +| `AWS_REGION` | from the AWS SDK | Read by the AWS SDK v3 default chain, not by VRT code. | +| `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` | from the AWS SDK | Same chain. An IAM role or shared AWS config works without them. | + +### Image comparison + +| Variable | Default | What it does | +|---|---|---| +| `MAX_IMAGE_PIXELS` | `100000000` | Largest accepted image in pixels (width times height), checked from the PNG header before decoding. `0` turns the limit off. | +| `MULTIPART_FILE_SIZE_LIMIT` | `104857600` (100 MiB) | Largest accepted multipart file upload, in bytes. | +| `DIFF_WORKERS_COUNT` | CPUs minus 1, between 1 and 8 | Number of diff worker threads. | +| `DIFF_QUEUE_LIMIT` | `256` | Diff jobs allowed to wait for a worker. When the queue is full the API answers `503` with `Retry-After: 5`. | +| `DIFF_WORKER_MAX_HEAP_MB` | `512` | Heap limit of each diff worker, in MB. | +| `OLLAMA_BASE_URL` | none | Ollama server for VLM comparisons; required when the Ollama provider is used. | +| `DINOV2_URL` | none | DINOv2 service for the `dinov2Check` noise filter. With the filter on and this unset, runs go to review. | +| `DINOV2_TIMEOUT_MS` | `60000` | Timeout for one DINOv2 request, in milliseconds. | + +### Auth + +| Variable | Default | What it does | +|---|---|---| +| `REGISTRATION_ENABLED` | `true` | `false` (also `0`, `no`, `off`) disables self-registration. | +| `LDAP_ENABLED` | `false` | Enables LDAP login. When OIDC is enabled too, OIDC is used. | +| `LDAP_URL` | none | LDAP server URL, e.g. `ldaps://ldap.example.com:636/`. Required when LDAP is enabled. | +| `LDAP_BIND_USER` | none | DN of the service account used for the user search. Required when LDAP is enabled. | +| `LDAP_BIND_PASSWORD` | none | Password of the service account. Required when LDAP is enabled. | +| `LDAP_SEARCH_DN` | none | Base DN of the user search. Required when LDAP is enabled. | +| `LDAP_USERS_SEARCH_FILTER` | none | User search filter; `{{email}}` is replaced by the login email. Required when LDAP is enabled. | +| `LDAP_ATTRIBUTE_MAIL`, `LDAP_ATTRIBUTE_FIRST_NAME`, `LDAP_ATTRIBUTE_LAST_NAME` | none | Attributes that hold the email, first name and last name. Required when LDAP is enabled. | +| `LDAP_TLS_NO_VERIFY` | `false` | Skip TLS certificate verification for LDAP. Test servers only. | +| `OIDC_ENABLED` | `false` | Enables OIDC (SSO) login; see the OIDC section. | +| `OIDC_ISSUER_URL` | none | Issuer URL of the identity provider. Required when OIDC is enabled. | +| `OIDC_CLIENT_ID` | none | OIDC client ID. Required when OIDC is enabled. | +| `OIDC_CLIENT_SECRET` | none | OIDC client secret. Required when OIDC is enabled. | +| `OIDC_REDIRECT_URI` | `http://localhost:4200/users/oidc/callback` | Callback URL registered at the identity provider. | +| `OIDC_SCOPES` | `openid profile email` | Requested scopes. | +| `OIDC_ATTRIBUTE_MAIL` | `email` | Claim that holds the email. | +| `OIDC_ATTRIBUTE_FIRST_NAME` | `given_name` | Claim that holds the first name. | +| `OIDC_ATTRIBUTE_LAST_NAME` | `family_name` | Claim that holds the last name. | +| `OIDC_ROLE_CLAIM` | `groups` | Claim, or dotted path, that holds the roles. | +| `OIDC_ADMIN_VALUES` | `vrt-admin,admin` | Comma-separated claim values that give the admin role. | +| `OIDC_DEFAULT_ROLE` | `editor` | Role for every other user: `admin` or `editor`. | +| `OIDC_REQUIRE_VERIFIED_EMAIL` | `true` | Reject logins whose email is not verified. | +| `OIDC_TLS_NO_VERIFY` | `false` | Skip TLS verification for the identity provider. Test servers only. | + +### Service variables + +| Variable | Default | What it does | +|---|---|---| +| `REDIS_URL` | unset, in-memory adapter | Redis for socket.io fan-out between instances (see Scaling to multiple instances). | + +### Seed + +Read by `prisma db seed`. + +| Variable | Default | What it does | +|---|---|---| +| `DEFAULT_USER_EMAIL` | `visual-regression-tracker@example.com` | Email of the default user. | +| `DEFAULT_USER_PASSWORD` | generated, printed once | Password of the default user. | +| `DEFAULT_USER_API_KEY` | generated, printed once | API key of the default user. | + +## Optional services + +- **Redis** runs several API instances behind one socket.io fan-out. Set `REDIS_URL`; see [Scaling to multiple instances](#scaling-to-multiple-instances-real-time-updates). +- **DINOv2** serves the `dinov2Check` rendering-noise filter (see `dinov2/README.md`). Start it with `docker compose --profile dinov2 up -d`. The service listens on port 8000 inside the compose network only, so the API container uses `DINOV2_URL=http://dinov2:8000`. +- **Ollama** runs the VLM comparison locally. Set `OLLAMA_BASE_URL`; from the API container, which is started by compose, the host is `http://host.docker.internal:11434`. +- **Gemini** runs the VLM comparison in the cloud. Its API key is set in the project's VLM settings, not in the environment. + ## Scaling to multiple instances (real-time updates) Real-time UI updates are delivered over socket.io. Running **more than one API diff --git a/docker-compose.ldap.yml b/docker-compose.ldap.yml index 83acfd7b..61f68e4d 100644 --- a/docker-compose.ldap.yml +++ b/docker-compose.ldap.yml @@ -18,9 +18,6 @@ services: condition: service_started ldapmock: container_name: ldapmock - # See https://github.com/docker-ThoTeam/slapd-server-mock/tree/main - # Default users in LDAP: https://github.com/docker-ThoTeam/slapd-server-mock/blob/main/bootstrap/data.ldif.TEMPLATE - # e.g.: developer.one@ldapmock.local:password image: thoteam/slapd-server-mock:latest restart: always ports: diff --git a/docker-compose.oidc.yml b/docker-compose.oidc.yml index 12c81f74..0cc4cd17 100644 --- a/docker-compose.oidc.yml +++ b/docker-compose.oidc.yml @@ -2,9 +2,6 @@ services: api: environment: OIDC_ENABLED: 'true' - # Issuer must be reachable both from the api container and the browser. - # On macOS/Windows Docker Desktop host.docker.internal works out of the box, - # on Linux add `127.0.0.1 host.docker.internal` to /etc/hosts. OIDC_ISSUER_URL: 'http://host.docker.internal:8090/default' OIDC_CLIENT_ID: 'vrt-client' OIDC_CLIENT_SECRET: 'vrt-secret' @@ -17,8 +14,6 @@ services: OIDC_ADMIN_VALUES: 'vrt-admin' OIDC_DEFAULT_ROLE: 'editor' OIDC_REQUIRE_VERIFIED_EMAIL: 'true' - # Docker Desktop provides host.docker.internal; on Linux map it to the host - # so the api container reaches the mock IdP through the published port. extra_hosts: - 'host.docker.internal:host-gateway' depends_on: @@ -28,9 +23,6 @@ services: condition: service_started oidcmock: container_name: oidcmock - # See https://github.com/navikt/mock-oauth2-server - # Interactive login: http://host.docker.internal:8090/default/debugger - # Default users: 'admin' (role admin) and any other username (role editor). image: ghcr.io/navikt/mock-oauth2-server:6.1.0 hostname: host.docker.internal restart: always diff --git a/docker-compose.yml b/docker-compose.yml index 32bbf9db..7c9f7959 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,6 +30,14 @@ services: condition: service_healthy redis: condition: service_healthy + migration: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:3000/health').then((r) => process.exit(r.ok ? 0 : 1), () => process.exit(1))"] + interval: 10s + timeout: 5s + retries: 12 + start_period: 30s migration: container_name: vrt_migration build: @@ -52,7 +60,7 @@ services: POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} POSTGRES_DB: ${POSTGRES_DB} ports: - - "${POSTGRES_PORT}:5432" + - "127.0.0.1:${POSTGRES_PORT}:5432" expose: - "${POSTGRES_PORT}" volumes: @@ -62,9 +70,6 @@ services: interval: 10s timeout: 120s retries: 10 - # The DINOv2 service behind an image-comparison config's "dinov2Check". Only - # with `docker compose --profile dinov2 up`, and the api reaches it once - # DINOV2_URL=http://dinov2:8000 is set. dinov2: container_name: vrt_dinov2 profiles: ["dinov2"] @@ -85,20 +90,5 @@ services: interval: 10s timeout: 5s retries: 10 - # Available at http://localhost:5050 - # pgadmin: - # container_name: pgadmin4 - # # https://hub.docker.com/r/dpage/pgadmin4 - # # https://www.pgadmin.org/docs/pgadmin4/latest/release_notes_7_4.html - # image: dpage/pgadmin4:7.4 - # restart: always - # environment: - # PGADMIN_DEFAULT_EMAIL: admin@admin.com - # PGADMIN_DEFAULT_PASSWORD: root - # ports: - # - "5050:80" - # depends_on: - # postgres: - # condition: service_healthy volumes: postgres: \ No newline at end of file diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100755 index 00000000..d5588965 --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,23 @@ +#!/bin/sh +set -e +# installations whose image volume or TLS files belong to root must keep running as root +as_node() { + setpriv --reuid=node --regid=node --init-groups "$@" +} + +can_drop_root() { + as_node test -w /imageUploads || return 1 + for file in "${HTTPS_KEY_PATH:-./secrets/ssl.key}" "${HTTPS_CERT_PATH:-./secrets/ssl.cert}"; do + if [ -e "$file" ] && ! as_node test -r "$file"; then + return 1 + fi + done +} + +if [ "$(id -u)" = "0" ]; then + if can_drop_root; then + exec setpriv --reuid=node --regid=node --init-groups "$@" + fi + echo "vrt-entrypoint: /imageUploads or the TLS files are not accessible to uid 1000 (node); running as root" >&2 +fi +exec "$@" diff --git a/eslint.config.mjs b/eslint.config.mjs index b9fd1aee..f89d2f5d 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -1,33 +1,55 @@ -import typescriptParser from '@typescript-eslint/parser'; -import typescriptPlugin from '@typescript-eslint/eslint-plugin'; +import tsParser from '@typescript-eslint/parser'; +import tsPlugin from '@typescript-eslint/eslint-plugin'; + +const TESTS = [ + '**/*.spec.ts', + '**/*.e2e-spec.ts', + 'test/**/*.ts', + 'test_*/**/*.ts', + 'src/_data_/**/*.ts', + 'prisma/test/**/*.ts', +]; export default [ { - files: ["**/*.ts"], + ignores: [ + 'dist/**', + 'coverage/**', + 'node_modules/**', + 'dinov2/**', + 'prisma/node_modules/**', + 'prisma/.generate/**', + ], + }, + ...tsPlugin.configs['flat/recommended'].map((config) => ({ ...config, files: ['**/*.ts'] })), + { + files: ['**/*.ts'], languageOptions: { - parser: typescriptParser, + parser: tsParser, parserOptions: { project: ['tsconfig.json', 'prisma/tsconfig.json'], + tsconfigRootDir: import.meta.dirname, sourceType: 'module', }, - globals: { - node: true, - jest: true, - }, - }, - plugins: { - '@typescript-eslint': typescriptPlugin, }, rules: { - '@typescript-eslint/interface-name-prefix': 'off', - '@typescript-eslint/explicit-function-return-type': 'off', '@typescript-eslint/no-explicit-any': 'off', + '@typescript-eslint/no-unused-vars': [ + 'error', + { argsIgnorePattern: '^_', varsIgnorePattern: '^_', caughtErrors: 'none' }, + ], + '@typescript-eslint/no-floating-promises': 'error', + '@typescript-eslint/no-misused-promises': 'error', + '@typescript-eslint/await-thenable': 'error', + '@typescript-eslint/only-throw-error': 'error', + '@typescript-eslint/no-unnecessary-type-assertion': 'error', }, }, { - files: ["**/*.spec.ts"], + files: TESTS, rules: { - '@typescript-eslint/explicit-function-return-type': 'off', + '@typescript-eslint/no-require-imports': 'off', + '@typescript-eslint/no-misused-promises': ['error', { checksVoidReturn: false }], }, }, ]; diff --git a/jest.config.ts b/jest.config.ts deleted file mode 100644 index 53903ecd..00000000 --- a/jest.config.ts +++ /dev/null @@ -1,12 +0,0 @@ -/** @returns {Promise} */ -module.exports = async () => { - return { - projects: [ - './src/jest.config.ts', - './test/jest.config.ts', - './test_acceptance/jest.config.ts' - ], - roots: ['./'], - testTimeout: 30000, - }; -}; diff --git a/package-lock.json b/package-lock.json index 049e75d9..de23fc8d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28,25 +28,24 @@ "@prisma/client": "^7.10.1", "@socket.io/redis-adapter": "^8.3.0", "bcryptjs": "^2.4.3", - "body-parser": "^2.2.0", "class-transformer": "^0.5.1", "class-validator": "^0.15.1", "compression": "^1.8.2", + "express": "^5.3.0", "ldapts": "^9.2.0", - "lodash": "^4.17.21", "looks-same": "^10.0.1", "odiff-bin": "^4.5.0", "ollama": "^0.6.4", "openid-client": "^5.7.1", "passport": "^0.7.0", "passport-jwt": "^4.0.1", - "passport-local": "^1.0.0", "pg": "^8.16.3", "pixelmatch": "^5.3.0", "pngjs": "^7.0.0", "redis": "^4.7.0", "reflect-metadata": "^0.2.2", "rxjs": "^7.8.2", + "socket.io": "^4.8.4", "zod": "^4.6.7", "zod-to-json-schema": "^3.25.0" }, @@ -61,11 +60,9 @@ "@types/compression": "^1.8.1", "@types/express": "^5.0.6", "@types/jest": "^30.0.0", - "@types/lodash": "^4.17.26", "@types/multer": "^2.0.0", "@types/node": "^24.19.2", "@types/passport-jwt": "^3.0.9", - "@types/passport-local": "^1.0.38", "@types/pixelmatch": "^5.2.6", "@types/pngjs": "^6.0.5", "@types/supertest": "^7.2.1", @@ -74,16 +71,12 @@ "axios": "^1.20.0", "dotenv": "^17.2.3", "eslint": "^9.39.1", - "eslint-config-prettier": "^10.1.8", "jest": "^30.5.2", - "jest-mock-extended": "^4.0.2", "prettier": "^3.9.10", "prisma": "^7.10.1", "socket.io-client": "^4.8.4", "supertest": "^7.3.1", - "ts-jest": "^29.4.15", "ts-node": "^10.9.2", - "tsconfig-paths": "^4.2.0", "typescript": "~6.0.3" }, "engines": { @@ -5358,18 +5351,6 @@ "@types/passport-strategy": "*" } }, - "node_modules/@types/passport-local": { - "version": "1.0.38", - "resolved": "https://registry.npmjs.org/@types/passport-local/-/passport-local-1.0.38.tgz", - "integrity": "sha512-nsrW4A963lYE7lNTv9cr5WmiUD1ibYJvWrpE13oxApFsRt77b0RdtZvKbCdNIY4v/QZ6TRQWaDDEwV1kCTmcXg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/express": "*", - "@types/passport": "*", - "@types/passport-strategy": "*" - } - }, "node_modules/@types/passport-strategy": { "version": "0.2.38", "resolved": "https://registry.npmjs.org/@types/passport-strategy/-/passport-strategy-0.2.38.tgz", @@ -7507,19 +7488,6 @@ "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" } }, - "node_modules/bs-logger": { - "version": "0.2.6", - "resolved": "https://registry.npmjs.org/bs-logger/-/bs-logger-0.2.6.tgz", - "integrity": "sha512-pd8DCoxmbgc7hyPKOvxtqNcjYoOsABPQdcCUjGp3d42VR2CX1ORhk2A87oqqu5R1kk+76nsxZupkmyd+MVtCog==", - "dev": true, - "license": "MIT", - "dependencies": { - "fast-json-stable-stringify": "2.x" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/bser": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/bser/-/bser-2.1.1.tgz", @@ -8980,22 +8948,6 @@ } } }, - "node_modules/eslint-config-prettier": { - "version": "10.1.8", - "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-10.1.8.tgz", - "integrity": "sha512-82GZUjRS0p/jganf6q1rEO25VSoHH0hKPCTrgillPjdI/3bgBhAE1QzHrHTizjpRvy6pGAvKjDJtk2pF9NDq8w==", - "dev": true, - "license": "MIT", - "bin": { - "eslint-config-prettier": "bin/cli.js" - }, - "funding": { - "url": "https://opencollective.com/eslint-config-prettier" - }, - "peerDependencies": { - "eslint": ">=7.0.0" - } - }, "node_modules/eslint-scope": { "version": "8.4.0", "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.4.0.tgz", @@ -10213,38 +10165,6 @@ "devOptional": true, "license": "MIT" }, - "node_modules/handlebars": { - "version": "4.7.10", - "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.10.tgz", - "integrity": "sha512-P5VJMVM7qgBn6vjXMw8WG9uVI+ncf2pi72j4de4yz5ZULLj2RGqLYaKOYGsgyrViQ0tePOVlN1tDCCXXtFqXKg==", - "dev": true, - "license": "MIT", - "dependencies": { - "minimist": "^1.2.8", - "neo-async": "^2.6.2", - "source-map": "^0.6.1", - "wordwrap": "^1.0.0" - }, - "bin": { - "handlebars": "bin/handlebars" - }, - "engines": { - "node": ">=0.4.7" - }, - "optionalDependencies": { - "uglify-js": "^3.1.4" - } - }, - "node_modules/handlebars/node_modules/source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", @@ -11058,22 +10978,6 @@ "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, - "node_modules/jest-mock-extended": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/jest-mock-extended/-/jest-mock-extended-4.0.2.tgz", - "integrity": "sha512-7uSea/mNCcTozEaIHWGATiPXcUnHwx5pXuP87KrHgMKmZXXi9Qf3CywBh3ALXdvRrehd3ahP/BTl1M1FUSJkyg==", - "dev": true, - "license": "MIT", - "dependencies": { - "fast-deep-equal": "^3.1.3", - "ts-essentials": "^10.1.1" - }, - "peerDependencies": { - "@jest/globals": "^28.0.0 || ^29.0.0 || ^30.0.0", - "jest": "^24.0.0 || ^25.0.0 || ^26.0.0 || ^27.0.0 || ^28.0.0 || ^29.0.0 || ^30.0.0", - "typescript": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0" - } - }, "node_modules/jest-regex-util": { "version": "30.5.0", "resolved": "https://registry.npmjs.org/jest-regex-util/-/jest-regex-util-30.5.0.tgz", @@ -11609,6 +11513,7 @@ "version": "4.18.1", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", + "devOptional": true, "license": "MIT" }, "node_modules/lodash.includes": { @@ -11647,13 +11552,6 @@ "integrity": "sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==", "license": "MIT" }, - "node_modules/lodash.memoize": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", - "integrity": "sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag==", - "dev": true, - "license": "MIT" - }, "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -12131,13 +12029,6 @@ "url": "https://opencollective.com/express" } }, - "node_modules/neo-async": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", - "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", - "dev": true, - "license": "MIT" - }, "node_modules/nested-error-stacks": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/nested-error-stacks/-/nested-error-stacks-2.1.1.tgz", @@ -12709,17 +12600,6 @@ "passport-strategy": "^1.0.0" } }, - "node_modules/passport-local": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/passport-local/-/passport-local-1.0.0.tgz", - "integrity": "sha512-9wCE6qKznvf9mQYYbgJ3sVOHmCWoUNMVFoZzNoznmISbhnNNPhN9xfY3sLmScHMetEJeoY7CXwfhCe7argfQow==", - "dependencies": { - "passport-strategy": "1.x.x" - }, - "engines": { - "node": ">= 0.4.0" - } - }, "node_modules/passport-strategy": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/passport-strategy/-/passport-strategy-1.0.0.tgz", @@ -14604,87 +14484,6 @@ "url": "https://github.com/sponsors/Borewit" } }, - "node_modules/ts-essentials": { - "version": "10.2.1", - "resolved": "https://registry.npmjs.org/ts-essentials/-/ts-essentials-10.2.1.tgz", - "integrity": "sha512-+Id1fRkuir+CsgK2x04/icS2b4V1hQmq7ObzIrDjhN0ozfRYivnP7aaKMVJfLApQm0trjR39A6NIMVchiB9Erw==", - "dev": true, - "license": "MIT", - "peerDependencies": { - "typescript": ">=4.5.0" - }, - "peerDependenciesMeta": { - "typescript": { - "optional": true - } - } - }, - "node_modules/ts-jest": { - "version": "29.4.15", - "resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-29.4.15.tgz", - "integrity": "sha512-s2qmEUYFRqbjz1wv0EmccNPVOb45HOFpFuhzQJLKShYVisuTbCFPIIAIV2EuK5rIxb4BejLDbH7dnBZfYjybUQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "bs-logger": "^0.2.6", - "fast-json-stable-stringify": "^2.1.0", - "handlebars": "^4.7.10", - "json5": "^2.2.3", - "lodash.memoize": "^4.1.2", - "make-error": "^1.3.6", - "semver": "^7.8.5", - "type-fest": "^4.41.0", - "yargs-parser": "^21.1.1" - }, - "bin": { - "ts-jest": "cli.js" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || ^18.0.0 || >=20.0.0" - }, - "peerDependencies": { - "@babel/core": ">=7.0.0-beta.0 <9", - "@jest/transform": "^29.0.0 || ^30.0.0", - "@jest/types": "^29.0.0 || ^30.0.0", - "babel-jest": "^29.0.0 || ^30.0.0", - "jest": "^29.0.0 || ^30.0.0", - "jest-util": "^29.0.0 || ^30.0.0", - "typescript": ">=4.3 <7" - }, - "peerDependenciesMeta": { - "@babel/core": { - "optional": true - }, - "@jest/transform": { - "optional": true - }, - "@jest/types": { - "optional": true - }, - "babel-jest": { - "optional": true - }, - "esbuild": { - "optional": true - }, - "jest-util": { - "optional": true - } - } - }, - "node_modules/ts-jest/node_modules/type-fest": { - "version": "4.41.0", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.41.0.tgz", - "integrity": "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==", - "dev": true, - "license": "(MIT OR CC0-1.0)", - "engines": { - "node": ">=16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/ts-node": { "version": "10.9.2", "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", @@ -14828,20 +14627,6 @@ "node": ">=14.17" } }, - "node_modules/uglify-js": { - "version": "3.19.3", - "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.19.3.tgz", - "integrity": "sha512-v3Xu+yuwBXisp6QYTcH4UbH+xYJXqnq2m/LtQVWKWzYc1iehYnLixoQDN9FH6/j9/oybfd6W9Ghwkl8+UMKTKQ==", - "dev": true, - "license": "BSD-2-Clause", - "optional": true, - "bin": { - "uglifyjs": "bin/uglifyjs" - }, - "engines": { - "node": ">=0.8.0" - } - }, "node_modules/uid": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/uid/-/uid-2.0.2.tgz", @@ -15102,13 +14887,6 @@ "node": ">=0.10.0" } }, - "node_modules/wordwrap": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", - "integrity": "sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==", - "dev": true, - "license": "MIT" - }, "node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", diff --git a/package.json b/package.json index 4b3dcc20..5af92327 100644 --- a/package.json +++ b/package.json @@ -8,17 +8,17 @@ "scripts": { "postinstall": "prisma generate", "build": "nest build", - "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\" \"prisma/**/*.ts\"", - "format:check": "prettier --check \"src/**/*.ts\" \"test/**/*.ts\" \"prisma/**/*.ts\"", + "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\" \"test_*/**/*.ts\" \"prisma/**/*.ts\" \"*.ts\"", + "format:check": "prettier --check \"src/**/*.ts\" \"test/**/*.ts\" \"test_*/**/*.ts\" \"prisma/**/*.ts\" \"*.ts\"", "start": "nest start", "start:dev": "nest start --watch", "start:debug": "nest start --debug --watch", "start:prod": "node dist/main", - "lint": "eslint \"{src,apps,libs,test,prisma}/**/*.ts\"", + "lint": "eslint \"{src,test,test_acceptance,test_ldap,test_oidc,prisma}/**/*.ts\" \"*.ts\"", "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js --projects src", "test:watch": "node --experimental-vm-modules node_modules/jest/bin/jest.js --projects src --watch", "test:cov": "node --experimental-vm-modules node_modules/jest/bin/jest.js --projects src --coverage", - "test:debug": "node --experimental-vm-modules --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --projects src --runInBand", + "test:debug": "node --experimental-vm-modules --inspect-brk node_modules/jest/bin/jest.js --projects src --runInBand", "test:e2e": "node --experimental-vm-modules node_modules/jest/bin/jest.js --projects test", "test:acceptance": "node --experimental-vm-modules node_modules/jest/bin/jest.js --projects test_acceptance", "test:ldap": "node --experimental-vm-modules node_modules/jest/bin/jest.js --projects test_ldap", @@ -47,25 +47,24 @@ "@prisma/client": "^7.10.1", "@socket.io/redis-adapter": "^8.3.0", "bcryptjs": "^2.4.3", - "body-parser": "^2.2.0", "class-transformer": "^0.5.1", "class-validator": "^0.15.1", "compression": "^1.8.2", + "express": "^5.3.0", "ldapts": "^9.2.0", - "lodash": "^4.17.21", "looks-same": "^10.0.1", "odiff-bin": "^4.5.0", "ollama": "^0.6.4", "openid-client": "^5.7.1", "passport": "^0.7.0", "passport-jwt": "^4.0.1", - "passport-local": "^1.0.0", "pg": "^8.16.3", "pixelmatch": "^5.3.0", "pngjs": "^7.0.0", "redis": "^4.7.0", "reflect-metadata": "^0.2.2", "rxjs": "^7.8.2", + "socket.io": "^4.8.4", "zod": "^4.6.7", "zod-to-json-schema": "^3.25.0" }, @@ -80,11 +79,9 @@ "@types/compression": "^1.8.1", "@types/express": "^5.0.6", "@types/jest": "^30.0.0", - "@types/lodash": "^4.17.26", "@types/multer": "^2.0.0", "@types/node": "^24.19.2", "@types/passport-jwt": "^3.0.9", - "@types/passport-local": "^1.0.38", "@types/pixelmatch": "^5.2.6", "@types/pngjs": "^6.0.5", "@types/supertest": "^7.2.1", @@ -93,16 +90,22 @@ "axios": "^1.20.0", "dotenv": "^17.2.3", "eslint": "^9.39.1", - "eslint-config-prettier": "^10.1.8", "jest": "^30.5.2", - "jest-mock-extended": "^4.0.2", "prettier": "^3.9.10", "prisma": "^7.10.1", "socket.io-client": "^4.8.4", "supertest": "^7.3.1", - "ts-jest": "^29.4.15", "ts-node": "^10.9.2", - "tsconfig-paths": "^4.2.0", "typescript": "~6.0.3" + }, + "allowScripts": { + "prisma": true, + "@prisma/engines": true, + "@swc/core": true, + "@scarf/scarf": false, + "protobufjs": false, + "unrs-resolver": false, + "@google/genai": false, + "@parcel/watcher": false } } diff --git a/prisma/Dockerfile b/prisma/Dockerfile index a88ac27a..0ea1f3c5 100644 --- a/prisma/Dockerfile +++ b/prisma/Dockerfile @@ -4,7 +4,6 @@ WORKDIR /app COPY . . -ENV NODE_TLS_REJECT_UNAUTHORIZED='0' RUN npm ci RUN chmod +x /app/wait-for-it.sh diff --git a/prisma/client.ts b/prisma/client.ts index 6122d966..8a91a8c0 100644 --- a/prisma/client.ts +++ b/prisma/client.ts @@ -8,8 +8,8 @@ const prisma = new PrismaClient({ // the adapter builds its connection from this object when Prisma first // connects, so prepareConnection can still settle TLS on it adapter: new PrismaPg(config, schema ? { schema } : undefined), - // 'info' | 'query' | 'warn' | 'error' - log: ['query'], + // 'query' would print every statement, including the ones that carry password hashes and API keys + log: ['warn', 'error'], }); /** Settles sslmode=prefer; call before the first query. */ diff --git a/prisma/migrations/20261024174953_add_test_run_test_variation_index/migration.sql b/prisma/migrations/20261024174953_add_test_run_test_variation_index/migration.sql new file mode 100644 index 00000000..6311aa02 --- /dev/null +++ b/prisma/migrations/20261024174953_add_test_run_test_variation_index/migration.sql @@ -0,0 +1,9 @@ +-- CreateIndex +-- Single statement on purpose: Prisma runs one-statement migrations outside a +-- transaction, which CREATE INDEX CONCURRENTLY requires. Concurrent build +-- avoids write-locking TestRun on large production tables. +-- +-- Deliberately without IF NOT EXISTS: a concurrent build that fails leaves an +-- invalid index behind, and skipping it on a retry would let the migration +-- succeed while the index stays unusable. Drop the invalid index, then retry. +CREATE INDEX CONCURRENTLY "TestRun_testVariationId_idx" ON "TestRun"("testVariationId"); diff --git a/prisma/migrations/20261024175021_add_baseline_test_variation_index/migration.sql b/prisma/migrations/20261024175021_add_baseline_test_variation_index/migration.sql new file mode 100644 index 00000000..f3f49699 --- /dev/null +++ b/prisma/migrations/20261024175021_add_baseline_test_variation_index/migration.sql @@ -0,0 +1,9 @@ +-- CreateIndex +-- Single statement on purpose: Prisma runs one-statement migrations outside a +-- transaction, which CREATE INDEX CONCURRENTLY requires. Concurrent build +-- avoids write-locking Baseline on large production tables. +-- +-- Deliberately without IF NOT EXISTS: a concurrent build that fails leaves an +-- invalid index behind, and skipping it on a retry would let the migration +-- succeed while the index stays unusable. Drop the invalid index, then retry. +CREATE INDEX CONCURRENTLY "Baseline_testVariationId_createdAt_idx" ON "Baseline"("testVariationId", "createdAt"); diff --git a/prisma/migrations/20261024175038_add_test_run_name_baseline_name_index/migration.sql b/prisma/migrations/20261024175038_add_test_run_name_baseline_name_index/migration.sql new file mode 100644 index 00000000..435613ea --- /dev/null +++ b/prisma/migrations/20261024175038_add_test_run_name_baseline_name_index/migration.sql @@ -0,0 +1,9 @@ +-- CreateIndex +-- Single statement on purpose: Prisma runs one-statement migrations outside a +-- transaction, which CREATE INDEX CONCURRENTLY requires. Concurrent build +-- avoids write-locking TestRun on large production tables. +-- +-- Deliberately without IF NOT EXISTS: a concurrent build that fails leaves an +-- invalid index behind, and skipping it on a retry would let the migration +-- succeed while the index stays unusable. Drop the invalid index, then retry. +CREATE INDEX CONCURRENTLY "TestRun_name_baselineName_idx" ON "TestRun"("name", "baselineName"); diff --git a/prisma/migrations/20261101165840_add_test_run_status_before_decision/migration.sql b/prisma/migrations/20261101165840_add_test_run_status_before_decision/migration.sql new file mode 100644 index 00000000..a36feb92 --- /dev/null +++ b/prisma/migrations/20261101165840_add_test_run_status_before_decision/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "TestRun" ADD COLUMN "statusBeforeDecision" "TestStatus"; diff --git a/prisma/package-lock.json b/prisma/package-lock.json index 7daacb5e..c6ca7ac4 100644 --- a/prisma/package-lock.json +++ b/prisma/package-lock.json @@ -7,6 +7,7 @@ "": { "name": "vrt-migration", "version": "1.2.0", + "hasInstallScript": true, "license": "Apache-2.0", "dependencies": { "@prisma/adapter-pg": "^7.10.1", @@ -23,7 +24,7 @@ "prisma": "^7.10.1", "ts-jest": "^29.4.15", "ts-node": "^10.9.2", - "typescript": "^5.9.3" + "typescript": "~6.0.3" }, "engines": { "node": ">=24.15" @@ -6468,9 +6469,9 @@ } }, "node_modules/typescript": { - "version": "5.9.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", - "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", + "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", "devOptional": true, "license": "Apache-2.0", "bin": { @@ -11045,9 +11046,9 @@ "dev": true }, "typescript": { - "version": "5.9.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", - "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", + "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", "devOptional": true }, "uglify-js": { diff --git a/prisma/package.json b/prisma/package.json index f7797bd9..24af60bf 100644 --- a/prisma/package.json +++ b/prisma/package.json @@ -19,15 +19,19 @@ "devDependencies": { "@types/bcryptjs": "^2.4.2", "@types/jest": "^30.0.0", + "dotenv": "^17.2.3", "jest": "^30.5.2", "jest-mock-extended": "^4.0.2", "prisma": "^7.10.1", "ts-jest": "^29.4.15", "ts-node": "^10.9.2", - "typescript": "^5.9.3", - "dotenv": "^17.2.3" + "typescript": "~6.0.3" }, "engines": { "node": ">=24.15" + }, + "allowScripts": { + "prisma": true, + "@prisma/engines": true } } diff --git a/prisma/pg-options.ts b/prisma/pg-options.ts index 1bb78c61..b8356ce1 100644 --- a/prisma/pg-options.ts +++ b/prisma/pg-options.ts @@ -2,12 +2,7 @@ import { readFileSync } from 'fs'; import { isAbsolute, resolve } from 'path'; import { Client, ClientConfig } from 'pg'; -/** - * The TLS and schema handling of the backend's getPgOptions - * (src/prisma/prisma.service.ts), so the seed reaches the database the way the - * API does and Prisma 6 did. This package is built on its own and cannot - * import from src/. - */ +/** Mirrors getPgOptions (src/prisma/prisma.service.ts); this package cannot import from src/. */ const PRISMA_SSL_PARAMS = ['sslmode', 'sslaccept', 'sslcert']; const PG_ONLY_SSL_PARAMS = ['sslrootcert', 'sslkey', 'uselibpqcompat']; @@ -26,12 +21,8 @@ export function getSeedPgOptions(databaseUrl: string | undefined): { const schema = params.get('schema') ?? undefined; const sslmode = params.get('sslmode') ?? 'prefer'; if (!['prefer', 'require', 'disable'].includes(sslmode) || PG_ONLY_SSL_PARAMS.some((name) => params.has(name))) { - // written for pg: its TLS settings are pg's to read return { config: { connectionString: databaseUrl }, schema, preferSsl: false }; } - // Prisma 6: no certificate check unless sslaccept=strict, which checks - // against sslcert (relative to the prisma/ folder, this file's, wherever the - // seed is started from) const sslcert = params.get('sslcert'); const ssl = params.get('sslaccept') === 'strict' @@ -45,7 +36,6 @@ export function getSeedPgOptions(databaseUrl: string | undefined): { }; } -/** sslmode=prefer: plaintext from now on if the server has no TLS at all. */ export async function negotiatePreferredSsl(config: ClientConfig): Promise { const client = new Client(config); try { diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 8190ef04..40901248 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -1,12 +1,9 @@ generator client { provider = "prisma-client-js" - // Pinned beside the schema rather than "the nearest @prisma/client": with - // prisma/node_modules installed that would be the migration package's copy. - // Every @prisma/client below this directory finds it by walking up. + // Pinned beside the schema: the nearest @prisma/client would be the migration package's copy output = "../node_modules/.prisma/client" } -// the connection URL lives in prisma.config.ts datasource db { provider = "postgresql" } @@ -44,7 +41,7 @@ model Project { bulkApproveVariations Boolean @default(false) bulkApproveGroupBy String @default("customTags") builds Build[] - TestRun TestRun[] + testRuns TestRun[] testVariations TestVariation[] favoritedBy User[] @relation("UserFavoriteProjects") } @@ -57,6 +54,7 @@ model TestRun { diffTollerancePercent Float @default(0) pixelMisMatchCount Int? status TestStatus + statusBeforeDecision TestStatus? buildId String testVariationId String? projectId String? @@ -76,18 +74,7 @@ model TestRun { ignoreAreas String @default("[]") tempIgnoreAreas String @default("[]") vlmDescription String? - // Position-independent colour signature of this run's change, as JSON, so the - // variations dialog can group a screen's locales without fetching and - // decoding every sibling's screenshots at review time. Written once, next to - // the diff that already decoded them. Null on runs that predate this, that - // have no diff, or that were compared by something other than pixelmatch — - // those fall back to computing it on demand. changeSignature String? - // Small copies of the checkpoint and of the diff, made at ingest from the - // pixels the comparison had already decoded. The card grids draw pictures a - // hundred-odd pixels wide, and pulling the full-size files for that cost - // megabytes per screen. Null on runs ingested before this, which fall back to - // the full-size file. imageThumbnailName String? diffThumbnailName String? baseline Baseline? @@ -96,6 +83,8 @@ model TestRun { testVariation TestVariation? @relation(fields: [testVariationId], references: [id]) @@index([buildId, branchName, name]) + @@index([testVariationId]) + @@index([name, baselineName]) } model TestVariation { @@ -131,6 +120,8 @@ model Baseline { testRun TestRun? @relation(fields: [testRunId], references: [id]) testVariation TestVariation @relation(fields: [testVariationId], references: [id]) user User? @relation(fields: [userId], references: [id]) + + @@index([testVariationId, createdAt]) } model User { diff --git a/prisma/seed.ts b/prisma/seed.ts index 3c004195..86f26073 100644 --- a/prisma/seed.ts +++ b/prisma/seed.ts @@ -1,12 +1,14 @@ -/** - * @see https://www.prisma.io/docs/guides/migrate/seed-database - */ -import type { Prisma } from '@prisma/client'; +import { randomBytes } from 'crypto'; import { Role } from '@prisma/client'; import { genSalt, hash } from 'bcryptjs'; import prisma, { prepareConnection } from './client'; +const DEFAULT_EMAIL = 'visual-regression-tracker@example.com'; +const PUBLIC_DEFAULT_PASSWORD = '123456'; +const PUBLIC_DEFAULT_API_KEY = 'DEFAULTUSERAPIKEYTOBECHANGED'; +const CROCKFORD_DIGITS = '0123456789ABCDEFGHJKMNPQRSTVWXYZ'; + export async function seed() { await prepareConnection(); await prisma.$connect(); @@ -15,81 +17,76 @@ export async function seed() { await prisma.$disconnect(); } -seed() - .catch((e) => console.error('e', e)) - .finally(async () => await prisma.$disconnect()); +if (require.main === module) { + seed() + .catch((error) => { + console.error('Seeding failed:', error); + process.exitCode = 1; + }) + .finally(() => prisma.$disconnect()); +} -async function createDefaultUser() { - let userList: Prisma.UserGetPayload[] = []; - try { - userList = await prisma.user.findMany(); - console.log(userList); - } catch (error) { - // Expected to see that "user" table does not exist - console.log(error.message); +// same format as src/utils/random-key.ts; this package cannot import from src/ +export function generateApiKey(): string { + let key = ''; + for (const byte of randomBytes(28)) { + key += CROCKFORD_DIGITS[byte & 31]; } + return key; +} - const defaultEmail = process.env.DEFAULT_USER_EMAIL || 'visual-regression-tracker@example.com'; - const defaultPassword = process.env.DEFAULT_USER_PASSWORD || '123456'; - const salt = await genSalt(10); - - // Only create default user if the db has no admin user - if (!userList.some(({ role, isActive }) => role === Role.admin && isActive)) { - await prisma.user - .upsert({ - where: { - email: defaultEmail, - }, - update: { - role: Role.admin, - }, - create: { - email: defaultEmail, - firstName: 'fname', - lastName: 'lname', - role: Role.admin, - apiKey: process.env.DEFAULT_USER_API_KEY || 'DEFAULTUSERAPIKEYTOBECHANGED', - password: await hash(defaultPassword, salt), - }, - }) - .then((user) => { - console.log('###########################'); - console.log('####### DEFAULT USER ######'); - console.log('###########################'); - console.log(''); - console.log( - `The user with the email "${defaultEmail}" and password "${defaultPassword}" was created (if not changed before)` - ); - console.log(`The Api key is: ${user.apiKey}`); - }); +export async function createDefaultUser(): Promise { + const activeAdmin = await prisma.user.findFirst({ + where: { role: Role.admin, isActive: true }, + select: { id: true }, + }); + if (activeAdmin) { + return; } -} + const email = process.env.DEFAULT_USER_EMAIL?.trim() || DEFAULT_EMAIL; + const existing = await prisma.user.findUnique({ where: { email }, select: { role: true, isActive: true } }); + if (existing) { + // promoting it would hand admin to whoever registered this email + console.warn( + `No active admin exists, and ${email} is already a ${existing.isActive ? '' : 'deactivated '}${existing.role}; ` + + 'it is not promoted. Give a user the admin role in the database.' + ); + return; + } + // an empty variable (compose passes unset ones as '') counts as not set + const passwordFromEnv = process.env.DEFAULT_USER_PASSWORD || undefined; + const apiKeyFromEnv = process.env.DEFAULT_USER_API_KEY?.trim() || undefined; + const password = passwordFromEnv ?? randomBytes(18).toString('base64url'); + const apiKey = apiKeyFromEnv ?? generateApiKey(); -async function createDefaultProject() { - let projectList: Prisma.ProjectGetPayload[] = []; - try { - projectList = await prisma.project.findMany(); - console.log(projectList); - } catch (error) { - // Expected to see that "project" table does not exist - console.log(error.message); + await prisma.user.create({ + data: { + email, + firstName: 'fname', + lastName: 'lname', + role: Role.admin, + apiKey, + password: await hash(password, await genSalt(10)), + }, + }); + + console.log('###########################'); + console.log('####### DEFAULT USER ######'); + console.log('###########################'); + console.log(`Created admin ${email}`); + console.log( + passwordFromEnv ? 'Password: from DEFAULT_USER_PASSWORD' : `Password (generated, shown once): ${password}` + ); + console.log(apiKeyFromEnv ? 'API key: from DEFAULT_USER_API_KEY' : `API key (generated, shown once): ${apiKey}`); + if (passwordFromEnv === PUBLIC_DEFAULT_PASSWORD || apiKeyFromEnv === PUBLIC_DEFAULT_API_KEY) { + console.warn('The password or API key is the publicly known default: change both after the first login.'); } +} - if (projectList.length === 0) { - await prisma.project - .create({ - data: { - name: 'Default project', - }, - }) - .then((project) => { - console.log('##############################'); - console.log('## CREATING DEFAULT PROJECT ##'); - console.log('##############################'); - console.log(''); - console.log(`Project key: ${project.id}`); - console.log(`Project name ${project.name}`); - console.log(`Project name ${project.mainBranchName}`); - }); +export async function createDefaultProject(): Promise { + if ((await prisma.project.count()) > 0) { + return; } + const project = await prisma.project.create({ data: { name: 'Default project' } }); + console.log(`Created the default project "${project.name}" (${project.id}), main branch ${project.mainBranchName}`); } diff --git a/prisma/test/seed.spec.ts b/prisma/test/seed.spec.ts index e7031eb4..fb637d5c 100644 --- a/prisma/test/seed.spec.ts +++ b/prisma/test/seed.spec.ts @@ -1,7 +1,6 @@ -/** - * @see https://www.prisma.io/docs/orm/prisma-client/testing/unit-testing - */ -import { seed } from '../seed'; +import { compareSync } from 'bcryptjs'; + +import { createDefaultProject, createDefaultUser, generateApiKey, seed } from '../seed'; import { prismaMock } from './client.mock'; const mockDefaultProject = { @@ -33,74 +32,194 @@ const mockDefaultUser = { createdAt: new Date(), } as const; -describe('Defaults get created', () => { +const API_KEY_FORMAT = /^[0-9A-HJKMNP-TV-Z]{28}$/; + +const ENV_KEYS = ['DEFAULT_USER_EMAIL', 'DEFAULT_USER_PASSWORD', 'DEFAULT_USER_API_KEY'] as const; + +describe('seed', () => { + const savedEnv: Record = {}; + let log: jest.SpyInstance; + let warn: jest.SpyInstance; + beforeEach(() => { - prismaMock.user.upsert.mockResolvedValue(mockDefaultUser); + for (const key of ENV_KEYS) { + savedEnv[key] = process.env[key]; + delete process.env[key]; + } + log = jest.spyOn(console, 'log').mockImplementation(); + warn = jest.spyOn(console, 'warn').mockImplementation(); + prismaMock.user.create.mockResolvedValue(mockDefaultUser); prismaMock.project.create.mockResolvedValue(mockDefaultProject); }); - test('Default user and project are created for empty database', async () => { - prismaMock.user.findMany.mockResolvedValue([]); - prismaMock.project.findMany.mockResolvedValue([]); - await seed(); - - expect(prismaMock.user.upsert).toHaveBeenCalled(); - expect(prismaMock.project.create).toHaveBeenCalled(); + afterEach(() => { + for (const key of ENV_KEYS) { + if (savedEnv[key] === undefined) { + delete process.env[key]; + } else { + process.env[key] = savedEnv[key]; + } + } + log.mockRestore(); + warn.mockRestore(); }); - test('Default user and project are created on error', async () => { - prismaMock.user.findMany.mockRejectedValue(new Error('"user" table does not exist')); - prismaMock.project.findMany.mockRejectedValue(new Error('"project" table does not exist')); + const noAdmin = (existing: object | null = null) => { + prismaMock.user.findFirst.mockResolvedValue(null); + prismaMock.user.findUnique.mockResolvedValue(existing as never); + }; - await seed(); + const loggedText = () => JSON.stringify([...log.mock.calls, ...warn.mock.calls]); - expect(prismaMock.user.upsert).toHaveBeenCalled(); - expect(prismaMock.project.create).toHaveBeenCalled(); - }); + describe('seed()', () => { + test('creates the default user and project in an empty database', async () => { + noAdmin(); + prismaMock.project.count.mockResolvedValue(0); - test('Default user is create if NO admin user exists', async () => { - prismaMock.user.findMany.mockResolvedValue([{ ...mockDefaultUser, email: 'foo@bar.com', role: 'editor' }]); - prismaMock.project.findMany.mockResolvedValue([]); + await seed(); - await seed(); + expect(prismaMock.user.create).toHaveBeenCalledTimes(1); + expect(prismaMock.project.create).toHaveBeenCalledTimes(1); + }); - expect(prismaMock.user.upsert).toHaveBeenCalled(); - expect(prismaMock.project.create).toHaveBeenCalled(); + test('rejects when the database cannot be read', async () => { + prismaMock.user.findFirst.mockRejectedValue(new Error('"user" table does not exist')); + prismaMock.project.count.mockRejectedValue(new Error('"project" table does not exist')); + + await expect(seed()).rejects.toThrow(); + }); }); - test('Default user is create if the only admin user is inActive', async () => { - prismaMock.user.findMany.mockResolvedValue([{ ...mockDefaultUser, email: 'foo@bar.com', isActive: false }]); - prismaMock.project.findMany.mockResolvedValue([]); + describe('createDefaultUser', () => { + test('active admin exists: nothing is created or changed', async () => { + prismaMock.user.findFirst.mockResolvedValue({ id: 'admin' } as never); + + await createDefaultUser(); + + expect(prismaMock.user.findUnique).not.toHaveBeenCalled(); + expect(prismaMock.user.create).not.toHaveBeenCalled(); + expect(prismaMock.user.update).not.toHaveBeenCalled(); + expect(prismaMock.user.upsert).not.toHaveBeenCalled(); + }); + + test('never logs user rows, password hashes or api keys of existing users', async () => { + prismaMock.user.findFirst.mockResolvedValue({ + id: 'admin', + apiKey: 'EXISTINGKEY', + password: 'hash', + } as never); + + await createDefaultUser(); + + expect(loggedText()).not.toContain('EXISTINGKEY'); + expect(loggedText()).not.toContain('hash'); + }); + + test('fresh database with env values: uses them and does not print the password or key', async () => { + process.env.DEFAULT_USER_EMAIL = 'admin@corp.test'; + process.env.DEFAULT_USER_PASSWORD = 'env-password-value'; + process.env.DEFAULT_USER_API_KEY = 'ENVAPIKEYVALUE'; + noAdmin(); + + await createDefaultUser(); + + expect(prismaMock.user.create).toHaveBeenCalledTimes(1); + const { data } = prismaMock.user.create.mock.calls[0][0]; + expect(data.email).toBe('admin@corp.test'); + expect(data.role).toBe('admin'); + expect(data.apiKey).toBe('ENVAPIKEYVALUE'); + expect(compareSync('env-password-value', data.password)).toBe(true); + expect(loggedText()).not.toContain('env-password-value'); + expect(loggedText()).not.toContain('ENVAPIKEYVALUE'); + }); + + test('fresh database without env values: generates the secrets and prints each exactly once', async () => { + noAdmin(); + + await createDefaultUser(); + + const { data } = prismaMock.user.create.mock.calls[0][0]; + expect(data.email).toBe('visual-regression-tracker@example.com'); + expect(data.apiKey).toMatch(API_KEY_FORMAT); + const printed = log.mock.calls.map((args) => args.join(' ')); + const passwordLine = printed.find((line) => line.startsWith('Password (generated, shown once): ')); + const password = passwordLine?.replace('Password (generated, shown once): ', ''); + expect(password).toMatch(/^[A-Za-z0-9_-]{24}$/); + expect(compareSync(password as string, data.password)).toBe(true); + expect(printed.filter((line) => line.includes(password as string))).toHaveLength(1); + expect(printed.filter((line) => line.includes(data.apiKey))).toHaveLength(1); + }); + + test('empty env values count as unset', async () => { + process.env.DEFAULT_USER_EMAIL = ''; + process.env.DEFAULT_USER_PASSWORD = ''; + process.env.DEFAULT_USER_API_KEY = ''; + noAdmin(); + + await createDefaultUser(); + + const { data } = prismaMock.user.create.mock.calls[0][0]; + expect(data.email).toBe('visual-regression-tracker@example.com'); + expect(data.apiKey).toMatch(API_KEY_FORMAT); + expect(compareSync('', data.password)).toBe(false); + expect(loggedText()).toContain('generated, shown once'); + }); + + test('no active admin, email taken by an editor: not promoted', async () => { + noAdmin({ role: 'editor', isActive: true }); + + await createDefaultUser(); + + expect(prismaMock.user.create).not.toHaveBeenCalled(); + expect(prismaMock.user.update).not.toHaveBeenCalled(); + expect(prismaMock.user.upsert).not.toHaveBeenCalled(); + expect(warn).toHaveBeenCalled(); + }); + + test('warns about the public defaults', async () => { + process.env.DEFAULT_USER_PASSWORD = '123456'; + noAdmin(); + + await createDefaultUser(); + + expect(warn).toHaveBeenCalledWith(expect.stringContaining('publicly known default')); + }); + + test('does not warn about values that are not the public defaults', async () => { + process.env.DEFAULT_USER_PASSWORD = 'something-private'; + noAdmin(); + + await createDefaultUser(); + + expect(warn).not.toHaveBeenCalled(); + }); + }); - await seed(); + describe('createDefaultProject', () => { + test('creates it when there are no projects', async () => { + prismaMock.project.count.mockResolvedValue(0); - expect(prismaMock.user.upsert).toHaveBeenCalled(); - expect(prismaMock.project.create).toHaveBeenCalled(); - }); -}); + await createDefaultProject(); -describe('Defaults do NOT get created', () => { - beforeEach(() => { - prismaMock.user.upsert.mockResolvedValue(mockDefaultUser); - prismaMock.project.create.mockResolvedValue(mockDefaultProject); - }); - test('Default project is NOT create if any project exists', async () => { - prismaMock.user.findMany.mockResolvedValue([]); - prismaMock.project.findMany.mockResolvedValue([mockDefaultProject]); + expect(prismaMock.project.create).toHaveBeenCalledTimes(1); + }); - await seed(); + test('does nothing when a project exists', async () => { + prismaMock.project.count.mockResolvedValue(1); - expect(prismaMock.user.upsert).toHaveBeenCalled(); - expect(prismaMock.project.create).not.toHaveBeenCalled(); - }); + await createDefaultProject(); - test('Default user is NOT create if any active admin user exists', async () => { - prismaMock.user.findMany.mockResolvedValue([{ ...mockDefaultUser, email: 'foo@bar.com' }]); - prismaMock.project.findMany.mockResolvedValue([]); + expect(prismaMock.project.create).not.toHaveBeenCalled(); + }); + }); - await seed(); + describe('generateApiKey', () => { + test('has the format of the API keys generated by the backend', () => { + expect(generateApiKey()).toMatch(API_KEY_FORMAT); + }); - expect(prismaMock.user.upsert).not.toHaveBeenCalled(); - expect(prismaMock.project.create).toHaveBeenCalled(); + test('differs between calls', () => { + expect(generateApiKey()).not.toBe(generateApiKey()); + }); }); }); diff --git a/src/_data_/index.ts b/src/_data_/index.ts index 9b0c46ed..431accbb 100644 --- a/src/_data_/index.ts +++ b/src/_data_/index.ts @@ -80,6 +80,7 @@ export const generateTestRun = (testRun?: Partial): TestRun => { diffTollerancePercent: 1, pixelMisMatchCount: 123, status: 'new', + statusBeforeDecision: null, buildId: '146e7a8d-89f0-4565-aa2c-e61efabb0afd', testVariationId: '3bc4a5bc-006e-4d43-8e4e-eaa132627fca', projectId: '3bc4a5bc-006e-4d43-8e4e-eaa132627fcc', @@ -111,10 +112,10 @@ export const generatePng = (width: number, height: number): PNG => { for (let y = 0; y < png.height; y++) { for (let x = 0; x < png.width; x++) { const idx = (png.width * y + x) << 2; - png.data[idx] = 255; // red - png.data[idx + 1] = 255; // green - png.data[idx + 2] = 255; // blue - png.data[idx + 3] = 255; // alpha (0 is transparent) + png.data[idx] = 255; + png.data[idx + 1] = 255; + png.data[idx + 2] = 255; + png.data[idx + 3] = 255; } } return png.pack(); diff --git a/src/app.module.spec.ts b/src/app.module.spec.ts index c8497998..74afd8a3 100644 --- a/src/app.module.spec.ts +++ b/src/app.module.spec.ts @@ -7,6 +7,7 @@ import { HealthCheckService } from '@nestjs/terminus'; import { Test, TestingModule } from '@nestjs/testing'; import request from 'supertest'; import { AppService } from './app.service'; +import { validateEnv } from './config/env.validation'; import { AuthService } from './auth/auth.service'; import { ApiGuard } from './auth/guards/api.guard'; import { JwtAuthGuard } from './auth/guards/auth.guard'; @@ -41,38 +42,39 @@ import { UsersController } from './users/users.controller'; import { UsersFactoryService } from './users/users.factory'; import { UsersService } from './users/users.service'; -// The point of this spec is to catch dependency-injection breakage when the -// framework is upgraded: every module, provider and controller of the real -// AppModule has to resolve, with only the database client swapped out. +jest.mock('./config/env.validation', () => { + const actual = jest.requireActual('./config/env.validation'); + return { ...actual, validateEnv: jest.fn(actual.validateEnv) }; +}); type PrismaMock = { onModuleInit: jest.Mock; onApplicationShutdown: jest.Mock; $connect: jest.Mock; $disconnect: jest.Mock; - $queryRaw: jest.Mock; + $runCommandRaw: jest.Mock; + $queryRawUnsafe: jest.Mock; user: { findUnique: jest.Mock }; project: { findMany: jest.Mock }; }; -// a plain object rather than a PrismaService, so the lifecycle hooks are -// stand-ins: asserting they are called proves Nest still drives them const createPrismaMock = (): PrismaMock => ({ onModuleInit: jest.fn().mockResolvedValue(undefined), onApplicationShutdown: jest.fn().mockResolvedValue(undefined), $connect: jest.fn().mockResolvedValue(undefined), $disconnect: jest.fn().mockResolvedValue(undefined), - $queryRaw: jest.fn().mockResolvedValue([{ '?column?': 1 }]), + $runCommandRaw: jest + .fn() + .mockRejectedValue(new Error('The postgresql provider does not support $runCommandRaw. Use the mongodb provider.')), + $queryRawUnsafe: jest.fn().mockResolvedValue([{ '?column?': 1 }]), user: { findUnique: jest.fn().mockResolvedValue(null) }, project: { findMany: jest.fn().mockResolvedValue([]) }, }); -// Pinned so a developer .env (S3 storage, OIDC/LDAP login) cannot make the -// wiring reach out to the network. ConfigModule never overwrites variables that -// are already set, so these win over the .env file. const TEST_ENV = { JWT_SECRET: 'app-module-spec-secret', JWT_LIFE_TIME: '1h', + APP_FRONTEND_URL: 'http://localhost:8080', STATIC_SERVICE: 'hdd', OIDC_ENABLED: 'false', LDAP_ENABLED: 'false', @@ -86,13 +88,10 @@ describe('AppModule', () => { const get = (token: Type): T => moduleRef.get(token, { strict: false }); const prismaOf = (instance: unknown): PrismaMock => (instance as { prismaService: PrismaMock }).prismaService; + const inner = (token: Type) => get(token) as Record; beforeAll(async () => { Object.assign(process.env, TEST_ENV); - // imported only after the env is pinned: ConfigModule.forRoot() reads the - // .env file at import time of the module that calls it - // a plain require: under `module: nodenext` an import() stays a real ESM import - // eslint-disable-next-line @typescript-eslint/no-require-imports const { AppModule } = require('./app.module') as typeof import('./app.module'); moduleRef = await Test.createTestingModule({ imports: [AppModule] }) @@ -111,7 +110,6 @@ describe('AppModule', () => { }); afterAll(async () => { - // a no-op when the shutdown test below already closed the app await app?.close(); process.env = originalEnv; }); @@ -164,16 +162,16 @@ describe('AppModule', () => { }); it('closes the forwardRef cycles between builds, projects, test runs and test variations', () => { - const builds = get(BuildsService) as unknown as Record; - const projects = get(ProjectsService) as unknown as Record; - const testRuns = get(TestRunsService) as unknown as Record; - const testVariations = get(TestVariationsService) as unknown as Record; - const tasks = get(TasksService) as unknown as Record; + const builds = inner(BuildsService); + const projects = inner(ProjectsService); + const testRuns = inner(TestRunsService); + const testVariations = inner(TestVariationsService); + const tasks = inner(TasksService); expect(builds.testRunsService).toBe(testRuns); expect(builds.eventsGateway).toBe(get(EventsGateway)); - expect(projects.buildsService).toBe(builds); - expect(projects.testVariationsService).toBe(testVariations); + expect(projects.staticService).toBe(get(StaticService)); + expect(projects.eventsGateway).toBe(get(EventsGateway)); expect(testRuns.testVariationService).toBe(testVariations); expect(testRuns.compareService).toBe(get(CompareService)); expect(testRuns.staticService).toBe(get(StaticService)); @@ -183,20 +181,15 @@ describe('AppModule', () => { }); it('shares one StaticModule between every importer', () => { - const vlm = get(VlmService) as unknown as Record; - const testVariations = get(TestVariationsService) as unknown as Record; - - expect(vlm.staticService).toBe(get(StaticService)); - expect(testVariations.staticService).toBe(get(StaticService)); + expect(inner(VlmService).staticService).toBe(get(StaticService)); + expect(inner(TestVariationsService).staticService).toBe(get(StaticService)); }); it('picks the HDD storage and the DB users backend under the default configuration', () => { - expect((get(StaticService) as unknown as Record).staticService).toBeInstanceOf(HddService); - expect((get(UsersService) as unknown as Record).usersService).toBeInstanceOf(DbUsersService); + expect(inner(StaticService).staticService).toBeInstanceOf(HddService); + expect(inner(UsersService).usersService).toBeInstanceOf(DbUsersService); }); - // PrismaModule is global: every consumer shares one client and so one - // connection pool per instance. it('shares one PrismaService instance across the whole app', () => { expect(prismaInstances).toHaveLength(1); @@ -208,13 +201,20 @@ describe('AppModule', () => { prismaOf(get(ProjectsService)), prismaOf(get(TestRunsService)), prismaOf(get(TestVariationsService)), - (get(CompareService) as unknown as Record).prismaService, + inner(CompareService).prismaService, prismaOf(get(TasksService)), ]; consumers.forEach((prisma) => expect(prisma).toBe(prismaInstances[0])); }); }); + describe('configuration', () => { + it('exposes the validated config through ConfigService', () => { + expect(validateEnv).toHaveBeenCalledWith(expect.objectContaining({ JWT_SECRET: TEST_ENV.JWT_SECRET })); + expect(get(ConfigService).get('JWT_LIFE_TIME')).toBe('1h'); + }); + }); + describe('lifecycle', () => { it('runs onModuleInit of every PrismaService on init', () => { prismaInstances.forEach((prisma) => expect(prisma.onModuleInit).toHaveBeenCalledTimes(1)); @@ -244,12 +244,11 @@ describe('AppModule', () => { const response = await request(app.getHttpServer()).get('/does-not-exist'); expect(response.status).toBe(404); - expect(response.body).toMatchObject({ + expect(response.body).toEqual({ + statusCode: 404, path: '/does-not-exist', name: 'NotFoundException', message: 'Cannot GET /does-not-exist', - exception: expect.objectContaining({ status: 404 }), - stack: expect.stringContaining('NotFoundException'), }); }); @@ -276,24 +275,31 @@ describe('AppModule', () => { it('serves the health check through the AppModule PrismaService', async () => { const response = await request(app.getHttpServer()).get('/health'); - expect(prismaOf(get(HealthController)).$queryRaw).toHaveBeenCalled(); + expect(prismaOf(get(HealthController)).$queryRawUnsafe).toHaveBeenCalledWith('SELECT 1'); expect(response.status).toBe(200); - expect(response.body).toEqual({ status: 'ok', info: {}, error: {}, details: {} }); + expect(response.body).toEqual({ + status: 'ok', + info: { database: { status: 'up', responseTime: expect.any(Number) } }, + error: {}, + details: { database: { status: 'up', responseTime: expect.any(Number) } }, + }); }); - // NOTE: current behavior — a database failure surfaces as a 500 from the - // global filter, not as terminus' 503 health report. - it('answers 500 from the health check when the database query fails', async () => { - prismaOf(get(HealthController)).$queryRaw.mockRejectedValueOnce(new Error('Can not reach database server')); + it('answers 503 from the health check when the database query fails', async () => { + prismaOf(get(HealthController)).$queryRawUnsafe.mockRejectedValueOnce(new Error('Can not reach database server')); const response = await request(app.getHttpServer()).get('/health'); - expect(response.status).toBe(500); - expect(response.body).toMatchObject({ path: '/health', name: 'Error', message: 'Can not reach database server' }); + expect(response.status).toBe(503); + expect(response.body).toEqual({ + statusCode: 503, + path: '/health', + name: 'ServiceUnavailableException', + message: 'Service Unavailable Exception', + }); }); }); - // must stay the last block: it closes the application describe('shutdown', () => { it('stops the cron jobs, the diff worker pool and every PrismaService on close', async () => { const [job] = [...get(SchedulerRegistry).getCronJobs().values()]; @@ -303,8 +309,6 @@ describe('AppModule', () => { await app.close(); - // the database and the workers go only after the HTTP server has - // drained, so no request still running finds them closed const httpClosedAt = closeHttp.mock.invocationCallOrder[0]; expect(httpClosedAt).toBeDefined(); expect(stopWorkers.mock.invocationCallOrder[0]).toBeGreaterThan(httpClosedAt); diff --git a/src/app.module.ts b/src/app.module.ts index 7e537007..c6c2d6f6 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -8,6 +8,7 @@ import { TestRunsModule } from './test-runs/test-runs.module'; import { TestVariationsModule } from './test-variations/test-variations.module'; import { PrismaModule } from './prisma/prisma.module'; import { ConfigModule } from '@nestjs/config'; +import { validateEnv } from './config/env.validation'; import { APP_FILTER } from '@nestjs/core'; import { HttpExceptionFilter } from './http-exception.filter'; import { CompareModule } from './compare/compare.module'; @@ -19,9 +20,8 @@ import { HealthController } from './health/health.controller'; imports: [ ConfigModule.forRoot({ isGlobal: true, - // Prisma 6 read DATABASE_URL from prisma/.env on its own; Prisma 7 reads - // no env files, so load it here for local runs (.env wins on conflicts) envFilePath: ['.env', 'prisma/.env'], + validate: (config) => validateEnv(config), }), PrismaModule, ScheduleModule.forRoot(), diff --git a/src/app.service.spec.ts b/src/app.service.spec.ts index 13123fc1..8fe3861a 100644 --- a/src/app.service.spec.ts +++ b/src/app.service.spec.ts @@ -1,30 +1,9 @@ import { Logger } from '@nestjs/common'; -import { Test, TestingModule } from '@nestjs/testing'; import { HttpAdapterHost } from '@nestjs/core'; import { AppService } from './app.service'; import { ConfigService } from '@nestjs/config'; describe('AppService', () => { - let service: AppService; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [ - AppService, - { - provide: ConfigService, - useValue: {}, - }, - ], - }).compile(); - - service = module.get(AppService); - }); - - it('should be defined', () => { - expect(service).toBeDefined(); - }); - describe('onApplicationBootstrap', () => { const ENV_KEYS = ['SERVER_TIMEOUT', 'SERVER_HEADERS_TIMEOUT', 'SERVER_KEEP_ALIVE_TIMEOUT']; const originalEnv = { ...process.env }; @@ -79,10 +58,25 @@ describe('AppService', () => { expect(bootstrap({ SERVER_TIMEOUT: '1500.9' }).timeout).toBe(1500); }); - // NOTE: current behavior — the value is parsed with parseInt and never - // validated, so a typo in the env sets the timeout to NaN. - it('sets NaN for a non-numeric override', () => { - expect(bootstrap({ SERVER_TIMEOUT: 'abc' }).timeout).toBeNaN(); + it('keeps the server default and warns for a non-numeric override', () => { + const warnSpy = jest.spyOn(Logger.prototype, 'warn').mockImplementation(); + + const server = bootstrap({ SERVER_TIMEOUT: 'abc', SERVER_HEADERS_TIMEOUT: '-5' }); + + expect(server).toEqual({ timeout: 0, headersTimeout: 60000, keepAliveTimeout: 5000 }); + expect(warnSpy).toHaveBeenCalledWith('SERVER_TIMEOUT="abc" is not a number of milliseconds; keeping 0'); + expect(warnSpy).toHaveBeenCalledWith( + 'SERVER_HEADERS_TIMEOUT="-5" is not a number of milliseconds; keeping 60000' + ); + warnSpy.mockRestore(); + }); + + it('treats an empty value as not set, without a warning', () => { + const warnSpy = jest.spyOn(Logger.prototype, 'warn').mockImplementation(); + + expect(bootstrap({ SERVER_TIMEOUT: ' ' }).timeout).toBe(0); + expect(warnSpy).not.toHaveBeenCalled(); + warnSpy.mockRestore(); }); it('logs every resulting timeout', () => { diff --git a/src/app.service.ts b/src/app.service.ts index 3859ba98..04463fc8 100644 --- a/src/app.service.ts +++ b/src/app.service.ts @@ -14,14 +14,22 @@ export class AppService implements OnApplicationBootstrap { onApplicationBootstrap() { const server: Server = this.refHost.httpAdapter.getHttpServer(); - const { timeout, headersTimeout, keepAliveTimeout } = server; - server.timeout = parseInt(this.configService.get('SERVER_TIMEOUT', timeout.toString())); + server.timeout = this.readTimeout('SERVER_TIMEOUT', server.timeout); this.logger.log(`server.timeout is ${server.timeout}`); - server.headersTimeout = parseInt(this.configService.get('SERVER_HEADERS_TIMEOUT', headersTimeout.toString())); + server.headersTimeout = this.readTimeout('SERVER_HEADERS_TIMEOUT', server.headersTimeout); this.logger.log(`server.headersTimeout is ${server.headersTimeout}`); - server.keepAliveTimeout = parseInt( - this.configService.get('SERVER_KEEP_ALIVE_TIMEOUT', keepAliveTimeout.toString()) - ); + server.keepAliveTimeout = this.readTimeout('SERVER_KEEP_ALIVE_TIMEOUT', server.keepAliveTimeout); this.logger.log(`server.keepAliveTimeout is ${server.keepAliveTimeout}`); } + + private readTimeout(name: string, fallback: number): number { + const raw = this.configService.get(name); + if (raw === undefined || String(raw).trim() === '') return fallback; + const value = parseInt(String(raw), 10); + if (Number.isNaN(value) || value < 0) { + this.logger.warn(`${name}="${raw}" is not a number of milliseconds; keeping ${fallback}`); + return fallback; + } + return value; + } } diff --git a/src/auth/auth.service.spec.ts b/src/auth/auth.service.spec.ts index 8a5c729c..69fc16b3 100644 --- a/src/auth/auth.service.spec.ts +++ b/src/auth/auth.service.spec.ts @@ -18,10 +18,6 @@ describe('AuthService', () => { service = module.get(AuthService); }); - it('should be defined', () => { - expect(service).toBeDefined(); - }); - describe('encryptPassword', () => { it('produces a salted bcrypt hash with 10 rounds that matches the password', async () => { const hashed = await service.encryptPassword('secret'); @@ -55,7 +51,6 @@ describe('AuthService', () => { it('returns a key in the format keys were always issued in', () => { const apiKey = service.generateApiKey(); - // 28 Crockford base32 digits, upper case, no dashes expect(apiKey).toMatch(/^[0-9A-HJKMNP-TV-Z]{28}$/); }); @@ -65,11 +60,25 @@ describe('AuthService', () => { }); describe('signToken', () => { - it('signs a payload carrying only the email of the user', () => { + it('signs the user id as sub next to the email, so older API versions still read the token', () => { const user = { id: 'user-id', email: 'user@example.com', password: 'hash', apiKey: 'key' } as User; expect(service.signToken(user)).toBe('signed-token'); - expect(jwtService.sign).toHaveBeenCalledWith({ email: 'user@example.com' }); + expect(jwtService.sign).toHaveBeenCalledWith({ sub: 'user-id', email: 'user@example.com' }); + }); + + it('produces a token that decodes to sub, email, iat and exp', () => { + const real = new JwtService({ secret: 'secret', signOptions: { expiresIn: '1h' } }); + const user = { id: 'user-id', email: 'user@example.com' } as User; + + const token = new AuthService(real).signToken(user); + + expect(real.decode(token)).toEqual({ + sub: 'user-id', + email: 'user@example.com', + iat: expect.any(Number), + exp: expect.any(Number), + }); }); }); }); diff --git a/src/auth/auth.service.ts b/src/auth/auth.service.ts index 352d3e75..ce05fbbe 100644 --- a/src/auth/auth.service.ts +++ b/src/auth/auth.service.ts @@ -18,16 +18,12 @@ export class AuthService { return generateRandomKey(); } - async compare(password: string, hashedPassword: string): Promise { - return await compare(password, hashedPassword); + compare(password: string, hashedPassword: string): Promise { + return compare(password, hashedPassword); } signToken(user: User): string { - const payload: JwtPayload = { - email: user.email, - }; - - const token = this.jwtService.sign(payload); - return token; + const payload: JwtPayload = { sub: user.id, email: user.email }; + return this.jwtService.sign(payload); } } diff --git a/src/auth/guards/api.guard.spec.ts b/src/auth/guards/api.guard.spec.ts index 2b4065e4..ce0a9e72 100644 --- a/src/auth/guards/api.guard.spec.ts +++ b/src/auth/guards/api.guard.spec.ts @@ -21,7 +21,7 @@ describe('ApiGuard', () => { }); it('allows a request whose apiKey header belongs to a user', async () => { - prismaService.user.findUnique.mockResolvedValue({ id: '1' } as User); + prismaService.user.findUnique.mockResolvedValue({ id: '1' }); const { executionContext, request } = context('valid-key'); await expect(guard.canActivate(executionContext)).resolves.toBe(true); @@ -35,19 +35,40 @@ describe('ApiGuard', () => { await expect(guard.canActivate(context('unknown-key').executionContext)).resolves.toBe(false); }); - // NOTE: current behavior — a missing header is still sent to Prisma as - // `apiKey: undefined`; it is the lookup failing (or returning nothing) that - // rejects the request, not a check in the guard. - it('still queries the database when the apiKey header is missing', async () => { + it('answers 401 without a database query when the apiKey header is missing', async () => { + await expect(guard.canActivate(context(undefined).executionContext)).rejects.toThrow(UnauthorizedException); + + expect(prismaService.user.findUnique).not.toHaveBeenCalled(); + }); + + it('rejects an empty apiKey header with false, as before, without a database query', async () => { + await expect(guard.canActivate(context('').executionContext)).resolves.toBe(false); + + expect(prismaService.user.findUnique).not.toHaveBeenCalled(); + }); + + it('stores the user on the request', async () => { + const found = { id: '1' } as User; + prismaService.user.findUnique.mockResolvedValue(found); + const { executionContext, request } = context('valid-key'); + + await guard.canActivate(executionContext); + + expect((request as { user?: User }).user).toBe(found); + }); + + it('leaves the request without a user for an unknown key', async () => { prismaService.user.findUnique.mockResolvedValue(null); + const { executionContext, request } = context('unknown-key'); + + await guard.canActivate(executionContext); - await expect(guard.canActivate(context(undefined).executionContext)).resolves.toBe(false); - expect(prismaService.user.findUnique).toHaveBeenCalledWith({ where: { apiKey: undefined } }); + expect((request as { user?: User }).user).toBeUndefined(); }); it('turns a failing lookup into an UnauthorizedException', async () => { - prismaService.user.findUnique.mockRejectedValue(new Error('Argument where of type UserWhereUniqueInput needs')); + prismaService.user.findUnique.mockRejectedValue(new Error('connection lost')); - await expect(guard.canActivate(context(undefined).executionContext)).rejects.toThrow(UnauthorizedException); + await expect(guard.canActivate(context('some-key').executionContext)).rejects.toThrow(UnauthorizedException); }); }); diff --git a/src/auth/guards/api.guard.ts b/src/auth/guards/api.guard.ts index 045c04c7..52c660e0 100644 --- a/src/auth/guards/api.guard.ts +++ b/src/auth/guards/api.guard.ts @@ -1,4 +1,5 @@ import { ExecutionContext, Injectable, CanActivate, UnauthorizedException } from '@nestjs/common'; +import { User } from '@prisma/client'; import { Request } from 'express'; import { PrismaService } from '../../prisma/prisma.service'; @@ -8,13 +9,24 @@ export class ApiGuard implements CanActivate { async canActivate(context: ExecutionContext): Promise { const request: Request = context.switchToHttp().getRequest(); + const apiKey = request.header('apiKey'); + if (apiKey === undefined) { + throw new UnauthorizedException(); + } + // an empty header never matches a key: 403, not 401 + if (apiKey === '') { + return false; + } + let user: User | null; try { - const user = await this.prismaService.user.findUnique({ - where: { apiKey: request.header('apiKey') }, - }); - return !!user; + user = await this.prismaService.user.findUnique({ where: { apiKey } }); } catch { throw new UnauthorizedException(); } + if (!user) { + return false; + } + request.user = user; + return true; } } diff --git a/src/auth/guards/mixed.guard.spec.ts b/src/auth/guards/mixed.guard.spec.ts index 9804a9b1..c245ec70 100644 --- a/src/auth/guards/mixed.guard.spec.ts +++ b/src/auth/guards/mixed.guard.spec.ts @@ -42,15 +42,20 @@ describe('MixedGuard', () => { await expect(guard.canActivate(context)).resolves.toBe(expected); }); - // NOTE: current behavior — both guards always run, even when the JWT guard - // has already let the request through, so an extra api-key lookup hits the - // database on every JWT-authenticated request. - it('always consults both guards with the same context', async () => { - const { guard, authGuard, apiGuard } = createGuard('pass', 'fail'); + it('does not consult the api-key guard when the JWT is valid', async () => { + const { guard, authGuard, apiGuard } = createGuard('pass', 'pass'); await guard.canActivate(context); expect(authGuard.canActivate).toHaveBeenCalledWith(context); + expect(apiGuard.canActivate).not.toHaveBeenCalled(); + }); + + it('falls back to the api-key guard with the same context when the JWT fails', async () => { + const { guard, apiGuard } = createGuard('fail', 'pass'); + + await guard.canActivate(context); + expect(apiGuard.canActivate).toHaveBeenCalledWith(context); }); @@ -64,12 +69,10 @@ describe('MixedGuard', () => { ); }); - // NOTE: current behavior — the JWT guard's result is cast, not checked, so - // a truthy non-boolean value is passed through as-is. - it('passes a truthy non-boolean JWT result straight through', async () => { + it('returns true for a truthy non-boolean JWT result', async () => { const { guard, authGuard } = createGuard('fail', 'fail'); authGuard.canActivate.mockResolvedValue('yes'); - await expect(guard.canActivate(context)).resolves.toBe('yes'); + await expect(guard.canActivate(context)).resolves.toBe(true); }); }); diff --git a/src/auth/guards/mixed.guard.ts b/src/auth/guards/mixed.guard.ts index ffdb7bd9..04cef71f 100644 --- a/src/auth/guards/mixed.guard.ts +++ b/src/auth/guards/mixed.guard.ts @@ -10,15 +10,17 @@ export class MixedGuard implements CanActivate { ) {} async canActivate(context: ExecutionContext): Promise { - let jwtAuth = false; try { - jwtAuth = (await this.authGuard.canActivate(context)) as boolean; - } catch {} - - let apiAuth = false; + if (await this.authGuard.canActivate(context)) { + return true; + } + } catch { + // not a valid JWT: the api key is tried next + } try { - apiAuth = await this.apiGuard.canActivate(context); - } catch {} - return jwtAuth || apiAuth; + return await this.apiGuard.canActivate(context); + } catch { + return false; + } } } diff --git a/src/auth/guards/role.guard.spec.ts b/src/auth/guards/role.guard.spec.ts index c00fb5e8..a0d16616 100644 --- a/src/auth/guards/role.guard.spec.ts +++ b/src/auth/guards/role.guard.spec.ts @@ -3,6 +3,7 @@ import { Reflector } from '@nestjs/core'; import { Role, User } from '@prisma/client'; import { PrismaService } from '../../prisma/prisma.service'; import { Roles } from '../../shared/roles.decorator'; +import { ApiGuard } from './api.guard'; import { RoleGuard } from './role.guard'; const userWithRole = (role: Role): User => ({ id: '1', role }) as User; @@ -12,11 +13,8 @@ describe('RoleGuard', () => { let prismaService: { user: { findUnique: jest.Mock } }; let guard: RoleGuard; - const context = (user?: User, apiKey?: string): ExecutionContext => { - const request = { - user, - header: jest.fn().mockReturnValue(apiKey), - }; + const context = (user?: User): ExecutionContext => { + const request = { user }; return { switchToHttp: () => ({ getRequest: () => request }), getHandler: () => ({}), @@ -27,60 +25,27 @@ describe('RoleGuard', () => { beforeEach(() => { reflector = { getAllAndOverride: jest.fn() }; prismaService = { user: { findUnique: jest.fn() } }; - guard = new RoleGuard(prismaService as unknown as PrismaService, reflector as unknown as Reflector); + guard = new RoleGuard(reflector as unknown as Reflector); }); - it('allows the request when no roles are required', async () => { + it('allows the request when no roles are required', () => { reflector.getAllAndOverride.mockReturnValue(undefined); - await expect(guard.canActivate(context())).resolves.toBe(true); + expect(guard.canActivate(context())).toBe(true); }); - it('allows an admin on an admin-only endpoint', async () => { - reflector.getAllAndOverride.mockReturnValue([Role.admin]); - - await expect(guard.canActivate(context(userWithRole(Role.admin)))).resolves.toBe(true); - }); - - it('rejects an editor on an admin-only endpoint', async () => { - reflector.getAllAndOverride.mockReturnValue([Role.admin]); + it.each<[string, Role[], Role | undefined, boolean]>([ + ['allows an admin on an admin-only endpoint', [Role.admin], Role.admin, true], + ['rejects an editor on an admin-only endpoint', [Role.admin], Role.editor, false], + ['allows an editor where editor is permitted', [Role.admin, Role.editor], Role.editor, true], + ['rejects a guest where only admin and editor are permitted', [Role.admin, Role.editor], Role.guest, false], + ['rejects when no user was authenticated', [Role.admin], undefined, false], + ])('%s', (_name, required, role, allowed) => { + reflector.getAllAndOverride.mockReturnValue(required); - await expect(guard.canActivate(context(userWithRole(Role.editor)))).resolves.toBe(false); + expect(guard.canActivate(context(role && userWithRole(role)))).toBe(allowed); }); - it('allows an editor where editor is permitted', async () => { - reflector.getAllAndOverride.mockReturnValue([Role.admin, Role.editor]); - - await expect(guard.canActivate(context(userWithRole(Role.editor)))).resolves.toBe(true); - }); - - it('rejects a guest where only admin and editor are permitted', async () => { - reflector.getAllAndOverride.mockReturnValue([Role.admin, Role.editor]); - - await expect(guard.canActivate(context(userWithRole(Role.guest)))).resolves.toBe(false); - }); - - it('rejects when neither a user nor an api key is present', async () => { - reflector.getAllAndOverride.mockReturnValue([Role.admin]); - - await expect(guard.canActivate(context())).resolves.toBe(false); - expect(prismaService.user.findUnique).not.toHaveBeenCalled(); - }); - - it('resolves the user from the apiKey header when request.user is absent', async () => { - reflector.getAllAndOverride.mockReturnValue([Role.admin]); - prismaService.user.findUnique.mockResolvedValue(userWithRole(Role.admin)); - - await expect(guard.canActivate(context(undefined, 'api-key'))).resolves.toBe(true); - expect(prismaService.user.findUnique).toHaveBeenCalledWith({ where: { apiKey: 'api-key' } }); - }); - - it('rejects when the api key does not belong to a user', async () => { - reflector.getAllAndOverride.mockReturnValue([Role.admin]); - prismaService.user.findUnique.mockResolvedValue(null); - - await expect(guard.canActivate(context(undefined, 'unknown'))).resolves.toBe(false); - }); describe('with the @Roles metadata', () => { @Roles(Role.admin) class AdminController { @@ -98,21 +63,37 @@ describe('RoleGuard', () => { }) as unknown as ExecutionContext; beforeEach(() => { - guard = new RoleGuard(prismaService as unknown as PrismaService, new Reflector()); + guard = new RoleGuard(new Reflector()); }); - it('applies the roles declared on the controller class', async () => { + it('applies the roles declared on the controller class', () => { const handler = AdminController.prototype.inherited; - await expect(guard.canActivate(decoratedContext(handler, userWithRole(Role.editor)))).resolves.toBe(false); - await expect(guard.canActivate(decoratedContext(handler, userWithRole(Role.admin)))).resolves.toBe(true); + expect(guard.canActivate(decoratedContext(handler, userWithRole(Role.editor)))).toBe(false); + expect(guard.canActivate(decoratedContext(handler, userWithRole(Role.admin)))).toBe(true); }); - it('lets the roles on the handler override the class', async () => { + it('lets the roles on the handler override the class', () => { const handler = AdminController.prototype.overridden; - await expect(guard.canActivate(decoratedContext(handler, userWithRole(Role.editor)))).resolves.toBe(true); - await expect(guard.canActivate(decoratedContext(handler, userWithRole(Role.admin)))).resolves.toBe(false); + expect(guard.canActivate(decoratedContext(handler, userWithRole(Role.editor)))).toBe(true); + expect(guard.canActivate(decoratedContext(handler, userWithRole(Role.admin)))).toBe(false); }); }); + + it('uses the user stored by ApiGuard without a second lookup', async () => { + reflector.getAllAndOverride.mockReturnValue([Role.admin]); + prismaService.user.findUnique.mockResolvedValue(userWithRole(Role.admin)); + const request = { header: jest.fn().mockReturnValue('admin-key') }; + const shared = { + switchToHttp: () => ({ getRequest: () => request }), + getHandler: () => ({}), + getClass: () => ({}), + } as unknown as ExecutionContext; + + await expect(new ApiGuard(prismaService as unknown as PrismaService).canActivate(shared)).resolves.toBe(true); + expect(guard.canActivate(shared)).toBe(true); + + expect(prismaService.user.findUnique).toHaveBeenCalledTimes(1); + }); }); diff --git a/src/auth/guards/role.guard.ts b/src/auth/guards/role.guard.ts index d8cee282..9bf32784 100644 --- a/src/auth/guards/role.guard.ts +++ b/src/auth/guards/role.guard.ts @@ -1,41 +1,21 @@ import { ExecutionContext, Injectable, CanActivate } from '@nestjs/common'; import { Role, User } from '@prisma/client'; import { Request } from 'express'; -import { PrismaService } from '../../prisma/prisma.service'; import { Reflector } from '@nestjs/core'; @Injectable() export class RoleGuard implements CanActivate { - constructor( - private readonly prismaService: PrismaService, - private reflector: Reflector - ) {} + constructor(private reflector: Reflector) {} - async canActivate(context: ExecutionContext): Promise { + canActivate(context: ExecutionContext): boolean { const requiredRoles = this.reflector.getAllAndOverride('roles', [context.getHandler(), context.getClass()]); if (!requiredRoles || requiredRoles.length === 0) { return true; } - const user = await this.getUser(context); + const request: Request = context.switchToHttp().getRequest(); + const user = request.user as User | undefined; return !!user && requiredRoles.includes(user.role); } - - private getUser = async (context: ExecutionContext): Promise => { - const request: Request = context.switchToHttp().getRequest(); - - if (request.user) { - return request.user as User; - } - - const apiKey = request.header('apiKey'); - if (!apiKey) { - return null; - } - - return this.prismaService.user.findUnique({ - where: { apiKey }, - }); - }; } diff --git a/src/auth/jwt-payload.model.ts b/src/auth/jwt-payload.model.ts index 831b1c07..df1efa7d 100644 --- a/src/auth/jwt-payload.model.ts +++ b/src/auth/jwt-payload.model.ts @@ -1,4 +1,4 @@ export interface JwtPayload { - email: string; - expire?: Date; + sub?: string; + email?: string; } diff --git a/src/auth/jwt.strategy.spec.ts b/src/auth/jwt.strategy.spec.ts index 84ac966e..52f22f11 100644 --- a/src/auth/jwt.strategy.spec.ts +++ b/src/auth/jwt.strategy.spec.ts @@ -1,4 +1,4 @@ -import { HttpException, HttpStatus } from '@nestjs/common'; +import { UnauthorizedException } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import { User } from '@prisma/client'; import { JwtService } from '@nestjs/jwt'; @@ -14,10 +14,13 @@ describe('JwtStrategy', () => { let configService: { get: jest.Mock }; let strategy: JwtStrategy; + const createStrategy = () => + new JwtStrategy(configService as unknown as ConfigService, prismaService as unknown as PrismaService); + beforeEach(() => { prismaService = { user: { findUnique: jest.fn() } }; configService = { get: jest.fn().mockReturnValue(SECRET) }; - strategy = new JwtStrategy(configService as unknown as ConfigService, prismaService as unknown as PrismaService); + strategy = createStrategy(); }); describe('constructor', () => { @@ -40,49 +43,32 @@ describe('JwtStrategy', () => { it('fails to construct without a secret', () => { configService.get.mockReturnValue(undefined); - expect( - () => new JwtStrategy(configService as unknown as ConfigService, prismaService as unknown as PrismaService) - ).toThrow('JwtStrategy requires a secret or key'); + expect(createStrategy).toThrow('JwtStrategy requires a secret or key'); }); }); describe('validate', () => { - it('looks the user up by the email in the payload and passes it to done', async () => { + it('returns the user the sub claim points to', async () => { prismaService.user.findUnique.mockResolvedValue(user); - const done = jest.fn().mockReturnValue('done-result'); - const result = await strategy.validate({ email: user.email }, done); + await expect(strategy.validate({ sub: 'user-id', email: 'old@example.com' })).resolves.toBe(user); - expect(prismaService.user.findUnique).toHaveBeenCalledWith({ where: { email: user.email } }); - expect(done).toHaveBeenCalledWith(null, user, undefined); - expect(result).toBe('done-result'); + expect(prismaService.user.findUnique).toHaveBeenCalledWith({ where: { id: 'user-id' } }); }); - it('passes payload.expire as the info argument', async () => { - prismaService.user.findUnique.mockResolvedValue(user); - const done = jest.fn(); - - await strategy.validate({ email: user.email, expire: 123 } as never, done); + it('throws UnauthorizedException when the user does not exist', async () => { + prismaService.user.findUnique.mockResolvedValue(null); - expect(done).toHaveBeenCalledWith(null, user, 123); + await expect(strategy.validate({ sub: 'ghost' })).rejects.toBeInstanceOf(UnauthorizedException); }); - it('passes a 401 HttpException and false to done when the user does not exist', async () => { - prismaService.user.findUnique.mockResolvedValue(null); - const done = jest.fn(); - - await strategy.validate({ email: 'ghost@example.com' }, done); + it('rejects a payload with neither sub nor email without a lookup', async () => { + await expect(strategy.validate({})).rejects.toBeInstanceOf(UnauthorizedException); - expect(done).toHaveBeenCalledTimes(1); - const [error, result] = done.mock.calls[0]; - expect(error).toBeInstanceOf(HttpException); - expect(error.getStatus()).toBe(HttpStatus.UNAUTHORIZED); - expect(error.getResponse()).toEqual({}); - expect(result).toBe(false); + expect(prismaService.user.findUnique).not.toHaveBeenCalled(); }); }); - // Drives the strategy the way passport does, with a real signed token. describe('authenticate', () => { const authenticate = (token?: string) => { const outcome = { success: jest.fn(), fail: jest.fn(), error: jest.fn() }; @@ -92,28 +78,45 @@ describe('JwtStrategy', () => { }; const settle = () => new Promise((resolve) => setTimeout(resolve, 20)); - // NOTE: current behavior — validate() calls `done` itself and the - // @nestjs/passport mixin calls it a second time with validate()'s return - // value (undefined), so passport sees success() followed by fail(). - it('authenticates a valid token, then also reports a failure from the second done call', async () => { + it('authenticates a valid token exactly once', async () => { prismaService.user.findUnique.mockResolvedValue(user); - const outcome = authenticate(sign({ email: user.email }, SECRET)); + const outcome = authenticate(sign({ sub: 'user-id', email: user.email }, SECRET)); await settle(); + expect(outcome.success).toHaveBeenCalledTimes(1); expect(outcome.success).toHaveBeenCalledWith(user, undefined); - expect(outcome.fail).toHaveBeenCalledTimes(1); - expect(outcome.success.mock.invocationCallOrder[0]).toBeLessThan(outcome.fail.mock.invocationCallOrder[0]); + expect(outcome.fail).not.toHaveBeenCalled(); expect(outcome.error).not.toHaveBeenCalled(); }); - it('errors with a 401 HttpException when the token belongs to an unknown user', async () => { + it('resolves a token with sub by user id', async () => { + prismaService.user.findUnique.mockResolvedValue(user); + + authenticate(sign({ sub: 'user-id', email: 'email-before-the-change@example.com' }, SECRET)); + await settle(); + + expect(prismaService.user.findUnique).toHaveBeenCalledTimes(1); + expect(prismaService.user.findUnique).toHaveBeenCalledWith({ where: { id: 'user-id' } }); + }); + + it('still accepts an email-only token issued before sub existed', async () => { + prismaService.user.findUnique.mockResolvedValue(user); + + const outcome = authenticate(sign({ email: user.email }, SECRET)); + await settle(); + + expect(prismaService.user.findUnique).toHaveBeenCalledWith({ where: { email: user.email } }); + expect(outcome.success).toHaveBeenCalledWith(user, undefined); + }); + + it('errors with a 401 when the token belongs to an unknown user', async () => { prismaService.user.findUnique.mockResolvedValue(null); - const outcome = authenticate(sign({ email: 'ghost@example.com' }, SECRET)); + const outcome = authenticate(sign({ sub: 'ghost' }, SECRET)); await settle(); - expect(outcome.error).toHaveBeenCalledWith(expect.any(HttpException)); + expect(outcome.error).toHaveBeenCalledWith(expect.any(UnauthorizedException)); expect(outcome.success).not.toHaveBeenCalled(); }); diff --git a/src/auth/jwt.strategy.ts b/src/auth/jwt.strategy.ts index 71fc617d..2b5c8fd6 100644 --- a/src/auth/jwt.strategy.ts +++ b/src/auth/jwt.strategy.ts @@ -1,6 +1,7 @@ -import { ExtractJwt, Strategy, VerifiedCallback } from 'passport-jwt'; +import { ExtractJwt, Strategy } from 'passport-jwt'; import { PassportStrategy } from '@nestjs/passport'; -import { Injectable, HttpException, HttpStatus } from '@nestjs/common'; +import { Injectable, UnauthorizedException } from '@nestjs/common'; +import { User } from '@prisma/client'; import { JwtPayload } from './jwt-payload.model'; import { ConfigService } from '@nestjs/config'; import { PrismaService } from '../prisma/prisma.service'; @@ -18,15 +19,23 @@ export class JwtStrategy extends PassportStrategy(Strategy) { }); } - async validate(payload: JwtPayload, done: VerifiedCallback) { - const user = await this.prismaService.user.findUnique({ - where: { email: payload.email }, - }); - + // @nestjs/passport calls `done` with whatever this returns, so it must not call `done` itself + async validate(payload: JwtPayload): Promise { + const user = await this.findUser(payload); if (!user) { - return done(new HttpException({}, HttpStatus.UNAUTHORIZED), false); + throw new UnauthorizedException(); } + return user; + } - return done(null, user, payload.expire); + private findUser(payload: JwtPayload): Promise { + if (typeof payload.sub === 'string' && payload.sub) { + return this.prismaService.user.findUnique({ where: { id: payload.sub } }); + } + // tokens issued before `sub` existed carry only the email + if (typeof payload.email === 'string' && payload.email) { + return this.prismaService.user.findUnique({ where: { email: payload.email } }); + } + return Promise.resolve(null); } } diff --git a/src/builds/build-stats.spec.ts b/src/builds/build-stats.spec.ts index 65b461d9..b42c1ff8 100644 --- a/src/builds/build-stats.spec.ts +++ b/src/builds/build-stats.spec.ts @@ -14,42 +14,36 @@ describe('getBuildsStats', () => { expect(groupByMock).not.toHaveBeenCalled(); }); - it('groups test runs by build and status, and by build for merge runs', async () => { + it('groups test runs by build, status and merge flag in one query', async () => { const groupByMock = jest.fn().mockResolvedValue([]); await getBuildsStats(prismaWith(groupByMock), ['b1', 'b2']); - expect(groupByMock).toHaveBeenNthCalledWith(1, { - by: ['buildId', 'status'], + expect(groupByMock).toHaveBeenCalledTimes(1); + expect(groupByMock).toHaveBeenCalledWith({ + by: ['buildId', 'status', 'merge'], where: { buildId: { in: ['b1', 'b2'] } }, _count: { _all: true }, }); - expect(groupByMock).toHaveBeenNthCalledWith(2, { - by: ['buildId'], - where: { buildId: { in: ['b1', 'b2'] }, merge: true }, - }); }); it('sums the status counts into passed, unresolved and failed per build', async () => { - const groupByMock = jest - .fn() - .mockResolvedValueOnce([ - { buildId: 'b1', status: TestStatus.ok, _count: { _all: 1 } }, - { buildId: 'b1', status: TestStatus.approved, _count: { _all: 2 } }, - { buildId: 'b1', status: TestStatus.autoApproved, _count: { _all: 4 } }, - { buildId: 'b1', status: TestStatus.new, _count: { _all: 8 } }, - { buildId: 'b1', status: TestStatus.unresolved, _count: { _all: 16 } }, - { buildId: 'b1', status: TestStatus.failed, _count: { _all: 32 } }, - { buildId: 'b2', status: TestStatus.failed, _count: { _all: 3 } }, - // a status outside the known groups is not counted anywhere - { buildId: 'b2', status: 'resolved' as TestStatus, _count: { _all: 100 } }, - ]) - .mockResolvedValueOnce([{ buildId: 'b2' }]); + const groupByMock = jest.fn().mockResolvedValueOnce([ + { buildId: 'b1', status: TestStatus.ok, merge: false, _count: { _all: 1 } }, + { buildId: 'b1', status: TestStatus.ok, merge: true, _count: { _all: 2 } }, + { buildId: 'b1', status: TestStatus.approved, merge: false, _count: { _all: 2 } }, + { buildId: 'b1', status: TestStatus.autoApproved, merge: false, _count: { _all: 4 } }, + { buildId: 'b1', status: TestStatus.new, merge: false, _count: { _all: 8 } }, + { buildId: 'b1', status: TestStatus.unresolved, merge: false, _count: { _all: 16 } }, + { buildId: 'b1', status: TestStatus.failed, merge: false, _count: { _all: 32 } }, + { buildId: 'b2', status: TestStatus.failed, merge: true, _count: { _all: 3 } }, + { buildId: 'b2', status: 'resolved' as TestStatus, merge: false, _count: { _all: 100 } }, + ]); const stats = await getBuildsStats(prismaWith(groupByMock), ['b1', 'b2', 'b3']); expect([...stats.entries()]).toEqual([ - ['b1', { passedCount: 7, unresolvedCount: 24, failedCount: 32, merge: false }], + ['b1', { passedCount: 9, unresolvedCount: 24, failedCount: 32, merge: true }], ['b2', { passedCount: 0, unresolvedCount: 0, failedCount: 3, merge: true }], ['b3', { passedCount: 0, unresolvedCount: 0, failedCount: 0, merge: false }], ]); @@ -58,8 +52,7 @@ describe('getBuildsStats', () => { it('keeps one entry per id when the same build id is asked for twice', async () => { const groupByMock = jest .fn() - .mockResolvedValueOnce([{ buildId: 'b1', status: TestStatus.ok, _count: { _all: 2 } }]) - .mockResolvedValueOnce([]); + .mockResolvedValueOnce([{ buildId: 'b1', status: TestStatus.ok, merge: false, _count: { _all: 2 } }]); const stats = await getBuildsStats(prismaWith(groupByMock), ['b1', 'b1']); diff --git a/src/builds/build-stats.ts b/src/builds/build-stats.ts index 5bf20fa6..8bbdde98 100644 --- a/src/builds/build-stats.ts +++ b/src/builds/build-stats.ts @@ -11,12 +11,6 @@ export interface BuildStats { const PASSED_STATUSES: TestStatus[] = [TestStatus.ok, TestStatus.approved, TestStatus.autoApproved]; const UNRESOLVED_STATUSES: TestStatus[] = [TestStatus.unresolved, TestStatus.new]; -/** - * Aggregates test-run statistics per build inside the database instead of - * loading every test-run row into the API process: builds can hold thousands - * of runs and these stats are recomputed on every build_updated event burst - * during ingestion. - */ export async function getBuildsStats(prisma: PrismaService, buildIds: string[]): Promise> { const stats = new Map( buildIds.map((id) => [id, { passedCount: 0, unresolvedCount: 0, failedCount: 0, merge: false }]) @@ -25,20 +19,13 @@ export async function getBuildsStats(prisma: PrismaService, buildIds: string[]): return stats; } - const [statusCounts, mergeBuilds] = await Promise.all([ - prisma.testRun.groupBy({ - by: ['buildId', 'status'], - where: { buildId: { in: buildIds } }, - _count: { _all: true }, - }), - prisma.testRun.groupBy({ - by: ['buildId'], - where: { buildId: { in: buildIds }, merge: true }, - }), - ]); + const rows = await prisma.testRun.groupBy({ + by: ['buildId', 'status', 'merge'], + where: { buildId: { in: buildIds } }, + _count: { _all: true }, + }); - for (const row of statusCounts) { - // every row belongs to one of buildIds, and the map holds an entry for each + for (const row of rows) { const buildStats = stats.get(row.buildId)!; const count = row._count._all; if (PASSED_STATUSES.includes(row.status)) { @@ -48,9 +35,9 @@ export async function getBuildsStats(prisma: PrismaService, buildIds: string[]): } else if (row.status === TestStatus.failed) { buildStats.failedCount += count; } - } - for (const row of mergeBuilds) { - stats.get(row.buildId)!.merge = true; + if (row.merge) { + buildStats.merge = true; + } } return stats; } diff --git a/src/builds/builds.controller.spec.ts b/src/builds/builds.controller.spec.ts index b7d05674..aa21b91d 100644 --- a/src/builds/builds.controller.spec.ts +++ b/src/builds/builds.controller.spec.ts @@ -1,5 +1,5 @@ -import { NotFoundException } from '@nestjs/common'; -import { Test, TestingModule } from '@nestjs/testing'; +import { BadRequestException, NotFoundException } from '@nestjs/common'; +import { Test } from '@nestjs/testing'; import { BuildsController } from './builds.controller'; import { BuildsService } from './builds.service'; import { PrismaService } from '../prisma/prisma.service'; @@ -7,130 +7,11 @@ import { ApiGuard } from '../auth/guards/api.guard'; import { JwtAuthGuard } from '../auth/guards/auth.guard'; import { ProjectsService } from '../projects/projects.service'; import { TEST_BUILD, TEST_PROJECT } from '../_data_'; -import { Build, User } from '@prisma/client'; +import { User } from '@prisma/client'; import { BuildDto } from './dto/build.dto'; -import { CreateBuildDto } from './dto/build-create.dto'; import { ModifyBuildDto } from './dto/build-modify.dto'; -import { EventsGateway } from '../shared/events/events.gateway'; - -const initController = async ({ - projectFindOneMock = jest.fn(), - buildFindOrCreateMock = jest.fn(), - buildIncrementBuildNumberMock = jest.fn(), - eventBuildCreatedMock = jest.fn(), - deleteOldBuilds = jest.fn(), - buildUpdateMock = jest.fn(), -}) => { - const module: TestingModule = await Test.createTestingModule({ - controllers: [BuildsController], - providers: [ - { - provide: ProjectsService, - useValue: { - findOne: projectFindOneMock, - }, - }, - { - provide: BuildsService, - useValue: { - findOrCreate: buildFindOrCreateMock, - incrementBuildNumber: buildIncrementBuildNumberMock, - deleteOldBuilds: deleteOldBuilds, - update: buildUpdateMock, - }, - }, - { provide: EventsGateway, useValue: { buildCreated: eventBuildCreatedMock } }, - { provide: PrismaService, useValue: {} }, - { provide: ApiGuard, useValue: {} }, - { provide: JwtAuthGuard, useValue: {} }, - ], - }).compile(); - - return module.get(BuildsController); -}; -describe('Builds Controller', () => { - let controller: BuildsController; - - const createBuildDto: CreateBuildDto = { - ciBuildId: 'ciBuildId', - branchName: 'branchName', - project: 'name', - }; - const project = TEST_PROJECT; - const buildWithNumber: Build = { - ...TEST_BUILD, - number: 12, - }; - - beforeEach(async () => {}); - - it('should be defined', async () => { - controller = await initController({}); - expect(controller).toBeDefined(); - }); - - it('should create new build', async () => { - const projectFindOneMock = jest.fn().mockResolvedValueOnce(project); - const buildFindOrCreateMock = jest.fn().mockResolvedValueOnce(buildWithNumber); - const deleteOldBuilds = jest.fn(); - controller = await initController({ - projectFindOneMock, - buildFindOrCreateMock, - deleteOldBuilds, - }); - - const result = await controller.create(createBuildDto); - - expect(result).toStrictEqual(new BuildDto(buildWithNumber)); - expect(deleteOldBuilds).toHaveBeenCalledWith(project.id, project.maxBuildAllowed, createBuildDto.ciBuildId); - }); - - it('should reuse build', async () => { - const eventBuildCreatedMock = jest.fn(); - const projectFindOneMock = jest.fn().mockResolvedValueOnce(project); - const buildFindOrCreateMock = jest.fn().mockResolvedValueOnce(buildWithNumber); - const deleteOldBuilds = jest.fn(); - controller = await initController({ - projectFindOneMock, - buildFindOrCreateMock, - eventBuildCreatedMock, - deleteOldBuilds, - }); - - const result = await controller.create(createBuildDto); - - expect(result).toStrictEqual(new BuildDto(buildWithNumber)); - }); - - describe('update', () => { - it.each([ - ['no body', undefined], - ['null body', null], - ['empty body', {}], - ])('should stop build when called with %s', async (_, body) => { - const buildUpdateMock = jest.fn().mockResolvedValueOnce(new BuildDto(TEST_BUILD)); - controller = await initController({ buildUpdateMock }); - - await controller.update(TEST_BUILD.id, body as ModifyBuildDto); - - expect(buildUpdateMock).toHaveBeenCalledWith(TEST_BUILD.id, { isRunning: false }); - }); - - it.each([[{ isRunning: false }], [{ isRunning: true }], [{ ciBuildId: 'ciBuildId' }]])( - 'should pass %o as is', - async (body) => { - const buildUpdateMock = jest.fn().mockResolvedValueOnce(new BuildDto(TEST_BUILD)); - controller = await initController({ buildUpdateMock }); - - await controller.update(TEST_BUILD.id, body); - expect(buildUpdateMock).toHaveBeenCalledWith(TEST_BUILD.id, body); - } - ); - }); -}); - -describe('Builds Controller handlers', () => { +describe('Builds Controller', () => { let controller: BuildsController; const buildsService = { findMany: jest.fn(), @@ -138,15 +19,16 @@ describe('Builds Controller handlers', () => { remove: jest.fn(), update: jest.fn(), approve: jest.fn(), - deleteOldBuilds: jest.fn(), + scheduleRetention: jest.fn(), findOrCreate: jest.fn(), }; const projectsService = { findOne: jest.fn() }; const buildDto = new BuildDto(TEST_BUILD); + const retentionArgs = [TEST_PROJECT.id, TEST_PROJECT.maxBuildAllowed, TEST_BUILD.id]; beforeEach(async () => { jest.resetAllMocks(); - const module: TestingModule = await Test.createTestingModule({ + const module = await Test.createTestingModule({ controllers: [BuildsController], providers: [ { provide: BuildsService, useValue: buildsService }, @@ -157,7 +39,7 @@ describe('Builds Controller handlers', () => { ], }).compile(); - controller = module.get(BuildsController); + controller = module.get(BuildsController); }); it('lists a page of builds with the filters it was given', async () => { @@ -178,6 +60,15 @@ describe('Builds Controller handlers', () => { expect(buildsService.findMany).toHaveBeenCalledWith('projectId', 10, 0, undefined, undefined); }); + it.each([ + [10, -1], + [-1, 0], + ])('answers 400 for take=%p skip=%p without querying', async (take, skip) => { + await expect(controller.get('projectId', take, skip)).rejects.toThrow(BadRequestException); + + expect(buildsService.findMany).not.toHaveBeenCalled(); + }); + it('returns the details of one build', async () => { buildsService.findOne.mockResolvedValue(buildDto); @@ -198,11 +89,29 @@ describe('Builds Controller handlers', () => { await expect(controller.remove('missing')).rejects.toThrow(new NotFoundException('Build not found: missing')); }); - it('updates a build with the body', async () => { - buildsService.update.mockResolvedValue(buildDto); + describe('update', () => { + it.each([ + ['no body', undefined], + ['null body', null], + ['empty body', {}], + ])('should stop build when called with %s', async (_, body) => { + buildsService.update.mockResolvedValue(buildDto); + + await expect(controller.update(TEST_BUILD.id, body as ModifyBuildDto)).resolves.toBe(buildDto); + + expect(buildsService.update).toHaveBeenCalledWith(TEST_BUILD.id, { isRunning: false }); + }); - await expect(controller.update(TEST_BUILD.id, { isRunning: false })).resolves.toBe(buildDto); - expect(buildsService.update).toHaveBeenCalledWith(TEST_BUILD.id, { isRunning: false }); + it.each([[{ isRunning: false }], [{ isRunning: true }], [{ ciBuildId: 'ciBuildId' }]])( + 'should pass %o as is', + async (body) => { + buildsService.update.mockResolvedValue(buildDto); + + await expect(controller.update(TEST_BUILD.id, body)).resolves.toBe(buildDto); + + expect(buildsService.update).toHaveBeenCalledWith(TEST_BUILD.id, body); + } + ); }); it.each([true, false])('approves a build with merge=%s', async (merge) => { @@ -214,32 +123,33 @@ describe('Builds Controller handlers', () => { it("creates the build on the project's main branch when no branch is given", async () => { projectsService.findOne.mockResolvedValue(TEST_PROJECT); - buildsService.findOrCreate.mockResolvedValue(TEST_BUILD); + buildsService.findOrCreate.mockResolvedValue({ ...TEST_BUILD, number: 12 }); - const result = await controller.create({ project: TEST_PROJECT.name, ciBuildId: 'ci-1' } as CreateBuildDto); + const result = await controller.create({ project: TEST_PROJECT.name, ciBuildId: 'ci-1' }); expect(projectsService.findOne).toHaveBeenCalledWith(TEST_PROJECT.name); - expect(buildsService.deleteOldBuilds).toHaveBeenCalledWith(TEST_PROJECT.id, TEST_PROJECT.maxBuildAllowed, 'ci-1'); + expect(buildsService.scheduleRetention).toHaveBeenCalledWith(...retentionArgs); expect(buildsService.findOrCreate).toHaveBeenCalledWith({ projectId: TEST_PROJECT.id, branchName: TEST_PROJECT.mainBranchName, ciBuildId: 'ci-1', }); - expect(result).toStrictEqual(new BuildDto(TEST_BUILD)); + expect(result).toStrictEqual(new BuildDto({ ...TEST_BUILD, number: 12 })); }); - it('deletes old builds before creating the new one', async () => { + it('schedules retention after creating the build, sparing it', async () => { const calls: string[] = []; projectsService.findOne.mockResolvedValue(TEST_PROJECT); - buildsService.deleteOldBuilds.mockImplementation(async () => calls.push('deleteOldBuilds')); + buildsService.scheduleRetention.mockImplementation(() => calls.push('scheduleRetention')); buildsService.findOrCreate.mockImplementation(async () => { calls.push('findOrCreate'); return TEST_BUILD; }); - await controller.create({ project: TEST_PROJECT.id, branchName: 'feature' } as CreateBuildDto); + await controller.create({ project: TEST_PROJECT.id, branchName: 'feature' }); - expect(calls).toEqual(['deleteOldBuilds', 'findOrCreate']); + expect(calls).toEqual(['findOrCreate', 'scheduleRetention']); + expect(buildsService.scheduleRetention).toHaveBeenCalledWith(...retentionArgs); expect(buildsService.findOrCreate).toHaveBeenCalledWith({ projectId: TEST_PROJECT.id, branchName: 'feature', diff --git a/src/builds/builds.controller.ts b/src/builds/builds.controller.ts index f455b953..f80aafbc 100644 --- a/src/builds/builds.controller.ts +++ b/src/builds/builds.controller.ts @@ -13,8 +13,8 @@ import { ParseBoolPipe, forwardRef, Inject, - ValidationPipe, NotFoundException, + BadRequestException, } from '@nestjs/common'; import { BuildsService } from './builds.service'; import { JwtAuthGuard } from '../auth/guards/auth.guard'; @@ -51,13 +51,16 @@ export class BuildsController { }) @ApiBearerAuth() @UseGuards(JwtAuthGuard) - get( + async get( @Query('projectId', new ParseUUIDPipe()) projectId: string, @Query('take', new ParseIntPipe()) take: number, @Query('skip', new ParseIntPipe()) skip: number, @Query('ciBuildId') ciBuildId?: string, @Query('aroundBuildId', new ParseUUIDPipe({ optional: true })) aroundBuildId?: string ): Promise { + if (skip < 0 || take < 0) { + throw new BadRequestException('take and skip must not be negative'); + } return this.buildsService.findMany(projectId, take, skip, ciBuildId, aroundBuildId); } @@ -75,9 +78,6 @@ export class BuildsController { @UseGuards(JwtAuthGuard, RoleGuard) @Roles(Role.admin, Role.editor) async remove(@Param('id', new ParseUUIDPipe()) id: string): Promise { - // remove() answers nothing for a build that is not there, and retention - // relies on that when two requests prune the same build; a client asking - // for it by id is told it does not exist const build = await this.buildsService.remove(id); if (!build) { throw new NotFoundException(`Build not found: ${id}`); @@ -91,13 +91,13 @@ export class BuildsController { @UseGuards(ApiGuard, RoleGuard) @Roles(Role.admin, Role.editor) async create(@Body() createBuildDto: CreateBuildDto): Promise { - const project = (await this.projectService.findOne(createBuildDto.project))!; - await this.buildsService.deleteOldBuilds(project.id, project.maxBuildAllowed, createBuildDto.ciBuildId); + const project = await this.projectService.findOne(createBuildDto.project); const build = await this.buildsService.findOrCreate({ projectId: project.id, branchName: createBuildDto.branchName ?? project.mainBranchName, ciBuildId: createBuildDto.ciBuildId, }); + this.buildsService.scheduleRetention(project.id, project.maxBuildAllowed, build.id); return new BuildDto(build); } @@ -107,14 +107,8 @@ export class BuildsController { @ApiBearerAuth() @UseGuards(MixedGuard, RoleGuard) @Roles(Role.admin, Role.editor) - update( - @Param('id', new ParseUUIDPipe()) id: string, - // forbidNonWhitelisted: a body naming any other column is refused, rather - // than written to the build or silently dropped (an emptied body would stop it) - @Body(new ValidationPipe({ whitelist: true, forbidNonWhitelisted: true })) modifyBuildDto?: ModifyBuildDto - ): Promise { - // Released SDKs (sdk-js, agent-cypress, sdk-python, sdk-java) stop a build with an empty PATCH body. - // Express 5 leaves req.body undefined when there is no body, so handle both cases. + update(@Param('id', new ParseUUIDPipe()) id: string, @Body() modifyBuildDto?: ModifyBuildDto): Promise { + // Released SDKs stop a build with an empty PATCH body; Express 5 leaves req.body undefined then. if (!modifyBuildDto || Object.keys(modifyBuildDto).length === 0) { modifyBuildDto = { isRunning: false }; } diff --git a/src/builds/builds.module.ts b/src/builds/builds.module.ts index 28853a1f..e8f4e057 100644 --- a/src/builds/builds.module.ts +++ b/src/builds/builds.module.ts @@ -6,9 +6,17 @@ import { TestRunsModule } from '../test-runs/test-runs.module'; import { SharedModule } from '../shared/shared.module'; import { AuthModule } from '../auth/auth.module'; import { ProjectsModule } from '../projects/projects.module'; +import { StaticModule } from '../static/static.module'; @Module({ - imports: [SharedModule, UsersModule, forwardRef(() => TestRunsModule), AuthModule, forwardRef(() => ProjectsModule)], + imports: [ + SharedModule, + UsersModule, + forwardRef(() => TestRunsModule), + AuthModule, + forwardRef(() => ProjectsModule), + StaticModule, + ], providers: [BuildsService], controllers: [BuildsController], exports: [BuildsService], diff --git a/src/builds/builds.service.spec.ts b/src/builds/builds.service.spec.ts index 83603c02..261fdbc4 100644 --- a/src/builds/builds.service.spec.ts +++ b/src/builds/builds.service.spec.ts @@ -1,4 +1,4 @@ -import { Test, TestingModule } from '@nestjs/testing'; +import { Test } from '@nestjs/testing'; import { BuildsService } from './builds.service'; import { PrismaService } from '../prisma/prisma.service'; import { TestRunsService } from '../test-runs/test-runs.service'; @@ -7,92 +7,72 @@ import { Build, Prisma, TestRun, TestStatus } from '@prisma/client'; import { mocked, MockedObject } from 'jest-mock'; import { BuildDto } from './dto/build.dto'; import { ModifyBuildDto } from './dto/build-modify.dto'; -import { NotFoundException } from '@nestjs/common'; -import { ProjectsService } from '../projects/projects.service'; -import { generateTestRun, TEST_PROJECT } from '../_data_'; -import { Prisma as PrismaRuntime } from '@prisma/client'; -const { PrismaClientKnownRequestError } = PrismaRuntime; +import { ConflictException, NotFoundException } from '@nestjs/common'; +import { StaticService } from '../static/static.service'; +import { generateTestRun } from '../_data_'; + +const { PrismaClientKnownRequestError } = Prisma; jest.mock('./dto/build.dto'); -const initService = async ({ - buildFindManyMock = jest.fn(), - buildCreateMock = jest.fn(), - buildUpdateMock = jest.fn(), - // a row for whatever id is asked: findOne answers 404 for a missing build - buildFindUniqueMock = jest.fn().mockImplementation(async ({ where }) => ({ id: where.id })), - buildFindFirstMock = jest.fn(), - buildDeleteMock = jest.fn(), - buildUpsertMock = jest.fn(), - buildCountMock = jest.fn(), - testRunDeleteMock = jest.fn(), - testRunApproveMock = jest.fn(), - testRunFindManyMock = jest.fn(), - testRunGroupByMock = jest.fn().mockResolvedValue([]), - eventsBuildUpdatedMock = jest.fn(), - eventsBuildCreatedMock = jest.fn(), - eventBuildDeletedMock = jest.fn(), - projectFindOneMock = jest.fn(), - projectUpdateMock = jest.fn(), -}) => { - const module: TestingModule = await Test.createTestingModule({ +const prismaError = (code: string) => new PrismaClientKnownRequestError(code, { code, clientVersion: '5' }); +const tick = () => new Promise((resolve) => setImmediate(resolve)); +const silentLogger = () => ({ warn: jest.fn(), debug: jest.fn(), log: jest.fn() }); + +const setup = async () => { + const prisma = { + project: { update: jest.fn() }, + build: { + findMany: jest.fn(), + create: jest.fn(), + update: jest.fn(), + findUnique: jest.fn(async ({ where }): Promise => ({ id: where.id })), + findUniqueOrThrow: jest.fn(), + findFirst: jest.fn(), + delete: jest.fn(), + upsert: jest.fn(), + count: jest.fn(), + updateManyAndReturn: jest.fn(), + }, + testRun: { + groupBy: jest.fn().mockResolvedValue([]), + findMany: jest.fn().mockResolvedValue([]), + deleteMany: jest.fn(), + }, + $transaction: jest.fn(), + }; + prisma.$transaction.mockImplementation((fn: (tx: typeof prisma) => unknown) => fn(prisma)); + const events = { + buildUpdated: jest.fn(), + buildCreated: jest.fn(), + buildDeleted: jest.fn(), + testRunDeleted: jest.fn(), + }; + const testRuns = { approve: jest.fn(), delete: jest.fn(), findMany: jest.fn() }; + const deleteImage = jest.fn().mockResolvedValue(true); + + const module = await Test.createTestingModule({ providers: [ BuildsService, + { provide: PrismaService, useValue: prisma }, + { provide: TestRunsService, useValue: testRuns }, + { provide: EventsGateway, useValue: events }, { - provide: PrismaService, + provide: StaticService, useValue: { - project: { - update: projectUpdateMock, - }, - build: { - findMany: buildFindManyMock, - create: buildCreateMock, - update: buildUpdateMock, - findUnique: buildFindUniqueMock, - findFirst: buildFindFirstMock, - delete: buildDeleteMock, - upsert: buildUpsertMock, - count: buildCountMock, - }, - testRun: { - groupBy: testRunGroupByMock, + deleteImages: async (names: Iterable) => { + for (const name of names) if (name) await deleteImage(name); }, }, }, - { - provide: TestRunsService, - useValue: { - approve: testRunApproveMock, - delete: testRunDeleteMock, - findMany: testRunFindManyMock, - }, - }, - { - provide: EventsGateway, - useValue: { - buildUpdated: eventsBuildUpdatedMock, - buildCreated: eventsBuildCreatedMock, - buildDeleted: eventBuildDeletedMock, - }, - }, - { - provide: ProjectsService, - useValue: { - findOne: projectFindOneMock, - }, - }, ], }).compile(); - return module.get(BuildsService); + return { service: module.get(BuildsService), prisma, events, testRuns, deleteImage }; }; describe('BuildsService', () => { - let service: BuildsService; - - const build: Build & { - testRuns: TestRun[]; - } = { + const build: Build & { testRuns: TestRun[] } = { id: 'a9385fc1-884d-4f9f-915e-40da0e7773d5', ciBuildId: null, number: null, @@ -105,6 +85,8 @@ describe('BuildsService', () => { isRunning: true, testRuns: [generateTestRun()], }; + const { testRuns: buildRuns, ...buildRow } = build; + const [run] = buildRuns; const buildDto: BuildDto = { id: 'a9385fc1-884d-4f9f-915e-40da0e7773d5', @@ -122,18 +104,22 @@ describe('BuildsService', () => { isRunning: true, merge: false, }; + const returnDtoOnce = () => mocked(BuildDto).mockReturnValueOnce(buildDto as MockedObject); + const buildOrder = [{ createdAt: 'desc' }, { id: 'desc' }]; + const unhandledRejections = () => { + const unhandled = jest.fn(); + process.on('unhandledRejection', unhandled); + return { unhandled, stop: () => process.off('unhandledRejection', unhandled) }; + }; it('findOne', async () => { - const buildFindUniqueMock = jest.fn().mockResolvedValueOnce(build); - const testRunGroupByMock = jest - .fn() - .mockResolvedValueOnce([ - { buildId: 'someId', status: TestStatus.ok, _count: { _all: 2 } }, - { buildId: 'someId', status: TestStatus.unresolved, _count: { _all: 1 } }, - ]) - .mockResolvedValueOnce([{ buildId: 'someId' }]); - mocked(BuildDto).mockReturnValueOnce(buildDto as MockedObject); - service = await initService({ buildFindUniqueMock, testRunGroupByMock }); + const { service, prisma } = await setup(); + prisma.build.findUnique.mockResolvedValueOnce(build); + prisma.testRun.groupBy.mockResolvedValueOnce([ + { buildId: 'someId', status: TestStatus.ok, merge: false, _count: { _all: 2 } }, + { buildId: 'someId', status: TestStatus.unresolved, merge: true, _count: { _all: 1 } }, + ]); + returnDtoOnce(); const result = await service.findOne('someId'); @@ -147,309 +133,417 @@ describe('BuildsService', () => { }); it('findOne throws NotFoundException for an unknown id', async () => { - const buildFindUniqueMock = jest.fn().mockResolvedValueOnce(null); - const testRunGroupByMock = jest.fn().mockResolvedValue([]); - service = await initService({ buildFindUniqueMock, testRunGroupByMock }); + const { service, prisma } = await setup(); + prisma.build.findUnique.mockResolvedValueOnce(null); await expect(service.findOne('missing')).rejects.toThrow(new NotFoundException('Build not found: missing')); }); it('update writes only the columns a client may change', async () => { - const buildUpdateMock = jest.fn(); - service = await initService({ buildUpdateMock, eventsBuildUpdatedMock: jest.fn() }); + const { service, prisma } = await setup(); service.findOne = jest.fn(); await service.update('id', { isRunning: false, ciBuildId: 'ci', branchName: 'other' } as ModifyBuildDto); - expect(buildUpdateMock.mock.calls[0][0].data).toStrictEqual({ ciBuildId: 'ci', isRunning: false }); + expect(prisma.build.update.mock.calls[0][0].data).toStrictEqual({ ciBuildId: 'ci', isRunning: false }); }); - it('findMany', async () => { - const buildFindManyMock = jest.fn().mockResolvedValueOnce([build]); - const buildCountMock = jest.fn().mockResolvedValueOnce(33); - const projectId = 'someId'; - mocked(BuildDto).mockReturnValueOnce(buildDto as MockedObject); - service = await initService({ buildFindManyMock, buildCountMock }); + it('update announces the change and returns the build', async () => { + const { service, prisma, events } = await setup(); + returnDtoOnce(); - const result = await service.findMany(projectId, 10, 20); + const result = await service.update('some id', { isRunning: false }); - expect(buildCountMock).toHaveBeenCalledWith({ - where: { projectId }, - }); - expect(buildFindManyMock).toHaveBeenCalledWith({ - take: 10, - skip: 20, - orderBy: [{ createdAt: 'desc' }, { id: 'desc' }], - where: { projectId }, - }); - expect(result).toEqual({ - data: [buildDto], - total: 33, - take: 10, - skip: 20, - }); + expect(prisma.build.update).toHaveBeenCalledWith({ where: { id: 'some id' }, data: { isRunning: false } }); + expect(events.buildUpdated).toHaveBeenCalledWith('some id'); + expect(result).toBe(buildDto); }); - it('findMany filtered by ciBuildId', async () => { - const buildFindManyMock = jest.fn().mockResolvedValueOnce([build]); - const buildCountMock = jest.fn().mockResolvedValueOnce(1); + describe('findMany', () => { const projectId = 'someId'; - mocked(BuildDto).mockReturnValueOnce(buildDto as MockedObject); - service = await initService({ buildFindManyMock, buildCountMock }); + const ciBuildId = { contains: 'Onboarding', mode: Prisma.QueryMode.insensitive }; - await service.findMany(projectId, 10, 0, 'Onboarding'); + it('pages the builds of a project', async () => { + const { service, prisma } = await setup(); + prisma.build.findMany.mockResolvedValueOnce([build]); + prisma.build.count.mockResolvedValueOnce(33); + returnDtoOnce(); - const where = { - projectId, - ciBuildId: { contains: 'Onboarding', mode: Prisma.QueryMode.insensitive }, - }; - expect(buildCountMock).toHaveBeenCalledWith({ where }); - expect(buildFindManyMock).toHaveBeenCalledWith({ - take: 10, - skip: 0, - orderBy: [{ createdAt: 'desc' }, { id: 'desc' }], - where, + const result = await service.findMany(projectId, 10, 20); + + expect(prisma.build.count).toHaveBeenCalledWith({ where: { projectId } }); + expect(prisma.build.findMany).toHaveBeenCalledWith({ + take: 10, + skip: 20, + orderBy: buildOrder, + where: { projectId }, + }); + expect(result).toEqual({ data: [buildDto], total: 33, take: 10, skip: 20 }); }); - }); - it('findMany ignores an empty ciBuildId', async () => { - const buildFindManyMock = jest.fn().mockResolvedValueOnce([build]); - const buildCountMock = jest.fn().mockResolvedValueOnce(33); - const projectId = 'someId'; - mocked(BuildDto).mockReturnValueOnce(buildDto as MockedObject); - service = await initService({ buildFindManyMock, buildCountMock }); + it.each([ + ['ciBuildId', 'Onboarding', { projectId, ciBuildId }], + ['an empty ciBuildId', '', { projectId }], + ])('filters by %s', async (_, search, where) => { + const { service, prisma } = await setup(); + prisma.build.findMany.mockResolvedValueOnce([build]); + prisma.build.count.mockResolvedValueOnce(1); + returnDtoOnce(); + + await service.findMany(projectId, 10, 0, search); + + expect(prisma.build.count).toHaveBeenCalledWith({ where }); + expect(prisma.build.findMany).toHaveBeenCalledWith({ take: 10, skip: 0, orderBy: buildOrder, where }); + }); + + describe('around a build', () => { + const aroundBuildId = '7428548b-863f-40e8-b26d-2187cc98afa1'; + const createdAt = new Date('2026-09-01T10:00:00.000Z'); + const ahead = [{ createdAt: { gt: createdAt } }, { createdAt, id: { gt: aroundBuildId } }]; + + const around = async (found: unknown, counts: number[]) => { + const ctx = await setup(); + ctx.prisma.build.findFirst.mockResolvedValueOnce(found); + ctx.prisma.build.findMany.mockResolvedValueOnce([build]); + counts.forEach((count) => ctx.prisma.build.count.mockResolvedValueOnce(count)); + returnDtoOnce(); + return ctx; + }; + + it('returns the page that holds the build rather than the one asked for', async () => { + const { service, prisma } = await around({ id: aroundBuildId, createdAt }, [35, 50]); + + const result = await service.findMany(projectId, 10, 0, undefined, aroundBuildId); + + expect(prisma.build.findFirst).toHaveBeenCalledWith({ + where: { projectId, id: aroundBuildId }, + select: { id: true, createdAt: true }, + }); + expect(prisma.build.count).toHaveBeenNthCalledWith(1, { where: { projectId, OR: ahead } }); + expect(prisma.build.findMany).toHaveBeenCalledWith({ + take: 10, + skip: 30, + orderBy: buildOrder, + where: { projectId }, + }); + expect(result.skip).toBe(30); + }); - await service.findMany(projectId, 10, 0, ''); + it('counts only what the search left, so the page matches the list on screen', async () => { + const { service, prisma } = await around({ id: aroundBuildId, createdAt }, [2, 4]); - expect(buildCountMock).toHaveBeenCalledWith({ where: { projectId } }); - expect(buildFindManyMock).toHaveBeenCalledWith({ - take: 10, - skip: 0, - orderBy: [{ createdAt: 'desc' }, { id: 'desc' }], - where: { projectId }, - }); - }); + await service.findMany(projectId, 2, 0, 'Onboarding', aroundBuildId); - describe('findMany around a build', () => { - const aroundBuildId = '7428548b-863f-40e8-b26d-2187cc98afa1'; - const createdAt = new Date('2026-09-01T10:00:00.000Z'); - const projectId = 'someId'; + expect(prisma.build.findFirst).toHaveBeenCalledWith({ + where: { projectId, ciBuildId, id: aroundBuildId }, + select: { id: true, createdAt: true }, + }); + expect(prisma.build.count).toHaveBeenNthCalledWith(1, { where: { projectId, ciBuildId, OR: ahead } }); + expect(prisma.build.findMany).toHaveBeenCalledWith(expect.objectContaining({ skip: 2 })); + }); - it('returns the page that holds the build rather than the one asked for', async () => { - const buildFindFirstMock = jest.fn().mockResolvedValueOnce({ id: aroundBuildId, createdAt }); - const buildFindManyMock = jest.fn().mockResolvedValueOnce([build]); - // 35 builds sit ahead of it, then the total - const buildCountMock = jest.fn().mockResolvedValueOnce(35).mockResolvedValueOnce(50); - mocked(BuildDto).mockReturnValueOnce(buildDto as MockedObject); - service = await initService({ buildFindFirstMock, buildFindManyMock, buildCountMock }); + it('falls back to the page asked for when the build is not in the list', async () => { + const { service, prisma } = await around(null, [50]); - const result = await service.findMany(projectId, 10, 0, undefined, aroundBuildId); + const result = await service.findMany(projectId, 10, 20, undefined, aroundBuildId); - expect(buildFindFirstMock).toHaveBeenCalledWith({ - where: { projectId, id: aroundBuildId }, - select: { id: true, createdAt: true }, - }); - expect(buildCountMock).toHaveBeenNthCalledWith(1, { - where: { - projectId, - OR: [{ createdAt: { gt: createdAt } }, { createdAt, id: { gt: aroundBuildId } }], - }, + expect(prisma.build.count).toHaveBeenCalledTimes(1); + expect(prisma.build.findMany).toHaveBeenCalledWith(expect.objectContaining({ skip: 20 })); + expect(result.skip).toBe(20); }); - // the fourth page of ten starts at 30, and 35 is on it - expect(buildFindManyMock).toHaveBeenCalledWith({ - take: 10, - skip: 30, - orderBy: [{ createdAt: 'desc' }, { id: 'desc' }], - where: { projectId }, - }); - expect(result.skip).toBe(30); - }); - it('counts only what the search left, so the page matches the list on screen', async () => { - const buildFindFirstMock = jest.fn().mockResolvedValueOnce({ id: aroundBuildId, createdAt }); - const buildFindManyMock = jest.fn().mockResolvedValueOnce([build]); - const buildCountMock = jest.fn().mockResolvedValueOnce(2).mockResolvedValueOnce(4); - mocked(BuildDto).mockReturnValueOnce(buildDto as MockedObject); - service = await initService({ buildFindFirstMock, buildFindManyMock, buildCountMock }); + it('does not divide by a page of no size', async () => { + const { service, prisma } = await around({ id: aroundBuildId, createdAt }, [0]); - await service.findMany(projectId, 2, 0, 'Onboarding', aroundBuildId); + const result = await service.findMany(projectId, 0, 0, undefined, aroundBuildId); - const ciBuildId = { contains: 'Onboarding', mode: Prisma.QueryMode.insensitive }; - expect(buildFindFirstMock).toHaveBeenCalledWith({ - where: { projectId, ciBuildId, id: aroundBuildId }, - select: { id: true, createdAt: true }, + expect(result.skip).toBe(0); + expect(prisma.build.findMany).toHaveBeenCalledWith(expect.objectContaining({ skip: 0 })); }); - expect(buildCountMock).toHaveBeenNthCalledWith(1, { - where: { - projectId, - ciBuildId, - OR: [{ createdAt: { gt: createdAt } }, { createdAt, id: { gt: aroundBuildId } }], - }, - }); - expect(buildFindManyMock).toHaveBeenCalledWith(expect.objectContaining({ skip: 2 })); }); + }); - it('falls back to the page asked for when the build is not in the list', async () => { - // deleted, another project's, or filtered out by the search: a stale link - // should still open the list rather than fail - const buildFindFirstMock = jest.fn().mockResolvedValueOnce(null); - const buildFindManyMock = jest.fn().mockResolvedValueOnce([build]); - const buildCountMock = jest.fn().mockResolvedValueOnce(50); - mocked(BuildDto).mockReturnValueOnce(buildDto as MockedObject); - service = await initService({ buildFindFirstMock, buildFindManyMock, buildCountMock }); + describe('remove', () => { + const removedBuild = { ...buildRow, testRuns: [run] }; - const result = await service.findMany(projectId, 10, 20, undefined, aroundBuildId); + it('removes a build, its runs, their files and announces it', async () => { + const { service, prisma, events, testRuns, deleteImage } = await setup(); + prisma.build.findUnique.mockResolvedValueOnce(buildRow); + prisma.build.delete.mockResolvedValueOnce(buildRow); + prisma.testRun.findMany.mockResolvedValueOnce([run]); - expect(buildCountMock).toHaveBeenCalledTimes(1); - expect(buildFindManyMock).toHaveBeenCalledWith(expect.objectContaining({ skip: 20 })); - expect(result.skip).toBe(20); - }); + const result = await service.remove(build.id); - it('does not divide by a page of no size', async () => { - const buildFindFirstMock = jest.fn().mockResolvedValueOnce({ id: aroundBuildId, createdAt }); - const buildFindManyMock = jest.fn().mockResolvedValueOnce([build]); - const buildCountMock = jest.fn().mockResolvedValueOnce(0); - mocked(BuildDto).mockReturnValueOnce(buildDto as MockedObject); - service = await initService({ buildFindFirstMock, buildFindManyMock, buildCountMock }); + expect(prisma.build.findUnique).toHaveBeenCalledWith({ where: { id: build.id } }); + expect(prisma.testRun.findMany).toHaveBeenCalledWith({ where: { buildId: build.id }, take: 500 }); + expect(prisma.testRun.deleteMany).toHaveBeenCalledWith({ where: { id: { in: [run.id] } } }); + expect(deleteImage).toHaveBeenCalledTimes(2); + expect(deleteImage).toHaveBeenCalledWith(run.imageName); + expect(deleteImage).toHaveBeenCalledWith(run.diffName); + expect(events.testRunDeleted).toHaveBeenCalledWith(run); + expect(prisma.build.delete).toHaveBeenCalledWith({ where: { id: build.id } }); + expect(mocked(BuildDto).mock.calls.at(-1)).toEqual([removedBuild]); + expect(events.buildDeleted).toHaveBeenCalledTimes(1); + expect(result).toEqual(removedBuild); + expect(testRuns.delete).not.toHaveBeenCalled(); + }); + + it('deletes the rows of a chunk before its files', async () => { + const { service, prisma, deleteImage } = await setup(); + const calls: string[] = []; + prisma.build.findUnique.mockResolvedValueOnce(buildRow); + prisma.testRun.findMany.mockResolvedValueOnce([run]); + prisma.testRun.deleteMany.mockImplementation(async () => void calls.push('rows')); + deleteImage.mockImplementation(async () => void calls.push('file')); - const result = await service.findMany(projectId, 0, 0, undefined, aroundBuildId); + await service.remove(build.id); - expect(result.skip).toBe(0); - expect(buildFindManyMock).toHaveBeenCalledWith(expect.objectContaining({ skip: 0 })); + expect(calls).toEqual(['rows', 'file', 'file']); }); - }); - describe('remove', () => { - it('should remove a build successfully', async () => { - const buildFindUniqueMock = jest.fn().mockResolvedValueOnce(build); - const buildFindManyMock = jest.fn().mockImplementation(() => Promise.resolve(build)); - const buildDeleteMock = jest.fn().mockImplementation(() => Promise.resolve(build)); - const testRunDeleteMock = jest.fn(); - const eventBuildDeletedMock = jest.fn(); - service = await initService({ - buildFindUniqueMock, - buildDeleteMock, - testRunDeleteMock, - eventBuildDeletedMock, - buildFindManyMock, - }); + it('removes the runs chunk by chunk', async () => { + const { service, prisma } = await setup(); + const a = generateTestRun({ id: 'a' }); + const b = generateTestRun({ id: 'b' }); + prisma.build.findUnique.mockResolvedValueOnce(buildRow); + prisma.testRun.findMany.mockResolvedValueOnce([a]).mockResolvedValueOnce([b]); - await service.remove(build.id); + const result = await service.remove(build.id); - expect(buildFindUniqueMock).toHaveBeenCalledWith({ - where: { id: build.id }, - include: { - testRuns: true, - }, - }); - expect(testRunDeleteMock).toHaveBeenCalledWith(build.testRuns[0].id); - expect(eventBuildDeletedMock).toHaveBeenCalledWith(new BuildDto(build)); - expect(buildDeleteMock).toHaveBeenCalledWith({ - where: { id: build.id }, - }); + expect(prisma.testRun.deleteMany).toHaveBeenCalledTimes(2); + expect(prisma.testRun.deleteMany).toHaveBeenNthCalledWith(1, { where: { id: { in: ['a'] } } }); + expect(prisma.testRun.deleteMany).toHaveBeenNthCalledWith(2, { where: { id: { in: ['b'] } } }); + expect(result?.testRuns).toEqual([a, b]); }); it('should handle undefined ID gracefully', async () => { - const buildFindUniqueMock = jest.fn(); - const loggerWarnMock = jest.fn(); - service = await initService({ buildFindUniqueMock }); - (service as any).logger = { warn: loggerWarnMock }; // Mock the logger + const { service, prisma } = await setup(); + const warn = jest.fn(); + (service as any).logger = { warn }; await service.remove(undefined as unknown as string); - expect(loggerWarnMock).toHaveBeenCalledWith('Attempted to remove build with undefined ID.'); - expect(buildFindUniqueMock).not.toHaveBeenCalled(); + expect(warn).toHaveBeenCalledWith('Attempted to remove build with undefined ID.'); + expect(prisma.build.findUnique).not.toHaveBeenCalled(); }); - }); - describe('remove (characterization)', () => { it('does nothing when the build does not exist', async () => { - const buildFindUniqueMock = jest.fn().mockResolvedValueOnce(null); - const buildDeleteMock = jest.fn(); - const eventBuildDeletedMock = jest.fn(); - service = await initService({ buildFindUniqueMock, buildDeleteMock, eventBuildDeletedMock }); + const { service, prisma, events } = await setup(); + prisma.build.findUnique.mockResolvedValueOnce(null); await expect(service.remove('missing')).resolves.toBeUndefined(); - expect(buildDeleteMock).not.toHaveBeenCalled(); - expect(eventBuildDeletedMock).not.toHaveBeenCalled(); + expect(prisma.build.delete).not.toHaveBeenCalled(); + expect(events.buildDeleted).not.toHaveBeenCalled(); }); it('treats a build deleted in the meantime (P2025) as done, without an event', async () => { - const buildFindUniqueMock = jest.fn().mockResolvedValueOnce(build); - const buildDeleteMock = jest - .fn() - .mockRejectedValueOnce(new PrismaClientKnownRequestError('gone', { code: 'P2025', clientVersion: '5' })); - const testRunDeleteMock = jest.fn(); - const eventBuildDeletedMock = jest.fn(); - service = await initService({ buildFindUniqueMock, buildDeleteMock, testRunDeleteMock, eventBuildDeletedMock }); + const { service, prisma, events } = await setup(); + prisma.build.findUnique.mockResolvedValueOnce(buildRow); + prisma.build.delete.mockRejectedValueOnce(prismaError('P2025')); + prisma.testRun.findMany.mockResolvedValueOnce([run]); await expect(service.remove(build.id)).resolves.toBeUndefined(); - expect(testRunDeleteMock).toHaveBeenCalledWith(build.testRuns[0].id); - expect(eventBuildDeletedMock).not.toHaveBeenCalled(); + expect(prisma.testRun.deleteMany).toHaveBeenCalledWith({ where: { id: { in: [run.id] } } }); + expect(events.buildDeleted).not.toHaveBeenCalled(); }); it.each([ - ['another Prisma error', new PrismaClientKnownRequestError('fk', { code: 'P2003', clientVersion: '5' })], + ['a Prisma error other than P2003 and P2025', prismaError('P2034')], ['any other error', new Error('connection lost')], - ])('swallows %s on delete and still reports the build as deleted', async (_, error) => { - const buildFindUniqueMock = jest.fn().mockResolvedValueOnce(build); - const buildDeleteMock = jest.fn().mockRejectedValueOnce(error); - const eventBuildDeletedMock = jest.fn(); - service = await initService({ buildFindUniqueMock, buildDeleteMock, eventBuildDeletedMock }); + ])('rethrows %s from build.delete and announces nothing', async (_, error) => { + const { service, prisma, events } = await setup(); + prisma.build.findUnique.mockResolvedValueOnce(buildRow); + prisma.build.delete.mockRejectedValueOnce(error); + + await expect(service.remove(build.id)).rejects.toBe(error); + + expect(prisma.build.delete).toHaveBeenCalledTimes(1); + expect(events.buildDeleted).not.toHaveBeenCalled(); + }); + + it('deletes a run that arrived meanwhile and retries (P2003)', async () => { + const { service, prisma, events } = await setup(); + const r1 = generateTestRun({ id: 'r1' }); + const r2 = generateTestRun({ id: 'r2' }); + prisma.build.findUnique.mockResolvedValueOnce(buildRow); + prisma.build.delete.mockRejectedValueOnce(prismaError('P2003')).mockResolvedValueOnce(buildRow); + prisma.testRun.findMany.mockResolvedValueOnce([r1]).mockResolvedValueOnce([]).mockResolvedValueOnce([r2]); const result = await service.remove(build.id); - // NOTE: current behavior — only P2025 is handled; every other delete error - // is swallowed and the build is announced as deleted although it is not. - expect(result).toBe(build); - expect(eventBuildDeletedMock).toHaveBeenCalledTimes(1); + expect(prisma.build.delete).toHaveBeenCalledTimes(2); + expect(result?.testRuns).toEqual([r1, r2]); + expect(events.buildDeleted).toHaveBeenCalledTimes(1); }); - }); - describe('incrementBuildNumber', () => { - it("bumps the project's build counter and gives the build the new number", async () => { - const projectUpdateMock = jest.fn().mockResolvedValueOnce({ ...TEST_PROJECT, buildsCounter: 43 }); - const buildUpdateMock = jest.fn().mockResolvedValueOnce({ ...build, number: 43 }); - service = await initService({ projectUpdateMock, buildUpdateMock }); + it('answers 409 when runs keep arriving', async () => { + const { service, prisma, events } = await setup(); + prisma.build.findUnique.mockResolvedValueOnce(buildRow); + prisma.build.delete.mockRejectedValue(prismaError('P2003')); - const result = await service.incrementBuildNumber(build.id, 'projectId'); + await expect(service.remove(build.id)).rejects.toThrow(ConflictException); - expect(projectUpdateMock).toHaveBeenCalledWith({ - where: { id: 'projectId' }, - data: { buildsCounter: { increment: 1 } }, + expect(prisma.build.delete).toHaveBeenCalledTimes(3); + expect(events.buildDeleted).not.toHaveBeenCalled(); + }); + + it('skips the file names a run does not have', async () => { + const { service, prisma, deleteImage } = await setup(); + prisma.build.findUnique.mockResolvedValueOnce(buildRow); + prisma.testRun.findMany.mockResolvedValueOnce([ + generateTestRun({ imageName: null as unknown as string, diffName: null }), + ]); + + await service.remove(build.id); + + expect(deleteImage).not.toHaveBeenCalled(); + }); + + it('does not look at activity without a cutoff', async () => { + const { service, prisma } = await setup(); + prisma.build.findUnique.mockResolvedValueOnce(buildRow); + + await service.remove(build.id); + + expect(prisma.build.findFirst).not.toHaveBeenCalled(); + }); + + describe('only if the build is idle', () => { + const since = new Date('2026-01-01T00:00:00Z'); + + it('keeps a build in use: no rows, files or events', async () => { + const { service, prisma, events, deleteImage } = await setup(); + prisma.build.findUnique.mockResolvedValueOnce(buildRow); + prisma.build.findFirst.mockResolvedValue(null); + prisma.testRun.findMany.mockResolvedValueOnce([run]); + + await expect(service.remove(build.id, { onlyIfIdleSince: since })).resolves.toBeUndefined(); + + expect(prisma.testRun.deleteMany).not.toHaveBeenCalled(); + expect(deleteImage).not.toHaveBeenCalled(); + expect(prisma.build.delete).not.toHaveBeenCalled(); + expect(events.testRunDeleted).not.toHaveBeenCalled(); + expect(events.buildDeleted).not.toHaveBeenCalled(); + }); + + it('stops when the build becomes active between chunks', async () => { + const { service, prisma, events } = await setup(); + prisma.build.findUnique.mockResolvedValueOnce(buildRow); + prisma.build.findFirst.mockResolvedValueOnce({ id: build.id }).mockResolvedValue(null); + prisma.testRun.findMany.mockResolvedValueOnce([generateTestRun({ id: 'r1' })]); + + await expect(service.remove(build.id, { onlyIfIdleSince: since })).resolves.toBeUndefined(); + + expect(prisma.testRun.deleteMany).toHaveBeenCalledTimes(1); + expect(prisma.build.delete).not.toHaveBeenCalled(); + expect(events.buildDeleted).not.toHaveBeenCalled(); + }); + + it('removes an idle build', async () => { + const { service, prisma, events } = await setup(); + prisma.build.findUnique.mockResolvedValueOnce(buildRow); + prisma.build.findFirst.mockResolvedValue({ id: build.id }); + prisma.build.delete.mockResolvedValueOnce(buildRow); + prisma.testRun.findMany.mockResolvedValueOnce([run]); + + const result = await service.remove(build.id, { onlyIfIdleSince: since }); + + expect(prisma.build.findFirst).toHaveBeenCalledWith({ + where: { id: build.id, updatedAt: { lt: since }, testRuns: { none: { createdAt: { gte: since } } } }, + select: { id: true }, + }); + expect(prisma.build.delete).toHaveBeenCalledWith({ where: { id: build.id } }); + expect(result).toEqual(removedBuild); + expect(events.buildDeleted).toHaveBeenCalledTimes(1); + }); + + it('does not retry P2003 when pruning', async () => { + const { service, prisma, events } = await setup(); + prisma.build.findUnique.mockResolvedValueOnce(buildRow); + prisma.build.findFirst.mockResolvedValue({ id: build.id }); + prisma.build.delete.mockRejectedValue(prismaError('P2003')); + + await expect(service.remove(build.id, { onlyIfIdleSince: since })).resolves.toBeUndefined(); + + expect(prisma.build.delete).toHaveBeenCalledTimes(1); + expect(events.buildDeleted).not.toHaveBeenCalled(); }); - expect(buildUpdateMock).toHaveBeenCalledWith({ where: { id: build.id }, data: { number: 43 } }); - expect(result).toEqual({ ...build, number: 43 }); }); }); - describe('findOrCreate (characterization)', () => { + describe('findOrCreate', () => { + const params = { projectId: 'projectId', branchName: 'develop', ciBuildId: 'ci' }; + it('numbers a new build and announces it', async () => { + const { service, prisma, events } = await setup(); const numbered = { ...build, number: 3 }; - const buildUpsertMock = jest.fn().mockResolvedValueOnce(build); - const eventsBuildCreatedMock = jest.fn(); - service = await initService({ buildUpsertMock, eventsBuildCreatedMock }); - service.incrementBuildNumber = jest.fn().mockResolvedValueOnce(numbered); + prisma.build.upsert.mockResolvedValueOnce(build); + prisma.project.update.mockResolvedValueOnce({ buildsCounter: 3 }); + prisma.build.updateManyAndReturn.mockResolvedValueOnce([numbered]); - const result = await service.findOrCreate({ projectId: 'projectId', branchName: 'develop', ciBuildId: 'ci' }); + const result = await service.findOrCreate(params); - expect(service.incrementBuildNumber).toHaveBeenCalledWith(build.id, 'projectId'); - expect(eventsBuildCreatedMock).toHaveBeenCalledWith(new BuildDto(numbered)); + expect(prisma.project.update).toHaveBeenCalledWith({ + where: { id: 'projectId' }, + data: { buildsCounter: { increment: 1 } }, + select: { buildsCounter: true }, + }); + expect(prisma.build.updateManyAndReturn).toHaveBeenCalledWith({ + where: { id: build.id, number: null }, + data: { number: 3 }, + }); + expect(mocked(BuildDto).mock.calls.at(-1)).toEqual([numbered]); + expect(events.buildCreated).toHaveBeenCalledTimes(1); expect(result).toBe(numbered); }); + it('numbers a new build once when parallel callers join it', async () => { + const { service, prisma, events } = await setup(); + prisma.build.upsert.mockResolvedValue({ ...build, number: null }); + prisma.project.update.mockResolvedValueOnce({ buildsCounter: 42 }).mockResolvedValueOnce({ buildsCounter: 43 }); + prisma.build.updateManyAndReturn.mockResolvedValueOnce([{ ...build, number: 42 }]).mockResolvedValueOnce([]); + prisma.build.findUniqueOrThrow.mockResolvedValue({ ...build, number: 42 }); + + const results = await Promise.all([service.findOrCreate(params), service.findOrCreate(params)]); + + expect(events.buildCreated).toHaveBeenCalledTimes(1); + expect(results.map((result) => result.number)).toEqual([42, 42]); + }); + + it('returns the build another caller numbered, without announcing it', async () => { + const { service, prisma, events } = await setup(); + prisma.build.upsert.mockResolvedValueOnce(build); + prisma.project.update.mockResolvedValueOnce({ buildsCounter: 8 }); + prisma.build.updateManyAndReturn.mockResolvedValueOnce([]); + prisma.build.findUniqueOrThrow.mockResolvedValueOnce({ ...build, number: 7 }); + + const result = await service.findOrCreate(params); + + expect(prisma.build.findUniqueOrThrow).toHaveBeenCalledWith({ where: { id: build.id } }); + expect(events.buildCreated).not.toHaveBeenCalled(); + expect(result).toEqual({ ...build, number: 7 }); + }); + + it('rethrows a failure of the numbering transaction', async () => { + const { service, prisma, events } = await setup(); + const error = new Error('connection lost'); + prisma.build.upsert.mockResolvedValueOnce(build); + prisma.project.update.mockRejectedValueOnce(error); + + await expect(service.findOrCreate(params)).rejects.toBe(error); + expect(events.buildCreated).not.toHaveBeenCalled(); + }); + it('retries a concurrent upsert (P2002) as an update with the same unique key', async () => { - const buildUpsertMock = jest - .fn() - .mockRejectedValueOnce(new PrismaClientKnownRequestError('dup', { code: 'P2002', clientVersion: '5' })); - const buildUpdateMock = jest.fn().mockResolvedValueOnce({ ...build, number: 9 }); - service = await initService({ buildUpsertMock, buildUpdateMock }); + const { service, prisma } = await setup(); + prisma.build.upsert.mockRejectedValueOnce(prismaError('P2002')); + prisma.build.update.mockResolvedValueOnce({ ...build, number: 9 }); - const result = await service.findOrCreate({ projectId: 'projectId', branchName: 'develop', ciBuildId: 'ci' }); + const result = await service.findOrCreate(params); - expect(buildUpdateMock).toHaveBeenCalledWith({ + expect(prisma.build.update).toHaveBeenCalledWith({ where: { projectId_ciBuildId: { projectId: 'projectId', ciBuildId: 'ci' } }, data: { isRunning: true }, }); @@ -457,254 +551,256 @@ describe('BuildsService', () => { }); it.each([ - ['another Prisma error', new PrismaClientKnownRequestError('x', { code: 'P2003', clientVersion: '5' })], + ['another Prisma error', prismaError('P2003')], ['any other error', new Error('connection lost')], - ])('turns %s on upsert into a TypeError', async (_, error) => { - const buildUpsertMock = jest.fn().mockRejectedValueOnce(error); - const buildUpdateMock = jest.fn(); - service = await initService({ buildUpsertMock, buildUpdateMock }); - - // NOTE: current behavior — the original error is swallowed and the - // undefined build is dereferenced afterwards - await expect( - service.findOrCreate({ projectId: 'projectId', branchName: 'develop', ciBuildId: 'ci' }) - ).rejects.toThrow(TypeError); - expect(buildUpdateMock).not.toHaveBeenCalled(); + ])('rethrows %s from upsert unchanged', async (_, error) => { + const { service, prisma } = await setup(); + prisma.build.upsert.mockRejectedValueOnce(error); + + await expect(service.findOrCreate(params)).rejects.toBe(error); + expect(prisma.build.update).not.toHaveBeenCalled(); + }); + + it.each([ + ['without ciBuildId', undefined, { id: '111' }], + ['with ciBuildId', '222', { projectId_ciBuildId: { projectId: '111', ciBuildId: '222' } }], + ])('upserts %s', async (_, ciBuildId, where) => { + const { service, prisma } = await setup(); + prisma.build.upsert.mockResolvedValueOnce({ ...build, number: 1 }); + + await service.findOrCreate({ projectId: '111', branchName: 'develop', ciBuildId }); + + expect(prisma.build.upsert).toHaveBeenCalledWith({ + where, + create: { + branchName: 'develop', + ...(ciBuildId && { ciBuildId }), + isRunning: true, + project: { connect: { id: '111' } }, + }, + update: { isRunning: true }, + }); + }); + + it('leaves an already numbered build alone', async () => { + const { service, prisma } = await setup(); + prisma.build.upsert.mockResolvedValueOnce({ ...build, number: 100 }); + + const result = await service.findOrCreate({ projectId: '111', branchName: 'develop' }); + + expect(prisma.project.update).not.toHaveBeenCalled(); + expect(result).toEqual({ ...build, number: 100 }); }); }); describe('deleteOldBuilds', () => { const projectId = 'someProjectId'; - const orderBy = [{ createdAt: 'desc' }, { id: 'desc' }]; + const ids = (...names: string[]) => names.map((id) => ({ id })); const init = async (oldBuilds: { id: string }[]) => { - const buildFindManyMock = jest.fn().mockResolvedValueOnce(oldBuilds); - service = await initService({ buildFindManyMock }); - service.remove = jest.fn().mockResolvedValue(undefined); - return buildFindManyMock; + const ctx = await setup(); + ctx.prisma.build.findMany.mockResolvedValueOnce(oldBuilds); + ctx.service.remove = jest.fn().mockResolvedValue(undefined); + return ctx; }; + afterEach(() => { + jest.useRealTimers(); + }); + it('removes every build past the newest maxBuildAllowed - 1, making room for the new one', async () => { - const buildFindManyMock = await init([{ id: 'build1' }, { id: 'build2' }]); + const { service, prisma } = await init(ids('build1', 'build2')); - await service.deleteOldBuilds(projectId, 3); + await service.deleteOldBuilds(projectId, 3, 'b-new'); - expect(buildFindManyMock).toHaveBeenCalledWith({ where: { projectId }, orderBy, skip: 2, select: { id: true } }); + expect(prisma.build.findMany).toHaveBeenCalledWith({ + where: { projectId, id: { not: 'b-new' } }, + orderBy: buildOrder, + skip: 2, + select: { id: true }, + }); expect(service.remove).toHaveBeenCalledTimes(2); - expect(service.remove).toHaveBeenCalledWith('build1'); - expect(service.remove).toHaveBeenCalledWith('build2'); + expect(service.remove).toHaveBeenCalledWith('build1', expect.anything()); + expect(service.remove).toHaveBeenCalledWith('build2', expect.anything()); }); - it.each([ - [1, 0], - [0, 0], - ])('with maxBuildAllowed=%p keeps no old build at all', async (max, skip) => { - const buildFindManyMock = await init([]); + it.each([[1], [0]])('with maxBuildAllowed=%p keeps no old build at all', async (max) => { + const { service, prisma } = await init([]); - await service.deleteOldBuilds(projectId, max); + await service.deleteOldBuilds(projectId, max, 'b-new'); - expect(buildFindManyMock).toHaveBeenCalledWith(expect.objectContaining({ skip })); + expect(prisma.build.findMany).toHaveBeenCalledWith(expect.objectContaining({ skip: 0 })); }); - it('never prunes the build a CI build id is about to reuse', async () => { - const buildFindManyMock = await init([]); + it('passes the grace cutoff to remove', async () => { + jest.useFakeTimers({ now: new Date('2026-05-01T12:00:00Z'), doNotFake: ['setImmediate', 'nextTick'] }); + const { service } = await init(ids('build1')); - await service.deleteOldBuilds(projectId, 1, 'ci-1'); + await service.deleteOldBuilds(projectId, 1, 'b-new'); - expect(buildFindManyMock).toHaveBeenCalledWith( - expect.objectContaining({ - where: { projectId, OR: [{ ciBuildId: null }, { ciBuildId: { not: 'ci-1' } }] }, - }) + expect(service.remove).toHaveBeenCalledWith('build1', { onlyIfIdleSince: new Date('2026-05-01T11:00:00Z') }); + }); + + it('runs a separate pruning for another kept build of the same project', async () => { + const { service, prisma } = await setup(); + prisma.build.findMany.mockResolvedValue([]); + + await Promise.all([service.deleteOldBuilds(projectId, 2, 'b1'), service.deleteOldBuilds(projectId, 2, 'b2')]); + + expect(prisma.build.findMany).toHaveBeenCalledTimes(2); + expect(prisma.build.findMany).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ where: { projectId, id: { not: 'b1' } } }) + ); + expect(prisma.build.findMany).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ where: { projectId, id: { not: 'b2' } } }) ); }); - it('should handle concurrent calls for the same project by returning the existing promise', async () => { - const buildFindManyMock = await init([{ id: 'build1' }]); + it('should handle concurrent calls for the same kept build by returning the existing promise', async () => { + const { service, prisma } = await init(ids('build1')); - const promise1 = service.deleteOldBuilds(projectId, 2); - const promise2 = service.deleteOldBuilds(projectId, 2); + const promise1 = service.deleteOldBuilds(projectId, 2, 'b-new'); + const promise2 = service.deleteOldBuilds(projectId, 2, 'b-new'); expect(promise1).toStrictEqual(promise2); await promise1; - expect(buildFindManyMock).toHaveBeenCalledTimes(1); + expect(prisma.build.findMany).toHaveBeenCalledTimes(1); expect(service.remove).toHaveBeenCalledTimes(1); }); it('should remove the promise from the map after completion (success)', async () => { - await init([{ id: 'build1' }]); + const { service } = await init(ids('build1')); - await service.deleteOldBuilds(projectId, 1); + await service.deleteOldBuilds(projectId, 1, 'b-new'); - expect(service['ongoingDeletions'].has(projectId)).toBe(false); + expect(service['ongoingDeletions'].has(`${projectId}|b-new`)).toBe(false); + }); + + it('one failing old build does not stop pruning the others', async () => { + const { service } = await init(ids('old1', 'old2')); + jest.mocked(service.remove).mockRejectedValueOnce(new Error('boom')).mockResolvedValueOnce(undefined); + + await expect(service.deleteOldBuilds(projectId, 1, 'b-new')).resolves.toBeUndefined(); + + expect(service.remove).toHaveBeenCalledWith('old2', expect.anything()); }); it('hands a failure to the caller and leaves no unhandled rejection behind', async () => { - const unhandled = jest.fn(); - process.on('unhandledRejection', unhandled); + const { unhandled, stop } = unhandledRejections(); try { - const buildFindManyMock = jest.fn().mockRejectedValueOnce(new Error('db down')); - service = await initService({ buildFindManyMock }); + const { service, prisma } = await setup(); + prisma.build.findMany.mockRejectedValueOnce(new Error('db down')); - await expect(service.deleteOldBuilds(projectId, 1)).rejects.toThrow('db down'); - await new Promise((resolve) => setImmediate(resolve)); + await expect(service.deleteOldBuilds(projectId, 1, 'b-new')).rejects.toThrow('db down'); + await tick(); expect(unhandled).not.toHaveBeenCalled(); - expect(service['ongoingDeletions'].has(projectId)).toBe(false); + expect(service['ongoingDeletions'].has(`${projectId}|b-new`)).toBe(false); } finally { - process.off('unhandledRejection', unhandled); + stop(); } }); }); - it('should stop', async () => { - const id = 'some id'; - const buildUpdateMock = jest.fn(); - const eventsBuildUpdatedMock = jest.fn(); - mocked(BuildDto).mockReturnValueOnce(buildDto as MockedObject); - service = await initService({ buildUpdateMock, eventsBuildUpdatedMock }); + describe('background retention', () => { + const projectId = 'someProjectId'; - const result = await service.update(id, { isRunning: false }); + it('scheduleRetention logs a failure instead of leaving it unhandled', async () => { + const { unhandled, stop } = unhandledRejections(); + try { + const { service, prisma } = await setup(); + prisma.build.findMany.mockRejectedValueOnce(new Error('db down')); + const logger = silentLogger(); + (service as any).logger = logger; - expect(buildUpdateMock).toHaveBeenCalledWith({ - where: { id }, - data: { isRunning: false }, - }); - expect(eventsBuildUpdatedMock).toHaveBeenCalledWith(id); - expect(result).toBe(buildDto); - }); + service.scheduleRetention(projectId, 1, 'b-new'); + await service.whenRetentionIdle(); + await tick(); - it('approve', async () => { - const buildFindUniqueMock = jest.fn().mockResolvedValueOnce(build); - const testRunApproveMock = jest.fn().mockResolvedValueOnce({ - ...build.testRuns[0], - status: TestStatus.approved, + expect(unhandled).not.toHaveBeenCalled(); + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('db down')); + } finally { + stop(); + } }); - service = await initService({ buildFindUniqueMock, testRunApproveMock }); - service.findOne = jest.fn(); - await service.approve('someId', true, 'user id'); + it('whenRetentionIdle resolves when nothing runs', async () => { + const { service } = await setup(); - expect(buildFindUniqueMock).toHaveBeenCalledWith({ - where: { id: 'someId' }, - include: { - testRuns: { - where: { - status: { - in: [TestStatus.new, TestStatus.unresolved], - }, - }, - }, - }, + await expect(service.whenRetentionIdle()).resolves.toBeUndefined(); }); - expect(testRunApproveMock).toHaveBeenCalledWith(build.testRuns[0].id, true, false, 'user id'); - }); - - it('approve throws NotFoundException for an unknown build', async () => { - const buildFindUniqueMock = jest.fn().mockResolvedValueOnce(null); - service = await initService({ buildFindUniqueMock }); - await expect(service.approve('missing', false)).rejects.toThrow(new NotFoundException('Build not found: missing')); - }); - - describe('findOsCreate', () => { - it('create without ciBuildId', async () => { - const buildUpsertMock = jest.fn().mockResolvedValueOnce(build); + it('whenRetentionIdle waits for a running pruning, even a failing one', async () => { + const { service, prisma } = await setup(); + let fail: (e: Error) => void = () => undefined; + prisma.build.findMany.mockReturnValueOnce(new Promise((_, reject) => (fail = reject))); + (service as any).logger = silentLogger(); + service.scheduleRetention(projectId, 1, 'b-new'); + let idle = false; + const waiting = service.whenRetentionIdle().then(() => (idle = true)); - service = await initService({ buildUpsertMock }); - service.incrementBuildNumber = jest.fn().mockResolvedValueOnce(build); + await tick(); + expect(idle).toBe(false); + fail(new Error('boom')); + await waiting; - await service.findOrCreate({ - projectId: '111', - branchName: 'develop', - }); + expect(idle).toBe(true); + }); - expect(buildUpsertMock).toHaveBeenCalledWith({ - where: { - id: '111', - }, - create: { - branchName: 'develop', - isRunning: true, - project: { - connect: { - id: '111', - }, - }, - }, - update: { - isRunning: true, - }, - }); + it('beforeApplicationShutdown stops between builds and waits', async () => { + const { service, prisma } = await setup(); + prisma.build.findMany.mockResolvedValueOnce([{ id: 'o1' }, { id: 'o2' }]); + let finishFirst: () => void = () => undefined; + service.remove = jest + .fn() + .mockImplementationOnce(() => new Promise((resolve) => (finishFirst = () => resolve(undefined)))) + .mockResolvedValue(undefined); + service.scheduleRetention(projectId, 1, 'b-new'); + await tick(); + expect(service.remove).toHaveBeenCalledWith('o1', expect.anything()); + + let stopped = false; + const shutdown = service.beforeApplicationShutdown().then(() => (stopped = true)); + await tick(); + expect(stopped).toBe(false); + finishFirst(); + await shutdown; + + expect(stopped).toBe(true); + expect(service.remove).toHaveBeenCalledTimes(1); + expect(service.remove).not.toHaveBeenCalledWith('o2', expect.anything()); }); + }); - it('create with ciBuildId', async () => { - const buildUpsertMock = jest.fn().mockResolvedValueOnce(build); - service = await initService({ buildUpsertMock }); - service.incrementBuildNumber = jest.fn().mockResolvedValueOnce(build); + describe('approve', () => { + it('approves the new and unresolved runs of the build', async () => { + const { service, prisma, testRuns } = await setup(); + prisma.build.findUnique.mockResolvedValueOnce(build); + testRuns.approve.mockResolvedValueOnce({ ...run, status: TestStatus.approved }); - await service.findOrCreate({ - projectId: '111', - branchName: 'develop', - ciBuildId: '222', - }); + await service.approve('someId', true, 'user id'); - expect(buildUpsertMock).toHaveBeenCalledWith({ - where: { - projectId_ciBuildId: { - projectId: '111', - ciBuildId: '222', - }, - }, - create: { - branchName: 'develop', - ciBuildId: '222', - isRunning: true, - project: { - connect: { - id: '111', - }, + expect(prisma.build.findUnique).toHaveBeenCalledWith({ + where: { id: 'someId' }, + include: { + testRuns: { + where: { status: { in: [TestStatus.new, TestStatus.unresolved] } }, + select: { id: true }, }, }, - update: { - isRunning: true, - }, }); + expect(testRuns.approve).toHaveBeenCalledWith(run.id, true, false, 'user id'); }); - it('create with retry', async () => { - const buildUpsertMock = jest - .fn() - .mockRejectedValueOnce(new PrismaClientKnownRequestError('mock error', { code: 'P2002', clientVersion: '5' })); - const buildUpdateMock = jest.fn().mockResolvedValueOnce(build); - service = await initService({ buildUpsertMock, buildUpdateMock }); - service.incrementBuildNumber = jest.fn().mockResolvedValueOnce(build); - - const result = await service.findOrCreate({ - projectId: '111', - branchName: 'develop', - }); - - expect(result).toEqual(build); - }); + it('throws NotFoundException for an unknown build', async () => { + const { service, prisma } = await setup(); + prisma.build.findUnique.mockResolvedValueOnce(null); - it('update already created', async () => { - const buildUpsertMock = jest.fn().mockResolvedValueOnce({ - ...build, - number: 100, - }); - service = await initService({ buildUpsertMock }); - service.incrementBuildNumber = jest.fn(); - - const result = await service.findOrCreate({ - projectId: '111', - branchName: 'develop', - }); - - expect(service.incrementBuildNumber).toHaveBeenCalledTimes(0); - expect(result).toEqual({ - ...build, - number: 100, - }); + await expect(service.approve('missing', false)).rejects.toThrow( + new NotFoundException('Build not found: missing') + ); }); }); }); diff --git a/src/builds/builds.service.ts b/src/builds/builds.service.ts index b8273e70..1ce092d1 100644 --- a/src/builds/builds.service.ts +++ b/src/builds/builds.service.ts @@ -1,30 +1,45 @@ -import { forwardRef, Inject, Injectable, Logger, NotFoundException } from '@nestjs/common'; +import { + BeforeApplicationShutdown, + ConflictException, + forwardRef, + Inject, + Injectable, + Logger, + NotFoundException, +} from '@nestjs/common'; import { PrismaService } from '../prisma/prisma.service'; -import { Build, Prisma, TestStatus } from '@prisma/client'; +import { isPrismaError } from '../prisma/prisma-errors'; +import { Build, Prisma, TestRun, TestStatus } from '@prisma/client'; import { TestRunsService } from '../test-runs/test-runs.service'; import { EventsGateway } from '../shared/events/events.gateway'; import { BuildDto } from './dto/build.dto'; import { getBuildsStats } from './build-stats'; +import { testRunImageNames } from '../utils'; import { PaginatedBuildDto } from './dto/build-paginated.dto'; import { ModifyBuildDto } from './dto/build-modify.dto'; import type { WrapperType } from '../shared/wrapper-type'; +import { StaticService } from '../static/static.service'; -// Newest first, with the id to break a tie: without a total order two builds -// made in the same instant have no fixed place, so an offset page could show -// one of them twice or not at all, and the page a build sits on would be a -// matter of chance. +const TEST_RUN_CHUNK = 500; +const MAX_REMOVE_ATTEMPTS = 3; const BUILD_ORDER: Prisma.BuildOrderByWithRelationInput[] = [{ createdAt: 'desc' }, { id: 'desc' }]; +const RETENTION_GRACE_MS = 60 * 60_000; + +class BuildAlreadyNumbered extends Error {} + @Injectable() -export class BuildsService { +export class BuildsService implements BeforeApplicationShutdown { private readonly logger: Logger = new Logger(BuildsService.name); private readonly ongoingDeletions = new Map>(); + private shuttingDown = false; constructor( private prismaService: PrismaService, private eventsGateway: EventsGateway, @Inject(forwardRef(() => TestRunsService)) - private testRunsService: WrapperType + private testRunsService: WrapperType, + private staticService: StaticService ) {} async findOne(id: string): Promise { @@ -42,8 +57,7 @@ export class BuildsService { async findMany( projectId: string, - // undefined reads every build, which is how deleteOldBuilds finds the ones past the limit - take: number | undefined, + take: number, skip: number, ciBuildId?: string, aroundBuildId?: string @@ -55,10 +69,7 @@ export class BuildsService { }), }; - // A link carries a build, not a page, so the page has to be worked out from - // the build before anything is read. - // the controller always sends take with aroundBuildId; deleteOldBuilds sends neither - const pageSkip = aroundBuildId ? ((await this.skipToPageWith(where, take!, aroundBuildId)) ?? skip) : skip; + const pageSkip = aroundBuildId ? ((await this.skipToPageWith(where, take, aroundBuildId)) ?? skip) : skip; const [total, buildList] = await Promise.all([ this.prismaService.build.count({ where }), @@ -70,21 +81,18 @@ export class BuildsService { }), ]); + const stats = await getBuildsStats( + this.prismaService, + buildList.map((build) => build.id) + ); return { - data: await Promise.all(buildList.map((build) => this.findOne(build.id))), + data: buildList.map((build) => new BuildDto(build, stats.get(build.id))), total, - // only deleteOldBuilds leaves take unset, and it reads just the data - take: take!, + take, skip: pageSkip, }; } - /** - * Where the page holding this build starts, or null when there is no such - * page: the build has been deleted, belongs elsewhere, or the search in force - * leaves it out. A stale link should then open the list it asked for rather - * than fail. - */ private async skipToPageWith(where: Prisma.BuildWhereInput, take: number, buildId: string): Promise { if (take < 1) { return null; @@ -98,9 +106,6 @@ export class BuildsService { return null; } - // BUILD_ORDER puts a build ahead of this one when it is newer, or was made - // in the same instant and carries the larger id. Counting those gives the - // build's place in the list, and the page follows from the page size. const ahead = await this.prismaService.build.count({ where: { ...where, @@ -114,97 +119,149 @@ export class BuildsService { async update(id: string, modifyBuildDto: ModifyBuildDto): Promise { await this.prismaService.build.update({ where: { id }, - // named one by one so nothing else a caller passes can reach the row data: { ciBuildId: modifyBuildDto.ciBuildId, isRunning: modifyBuildDto.isRunning }, }); this.eventsGateway.buildUpdated(id); return this.findOne(id); } - async remove(id: string): Promise { + async remove( + id: string, + { onlyIfIdleSince }: { onlyIfIdleSince?: Date } = {} + ): Promise<(Build & { testRuns: TestRun[] }) | undefined> { if (!id) { this.logger.warn(`Attempted to remove build with undefined ID.`); return; } this.logger.debug(`Going to remove Build ${id}`); - const build = await this.prismaService.build.findUnique({ - where: { id }, - include: { - testRuns: true, - }, - }); - + const build = await this.prismaService.build.findUnique({ where: { id } }); if (!build) { this.logger.warn(`Build not found ${id}`); return; } - await Promise.all(build.testRuns.map((testRun) => this.testRunsService.delete(testRun.id))); - - try { - await this.prismaService.build.delete({ where: { id } }); - } catch (e) { - if (e instanceof Prisma.PrismaClientKnownRequestError) { + const testRuns: TestRun[] = []; + for (let attempt = 1; ; attempt++) { + const removedRuns = await this.removeTestRuns(id, onlyIfIdleSince); + if (!removedRuns || (onlyIfIdleSince && !(await this.isIdle(id, onlyIfIdleSince)))) { + this.logger.log(`Keeping build ${id}: it is in use`); + return; + } + testRuns.push(...removedRuns); + try { + await this.prismaService.build.delete({ where: { id } }); + break; + } catch (e) { // workaround https://github.com/Visual-Regression-Tracker/Visual-Regression-Tracker/issues/435 - if (e.code === 'P2025') { + if (isPrismaError(e, 'P2025')) { this.logger.warn(`Build already deleted ${id}`); return; } + if (!isPrismaError(e, 'P2003')) { + throw e; + } + if (onlyIfIdleSince) { + this.logger.log(`Keeping build ${id}: it received new test runs`); + return; + } + if (attempt === MAX_REMOVE_ATTEMPTS) { + throw new ConflictException(`Build ${id} keeps receiving test runs, try again later`); + } } } this.logger.log(`Build deleted ${id}`); - this.eventsGateway.buildDeleted(new BuildDto({ ...build })); - return build; + const removed = { ...build, testRuns }; + this.eventsGateway.buildDeleted(new BuildDto(removed)); + return removed; } - /** - * Makes room for the build about to be started: keeps the newest - * `keepBuilds - 1` builds, so that with it the project holds `keepBuilds`. - * The build that a POST with `ciBuildId` is about to reuse (parallel CI jobs - * share one) is never pruned, and does not count among the ones kept. - */ - async deleteOldBuilds(projectId: string, keepBuilds: number, ciBuildId?: string): Promise { - if (this.ongoingDeletions.has(projectId)) { - this.logger.debug(`Deletion for project ${projectId} is already in progress. Returning existing promise.`); - return this.ongoingDeletions.get(projectId); + private async isIdle(buildId: string, since: Date): Promise { + const idle = await this.prismaService.build.findFirst({ + where: { id: buildId, updatedAt: { lt: since }, testRuns: { none: { createdAt: { gte: since } } } }, + select: { id: true }, + }); + return idle !== null; + } + + // rows before files: a failure leaks bytes instead of leaving runs that point at deleted images + private async removeTestRuns(buildId: string, onlyIfIdleSince?: Date): Promise { + const removed: TestRun[] = []; + for (;;) { + if (onlyIfIdleSince && !(await this.isIdle(buildId, onlyIfIdleSince))) { + return null; + } + const chunk = await this.prismaService.testRun.findMany({ where: { buildId }, take: TEST_RUN_CHUNK }); + if (chunk.length === 0) { + return removed; + } + await this.prismaService.testRun.deleteMany({ where: { id: { in: chunk.map((testRun) => testRun.id) } } }); + await this.staticService.deleteImages(chunk.flatMap(testRunImageNames)); + chunk.forEach((testRun) => this.eventsGateway.testRunDeleted(testRun)); + removed.push(...chunk); + } + } + + async deleteOldBuilds(projectId: string, keepBuilds: number, keepBuildId: string): Promise { + const key = `${projectId}|${keepBuildId}`; + const ongoing = this.ongoingDeletions.get(key); + if (ongoing) { + this.logger.debug(`Deletion for ${key} is already in progress. Returning existing promise.`); + return ongoing; } const deletionPromise = (async () => { this.logger.debug(`Starting to delete old builds for project ${projectId}, keeping ${keepBuilds} builds.`); + const onlyIfIdleSince = new Date(Date.now() - RETENTION_GRACE_MS); const oldBuilds = await this.prismaService.build.findMany({ - where: { - projectId, - // spelled out with the null case: NOT { ciBuildId } alone would also - // leave out every build without a CI build id - ...(ciBuildId && { OR: [{ ciBuildId: null }, { ciBuildId: { not: ciBuildId } }] }), - }, + where: { projectId, id: { not: keepBuildId } }, orderBy: BUILD_ORDER, skip: Math.max(keepBuilds - 1, 0), select: { id: true }, }); - for (const eachBuild of oldBuilds) { - await this.remove(eachBuild.id); + for (const { id } of oldBuilds) { + if (this.shuttingDown) { + break; + } + try { + await this.remove(id, { onlyIfIdleSince }); + } catch (e) { + this.logger.warn(`Cannot remove old build ${id}: ${e}`); + } } this.logger.debug(`Finished deleting old builds for project ${projectId}.`); })(); - this.ongoingDeletions.set(projectId, deletionPromise); + this.ongoingDeletions.set(key, deletionPromise); deletionPromise .finally(() => { - this.ongoingDeletions.delete(projectId); - this.logger.debug(`Deletion promise for project ${projectId} resolved/rejected and removed from map.`); + this.ongoingDeletions.delete(key); + this.logger.debug(`Deletion promise for ${key} resolved/rejected and removed from map.`); }) - // finally() hands on the rejection in a promise nobody awaits: unhandled, - // it would take the whole process down. The caller gets the error from - // deletionPromise itself. + // finally() would pass the rejection to a promise nobody awaits and crash the process .catch((): void => undefined); return deletionPromise; } + scheduleRetention(projectId: string, keepBuilds: number, keepBuildId: string): void { + this.deleteOldBuilds(projectId, keepBuilds, keepBuildId).catch((e) => + this.logger.warn(`Cannot remove old builds of project ${projectId}: ${e}`) + ); + } + + async whenRetentionIdle(): Promise { + await Promise.allSettled([...this.ongoingDeletions.values()]); + } + + // Runs before Prisma disconnects in onApplicationShutdown + async beforeApplicationShutdown(): Promise { + this.shuttingDown = true; + await this.whenRetentionIdle(); + } + async approve(id: string, merge: boolean, userId?: string): Promise { const build = await this.prismaService.build.findUnique({ where: { id }, @@ -215,6 +272,7 @@ export class BuildsService { in: [TestStatus.new, TestStatus.unresolved], }, }, + select: { id: true }, }, }, }); @@ -224,7 +282,6 @@ export class BuildsService { } for (const testRun of build.testRuns) { - // recorded on each new baseline, as POST /test-runs/approve does await this.testRunsService.approve(testRun.id, merge, false, userId); } } @@ -247,8 +304,7 @@ export class BuildsService { } : { id: projectId }; - // left unset when the upsert fails with anything but P2002, and then reading it throws - let build!: Build; + let build: Build; try { build = await this.prismaService.build.upsert({ where, @@ -267,42 +323,52 @@ export class BuildsService { }, }); } catch (e) { - if (e instanceof Prisma.PrismaClientKnownRequestError) { - if (e.code === 'P2002') { - // cuncurent upsert workaround https://www.prisma.io/docs/reference/api-reference/prisma-client-reference#unique-key-constraint-errors-on-upserts - build = await this.prismaService.build.update({ - where, - data: { - isRunning: true, - }, - }); - } + if (!isPrismaError(e, 'P2002')) { + throw e; } + // concurrent upsert workaround https://www.prisma.io/docs/reference/api-reference/prisma-client-reference#unique-key-constraint-errors-on-upserts + build = await this.prismaService.build.update({ + where, + data: { + isRunning: true, + }, + }); } - // assigne build number - if (!build.number) { - build = await this.incrementBuildNumber(build.id, projectId); - this.eventsGateway.buildCreated(new BuildDto(build)); + if (build.number === null) { + const numbered = await this.assignNumber(build.id, projectId); + if (numbered) { + this.eventsGateway.buildCreated(new BuildDto(numbered)); + return numbered; + } + return this.prismaService.build.findUniqueOrThrow({ where: { id: build.id } }); } return build; } - async incrementBuildNumber(buildId: string, projectId: string): Promise { - const project = await this.prismaService.project.update({ - where: { - id: projectId, - }, - data: { - buildsCounter: { - increment: 1, - }, - }, - }); - return this.prismaService.build.update({ - where: { id: buildId }, - data: { number: project.buildsCounter }, - }); + private async assignNumber(buildId: string, projectId: string): Promise { + try { + return await this.prismaService.$transaction(async (tx) => { + const { buildsCounter } = await tx.project.update({ + where: { id: projectId }, + data: { buildsCounter: { increment: 1 } }, + select: { buildsCounter: true }, + }); + const [numbered] = await tx.build.updateManyAndReturn({ + where: { id: buildId, number: null }, + data: { number: buildsCounter }, + }); + if (!numbered) { + throw new BuildAlreadyNumbered(); + } + return numbered; + }); + } catch (e) { + if (e instanceof BuildAlreadyNumbered) { + return null; + } + throw e; + } } } diff --git a/src/builds/dto/build-modify.dto.ts b/src/builds/dto/build-modify.dto.ts index 2777dd07..d84fc70e 100644 --- a/src/builds/dto/build-modify.dto.ts +++ b/src/builds/dto/build-modify.dto.ts @@ -1,9 +1,8 @@ import { ApiPropertyOptional } from '@nestjs/swagger'; import { IsBoolean, IsNotEmpty, IsOptional, IsString } from 'class-validator'; +import { ValidationOptions } from '../../shared/validation'; -// The only build columns a client may change: the SDKs stop a build (an empty -// body, see BuildsController.update) and a CI may attach its own build id. -// Everything else — branch, number, status — belongs to the server. +@ValidationOptions({ forbidNonWhitelisted: true }) export class ModifyBuildDto { @ApiPropertyOptional() @IsOptional() diff --git a/src/builds/dto/build.dto.spec.ts b/src/builds/dto/build.dto.spec.ts index 8ba5ead4..23c420c4 100644 --- a/src/builds/dto/build.dto.spec.ts +++ b/src/builds/dto/build.dto.spec.ts @@ -2,220 +2,84 @@ import { Build, TestRun, TestStatus } from '@prisma/client'; import { generateTestRun, TEST_BUILD } from '../../_data_'; import { BuildDto } from './build.dto'; +const makeBuild = (testRuns?: TestRun[]): Build & { testRuns?: TestRun[] } => ({ + id: 'a9385fc1-884d-4f9f-915e-40da0e7773d5', + ciBuildId: 'ciBuildId', + number: null, + branchName: 'develop', + status: null, + projectId: 'e0a37894-6f29-478d-b13e-6182fecc715e', + updatedAt: new Date(), + createdAt: new Date(), + userId: null, + isRunning: true, + testRuns, +}); + describe('BuildDto', () => { it.each([ + ['new', 'undefined testRuns', undefined, { passedCount: 0, unresolvedCount: 0, failedCount: 0, merge: false }], + ['new', 'empty testRuns', [], { passedCount: 0, unresolvedCount: 0, failedCount: 0, merge: false }], [ - 'undefined testRuns', - { - id: 'a9385fc1-884d-4f9f-915e-40da0e7773d5', - ciBuildId: 'ciBuildId', - number: null, - branchName: 'develop', - status: null, - projectId: 'e0a37894-6f29-478d-b13e-6182fecc715e', - updatedAt: new Date(), - createdAt: new Date(), - userId: null, - isRunning: true, - testRuns: undefined, - merge: false, - }, - ], - [ - 'empty testRuns', - { - id: 'a9385fc1-884d-4f9f-915e-40da0e7773d5', - ciBuildId: 'ciBuildId', - number: null, - branchName: 'develop', - status: null, - projectId: 'e0a37894-6f29-478d-b13e-6182fecc715e', - updatedAt: new Date(), - createdAt: new Date(), - userId: null, - isRunning: true, - testRuns: [], - merge: false, - }, - ], - ])('new with %s', (_, build) => { - const buildDto: BuildDto = { - id: build.id, - ciBuildId: build.ciBuildId, - number: build.number, - branchName: build.branchName, - status: 'new', - projectId: build.projectId, - updatedAt: build.updatedAt, - createdAt: build.createdAt, - userId: null, - passedCount: 0, - unresolvedCount: 0, - failedCount: 0, - isRunning: true, - merge: false, - }; - - const result = new BuildDto(build); - - expect(result).toEqual(buildDto); - }); - - it('passed', () => { - const build: Build & { - testRuns: TestRun[]; - } = { - id: 'a9385fc1-884d-4f9f-915e-40da0e7773d5', - ciBuildId: 'ciBuildId', - number: null, - branchName: 'develop', - status: null, - projectId: 'e0a37894-6f29-478d-b13e-6182fecc715e', - updatedAt: new Date(), - createdAt: new Date(), - userId: null, - isRunning: true, - testRuns: [ + 'passed', + 'passed and approved merge runs', + [ generateTestRun({ status: TestStatus.ok, merge: true }), generateTestRun({ status: TestStatus.approved, merge: true }), ], - }; - const buildDto: BuildDto = { - id: build.id, - ciBuildId: build.ciBuildId, - number: build.number, - branchName: build.branchName, - status: 'passed', - projectId: build.projectId, - updatedAt: build.updatedAt, - createdAt: build.createdAt, - userId: null, - passedCount: 2, - unresolvedCount: 0, - failedCount: 0, - isRunning: true, - merge: true, - }; - - const result = new BuildDto(build); - - expect(result).toEqual(buildDto); - }); - - it('failed', () => { - const build: Build & { testRuns: TestRun[] } = { - id: 'a9385fc1-884d-4f9f-915e-40da0e7773d5', - ciBuildId: 'ciBuildId', - number: null, - branchName: 'develop', - status: null, - projectId: 'e0a37894-6f29-478d-b13e-6182fecc715e', - updatedAt: new Date(), - createdAt: new Date(), - userId: null, - isRunning: true, - testRuns: [ + { passedCount: 2, unresolvedCount: 0, failedCount: 0, merge: true }, + ], + [ + 'failed', + 'a failed run', + [ generateTestRun({ status: TestStatus.ok }), generateTestRun({ status: TestStatus.approved }), generateTestRun({ status: TestStatus.failed }), ], - }; - const buildDto: BuildDto = { - id: build.id, - ciBuildId: build.ciBuildId, - number: build.number, - branchName: build.branchName, - status: 'failed', - projectId: build.projectId, - updatedAt: build.updatedAt, - createdAt: build.createdAt, - userId: null, - passedCount: 2, - unresolvedCount: 0, - failedCount: 1, - isRunning: true, - merge: false, - }; - - const result = new BuildDto(build); - - expect(result).toEqual(buildDto); - }); - - it('unresolved', () => { - const build: Build & { testRuns: TestRun[] } = { - id: 'a9385fc1-884d-4f9f-915e-40da0e7773d5', - ciBuildId: 'ciBuildId', - number: null, - branchName: 'develop', - status: null, - projectId: 'e0a37894-6f29-478d-b13e-6182fecc715e', - updatedAt: new Date(), - createdAt: new Date(), - userId: null, - isRunning: true, - testRuns: [ + { passedCount: 2, unresolvedCount: 0, failedCount: 1, merge: false }, + ], + [ + 'unresolved', + 'new and unresolved runs', + [ generateTestRun({ status: TestStatus.ok }), generateTestRun({ status: TestStatus.approved }), generateTestRun({ status: TestStatus.failed }), generateTestRun({ status: TestStatus.new }), generateTestRun({ status: TestStatus.unresolved }), ], - }; - const buildDto: BuildDto = { + { passedCount: 2, unresolvedCount: 2, failedCount: 1, merge: false }, + ], + ])('is %s with %s', (status, _, testRuns, counts) => { + const build = makeBuild(testRuns); + + expect(new BuildDto(build)).toEqual({ id: build.id, ciBuildId: build.ciBuildId, - number: build.number, - branchName: build.branchName, - status: 'unresolved', + number: null, + branchName: 'develop', + status, projectId: build.projectId, updatedAt: build.updatedAt, createdAt: build.createdAt, userId: null, - passedCount: 2, - unresolvedCount: 2, - failedCount: 1, isRunning: true, - merge: false, - }; - - const result = new BuildDto(build); - - expect(result).toEqual(buildDto); - }); - - it('counts auto-approved test runs as passed', () => { - const result = new BuildDto({ - ...TEST_BUILD, - testRuns: [generateTestRun({ status: TestStatus.autoApproved })], - } as Build & { testRuns: TestRun[] }); - - expect(result).toMatchObject({ status: 'passed', passedCount: 1, unresolvedCount: 0, failedCount: 0 }); + ...counts, + }); }); - it('is passed when its loaded test runs have a status outside the known groups', () => { - const result = new BuildDto({ - ...TEST_BUILD, - testRuns: [generateTestRun({ status: 'resolved' as TestStatus })], - } as Build & { testRuns: TestRun[] }); + it.each([ + ['counts auto-approved test runs as passed', TestStatus.autoApproved, 1], + ['is passed when it has runs but none in a known status group', 'resolved' as TestStatus, 0], + ])('%s', (_, status, passedCount) => { + const result = new BuildDto({ ...TEST_BUILD, testRuns: [generateTestRun({ status })] }); - // NOTE: current behavior — the run counts nowhere, but having runs at all makes it "passed" - expect(result).toMatchObject({ status: 'passed', passedCount: 0, unresolvedCount: 0, failedCount: 0 }); + expect(result).toMatchObject({ status: 'passed', passedCount, unresolvedCount: 0, failedCount: 0 }); }); describe('with stats aggregated by the database', () => { - const build: Build & { testRuns?: TestRun[] } = { - id: 'a9385fc1-884d-4f9f-915e-40da0e7773d5', - ciBuildId: 'ciBuildId', - number: 7, - branchName: 'develop', - status: 'stored status', - projectId: 'e0a37894-6f29-478d-b13e-6182fecc715e', - updatedAt: new Date(), - createdAt: new Date(), - userId: 'userId', - isRunning: false, - }; + const build: Build = { ...makeBuild(), number: 7, status: 'stored status', userId: 'userId', isRunning: false }; it.each([ ['new', { passedCount: 0, unresolvedCount: 0, failedCount: 0, merge: false }], @@ -249,8 +113,7 @@ describe('BuildDto', () => { expect(result).toMatchObject({ status: 'new', passedCount: 0, failedCount: 0, merge: false }); }); - it('never keeps the status stored on the build', () => { - // NOTE: current behavior — build.status is copied and then always overwritten + it('derives the status from the runs, never from the stored column', () => { expect(new BuildDto(build).status).toBe('new'); }); }); diff --git a/src/builds/dto/build.dto.ts b/src/builds/dto/build.dto.ts index af97eb7a..51d74d90 100644 --- a/src/builds/dto/build.dto.ts +++ b/src/builds/dto/build.dto.ts @@ -64,10 +64,8 @@ export class BuildDto { this.failedCount = stats.failedCount; this.merge = stats.merge; } else if (build.testRuns) { - // determine if merge this.merge = build.testRuns.some((testRun) => testRun.merge); - // calculate statistics build.testRuns.forEach((testRun) => { switch (testRun.status) { case TestStatus.autoApproved: diff --git a/src/compare/compare.service.spec.ts b/src/compare/compare.service.spec.ts index aab21bb6..aa993390 100644 --- a/src/compare/compare.service.spec.ts +++ b/src/compare/compare.service.spec.ts @@ -1,4 +1,4 @@ -import { Test, TestingModule } from '@nestjs/testing'; +import { Test } from '@nestjs/testing'; import { ConfigService } from '@nestjs/config'; import { PrismaService } from '../prisma/prisma.service'; import { CompareService } from './compare.service'; @@ -12,79 +12,61 @@ import { GeminiService } from './libs/vlm/providers/gemini/gemini.service'; import { StaticModule } from '../static/static.module'; import { StaticService } from '../static/static.service'; import { Dinov2Service } from './libs/dinov2/dinov2.service'; -import { ImageComparison, TestStatus } from '@prisma/client'; +import { ImageComparison, Project, TestStatus } from '@prisma/client'; import { isHddStaticServiceConfigured } from '../static/utils'; import { ImageCompareInput } from './libs/ImageCompareInput'; -import { BlurredDifference, DEFAULT_BLUR_SIGMA, MAX_SOLID_SHARE } from './libs/pixelmatch/blur-amplitude.core'; +import { BlurredDifference, MAX_SOLID_SHARE } from './libs/pixelmatch/blur-amplitude.core'; -// a mock function in place of the export rather than a spy on it: SWC's -// CommonJS exports are getters, which jest.spyOn cannot replace jest.mock('../static/utils', () => { const actual = jest.requireActual('../static/utils'); return { ...actual, isHddStaticServiceConfigured: jest.fn(actual.isHddStaticServiceConfigured) }; }); -jest.mock('zod/v3', () => { - const actualZod = jest.requireActual('zod'); - return actualZod; -}); - describe('CompareService', () => { - let service: CompareService; - let pixelmatchService: PixelmatchService; - let lookSameService: LookSameService; - let odiffService: OdiffService; - let vlmService: VlmService; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [ - CompareService, - DiffWorkerPool, - Dinov2Service, - OdiffService, - PixelmatchService, - LookSameService, - VlmService, - OllamaService, - GeminiService, - PrismaService, - { - provide: ConfigService, - useValue: { - getOrThrow: jest.fn(), - }, - }, - ], - imports: [StaticModule], - }).compile(); - - service = module.get(CompareService); - pixelmatchService = module.get(PixelmatchService); - lookSameService = module.get(LookSameService); - odiffService = module.get(OdiffService); - vlmService = module.get(VlmService); - }); - - it('should be defined', () => { - expect(service).toBeDefined(); - }); - describe('getComparator', () => { - it('should return pixelmatchService', () => { - const result = service.getComparator(ImageComparison.pixelmatch); - expect(result).toBe(pixelmatchService); + let service: CompareService; + let comparators: Record; + + beforeEach(async () => { + const module = await Test.createTestingModule({ + providers: [ + CompareService, + DiffWorkerPool, + Dinov2Service, + OdiffService, + PixelmatchService, + LookSameService, + VlmService, + OllamaService, + GeminiService, + PrismaService, + { provide: ConfigService, useValue: { getOrThrow: jest.fn() } }, + ], + imports: [StaticModule], + }).compile(); + + service = module.get(CompareService); + comparators = { + pixelmatch: module.get(PixelmatchService), + lookSame: module.get(LookSameService), + odiff: module.get(OdiffService), + vlm: module.get(VlmService), + }; }); - it('should return lookSameService', () => { - const result = service.getComparator(ImageComparison.lookSame); - expect(result).toBe(lookSameService); + it.each([ + [ImageComparison.pixelmatch, 'pixelmatch'], + [ImageComparison.lookSame, 'lookSame'], + [ImageComparison.vlm, 'vlm'], + ['unknown' as ImageComparison, 'pixelmatch'], + ])('should return the comparator for %s', (imageComparison, name) => { + expect(service.getComparator(imageComparison)).toBe(comparators[name]); }); it('should return odiffService', () => { jest.mocked(isHddStaticServiceConfigured).mockReturnValue(true); - expect(service.getComparator(ImageComparison.odiff)).toBe(odiffService); + expect(service.getComparator(ImageComparison.odiff)).toBe(comparators.odiff); }); it('should throw if not HDD for Odiff', () => { @@ -94,52 +76,86 @@ describe('CompareService', () => { 'Odiff can only be used with HDD static service. Please use another image comparison lib in project settings or switch STATIC_SERVICE envitonmental variable to HDD.' ); }); - - it('should return vlmService', () => { - expect(service.getComparator(ImageComparison.vlm)).toBe(vlmService); - }); - - it('should return pixelmatchService for unknown value', () => { - const result = service.getComparator('unknown' as ImageComparison); - expect(result).toBe(pixelmatchService); - }); }); }); describe('CompareService with mocked collaborators', () => { - // pixelmatch applies the rendering-noise filter in its own worker pass, and - // VLM through the pixelmatch it runs first; the others leave it to the service - const comparator = (name: string, appliesBlurAmplitudeFilter = false) => ({ - parseConfig: jest.fn().mockReturnValue({ parsedBy: name }), - getDiff: jest.fn().mockResolvedValue({ comparedBy: name }), + const comparator = (appliesBlurAmplitudeFilter = false) => ({ + parseConfig: jest.fn().mockImplementation((json: string) => JSON.parse(json)), + getDiff: jest.fn(), appliesBlurAmplitudeFilter, }); - const initService = async ({ - projectFindUniqueMock = jest.fn(), - diffWorkerPoolRunMock = jest.fn(), - staticServiceMock = { getImageBuffer: jest.fn(), deleteImage: jest.fn() }, - confirmsNoiseMock = jest.fn(), - }) => { - const pixelmatch = comparator('pixelmatch', true); - const lookSame = comparator('lookSame'); - const odiff = comparator('odiff'); - const vlm = comparator('vlm', true); - const module: TestingModule = await Test.createTestingModule({ + const unresolved = { + status: TestStatus.unresolved, + diffName: 'diff.png', + pixelMisMatchCount: 10, + diffPercent: 5, + isSameDimension: true, + }; + + interface Options { + imageComparison?: ImageComparison; + config?: Record; + project?: Project | null; + result?: object; + measure?: BlurredDifference | null; + gate?: object; + run?: jest.Mock; + confirmsNoiseForGate?: jest.Mock; + } + + const setup = async ({ + imageComparison = ImageComparison.lookSame, + config = {}, + project, + result = unresolved, + measure = { amplitude: 40, solidPixels: 0, solidShare: 0 }, + gate, + run = jest.fn().mockResolvedValue({ measure, gate }), + confirmsNoiseForGate = jest.fn(), + }: Options = {}) => { + const staticService = { + getImageBuffer: jest.fn().mockImplementation(async (name: string) => Buffer.from(name)), + deleteImage: jest.fn(), + }; + const findUnique = jest + .fn() + .mockResolvedValue( + project === undefined + ? { id: 'projectId', imageComparison, imageComparisonConfig: JSON.stringify(config) } + : project + ); + const pixelmatch = comparator(true); + const lookSame = comparator(); + const odiff = comparator(); + const vlm = comparator(true); + [pixelmatch, lookSame, odiff, vlm].forEach((c) => c.getDiff.mockResolvedValue(result)); + const module = await Test.createTestingModule({ providers: [ CompareService, { provide: PixelmatchService, useValue: pixelmatch }, { provide: LookSameService, useValue: lookSame }, { provide: OdiffService, useValue: odiff }, { provide: VlmService, useValue: vlm }, - { provide: PrismaService, useValue: { project: { findUnique: projectFindUniqueMock } } }, - { provide: DiffWorkerPool, useValue: { run: diffWorkerPoolRunMock } }, - { provide: StaticService, useValue: staticServiceMock }, - { provide: Dinov2Service, useValue: { confirmsNoise: confirmsNoiseMock } }, + { provide: PrismaService, useValue: { project: { findUnique } } }, + { provide: DiffWorkerPool, useValue: { run } }, + { provide: StaticService, useValue: staticService }, + { provide: Dinov2Service, useValue: { confirmsNoiseForGate } }, ], }).compile(); - return { service: module.get(CompareService), pixelmatch, lookSame, odiff, vlm }; + return { + service: module.get(CompareService), + pixelmatch, + lookSame, + odiff, + vlm, + run, + staticService, + findUnique, + confirmsNoiseForGate, + }; }; const data: ImageCompareInput = { @@ -150,18 +166,30 @@ describe('CompareService with mocked collaborators', () => { saveDiffAsFile: true, }; + const diffProject = (service: CompareService, input: ImageCompareInput = data) => + service.getDiff({ projectId: 'projectId', data: input }); + describe('getDiff', () => { - it("diffs with the project's comparator and the config it parses", async () => { - const projectFindUniqueMock = jest.fn().mockResolvedValue({ + it.each([ + ['reads the project and', false], + ['uses the project it is given without reading it again, and', true], + ])("%s diffs with the project's comparator and the config it parses", async (_, given) => { + const { service, lookSame, pixelmatch, findUnique } = await setup({ config: { strict: true } }); + lookSame.parseConfig.mockReturnValue({ parsedBy: 'lookSame' }); + lookSame.getDiff.mockResolvedValue({ comparedBy: 'lookSame' }); + const project = { id: 'projectId', imageComparison: ImageComparison.lookSame, imageComparisonConfig: '{"strict":true}', - }); - const { service, lookSame, pixelmatch } = await initService({ projectFindUniqueMock }); + } as Project; - const result = await service.getDiff({ projectId: 'projectId', data }); + const result = await service.getDiff({ projectId: 'projectId', ...(given ? { project } : {}), data }); - expect(projectFindUniqueMock).toHaveBeenCalledWith({ where: { id: 'projectId' } }); + if (given) { + expect(findUnique).not.toHaveBeenCalled(); + } else { + expect(findUnique).toHaveBeenCalledWith({ where: { id: 'projectId' } }); + } expect(lookSame.parseConfig).toHaveBeenCalledWith('{"strict":true}'); expect(lookSame.getDiff).toHaveBeenCalledWith(data, { parsedBy: 'lookSame' }); expect(pixelmatch.getDiff).not.toHaveBeenCalled(); @@ -169,310 +197,249 @@ describe('CompareService with mocked collaborators', () => { }); it('throws a TypeError when the project does not exist', async () => { - const projectFindUniqueMock = jest.fn().mockResolvedValue(null); - const { service } = await initService({ projectFindUniqueMock }); + const { service } = await setup({ project: null }); - // NOTE: current behavior — no not-found handling, the null project is dereferenced await expect(service.getDiff({ projectId: 'missing', data })).rejects.toThrow(TypeError); }); }); describe('rendering-noise filter', () => { - const unresolved = { - status: TestStatus.unresolved, - diffName: 'diff.png', - pixelMisMatchCount: 10, - diffPercent: 5, - isSameDimension: true, - }; - - // the filter's settings ride in the library's config, which CompareService - // reads and strips before the comparator sees it const filterOn = { blurAmplitudeFilter: true, blurAmplitudeThreshold: 80 }; - - const setup = async ({ - imageComparison = ImageComparison.lookSame as ImageComparison, - config = filterOn as Record, - result = unresolved as object, - measure = { amplitude: 40, solidPixels: 0, solidShare: 0 } as BlurredDifference | null, - run = jest.fn().mockResolvedValue({ measure }), - confirmsNoiseMock = jest.fn(), - } = {}) => { - const staticServiceMock = { - getImageBuffer: jest.fn().mockImplementation(async (name: string) => Buffer.from(name)), - deleteImage: jest.fn(), - }; - const projectFindUniqueMock = jest.fn().mockResolvedValue({ - id: 'projectId', - imageComparison, - imageComparisonConfig: JSON.stringify({ libraryOption: 1, ...config }), - }); - const context = await initService({ - projectFindUniqueMock, - diffWorkerPoolRunMock: run, - staticServiceMock, - confirmsNoiseMock, - }); - [context.pixelmatch, context.lookSame, context.odiff, context.vlm].forEach((c) => { - c.parseConfig.mockImplementation((json: string) => JSON.parse(json)); - c.getDiff.mockResolvedValue(result); - }); - return { ...context, run, staticServiceMock, confirmsNoiseMock }; - }; - - const filter = { sigma: DEFAULT_BLUR_SIGMA, threshold: 80 }; + const filtered = (options: Options = {}) => + setup({ ...options, config: { libraryOption: 1, ...(options.config ?? filterOn) } }); it('leaves a config that does not turn it on alone', async () => { - const { service, lookSame, run } = await setup({ config: {} }); + const { service, lookSame, run } = await filtered({ config: {} }); - const result = await service.getDiff({ projectId: 'projectId', data }); + const result = await diffProject(service); expect(lookSame.getDiff).toHaveBeenCalledWith(data, { libraryOption: 1 }); expect(run).not.toHaveBeenCalled(); expect(result).toEqual(unresolved); }); - // decoded already inside its own worker pass, where a run that is about to - // pass also never writes a diff - it('hands it to pixelmatch rather than measuring a second time', async () => { - const { service, pixelmatch, run } = await setup({ imageComparison: ImageComparison.pixelmatch }); - - await service.getDiff({ projectId: 'projectId', data }); - - expect(pixelmatch.getDiff).toHaveBeenCalledWith({ ...data, blurAmplitude: filter }, { libraryOption: 1 }); - expect(run).not.toHaveBeenCalled(); - }); - - // VLM runs pixelmatch first and stops when it passes, so noise is settled - // before a model is ever asked about it - it('hands it to VLM as a prefilter', async () => { - const { service, vlm, run } = await setup({ imageComparison: ImageComparison.vlm }); + it.each([ + ['pixelmatch rather than measuring a second time', ImageComparison.pixelmatch, 'pixelmatch'], + ['VLM as a prefilter', ImageComparison.vlm, 'vlm'], + ] as const)('hands it to %s', async (_, imageComparison, key) => { + const context = await filtered({ imageComparison }); - await service.getDiff({ projectId: 'projectId', data }); + await diffProject(context.service); - expect(vlm.getDiff).toHaveBeenCalledWith({ ...data, blurAmplitude: filter }, { libraryOption: 1 }); - expect(run).not.toHaveBeenCalled(); + expect(context[key].getDiff).toHaveBeenCalledWith( + { ...data, blurAmplitude: { threshold: 80 } }, + { libraryOption: 1 } + ); + expect(context.run).not.toHaveBeenCalled(); }); it('passes a looks-same failure that is only rendering noise', async () => { - const { service, lookSame, run, staticServiceMock } = await setup(); + const { service, lookSame, run, staticService } = await filtered(); - const result = await service.getDiff({ projectId: 'projectId', data }); + const result = await diffProject(service); - // looks-same gets its own options, not the filter's expect(lookSame.getDiff).toHaveBeenCalledWith(data, { libraryOption: 1 }); - expect(run).toHaveBeenCalledWith({ kind: 'blur-amplitude', baseline: Buffer.from('baseline.png'), image: Buffer.from('image.png'), ignoreAreas: data.ignoreAreas, - sigma: DEFAULT_BLUR_SIGMA, }); - // the measured mismatch stays, only the verdict changes expect(result).toEqual({ ...unresolved, status: TestStatus.ok, diffName: null }); - // nothing will ever point at the diff of a run that passed - expect(staticServiceMock.deleteImage).toHaveBeenCalledWith('diff.png'); + expect(staticService.deleteImage).toHaveBeenCalledWith('diff.png'); + }); + + it('measures with the image it is given', async () => { + const { service, run, staticService } = await filtered(); + const imageBuffer = Buffer.from('already read'); + + await diffProject(service, { ...data, imageBuffer }); + + expect(staticService.getImageBuffer).toHaveBeenCalledTimes(1); + expect(staticService.getImageBuffer).toHaveBeenCalledWith('baseline.png'); + expect(run).toHaveBeenCalledWith(expect.objectContaining({ kind: 'blur-amplitude', image: imageBuffer })); + expect(run.mock.calls[0][0].image).toBe(imageBuffer); }); it('passes an odiff failure that is only rendering noise', async () => { jest.mocked(isHddStaticServiceConfigured).mockReturnValue(true); - const { service } = await setup({ imageComparison: ImageComparison.odiff }); - - const result = await service.getDiff({ projectId: 'projectId', data }); + const { service } = await filtered({ imageComparison: ImageComparison.odiff }); - expect(result.status).toBe(TestStatus.ok); + expect((await diffProject(service)).status).toBe(TestStatus.ok); }); it('keeps a change stronger than rendering noise unresolved', async () => { - const { service, staticServiceMock } = await setup({ + const { service, staticService } = await filtered({ measure: { amplitude: 200, solidPixels: 0, solidShare: 0 }, }); - const result = await service.getDiff({ projectId: 'projectId', data }); - - expect(result).toEqual(unresolved); - expect(staticServiceMock.deleteImage).not.toHaveBeenCalled(); + expect(await diffProject(service)).toEqual(unresolved); + expect(staticService.deleteImage).not.toHaveBeenCalled(); }); - // a panel turned light grey is faint, but too solid to be noise it('keeps a faint but solid change unresolved', async () => { - const { service } = await setup({ + const { service } = await filtered({ measure: { amplitude: 31, solidPixels: 9300, solidShare: MAX_SOLID_SHARE + 0.001 }, }); - expect(await service.getDiff({ projectId: 'projectId', data })).toEqual(unresolved); + expect(await diffProject(service)).toEqual(unresolved); }); it("measures against the config's own threshold", async () => { - const { service } = await setup({ config: { ...filterOn, blurAmplitudeThreshold: 30 } }); + const { service } = await filtered({ config: { ...filterOn, blurAmplitudeThreshold: 30 } }); - expect((await service.getDiff({ projectId: 'projectId', data })).status).toBe(TestStatus.unresolved); + expect((await diffProject(service)).status).toBe(TestStatus.unresolved); }); it('does not second-guess a run that already passed', async () => { const ok = { ...unresolved, status: TestStatus.ok, diffName: null as string | null }; - const { service, run } = await setup({ result: ok }); + const { service, run } = await filtered({ result: ok }); - expect(await service.getDiff({ projectId: 'projectId', data })).toEqual(ok); + expect(await diffProject(service)).toEqual(ok); expect(run).not.toHaveBeenCalled(); }); - // nothing to line up, and the size change is a finding of its own it('leaves screenshots of different sizes to the comparator', async () => { const resized = { ...unresolved, isSameDimension: false }; - const { service, run } = await setup({ result: resized }); + const { service, run } = await filtered({ result: resized }); - expect(await service.getDiff({ projectId: 'projectId', data })).toEqual(resized); + expect(await diffProject(service)).toEqual(resized); expect(run).not.toHaveBeenCalled(); }); it('keeps the verdict when the change cannot be measured', async () => { - const { service } = await setup({ measure: null }); + const { service } = await filtered({ measure: null }); - expect(await service.getDiff({ projectId: 'projectId', data })).toEqual(unresolved); + expect(await diffProject(service)).toEqual(unresolved); }); - // a filter that can only ever pass runs must never be the reason one fails - it('keeps the verdict when measuring fails', async () => { - const run = jest.fn().mockRejectedValue(new Error('Image diff queue is full (256 jobs)')); - const { service } = await setup({ run }); - - expect(await service.getDiff({ projectId: 'projectId', data })).toEqual(unresolved); - }); - - it('keeps the verdict when measuring fails with something that is not an Error', async () => { - const { service } = await setup({ run: jest.fn().mockRejectedValue('worker gone') }); + it.each([ + ['an Error', new Error('Image diff queue is full (256 jobs)')], + ['something that is not an Error', 'worker gone'], + ])('keeps the verdict when measuring fails with %s', async (_, failure) => { + const { service } = await filtered({ run: jest.fn().mockRejectedValue(failure) }); - expect(await service.getDiff({ projectId: 'projectId', data })).toEqual(unresolved); + expect(await diffProject(service)).toEqual(unresolved); }); it('keeps the verdict when a screenshot cannot be read', async () => { - const { service, staticServiceMock, run } = await setup(); - staticServiceMock.getImageBuffer.mockResolvedValueOnce(undefined); + const { service, staticService, run } = await filtered(); + staticService.getImageBuffer.mockResolvedValueOnce(undefined); - expect(await service.getDiff({ projectId: 'projectId', data })).toEqual(unresolved); + expect(await diffProject(service)).toEqual(unresolved); expect(run).not.toHaveBeenCalled(); }); it('has no diff to delete when none was saved', async () => { - const { service, staticServiceMock } = await setup({ result: { ...unresolved, diffName: null } }); + const { service, staticService } = await filtered({ result: { ...unresolved, diffName: null } }); - expect((await service.getDiff({ projectId: 'projectId', data })).status).toBe(TestStatus.ok); - expect(staticServiceMock.deleteImage).not.toHaveBeenCalled(); + expect((await diffProject(service)).status).toBe(TestStatus.ok); + expect(staticService.deleteImage).not.toHaveBeenCalled(); }); }); describe('rendering-noise filter with the DINOv2 check', () => { - const unresolved = { - status: TestStatus.unresolved, - diffName: 'diff.png', - pixelMisMatchCount: 10, - diffPercent: 5, - isSameDimension: true, - }; - const checked = { blurAmplitudeFilter: true, dinov2Check: true }; - - const setup = async (imageComparison: ImageComparison, confirmsNoiseMock = jest.fn()) => { - const staticServiceMock = { - getImageBuffer: jest.fn().mockImplementation(async (name: string) => Buffer.from(name)), - deleteImage: jest.fn(), - }; - const context = await initService({ - projectFindUniqueMock: jest.fn().mockResolvedValue({ - id: 'projectId', - imageComparison, - imageComparisonConfig: JSON.stringify(checked), - }), - diffWorkerPoolRunMock: jest - .fn() - .mockResolvedValue({ measure: { amplitude: 40, solidPixels: 0, solidShare: 0 } }), - staticServiceMock, - confirmsNoiseMock, - }); - [context.pixelmatch, context.lookSame, context.odiff, context.vlm].forEach((c) => { - c.parseConfig.mockImplementation((json: string) => JSON.parse(json)); - c.getDiff.mockResolvedValue(unresolved); - }); - return { ...context, staticServiceMock }; - }; - + const gate = { width: 5, height: 3, cells: [7] }; + const checked = (options: Options = {}) => + setup({ gate, config: { blurAmplitudeFilter: true, dinov2Check: true }, ...options }); const dinov2 = { threshold: 0.15 }; - it('passes a looks-same failure DINOv2 confirms is only noise', async () => { - const confirmsNoise = jest.fn().mockResolvedValue({ noise: true, score: 0.04 }); - const { service, staticServiceMock } = await setup(ImageComparison.lookSame, confirmsNoise); + it('passes a looks-same failure DINOv2 confirms is only noise, with the gate the blur job returned', async () => { + const confirmsNoiseForGate = jest.fn().mockResolvedValue({ noise: true, score: 0.04 }); + const { service, staticService, run } = await checked({ confirmsNoiseForGate }); - const result = await service.getDiff({ projectId: 'projectId', data }); + const result = await diffProject(service); - expect(confirmsNoise).toHaveBeenCalledWith( + expect(run).toHaveBeenCalledTimes(1); + expect(run).toHaveBeenCalledWith({ + kind: 'blur-amplitude', + baseline: Buffer.from('baseline.png'), + image: Buffer.from('image.png'), + ignoreAreas: data.ignoreAreas, + gateIfNoise: { threshold: 80 }, + }); + expect(confirmsNoiseForGate).toHaveBeenCalledWith( + gate, Buffer.from('baseline.png'), Buffer.from('image.png'), - data.ignoreAreas, 0.15 ); expect(result.status).toBe(TestStatus.ok); - expect(staticServiceMock.deleteImage).toHaveBeenCalledWith('diff.png'); + expect(staticService.deleteImage).toHaveBeenCalledWith('diff.png'); }); it('keeps a looks-same failure in which DINOv2 sees a change of content', async () => { - const confirmsNoise = jest.fn().mockResolvedValue({ noise: false, score: 0.56 }); - const { service, staticServiceMock } = await setup(ImageComparison.lookSame, confirmsNoise); + const { service, staticService } = await checked({ + confirmsNoiseForGate: jest.fn().mockResolvedValue({ noise: false, score: 0.56 }), + }); + + expect(await diffProject(service)).toEqual(unresolved); + expect(staticService.deleteImage).not.toHaveBeenCalled(); + }); + + it('keeps the failure and does not ask DINOv2 when the blur job returned no gate', async () => { + const confirmsNoiseForGate = jest.fn(); + const { service, staticService } = await checked({ + gate: undefined, + confirmsNoiseForGate, + }); + + expect(await diffProject(service)).toEqual(unresolved); + expect(confirmsNoiseForGate).not.toHaveBeenCalled(); + expect(staticService.deleteImage).not.toHaveBeenCalled(); + }); - expect(await service.getDiff({ projectId: 'projectId', data })).toEqual(unresolved); - expect(staticServiceMock.deleteImage).not.toHaveBeenCalled(); + it.each([ + [ + 'a change the blur already fails', + { run: jest.fn().mockResolvedValue({ measure: { amplitude: 200, solidPixels: 0, solidShare: 0 } }) }, + ], + ['a pair when the blur job fails', { run: jest.fn().mockRejectedValue(new Error('worker crashed')) }], + ])('does not ask DINOv2 about %s', async (_, options) => { + const confirmsNoiseForGate = jest.fn(); + const { service } = await checked({ ...options, confirmsNoiseForGate }); + + expect(await diffProject(service)).toEqual(unresolved); + expect(confirmsNoiseForGate).not.toHaveBeenCalled(); }); - // pixelmatch — and VLM through it — ask DINOv2 themselves, inside their - // own pass, where the diff is still at hand it.each([ [ImageComparison.pixelmatch, 'pixelmatch'], [ImageComparison.vlm, 'vlm'], ] as const)('hands the check to %s', async (imageComparison, key) => { - const confirmsNoise = jest.fn(); - const context = await setup(imageComparison, confirmsNoise); + const context = await checked({ imageComparison }); - await context.service.getDiff({ projectId: 'projectId', data }); + await diffProject(context.service); - expect(context[key].getDiff).toHaveBeenCalledWith( - { ...data, blurAmplitude: { sigma: DEFAULT_BLUR_SIGMA, threshold: 80, dinov2 } }, - {} - ); - expect(confirmsNoise).not.toHaveBeenCalled(); + expect(context[key].getDiff).toHaveBeenCalledWith({ ...data, blurAmplitude: { threshold: 80, dinov2 } }, {}); + expect(context.confirmsNoiseForGate).not.toHaveBeenCalled(); }); }); describe('getChangeSignature', () => { + const input = { + baseline: Buffer.from('a'), + image: Buffer.from('b'), + ignoreAreas: [{ x: 1, y: 2, width: 3, height: 4 }], + threshold: 0.2, + includeAA: true, + }; + it('runs a signature job on the diff worker pool and returns its answer', async () => { - const diffWorkerPoolRunMock = jest.fn().mockResolvedValue({ signature: [0.5, 0.5] }); - const { service } = await initService({ diffWorkerPoolRunMock }); - const input = { - baseline: Buffer.from('a'), - image: Buffer.from('b'), - ignoreAreas: [{ x: 1, y: 2, width: 3, height: 4 }], - threshold: 0.2, - includeAA: true, - }; + const { service, run } = await setup({ run: jest.fn().mockResolvedValue({ signature: [0.5, 0.5] }) }); const result = await service.getChangeSignature(input); - expect(diffWorkerPoolRunMock).toHaveBeenCalledWith({ kind: 'signature', ...input }); + expect(run).toHaveBeenCalledWith({ kind: 'signature', ...input }); expect(result).toEqual({ signature: [0.5, 0.5] }); }); it('passes a pool rejection through', async () => { - const diffWorkerPoolRunMock = jest.fn().mockRejectedValue(new Error('Image diff queue is full (256 jobs)')); - const { service } = await initService({ diffWorkerPoolRunMock }); - - await expect( - service.getChangeSignature({ - baseline: Buffer.from('a'), - image: Buffer.from('b'), - ignoreAreas: [], - threshold: 0.1, - includeAA: false, - }) - ).rejects.toThrow('Image diff queue is full (256 jobs)'); + const { service } = await setup({ + run: jest.fn().mockRejectedValue(new Error('Image diff queue is full (256 jobs)')), + }); + + await expect(service.getChangeSignature(input)).rejects.toThrow('Image diff queue is full (256 jobs)'); }); }); }); diff --git a/src/compare/compare.service.ts b/src/compare/compare.service.ts index 20b661f8..c8b5fba1 100644 --- a/src/compare/compare.service.ts +++ b/src/compare/compare.service.ts @@ -31,11 +31,18 @@ export class CompareService { private readonly dinov2Service: Dinov2Service ) {} - async getDiff({ projectId, data }: { projectId: string; data: ImageCompareInput }): Promise { - const project: Project = (await this.prismaService.project.findUnique({ where: { id: projectId } }))!; + async getDiff({ + projectId, + project: loadedProject, + data, + }: { + projectId: string; + project?: Project; + data: ImageCompareInput; + }): Promise { + const project: Project = + loadedProject ?? (await this.prismaService.project.findUnique({ where: { id: projectId } }))!; const comparator = this.getComparator(project.imageComparison); - // the rendering-noise filter is switched on in the library's config but is - // applied here, for every library alike; the library gets the rest const { blurAmplitude, comparatorConfig } = splitBlurAmplitudeConfig( comparator.parseConfig(project.imageComparisonConfig), this.logger @@ -51,68 +58,54 @@ export class CompareService { return this.filterRenderingNoise(data, result, blurAmplitude); } - /** - * Overturns a failing verdict whose change, blurred, is no stronger than - * anti-aliasing and sub-pixel rendering noise. The filter can only ever pass - * a run, never fail one, so anything that keeps it from measuring — a missing - * screenshot, a full queue, different sizes — leaves the verdict as it was. - */ private async filterRenderingNoise( data: ImageCompareInput, result: DiffResult, - { sigma, threshold, dinov2 }: BlurAmplitudeOptions + { threshold, dinov2 }: BlurAmplitudeOptions ): Promise { - // a size change is a finding of its own, and there is nothing to line up if (result.status !== TestStatus.unresolved || !result.isSameDimension) { return result; } try { const [baseline, image] = await Promise.all([ this.staticService.getImageBuffer(data.baseline), - this.staticService.getImageBuffer(data.image), + data.imageBuffer ?? this.staticService.getImageBuffer(data.image), ]); if (!baseline || !image) { return result; } - const { measure } = await this.diffWorkerPool.run({ + const { measure, gate } = await this.diffWorkerPool.run({ kind: 'blur-amplitude', baseline, image, ignoreAreas: data.ignoreAreas, - sigma, + ...(dinov2 ? { gateIfNoise: { threshold } } : {}), }); if (!measure || !isRenderingNoise(measure, threshold)) { return result; } - // the project's DINOv2 check, when it has one, gets the last word - if ( - dinov2 && - !(await this.dinov2Service.confirmsNoise(baseline, image, data.ignoreAreas, dinov2.threshold)).noise - ) { - return result; + if (dinov2) { + if (!gate) { + return result; + } + if (!(await this.dinov2Service.confirmsNoiseForGate(gate, baseline, image, dinov2.threshold)).noise) { + return result; + } } this.logger.debug( `Diff of ${result.diffPercent}% passed as rendering noise (blur amplitude ${measure.amplitude.toFixed(1)}, ${measure.solidPixels} solid pixels)` ); - // nothing will ever point at the diff of a run that passed if (result.diffName) { - this.staticService.deleteImage(result.diffName); + void this.staticService.deleteImage(result.diffName); } return { ...result, status: TestStatus.ok, diffName: null }; } catch (error) { - // read as an Error, not checked with instanceof: fetch errors come from - // another realm under Jest + // a cast, not instanceof: fetch errors come from another realm under Jest this.logger.warn(`Rendering-noise filter skipped: ${(error as Error)?.message ?? error}`); return result; } } - /** - * Position-independent signature of what changed between a baseline and a - * screenshot, used to tell whether two screens carry the same change. Runs on - * the diff worker pool: decoding a pair of full-size screenshots takes long - * enough that doing it on the event loop stalls every other request. - */ async getChangeSignature(input: Omit): Promise { return this.diffWorkerPool.run({ kind: 'signature', ...input }); } diff --git a/src/compare/diff-worker-pool.inline.spec.ts b/src/compare/diff-worker-pool.inline.spec.ts new file mode 100644 index 00000000..e5788e50 --- /dev/null +++ b/src/compare/diff-worker-pool.inline.spec.ts @@ -0,0 +1,128 @@ +import { readFileSync } from 'fs'; +import { join } from 'path'; +import { PNG } from 'pngjs'; +import { DiffWorkerPool } from './diff-worker-pool'; +import { ImageTooLargeException } from '../static/utils'; +import { PixelmatchJobInput } from './libs/pixelmatch/pixelmatch.core'; +import { SignatureJobInput } from './libs/pixelmatch/signature.core'; +import { pngBytes, solidPng } from './test-utils'; + +const png = (width: number, height: number, level: number): Buffer => + pngBytes(solidPng(width, height, [level, level, level])); + +const diffJob = (baseline: Buffer, image: Buffer, allowDiffDimensions = false): PixelmatchJobInput => ({ + kind: 'diff', + baseline, + image, + ignoreAreas: [], + threshold: 0.1, + includeAA: false, + allowDiffDimensions, + diffTolerancePercent: 0, + saveDiff: false, +}); + +const signatureJob = (baseline: Buffer, image: Buffer): SignatureJobInput => ({ + kind: 'signature', + baseline, + image, + ignoreAreas: [], + threshold: 0.1, + includeAA: false, +}); + +describe('DiffWorkerPool', () => { + let pool: DiffWorkerPool; + + beforeEach(() => { + pool = new DiffWorkerPool(); + }); + + afterEach(async () => { + await pool.onApplicationShutdown(); + }); + + it('answers a diff job with the pixel mismatch it found', async () => { + const output = await pool.run(diffJob(png(10, 10, 0), png(10, 10, 255))); + + expect(output).toMatchObject({ equal: false, pixelMisMatchCount: 100 }); + }); + + it('answers a signature job with a change signature', async () => { + const output = await pool.run(signatureJob(png(10, 10, 0), png(10, 10, 255))); + + expect(output.signature).toHaveLength(64); + expect(output.signature!.reduce((sum, value) => sum + value, 0)).toBeCloseTo(1); + }); + + it('answers a looks-same job', async () => { + const fixture = (name: string) => readFileSync(join(__dirname, '../../test', name)); + + const output = await pool.run({ + kind: 'looks-same', + baseline: fixture('image.png'), + image: fixture('image_edited.png'), + ignoreAreas: [], + config: { strict: false, ignoreAntialiasing: true, ignoreCaret: true }, + saveDiff: false, + }); + + expect(output).toMatchObject({ outcome: 'compared', result: { equal: false, differentPixels: 31068 } }); + }); + + describe('MAX_IMAGE_PIXELS', () => { + const originalLimit = process.env.MAX_IMAGE_PIXELS; + + beforeEach(() => { + process.env.MAX_IMAGE_PIXELS = '50'; + }); + + afterEach(() => { + if (originalLimit === undefined) { + delete process.env.MAX_IMAGE_PIXELS; + } else { + process.env.MAX_IMAGE_PIXELS = originalLimit; + } + jest.restoreAllMocks(); + }); + + it('rejects a job whose image is above MAX_IMAGE_PIXELS with 422, before decoding', async () => { + const read = jest.spyOn(PNG.sync, 'read'); + + const rejection = pool.run(diffJob(png(10, 10, 0), png(5, 5, 0))); + + await expect(rejection).rejects.toBeInstanceOf(ImageTooLargeException); + await expect(rejection).rejects.toMatchObject({ status: 422 }); + expect(read).not.toHaveBeenCalled(); + }); + + it('checks the baseline as well as the new image', async () => { + await expect(pool.run(diffJob(png(5, 5, 0), png(10, 10, 0)))).rejects.toBeInstanceOf(ImageTooLargeException); + }); + + it('checks every job kind', async () => { + await expect(pool.run(signatureJob(png(10, 10, 0), png(10, 10, 255)))).rejects.toBeInstanceOf( + ImageTooLargeException + ); + }); + + it('rejects a diff whose canvas would exceed the limit when other sizes are allowed', async () => { + process.env.MAX_IMAGE_PIXELS = '60'; + const wide = png(10, 5, 0); + const tall = png(5, 10, 0); + + await expect(pool.run(diffJob(wide, tall, true))).rejects.toBeInstanceOf(ImageTooLargeException); + await expect(pool.run(diffJob(wide, tall, false))).resolves.toMatchObject({ + equal: false, + isSameDimension: false, + }); + }); + + it('leaves a malformed header to the decoder', async () => { + const rejection = pool.run(diffJob(Buffer.from('png'), png(5, 5, 0))); + + await expect(rejection).rejects.toThrow(); + await expect(rejection).rejects.not.toBeInstanceOf(ImageTooLargeException); + }); + }); +}); diff --git a/src/compare/diff-worker-pool.spec.ts b/src/compare/diff-worker-pool.spec.ts deleted file mode 100644 index aa3c6e7c..00000000 --- a/src/compare/diff-worker-pool.spec.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { PNG } from 'pngjs'; -import { DiffWorkerPool } from './diff-worker-pool'; - -// Opaque throughout: pixelmatch composites transparent pixels onto white, so a -// fully transparent image would read as equal to a white one. -const png = (width: number, height: number, rgb: number): Buffer => { - const image = new PNG({ width, height }); - for (let i = 0; i < width * height; i++) { - image.data[i * 4] = rgb; - image.data[i * 4 + 1] = rgb; - image.data[i * 4 + 2] = rgb; - image.data[i * 4 + 3] = 255; - } - return PNG.sync.write(image); -}; - -describe('DiffWorkerPool', () => { - let pool: DiffWorkerPool; - - beforeEach(() => { - pool = new DiffWorkerPool(); - }); - - afterEach(async () => { - await pool.onApplicationShutdown(); - }); - - it('answers a diff job with the pixel mismatch it found', async () => { - const output = await pool.run({ - kind: 'diff', - baseline: png(10, 10, 0), - image: png(10, 10, 255), - ignoreAreas: [], - threshold: 0.1, - includeAA: false, - allowDiffDimensions: false, - diffTolerancePercent: 0, - saveDiff: false, - }); - - expect(output).toMatchObject({ equal: false, pixelMisMatchCount: 100 }); - }); - - it('answers a signature job with a change signature', async () => { - const output = await pool.run({ - kind: 'signature', - baseline: png(10, 10, 0), - image: png(10, 10, 255), - ignoreAreas: [], - threshold: 0.1, - includeAA: false, - }); - - expect(output.signature).toHaveLength(64); - expect(output.signature!.reduce((sum, value) => sum + value, 0)).toBeCloseTo(1); - }); -}); diff --git a/src/compare/diff-worker-pool.ts b/src/compare/diff-worker-pool.ts index f6562c71..0454d4a5 100644 --- a/src/compare/diff-worker-pool.ts +++ b/src/compare/diff-worker-pool.ts @@ -1,35 +1,55 @@ -import { Injectable, Logger, OnApplicationShutdown } from '@nestjs/common'; +import { Injectable, Logger, OnApplicationShutdown, ServiceUnavailableException } from '@nestjs/common'; import { Worker } from 'worker_threads'; import { availableParallelism } from 'os'; import { existsSync } from 'fs'; import { join } from 'path'; -import { PixelmatchJobInput, PixelmatchJobOutput } from './libs/pixelmatch/pixelmatch.core'; -import { SignatureJobInput, SignatureJobOutput } from './libs/pixelmatch/signature.core'; -import { BlurAmplitudeJobInput, BlurAmplitudeJobOutput } from './libs/pixelmatch/blur-amplitude.core'; -import { PatchGateJobInput, PatchGateJobOutput } from './libs/pixelmatch/patch-gate.core'; -import { runWorkerJob, WorkerJobInput, WorkerJobOutput } from './libs/pixelmatch/worker-job'; +import { nonNegativeNumber } from '../utils/env'; +import { assertWithinPixelLimit, readPngDimensions } from '../static/utils'; +import { + runWorkerJob, + WorkerJobInput, + WorkerJobInputOf, + WorkerJobKind, + WorkerJobOutput, + WorkerJobResult, +} from './libs/pixelmatch/worker-job'; const WORKER_FILE = join(__dirname, 'libs', 'pixelmatch', 'pixelmatch.worker.js'); -// consecutive worker failures after which the pool stops respawning const MAX_SPAWN_FAILURES = 3; -// how many times a job may be handed to a worker before it is given up on -const MAX_JOB_ATTEMPTS = 2; +const MAX_JOB_CRASHES = 2; + +export const QUEUE_FULL_RETRY_AFTER_SECONDS = 5; + +export class ImageDiffQueueFullException extends ServiceUnavailableException { + readonly retryAfterSeconds = QUEUE_FULL_RETRY_AFTER_SECONDS; + constructor(queueLimit: number) { + super(`Image diff queue is full (${queueLimit} jobs)`); + } +} + +export class ImageDiffPoolStoppedException extends ServiceUnavailableException { + readonly retryAfterSeconds = QUEUE_FULL_RETRY_AFTER_SECONDS; +} + +export class ImageDiffJobTimeoutError extends Error { + constructor(timeoutMs: number) { + super(`Image diff job timed out after ${timeoutMs} ms`); + this.name = 'ImageDiffJobTimeoutError'; + } +} + +const JOB_TIMEOUT_MS = 120_000; +const DEFAULT_WORKER_MAX_HEAP_MB = 512; interface Job { input: WorkerJobInput; resolve: (output: WorkerJobOutput) => void; reject: (error: Error) => void; - attempts: number; + crashes: number; + timer?: NodeJS.Timeout; } -/** - * Fixed pool of worker threads for CPU-bound image work — diffing a screenshot - * against its baseline, the change signatures the variations dialog compares, - * and the rendering-noise measurement behind a project's blur-amplitude filter. Keeps the event loop free so the API stays responsive while - * screenshots are decoded and compared. Pool size: DIFF_WORKERS_COUNT env var, - * defaulting to cores - 1 (capped) so the main thread always has a core left. - */ @Injectable() export class DiffWorkerPool implements OnApplicationShutdown { private readonly logger: Logger = new Logger(DiffWorkerPool.name); @@ -37,41 +57,48 @@ export class DiffWorkerPool implements OnApplicationShutdown { 1, Math.min(Number(process.env.DIFF_WORKERS_COUNT) || availableParallelism() - 1, 8) ); - // Queued jobs hold both full image buffers, so an unbounded queue could - // exhaust memory under a flood of concurrent uploads. private readonly queueLimit = Number(process.env.DIFF_QUEUE_LIMIT) || 256; - // The compiled worker file only exists in the built app (dist). Under - // ts-jest / ts-node run the job inline instead. + private readonly maxHeapMb = nonNegativeNumber(process.env.DIFF_WORKER_MAX_HEAP_MB, DEFAULT_WORKER_MAX_HEAP_MB); private inline = !existsSync(WORKER_FILE); private workers: Worker[] = []; private idle: Worker[] = []; private queue: Job[] = []; + private ready = new Set(); private inFlight = new Map(); private started = false; private destroyed = false; private spawnFailures = 0; - async run(input: PixelmatchJobInput): Promise; - async run(input: SignatureJobInput): Promise; - async run(input: BlurAmplitudeJobInput): Promise; - async run(input: PatchGateJobInput): Promise; - async run(input: WorkerJobInput): Promise { + async run(input: WorkerJobInputOf): Promise> { + this.assertWithinPixelLimit(input); if (this.inline) { - return runWorkerJob(input); + return (await runWorkerJob(input)) as WorkerJobResult; } if (this.destroyed) { - throw new Error('Image diff worker pool is shut down'); + throw new ImageDiffPoolStoppedException('Image diff worker pool is shut down'); } if (this.queue.length >= this.queueLimit) { - throw new Error(`Image diff queue is full (${this.queueLimit} jobs)`); + throw new ImageDiffQueueFullException(this.queueLimit); } this.start(); - return new Promise((resolve, reject) => { - this.queue.push({ input, resolve, reject, attempts: 0 }); + return new Promise>((resolve, reject) => { + this.queue.push({ input, resolve: resolve as Job['resolve'], reject, crashes: 0 }); this.dispatch(); }); } + // a 69-byte PNG can claim 30000x30000: check IHDR before decoding + private assertWithinPixelLimit(input: WorkerJobInput): void { + const sizes = [input.baseline, input.image].map((bytes) => readPngDimensions(bytes)); + for (const size of sizes) { + if (size) assertWithinPixelLimit(size.width, size.height); + } + const [baseline, image] = sizes; + if (input.kind === 'diff' && input.allowDiffDimensions && baseline && image) { + assertWithinPixelLimit(Math.max(baseline.width, image.width), Math.max(baseline.height, image.height)); + } + } + private start(): void { if (this.started) return; this.started = true; @@ -82,18 +109,31 @@ export class DiffWorkerPool implements OnApplicationShutdown { } private spawn(): void { - const worker = new Worker(WORKER_FILE); + const worker = new Worker( + WORKER_FILE, + this.maxHeapMb > 0 ? { resourceLimits: { maxOldGenerationSizeMb: this.maxHeapMb } } : {} + ); worker.on('message', (output: WorkerJobOutput & { error?: string }) => { + if ((output as { ready?: unknown }).ready === true) { + if (this.workers.includes(worker)) { + this.ready.add(worker); + this.spawnFailures = 0; + } + return; + } const job = this.inFlight.get(worker); this.inFlight.delete(worker); - // a message can arrive after the worker was dropped or the pool shut - // down; taking it back would hand it a job it will never answer + clearTimeout(job?.timer); if (this.workers.includes(worker)) { this.idle.push(worker); this.spawnFailures = 0; } if (job) { - output.error ? job.reject(new Error(output.error)) : job.resolve(output); + if (output.error) { + job.reject(new Error(output.error)); + } else { + job.resolve(output); + } } this.dispatch(); }); @@ -101,26 +141,18 @@ export class DiffWorkerPool implements OnApplicationShutdown { this.logger.error(`Image diff worker crashed: ${error}`); this.replace(worker, error); }); - worker.on('exit', (code) => { - if (code !== 0) { - this.replace(worker, new Error(`Image diff worker exited with code ${code}`)); - } - }); + worker.on('exit', (code) => this.replace(worker, new Error(`Image diff worker exited with code ${code}`))); this.workers.push(worker); this.idle.push(worker); } private replace(worker: Worker, error: Error): void { if (!this.workers.includes(worker)) return; - this.workers = this.workers.filter((w) => w !== worker); - this.idle = this.idle.filter((w) => w !== worker); - const job = this.inFlight.get(worker); - this.inFlight.delete(worker); + const loaded = this.ready.has(worker); + const job = this.forget(worker); if (job) { - // a worker dying takes its job down with it; give the job one more run - // rather than failing the upload waiting behind it - if (!this.destroyed && job.attempts < MAX_JOB_ATTEMPTS) { - job.attempts += 1; + if (loaded) job.crashes += 1; + if (!this.destroyed && job.crashes < MAX_JOB_CRASHES) { this.queue.unshift(job); } else { job.reject(error); @@ -130,62 +162,79 @@ export class DiffWorkerPool implements OnApplicationShutdown { return; } - // a worker script that fails to load fails the same way every time, so - // respawning on each error would spin. Fall back to comparing on this - // thread instead: slower, but the uploads still go through. - this.spawnFailures += 1; - if (this.spawnFailures >= MAX_SPAWN_FAILURES) { - this.inline = true; - this.logger.error( - `Image diff workers failed to run ${this.spawnFailures} times, comparing on the main thread from now on` - ); - this.drainQueueInline(); - return; + if (!loaded) { + this.spawnFailures += 1; + if (this.spawnFailures >= MAX_SPAWN_FAILURES) { + this.inline = true; + this.logger.error( + `Image diff workers failed to load ${this.spawnFailures} times, comparing on the main thread from now on` + ); + this.drainQueueInline(); + return; + } } this.spawn(); this.dispatch(); } - // the pool gave up on workers: answer what is already queued here + private forget(worker: Worker): Job | undefined { + this.ready.delete(worker); + this.workers = this.workers.filter((w) => w !== worker); + this.idle = this.idle.filter((w) => w !== worker); + const job = this.inFlight.get(worker); + this.inFlight.delete(worker); + clearTimeout(job?.timer); + return job; + } + private drainQueueInline(): void { const queued = this.queue; this.queue = []; for (const job of queued) { - try { - job.resolve(runWorkerJob(job.input)); - } catch (error) { - job.reject(error instanceof Error ? error : new Error(String(error))); - } + Promise.resolve() + .then(() => runWorkerJob(job.input)) + .then(job.resolve, (error) => job.reject(error instanceof Error ? error : new Error(String(error)))); } } private dispatch(): void { while (this.idle.length > 0 && this.queue.length > 0) { - // the loop condition guarantees both are there const worker = this.idle.pop()!; const job = this.queue.shift()!; this.inFlight.set(worker, job); worker.postMessage(job.input); + job.timer = setTimeout(() => this.timeOut(worker, job), JOB_TIMEOUT_MS); + job.timer.unref(); } } - // After the HTTP server has stopped and drained its requests, so an upload - // still being compared at shutdown gets its result instead of an error + private timeOut(worker: Worker, job: Job): void { + if (this.inFlight.get(worker) !== job) return; + this.forget(worker); + this.logger.error(`Image diff job ran over ${JOB_TIMEOUT_MS} ms, replacing its worker`); + job.reject(new ImageDiffJobTimeoutError(JOB_TIMEOUT_MS)); + worker.terminate().catch(() => undefined); + this.spawn(); + this.dispatch(); + } + async onApplicationShutdown(): Promise { this.destroyed = true; - const shutdownError = new Error('Image diff worker pool is shutting down'); + const shutdownError = new ImageDiffPoolStoppedException('Image diff worker pool is shutting down'); for (const job of this.queue) { job.reject(shutdownError); } this.queue = []; for (const job of this.inFlight.values()) { + clearTimeout(job.timer); job.reject(shutdownError); } this.inFlight.clear(); const workers = this.workers; this.workers = []; this.idle = []; + this.ready.clear(); await Promise.all(workers.map((worker) => worker.terminate())); } } diff --git a/src/compare/diff-worker-pool.workers.spec.ts b/src/compare/diff-worker-pool.workers.spec.ts index f2526c39..cb1f0fda 100644 --- a/src/compare/diff-worker-pool.workers.spec.ts +++ b/src/compare/diff-worker-pool.workers.spec.ts @@ -4,13 +4,17 @@ import { Logger } from '@nestjs/common'; import { Worker } from 'worker_threads'; import { availableParallelism } from 'os'; import { existsSync } from 'fs'; -import { DiffWorkerPool } from './diff-worker-pool'; +import { + DiffWorkerPool, + ImageDiffJobTimeoutError, + ImageDiffPoolStoppedException, + ImageDiffQueueFullException, +} from './diff-worker-pool'; +import { PNG } from 'pngjs'; +import { ImageTooLargeException } from '../static/utils'; import { runWorkerJob } from './libs/pixelmatch/worker-job'; import { PixelmatchJobInput } from './libs/pixelmatch/pixelmatch.core'; -// The pool only talks to real worker threads once the compiled worker file -// exists (dist). These tests stand in for both: the file "exists", and every -// Worker is an in-process fake the test drives by emitting its events. class FakeWorker extends EventEmitter { postMessage = jest.fn(); terminate = jest.fn().mockResolvedValue(0); @@ -67,7 +71,8 @@ const output = (pixelMisMatchCount: number) => ({ diffPercent: pixelMisMatchCount, }); -// settles a promise into a value the test can inspect without an unhandled rejection +const ready = (worker: FakeWorker) => worker.emit('message', { ready: true }); + const settle = (promise: Promise) => promise.then( (value) => ({ value, error: undefined as Error | undefined }), @@ -80,12 +85,19 @@ describe('DiffWorkerPool with worker threads', () => { let loggerErrorSpy: jest.SpyInstance; let loggerLogSpy: jest.SpyInstance; - const createPool = ({ workers = '1', queueLimit }: { workers?: string; queueLimit?: string } = {}) => { - process.env.DIFF_WORKERS_COUNT = workers; - if (queueLimit === undefined) { - delete process.env.DIFF_QUEUE_LIMIT; - } else { - process.env.DIFF_QUEUE_LIMIT = queueLimit; + const createPool = ({ + workers = '1', + queueLimit, + maxHeap, + }: { workers?: string; queueLimit?: string; maxHeap?: string } = {}) => { + const env = { + DIFF_WORKER_MAX_HEAP_MB: maxHeap, + DIFF_WORKERS_COUNT: workers, + DIFF_QUEUE_LIMIT: queueLimit, + }; + for (const [name, value] of Object.entries(env)) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; } pool = new DiffWorkerPool(); return pool; @@ -103,7 +115,6 @@ describe('DiffWorkerPool with worker threads', () => { afterEach(async () => { await pool?.onApplicationShutdown(); - // each test makes its own pool; the shutdown above skips a test that made none pool = undefined as unknown as DiffWorkerPool; process.env = { ...originalEnv }; jest.restoreAllMocks(); @@ -114,15 +125,14 @@ describe('DiffWorkerPool with worker threads', () => { ['DIFF_WORKERS_COUNT when set', '3', 4, 3], ['DIFF_WORKERS_COUNT capped at 8', '20', 4, 8], ['cores - 1 when DIFF_WORKERS_COUNT is not a number', 'abc', 4, 3], - // NOTE: current behavior — an explicit "0" is falsy and treated as unset - ['cores - 1 when DIFF_WORKERS_COUNT is 0', '0', 6, 5], + ['cores - 1 when DIFF_WORKERS_COUNT is 0 (no workers is not an option)', '0', 6, 5], ['cores - 1 capped at 8', '', 32, 8], ['at least one worker on a single core', '', 1, 1], ])('uses %s', async (_, workers, cores, expected) => { availableParallelismMock.mockReturnValue(cores); createPool({ workers }); - settle(pool.run(job(1))); + void settle(pool.run(job(1))); expect(WorkerMock).toHaveBeenCalledTimes(expected); }); @@ -132,15 +142,31 @@ describe('DiffWorkerPool with worker threads', () => { expect(WorkerMock).not.toHaveBeenCalled(); - settle(pool.run(job(1))); - settle(pool.run(job(2))); + void settle(pool.run(job(1))); + void settle(pool.run(job(2))); expect(WorkerMock).toHaveBeenCalledTimes(2); - expect(WorkerMock).toHaveBeenCalledWith(join(__dirname, 'libs', 'pixelmatch', 'pixelmatch.worker.js')); + expect(WorkerMock).toHaveBeenCalledWith(join(__dirname, 'libs', 'pixelmatch', 'pixelmatch.worker.js'), { + resourceLimits: { maxOldGenerationSizeMb: 512 }, + }); expect(loggerLogSpy).toHaveBeenCalledWith('Started 2 image diff workers'); }); }); + describe('worker heap limit', () => { + it.each([ + ['takes the limit from DIFF_WORKER_MAX_HEAP_MB', '256', { resourceLimits: { maxOldGenerationSizeMb: 256 } }], + ['drops the limit for 0', '0', {}], + ['falls back to 512 for garbage', 'lots', { resourceLimits: { maxOldGenerationSizeMb: 512 } }], + ])('%s', async (_, maxHeap, options) => { + createPool({ maxHeap }); + + void settle(pool.run(job(1))); + + expect(WorkerMock).toHaveBeenCalledWith(expect.any(String), options); + }); + }); + describe('inline mode', () => { it('runs the job on the calling thread when the compiled worker file is missing', async () => { existsSyncMock.mockReturnValue(false); @@ -170,8 +196,8 @@ describe('DiffWorkerPool with worker threads', () => { it('hands the first job to the most recently spawned idle worker', async () => { createPool({ workers: '2' }); - settle(pool.run(job(1))); - settle(pool.run(job(2))); + void settle(pool.run(job(1))); + void settle(pool.run(job(2))); expect(mockWorkers[1].postMessage).toHaveBeenCalledWith(job(1)); expect(mockWorkers[0].postMessage).toHaveBeenCalledWith(job(2)); @@ -215,13 +241,17 @@ describe('DiffWorkerPool with worker threads', () => { }); describe('queue limit', () => { - it('rejects a job once the waiting queue holds DIFF_QUEUE_LIMIT jobs, not counting the ones in flight', async () => { + it('rejects a job once the waiting queue holds DIFF_QUEUE_LIMIT jobs, not counting the ones in flight, with 503 and a retry hint', async () => { createPool({ queueLimit: '1' }); const inFlight = settle(pool.run(job(1))); const queued = settle(pool.run(job(2))); - await expect(pool.run(job(3))).rejects.toThrow('Image diff queue is full (1 jobs)'); + const { error } = await settle(pool.run(job(3))); + expect(error).toBeInstanceOf(ImageDiffQueueFullException); + expect((error as ImageDiffQueueFullException).getStatus()).toBe(503); + expect((error as ImageDiffQueueFullException).retryAfterSeconds).toBe(5); + expect(error?.message).toBe('Image diff queue is full (1 jobs)'); expect(mockWorkers[0].postMessage).toHaveBeenCalledTimes(1); mockWorkers[0].emit('message', output(1)); @@ -230,6 +260,17 @@ describe('DiffWorkerPool with worker threads', () => { expect((await queued).value).toEqual(output(2)); }); + it('does not queue an oversized job', async () => { + process.env.MAX_IMAGE_PIXELS = '50'; + createPool(); + const big = PNG.sync.write(new PNG({ width: 10, height: 10 })); + + await expect(pool.run({ ...job(1), baseline: big, image: big })).rejects.toBeInstanceOf(ImageTooLargeException); + + expect(WorkerMock).not.toHaveBeenCalled(); + expect(runWorkerJobMock).not.toHaveBeenCalled(); + }); + it('defaults to a queue of 256 waiting jobs', async () => { createPool(); @@ -282,50 +323,107 @@ describe('DiffWorkerPool with worker threads', () => { await expect(result).resolves.toEqual(output(1)); }); - it('keeps a worker that exited with code 0 in the pool', async () => { + it('replaces a worker that exited with code 0', async () => { createPool(); - pool.run(job(1)); + const first = pool.run(job(1)); + ready(mockWorkers[0]); mockWorkers[0].emit('message', output(1)); + await expect(first).resolves.toEqual(output(1)); mockWorkers[0].emit('exit', 0); - const pending = settle(pool.run(job(2))); - - // NOTE: current behavior — a clean exit is ignored, so the exited worker - // stays idle and is handed the next job, which would never be answered. - expect(mockWorkers).toHaveLength(1); - expect(mockWorkers[0].postMessage).toHaveBeenLastCalledWith(job(2)); + const second = pool.run(job(2)); - await pool.onApplicationShutdown(); - expect((await pending).error).toEqual(new Error('Image diff worker pool is shutting down')); + expect(mockWorkers).toHaveLength(2); + expect(mockWorkers[1].postMessage).toHaveBeenCalledWith(job(2)); + expect(mockWorkers[0].postMessage).toHaveBeenCalledTimes(1); + mockWorkers[1].emit('message', output(2)); + await expect(second).resolves.toEqual(output(2)); }); - it('gives a job up with the error of the last worker after it was handed out three times', async () => { + it('runs a job at most twice on loaded workers', async () => { createPool({ workers: '2' }); const target = settle(pool.run(job(1))); const other = pool.run(job(2)); - const [, first] = mockWorkers; + const [first0, first] = mockWorkers; + ready(first0); + ready(first); expect(first.postMessage).toHaveBeenCalledWith(job(1)); first.emit('error', new Error('first')); const second = mockWorkers[2]; + ready(second); expect(second.postMessage).toHaveBeenCalledWith(job(1)); - // a healthy answer in between resets the consecutive failure count - mockWorkers[0].emit('message', output(2)); + first0.emit('message', output(2)); await expect(other).resolves.toEqual(output(2)); second.emit('error', new Error('second')); - const third = mockWorkers[3]; - expect(third.postMessage).toHaveBeenCalledWith(job(1)); - third.emit('error', new Error('third')); + expect((await target).error).toEqual(new Error('second')); + expect(mockWorkers).toHaveLength(4); + expect(mockWorkers[3].postMessage).not.toHaveBeenCalled(); + }); + + it('does not switch to inline mode when a job keeps crashing loaded workers', async () => { + createPool(); + + const poison = settle(pool.run(job(1))); + ready(mockWorkers[0]); + mockWorkers[0].emit('error', new Error('oom 1')); + ready(mockWorkers[1]); + mockWorkers[1].emit('error', new Error('oom 2')); + + expect((await poison).error).toEqual(new Error('oom 2')); + + const next = settle(pool.run(job(2))); + expect(runWorkerJobMock).not.toHaveBeenCalled(); + expect(mockWorkers[2].postMessage).toHaveBeenCalledWith(job(2)); + ready(mockWorkers[2]); + mockWorkers[2].emit('message', output(2)); + expect((await next).value).toEqual(output(2)); + }); + + it('does not count a load failure against the job', async () => { + createPool(); + + const result = pool.run(job(1)); + mockWorkers[0].emit('error', new Error('cannot load')); + + expect(mockWorkers[1].postMessage).toHaveBeenCalledWith(job(1)); + ready(mockWorkers[1]); + mockWorkers[1].emit('message', output(1)); + await expect(result).resolves.toEqual(output(1)); + }); + + it('resets the load-failure count once a worker has loaded', async () => { + createPool(); + + void settle(pool.run(job(1))); + mockWorkers[0].emit('error', new Error('load 1')); + mockWorkers[1].emit('error', new Error('load 2')); + ready(mockWorkers[2]); + mockWorkers[2].emit('message', output(1)); + mockWorkers[2].emit('error', new Error('idle crash')); + mockWorkers[3].emit('error', new Error('load 3')); + mockWorkers[4].emit('error', new Error('load 4')); - // NOTE: current behavior — MAX_JOB_ATTEMPTS = 2 reads as "handed to a worker - // at most twice", but the job runs on three workers before it is rejected. - expect((await target).error).toEqual(new Error('third')); - expect(mockWorkers).toHaveLength(5); - expect(mockWorkers[4].postMessage).not.toHaveBeenCalled(); + expect(runWorkerJobMock).not.toHaveBeenCalled(); + expect(loggerErrorSpy).not.toHaveBeenCalledWith(expect.stringContaining('comparing on the main thread')); + }); + + it('ignores a ready message from a dropped worker', async () => { + createPool(); + + void settle(pool.run(job(1))); + const dropped = mockWorkers[0]; + dropped.emit('error', new Error('load 1')); + dropped.emit('message', { ready: true }); + mockWorkers[1].emit('error', new Error('load 2')); + dropped.emit('message', { ready: true }); + mockWorkers[2].emit('error', new Error('load 3')); + + expect(loggerErrorSpy).toHaveBeenCalledWith(expect.stringContaining('comparing on the main thread')); }); it('rejects with the exit code when the last failure was a non-zero exit', async () => { @@ -333,28 +431,33 @@ describe('DiffWorkerPool with worker threads', () => { const target = settle(pool.run(job(1))); const other = pool.run(job(2)); + ready(mockWorkers[0]); + ready(mockWorkers[1]); mockWorkers[1].emit('exit', 2); mockWorkers[0].emit('message', output(2)); await other; + ready(mockWorkers[2]); mockWorkers[2].emit('exit', 3); - mockWorkers[3].emit('exit', 4); - expect((await target).error).toEqual(new Error('Image diff worker exited with code 4')); + expect((await target).error).toEqual(new Error('Image diff worker exited with code 3')); + expect(mockWorkers[3].postMessage).not.toHaveBeenCalled(); }); - it('falls back to inline after three consecutive failures and answers the queued jobs on this thread', async () => { + it('falls back to inline after three workers fail to load and answers every job on this thread', async () => { createPool(); runWorkerJobMock + .mockReturnValueOnce(output(1)) .mockReturnValueOnce(output(2)) .mockImplementationOnce(() => { throw new Error('inline failed'); }) .mockImplementationOnce(() => { + // eslint-disable-next-line @typescript-eslint/only-throw-error -- pool must survive non-Error throws throw 'not an error'; }) .mockReturnValueOnce(output(5)); - const inFlight = settle(pool.run(job(1))); + const inFlight = pool.run(job(1)); const queued2 = pool.run(job(2)); const queued3 = settle(pool.run(job(3))); const queued4 = settle(pool.run(job(4))); @@ -365,22 +468,40 @@ describe('DiffWorkerPool with worker threads', () => { mockWorkers[2].emit('error', new Error('load failed 3')); expect(loggerErrorSpy).toHaveBeenCalledWith( - 'Image diff workers failed to run 3 times, comparing on the main thread from now on' + 'Image diff workers failed to load 3 times, comparing on the main thread from now on' ); - // the job that was on the failing worker is rejected, the waiting ones run inline - expect((await inFlight).error).toEqual(new Error('load failed 3')); + await expect(inFlight).resolves.toEqual(output(1)); await expect(queued2).resolves.toEqual(output(2)); expect((await queued3).error).toEqual(new Error('inline failed')); expect((await queued4).error).toEqual(new Error('not an error')); - expect(runWorkerJobMock.mock.calls).toEqual([[job(2)], [job(3)], [job(4)]]); + expect(runWorkerJobMock.mock.calls).toEqual([[job(1)], [job(2)], [job(3)], [job(4)]]); expect(mockWorkers).toHaveLength(3); - // and every later job runs inline without spawning anything await expect(pool.run(job(5))).resolves.toEqual(output(5)); expect(runWorkerJobMock).toHaveBeenLastCalledWith(job(5)); expect(mockWorkers).toHaveLength(3); }); + it('answers jobs that finish asynchronously once it runs inline', async () => { + createPool(); + runWorkerJobMock + .mockReturnValueOnce(Promise.reject(new Error('async failed'))) + .mockReturnValueOnce(Promise.resolve(output(9))) + .mockReturnValueOnce(Promise.reject('not an error')); + + const failing = settle(pool.run(job(1))); + const succeeding = pool.run(job(2)); + const rejectingNonError = settle(pool.run(job(3))); + + mockWorkers[0].emit('error', new Error('load failed 1')); + mockWorkers[1].emit('error', new Error('load failed 2')); + mockWorkers[2].emit('error', new Error('load failed 3')); + + expect((await failing).error).toEqual(new Error('async failed')); + await expect(succeeding).resolves.toEqual(output(9)); + expect((await rejectingNonError).error).toEqual(new Error('not an error')); + }); + it('does not run a job again when the worker that already answered it crashes later', async () => { createPool(); @@ -397,19 +518,18 @@ describe('DiffWorkerPool with worker threads', () => { it('takes only the crashed worker out of the idle ones', async () => { createPool({ workers: '3' }); - settle(pool.run(job(1))); + void settle(pool.run(job(1))); const [crashed, idle, busy] = mockWorkers; expect(busy.postMessage).toHaveBeenCalledWith(job(1)); crashed.emit('error', new Error('boom')); const replacement = mockWorkers[3]; - settle(pool.run(job(2))); - settle(pool.run(job(3))); - settle(pool.run(job(4))); + void settle(pool.run(job(2))); + void settle(pool.run(job(3))); + void settle(pool.run(job(4))); expect(replacement.postMessage).toHaveBeenCalledWith(job(2)); expect(idle.postMessage).toHaveBeenCalledWith(job(3)); - // job 4 waits: the crashed worker is never handed anything expect(crashed.postMessage).not.toHaveBeenCalled(); }); @@ -424,7 +544,6 @@ describe('DiffWorkerPool with worker threads', () => { dropped.emit('message', output(99)); const next = pool.run(job(2)); - // the late answer neither settles the retried job nor brings the worker back expect(dropped.postMessage).toHaveBeenCalledTimes(1); expect(replacement.postMessage).toHaveBeenCalledTimes(1); @@ -437,6 +556,85 @@ describe('DiffWorkerPool with worker threads', () => { }); }); + describe('job timeout', () => { + beforeEach(() => { + jest.useFakeTimers(); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + it('rejects a job that runs longer than 120 s and replaces its worker', async () => { + createPool(); + + const hung = settle(pool.run(job(1))); + ready(mockWorkers[0]); + jest.advanceTimersByTime(120_000); + + expect((await hung).error).toBeInstanceOf(ImageDiffJobTimeoutError); + expect((await hung).error?.message).toBe('Image diff job timed out after 120000 ms'); + expect(mockWorkers[0].terminate).toHaveBeenCalledTimes(1); + expect(mockWorkers).toHaveLength(2); + expect(loggerErrorSpy).toHaveBeenCalledWith('Image diff job ran over 120000 ms, replacing its worker'); + + void settle(pool.run(job(2))); + expect(mockWorkers[1].postMessage).toHaveBeenCalledWith(job(2)); + }); + + it('does not time out a job answered in time', async () => { + createPool(); + + const result = pool.run(job(1)); + ready(mockWorkers[0]); + jest.advanceTimersByTime(119_999); + mockWorkers[0].emit('message', output(1)); + jest.advanceTimersByTime(5000); + + await expect(result).resolves.toEqual(output(1)); + expect(mockWorkers[0].terminate).not.toHaveBeenCalled(); + expect(mockWorkers).toHaveLength(1); + }); + + it('ignores the exit of the worker it terminated', async () => { + createPool(); + + const hung = settle(pool.run(job(1))); + jest.advanceTimersByTime(120_000); + await hung; + mockWorkers[0].emit('exit', 1); + mockWorkers[0].emit('error', new Error('terminated')); + + expect(mockWorkers).toHaveLength(2); + }); + + it('does not retry the timed-out job and does not count it as a load failure', async () => { + createPool(); + + for (const id of [1, 2, 3]) { + const hung = settle(pool.run(job(id))); + jest.advanceTimersByTime(120_000); + expect((await hung).error).toBeInstanceOf(ImageDiffJobTimeoutError); + } + + expect(runWorkerJobMock).not.toHaveBeenCalled(); + expect(mockWorkers).toHaveLength(4); + expect(mockWorkers.map((worker) => worker.postMessage.mock.calls.length)).toEqual([1, 1, 1, 0]); + }); + + it('does not spawn a replacement for a pool that is shut down while the job runs', async () => { + createPool(); + + const pending = settle(pool.run(job(1))); + await pool.onApplicationShutdown(); + jest.advanceTimersByTime(5000); + + expect((await pending).error).toBeInstanceOf(ImageDiffPoolStoppedException); + expect(mockWorkers).toHaveLength(1); + expect(mockWorkers[0].terminate).toHaveBeenCalledTimes(1); + }); + }); + describe('onApplicationShutdown', () => { it('rejects queued and in-flight jobs and terminates every worker', async () => { createPool({ workers: '2' }); @@ -455,13 +653,20 @@ describe('DiffWorkerPool with worker threads', () => { expect(mockWorkers.map((worker) => worker.terminate.mock.calls.length)).toEqual([1, 1]); }); - it('refuses new jobs once shut down', async () => { + it('rejects with 503 once shut down', async () => { createPool(); - settle(pool.run(job(1))); + const queued = [1, 2].map((id) => settle(pool.run(job(id)))); await pool.onApplicationShutdown(); - await expect(pool.run(job(2))).rejects.toThrow('Image diff worker pool is shut down'); + for (const { error } of await Promise.all(queued)) { + expect(error).toBeInstanceOf(ImageDiffPoolStoppedException); + } + const { error } = await settle(pool.run(job(3))); + expect(error).toBeInstanceOf(ImageDiffPoolStoppedException); + expect(error?.message).toBe('Image diff worker pool is shut down'); + expect((error as ImageDiffPoolStoppedException).getStatus()).toBe(503); + expect((error as ImageDiffPoolStoppedException).retryAfterSeconds).toBe(5); expect(mockWorkers).toHaveLength(1); }); diff --git a/src/compare/libs/ImageCompareInput.ts b/src/compare/libs/ImageCompareInput.ts index 48dee42a..1f8a7c0e 100644 --- a/src/compare/libs/ImageCompareInput.ts +++ b/src/compare/libs/ImageCompareInput.ts @@ -2,16 +2,11 @@ import { IgnoreAreaDto } from '../../test-runs/dto/ignore-area.dto'; import { BlurAmplitudeOptions } from './pixelmatch/blur-amplitude.core'; export interface ImageCompareInput { - // null when the variation has no baseline yet; comparators then report NO_BASELINE_RESULT baseline: string | null; image: string; + imageBuffer?: Buffer; diffTollerancePercent: number; ignoreAreas: IgnoreAreaDto[]; saveDiffAsFile: boolean; - /** - * The project's rendering-noise filter, when it has one on. Read only by a - * comparator that applies it itself (see ImageComparator); for the others - * CompareService applies it to their verdict. - */ blurAmplitude?: BlurAmplitudeOptions; } diff --git a/src/compare/libs/blur-amplitude.config.spec.ts b/src/compare/libs/blur-amplitude.config.spec.ts index f5e41c8b..3a49fa39 100644 --- a/src/compare/libs/blur-amplitude.config.spec.ts +++ b/src/compare/libs/blur-amplitude.config.spec.ts @@ -1,5 +1,5 @@ import { Logger } from '@nestjs/common'; -import { DEFAULT_BLUR_AMPLITUDE_THRESHOLD, DEFAULT_BLUR_SIGMA } from './pixelmatch/blur-amplitude.core'; +import { DEFAULT_BLUR_AMPLITUDE_THRESHOLD } from './pixelmatch/blur-amplitude.core'; import { DEFAULT_DINOV2_THRESHOLD, MAX_BLUR_AMPLITUDE_THRESHOLD, @@ -24,10 +24,9 @@ describe('splitBlurAmplitudeConfig', () => { it('turns it on at the default threshold', () => { const { blurAmplitude } = splitBlurAmplitudeConfig({ blurAmplitudeFilter: true }, logger); - expect(blurAmplitude).toEqual({ sigma: DEFAULT_BLUR_SIGMA, threshold: DEFAULT_BLUR_AMPLITUDE_THRESHOLD }); + expect(blurAmplitude).toEqual({ threshold: DEFAULT_BLUR_AMPLITUDE_THRESHOLD }); }); - // anything near 255 would pass almost every change of one size it('accepts a threshold up to half the contrast range', () => { const { blurAmplitude } = splitBlurAmplitudeConfig( { blurAmplitudeFilter: true, blurAmplitudeThreshold: MAX_BLUR_AMPLITUDE_THRESHOLD }, @@ -44,11 +43,9 @@ describe('splitBlurAmplitudeConfig', () => { logger ); - expect(blurAmplitude).toEqual({ sigma: DEFAULT_BLUR_SIGMA, threshold: 60.5 }); + expect(blurAmplitude).toEqual({ threshold: 60.5 }); }); - // the config is a JSON string nothing validates on the way in; only a real - // `true` turns on something that can pass runs it.each([['true'], [1], [false]])('stays off for blurAmplitudeFilter %p', (value) => { expect(splitBlurAmplitudeConfig({ blurAmplitudeFilter: value as boolean }, logger).blurAmplitude).toBeUndefined(); }); @@ -66,8 +63,6 @@ describe('splitBlurAmplitudeConfig', () => { } ); - // the keys are CompareService's, not the library's: looks-same, odiff and the - // VLM providers get their options passed straight through it("keeps the filter's keys out of the comparator's config", () => { const { comparatorConfig } = splitBlurAmplitudeConfig( { strict: false, blurAmplitudeFilter: false, blurAmplitudeThreshold: 70 }, @@ -98,8 +93,6 @@ describe('splitBlurAmplitudeConfig with the DINOv2 check', () => { expect(blurAmplitude?.dinov2).toEqual({ threshold: 0.3 }); }); - // it double-checks what the blur would pass; with no blur filter there is - // nothing for it to check it('has no effect without the blur filter', () => { const { blurAmplitude } = splitBlurAmplitudeConfig({ dinov2Check: true }, logger); diff --git a/src/compare/libs/blur-amplitude.config.ts b/src/compare/libs/blur-amplitude.config.ts index 612c3659..aa34d443 100644 --- a/src/compare/libs/blur-amplitude.config.ts +++ b/src/compare/libs/blur-amplitude.config.ts @@ -1,42 +1,19 @@ import { Logger } from '@nestjs/common'; -import { - BlurAmplitudeOptions, - DEFAULT_BLUR_AMPLITUDE_THRESHOLD, - DEFAULT_BLUR_SIGMA, -} from './pixelmatch/blur-amplitude.core'; +import { BlurAmplitudeOptions, DEFAULT_BLUR_AMPLITUDE_THRESHOLD } from './pixelmatch/blur-amplitude.core'; -/** - * The keys an image-comparison config carries to switch on the rendering-noise - * filter (see pixelmatch/blur-amplitude.core.ts). A library that supports the - * filter extends its config type with these; CompareService reads them and - * strips them, so the library itself never sees them. - */ export interface BlurAmplitudeConfig { - // pass a failing comparison whose blurred change is no stronger than noise blurAmplitudeFilter?: boolean; - // strongest blurred difference (0-128) still treated as noise blurAmplitudeThreshold?: number; - // ask the DINOv2 service before passing a change the blur calls noise dinov2Check?: boolean; - // largest content distance (0-2, cosine) still treated as noise dinov2Threshold?: number; } -// The blurred difference never exceeds 255, so a threshold near it would pass -// almost any change of one size; half the contrast range is the useful ceiling. export const MAX_BLUR_AMPLITUDE_THRESHOLD = 128; -// Above the strongest of 57 anti-aliasing-only pairs of real VRT pages (0.133) -// in the research dataset, measured the way Dinov2Service measures. +// strongest of 57 anti-aliasing-only real pairs: 0.133 export const DEFAULT_DINOV2_THRESHOLD = 0.15; const MAX_DINOV2_THRESHOLD = 2; -/** - * Splits a parsed config into the filter's settings — absent when it is off — - * and the config the comparator gets. The config is a JSON string nothing - * validates on the way in, so a threshold that is not a number in range falls - * back to the default rather than letting a typo pass everything. - */ export function splitBlurAmplitudeConfig( config: T, logger: Logger @@ -60,8 +37,7 @@ export function splitBlurAmplitudeConfig( ); } } - // the threshold was calibrated at this sigma; it is not a setting of its own - const blurAmplitude: BlurAmplitudeOptions = { sigma: DEFAULT_BLUR_SIGMA, threshold }; + const blurAmplitude: BlurAmplitudeOptions = { threshold }; if (dinov2Check === true) { blurAmplitude.dinov2 = { threshold: dinov2ThresholdOf(dinov2Threshold, logger) }; } @@ -72,7 +48,6 @@ function dinov2ThresholdOf(value: unknown, logger: Logger): number { if (value === undefined) { return DEFAULT_DINOV2_THRESHOLD; } - // a distance of 0 would treat every change as content and pass nothing if (typeof value === 'number' && value > 0 && value <= MAX_DINOV2_THRESHOLD) { return value; } diff --git a/src/compare/libs/dinov2/dinov2.controller.spec.ts b/src/compare/libs/dinov2/dinov2.controller.spec.ts index 73dcba80..12aca2ef 100644 --- a/src/compare/libs/dinov2/dinov2.controller.spec.ts +++ b/src/compare/libs/dinov2/dinov2.controller.spec.ts @@ -35,7 +35,6 @@ describe('Dinov2Controller', () => { expect(guards()).toContain(JwtAuthGuard); }); - // the people who can edit a project's config it('is limited to admins and editors', () => { expect(guards()).toContain(RoleGuard); expect(new Reflector().getAllAndOverride('roles', [Dinov2Controller.prototype.status, Dinov2Controller])).toEqual([ diff --git a/src/compare/libs/dinov2/dinov2.controller.ts b/src/compare/libs/dinov2/dinov2.controller.ts index 470399e4..1884d8a4 100644 --- a/src/compare/libs/dinov2/dinov2.controller.ts +++ b/src/compare/libs/dinov2/dinov2.controller.ts @@ -14,7 +14,6 @@ import { Dinov2Service, Dinov2Status } from './dinov2.service'; export class Dinov2Controller { constructor(private readonly dinov2Service: Dinov2Service) {} - // what the project form offers: the check only where a service is set up @Get('status') status(): Promise { return this.dinov2Service.status(); diff --git a/src/compare/libs/dinov2/dinov2.service.spec.ts b/src/compare/libs/dinov2/dinov2.service.spec.ts index c510ba32..e860641e 100644 --- a/src/compare/libs/dinov2/dinov2.service.spec.ts +++ b/src/compare/libs/dinov2/dinov2.service.spec.ts @@ -1,9 +1,7 @@ import { Test, TestingModule } from '@nestjs/testing'; import { ConfigService } from '@nestjs/config'; -import { DiffWorkerPool } from '../../diff-worker-pool'; import { blobScore, Dinov2Service, tilesOf } from './dinov2.service'; -// a 32 x 20 patch grid: two tile columns, two tile rows (16 x 16 patches each) const WIDTH = 32; const HEIGHT = 20; const at = (row: number, col: number) => row * WIDTH + col; @@ -11,7 +9,6 @@ const at = (row: number, col: number) => row * WIDTH + col; const grid = (values: Record) => Array.from({ length: WIDTH * HEIGHT }, (_, i) => values[i] ?? 0); describe('blobScore', () => { - // a lone patch is how noise looks: one changed patch, nothing beside it it('ignores a single changed patch', () => { const cells = [at(3, 3)]; @@ -24,8 +21,6 @@ describe('blobScore', () => { expect(blobScore(grid({ [at(3, 3)]: 0.9, [at(3, 4)]: 0.4 }), WIDTH, HEIGHT, cells)).toBeCloseTo(0.4); }); - // only where pixelmatch saw a change: elsewhere the distance is context - // bleeding across a tile, not a change it('counts only the patches pixelmatch marked', () => { const distances = grid({ [at(3, 3)]: 0.9, [at(3, 4)]: 0.9 }); @@ -69,53 +64,47 @@ describe('Dinov2Service', () => { json: async () => ({ patch: 14, width: WIDTH, height: HEIGHT, distances: grid(values) }), }); - const init = async ({ url = 'http://dinov2:8000', poolOutput = { gate } as object } = {}) => { - const run = jest.fn().mockResolvedValue(poolOutput); + const init = async (env: Record = { DINOV2_URL: 'http://dinov2:8000' }) => { const module: TestingModule = await Test.createTestingModule({ - providers: [ - Dinov2Service, - { provide: ConfigService, useValue: { get: jest.fn().mockReturnValue(url) } }, - { provide: DiffWorkerPool, useValue: { run } }, - ], + providers: [Dinov2Service, { provide: ConfigService, useValue: { get: (key: string) => env[key] } }], }).compile(); - return { service: module.get(Dinov2Service), run }; + return { service: module.get(Dinov2Service) }; }; const fetchMock = jest.spyOn(global, 'fetch'); afterEach(() => fetchMock.mockReset()); - const confirm = (service: Dinov2Service) => - service.confirmsNoise(Buffer.from('baseline'), Buffer.from('image'), [{ x: 1, y: 2, width: 3, height: 4 }], 0.15); + const confirm = (service: Dinov2Service, given = gate) => + service.confirmsNoiseForGate(given, Buffer.from('baseline'), Buffer.from('image'), 0.15); - it('confirms noise when the content did not change', async () => { - const { service, run } = await init(); - fetchMock.mockResolvedValue(answer({ [at(3, 3)]: 0.05, [at(3, 4)]: 0.04 }) as Response); - - expect(await confirm(service)).toEqual({ noise: true, score: 0.04 }); - expect(run).toHaveBeenCalledWith({ - kind: 'patch-gate', - baseline: Buffer.from('baseline'), - image: Buffer.from('image'), - ignoreAreas: [{ x: 1, y: 2, width: 3, height: 4 }], - patch: 14, - }); - }); + it('takes the service address with or without a trailing slash, and its own timeout', async () => { + const { service } = await init({ DINOV2_URL: 'http://dinov2:8000/', DINOV2_TIMEOUT_MS: '5000' }); + const timeout = jest.spyOn(AbortSignal, 'timeout'); + fetchMock.mockResolvedValue(answer({}) as Response); - it('sees a change in content', async () => { - const { service } = await init(); - fetchMock.mockResolvedValue(answer({ [at(3, 3)]: 0.6, [at(3, 4)]: 0.5 }) as Response); + await confirm(service); - expect(await confirm(service)).toEqual({ noise: false, score: 0.5 }); + expect(fetchMock.mock.calls[0][0]).toBe('http://dinov2:8000/v1/patch-distances'); + expect(timeout).toHaveBeenCalledWith(5000); + timeout.mockRestore(); }); - // only the tiles holding a marked patch are encoded: identical result, - // a fraction of the time - it('asks the service for the marked tiles only', async () => { - const { service } = await init(); + it('falls back to its default timeout for a negative DINOV2_TIMEOUT_MS', async () => { + const { service } = await init({ DINOV2_URL: 'http://dinov2:8000', DINOV2_TIMEOUT_MS: '-1' }); + const timeout = jest.spyOn(AbortSignal, 'timeout'); fetchMock.mockResolvedValue(answer({}) as Response); await confirm(service); + expect(timeout).toHaveBeenCalledWith(60_000); + timeout.mockRestore(); + }); + + it('confirms noise when the content did not change', async () => { + const { service } = await init(); + fetchMock.mockResolvedValue(answer({ [at(3, 3)]: 0.05, [at(3, 4)]: 0.04 }) as Response); + + expect(await confirm(service)).toEqual({ noise: true, score: 0.04 }); const [url, request] = fetchMock.mock.calls[0]; expect(url).toBe('http://dinov2:8000/v1/patch-distances'); expect(JSON.parse(request?.body as string)).toEqual({ @@ -125,37 +114,50 @@ describe('Dinov2Service', () => { }); }); - it('takes the service address with or without a trailing slash, and its own timeout', async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [ - Dinov2Service, - { - provide: ConfigService, - useValue: { get: (key: string) => ({ DINOV2_URL: 'http://dinov2:8000/', DINOV2_TIMEOUT_MS: '5000' })[key] }, - }, - { provide: DiffWorkerPool, useValue: { run: jest.fn().mockResolvedValue({ gate }) } }, - ], - }).compile(); - const timeout = jest.spyOn(AbortSignal, 'timeout'); - fetchMock.mockResolvedValue(answer({}) as Response); + it('sees a change in content', async () => { + const { service } = await init(); + fetchMock.mockResolvedValue(answer({ [at(3, 3)]: 0.6, [at(3, 4)]: 0.5 }) as Response); - await confirm(module.get(Dinov2Service)); + expect(await confirm(service)).toEqual({ noise: false, score: 0.5 }); + }); - expect(fetchMock.mock.calls[0][0]).toBe('http://dinov2:8000/v1/patch-distances'); - expect(timeout).toHaveBeenCalledWith(5000); - timeout.mockRestore(); + it('confirms noise without a call when no patch is marked', async () => { + const { service } = await init(); + + expect(await confirm(service, { ...gate, cells: [] })).toEqual({ noise: true, score: 0 }); + expect(fetchMock).not.toHaveBeenCalled(); }); - // nothing but anti-aliasing changed: nothing for DINOv2 to look at - it('confirms noise without a call when pixelmatch marked no patch', async () => { - const { service } = await init({ poolOutput: { gate: { ...gate, cells: [] } } }); + it('does not confirm noise without a service configured', async () => { + const { service } = await init({ DINOV2_URL: '' }); - expect(await confirm(service)).toEqual({ noise: true, score: 0 }); + expect((await confirm(service)).noise).toBe(false); expect(fetchMock).not.toHaveBeenCalled(); }); + it('does not confirm noise when the answer does not fit the screenshots', async () => { + const { service } = await init(); + fetchMock.mockResolvedValue({ + ok: true, + json: async () => ({ patch: 14, width: 3, height: 3, distances: [0, 0, 0, 0, 0, 0, 0, 0, 0] }), + } as Response); + + expect((await confirm(service)).noise).toBe(false); + }); + + it('does not confirm noise when the service fails or is unreachable', async () => { + const { service } = await init(); + fetchMock.mockResolvedValueOnce({ ok: false, status: 500, json: async () => ({}) } as Response); + fetchMock.mockRejectedValueOnce(new TypeError('fetch failed')); + fetchMock.mockRejectedValueOnce('socket hang up'); + + expect((await confirm(service)).noise).toBe(false); + expect((await confirm(service)).noise).toBe(false); + expect((await confirm(service)).noise).toBe(false); + }); + describe('status', () => { - const status = async (url: string) => (await init({ url })).service.status(); + const status = async (url: string) => (await init({ DINOV2_URL: url })).service.status(); it('is not configured without a service address', async () => { expect(await status('')).toEqual({ configured: false, available: false }); @@ -175,66 +177,13 @@ describe('Dinov2Service', () => { expect(await status('http://dinov2:8000')).toEqual({ configured: true, available: false }); }); - it('is configured but not available when the service is unreachable', async () => { - fetchMock.mockRejectedValue(new TypeError('fetch failed')); + it.each([ + ['the service is unreachable', new TypeError('fetch failed')], + ['the check throws something that is not an Error', 'socket hang up'], + ])('is configured but not available when %s', async (_, failure) => { + fetchMock.mockRejectedValue(failure); expect(await status('http://dinov2:8000')).toEqual({ configured: true, available: false }); }); - - it('is configured but not available when the check throws something that is not an Error', async () => { - fetchMock.mockRejectedValue('socket hang up'); - - expect(await status('http://dinov2:8000')).toEqual({ configured: true, available: false }); - }); - }); - - // the check exists to stop the blur passing a real change; when it cannot - // check, the run goes to review rather than through - describe('when it cannot check', () => { - it('does not confirm noise without a service configured', async () => { - // an unset variable reads as empty - const { service } = await init({ url: '' }); - - expect((await confirm(service)).noise).toBe(false); - expect(fetchMock).not.toHaveBeenCalled(); - }); - - it('does not confirm noise when the service fails', async () => { - const { service } = await init(); - fetchMock.mockResolvedValue({ ok: false, status: 500, json: async () => ({}) } as Response); - - expect((await confirm(service)).noise).toBe(false); - }); - - it('does not confirm noise when the check throws something that is not an Error', async () => { - const { service } = await init(); - fetchMock.mockRejectedValue('socket hang up'); - - expect((await confirm(service)).noise).toBe(false); - }); - - it('does not confirm noise when the service is unreachable', async () => { - const { service } = await init(); - fetchMock.mockRejectedValue(new TypeError('fetch failed')); - - expect((await confirm(service)).noise).toBe(false); - }); - - it('does not confirm noise when the answer does not fit the screenshots', async () => { - const { service } = await init(); - fetchMock.mockResolvedValue({ - ok: true, - json: async () => ({ patch: 14, width: 3, height: 3, distances: [0, 0, 0, 0, 0, 0, 0, 0, 0] }), - } as Response); - - expect((await confirm(service)).noise).toBe(false); - }); - - it('does not confirm noise when the screenshots cannot be lined up', async () => { - const { service } = await init({ poolOutput: { gate: null } }); - - expect((await confirm(service)).noise).toBe(false); - expect(fetchMock).not.toHaveBeenCalled(); - }); }); }); diff --git a/src/compare/libs/dinov2/dinov2.service.ts b/src/compare/libs/dinov2/dinov2.service.ts index 9d395341..30d6ef53 100644 --- a/src/compare/libs/dinov2/dinov2.service.ts +++ b/src/compare/libs/dinov2/dinov2.service.ts @@ -1,36 +1,22 @@ import { Injectable, Logger } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; -import { IgnoreAreaDto } from '../../../test-runs/dto/ignore-area.dto'; -import { DiffWorkerPool } from '../../diff-worker-pool'; +import { PatchGate } from '../pixelmatch/patch-gate.core'; +import { positiveNumber } from '../../../utils/env'; -// the model's patch size, and the patches along one side of a 224 px tile -const PATCH = 14; const TILE_PATCHES = 16; const DEFAULT_TIMEOUT_MS = 60_000; -// the status is asked for while a form opens: a service this slow to say it -// is up counts as down const STATUS_TIMEOUT_MS = 3_000; export interface Dinov2Status { - // DINOV2_URL is set: a project may turn the check on configured: boolean; - // the service answered its health check just now available: boolean; } export interface Dinov2Verdict { - // true only when DINOv2 looked and saw no change of content noise: boolean; - // the strongest content change where pixelmatch saw one, when it looked score?: number; } -/** - * The strongest content distance among the patches pixelmatch marked, counting - * a patch only when a marked neighbour (up, down, left, right) changed too: - * a change worth a reviewer covers more than one 14 px patch, a stray patch is - * how anti-aliasing shows up. The score is the weaker of the pair. - */ export function blobScore(distances: number[], width: number, height: number, cells: number[]): number { const marked = new Set(cells); const at = (row: number, col: number) => @@ -47,7 +33,6 @@ export function blobScore(distances: number[], width: number, height: number, ce return score; } -/** The 224 px tiles, as [row, col], that hold at least one marked patch. */ export function tilesOf(cells: number[], width: number): [number, number][] { const tiles = new Map(); for (const cell of cells) { @@ -60,30 +45,12 @@ export function tilesOf(cells: number[], width: number): [number, number][] { return [...tiles.values()].sort((a, b) => a[0] - b[0] || a[1] - b[1]); } -/** - * A second look at a change the blur filter would pass as rendering noise: - * the DINOv2 service compares the two screenshots' content patch by patch, - * which catches what is faint in pixels but clear in meaning — a missing light - * icon, a removed input border. Calibrated in research/jepa-screenshot-diff. - * - * The check exists to stop the filter passing a real change, so whenever it - * cannot look — no service configured, the service down, screenshots that do - * not line up — it does not confirm noise and the run goes to review. - */ @Injectable() export class Dinov2Service { private readonly logger: Logger = new Logger(Dinov2Service.name); - constructor( - private readonly configService: ConfigService, - private readonly diffWorkerPool: DiffWorkerPool - ) {} + constructor(private readonly configService: ConfigService) {} - /** - * Whether this server can run the check, for the project form: a project - * that turns it on without a service sends every run the filter would pass - * to review. - */ async status(): Promise { const url = this.serviceUrl(); if (!url) { @@ -98,10 +65,10 @@ export class Dinov2Service { } } - async confirmsNoise( + async confirmsNoiseForGate( + gate: PatchGate, baseline: Buffer, image: Buffer, - ignoreAreas: IgnoreAreaDto[], threshold: number ): Promise { const url = this.serviceUrl(); @@ -110,17 +77,6 @@ export class Dinov2Service { return { noise: false }; } try { - const { gate } = await this.diffWorkerPool.run({ - kind: 'patch-gate', - baseline, - image, - ignoreAreas, - patch: PATCH, - }); - if (!gate) { - return { noise: false }; - } - // nothing but anti-aliasing changed: nothing for the model to look at if (gate.cells.length === 0) { return { noise: true, score: 0 }; } @@ -131,11 +87,9 @@ export class Dinov2Service { body: JSON.stringify({ baseline: baseline.toString('base64'), image: image.toString('base64'), - // only the tiles that hold a marked patch: tiles are encoded - // independently, so the others would cost time and change nothing tiles: tilesOf(gate.cells, gate.width), }), - signal: AbortSignal.timeout(Number(this.configService.get('DINOV2_TIMEOUT_MS')) || DEFAULT_TIMEOUT_MS), + signal: AbortSignal.timeout(positiveNumber(this.configService.get('DINOV2_TIMEOUT_MS'), DEFAULT_TIMEOUT_MS)), }); if (!response.ok) { throw new Error(`service answered ${response.status}`); @@ -150,16 +104,16 @@ export class Dinov2Service { const score = blobScore(answer.distances, gate.width, gate.height, gate.cells); return { noise: score <= threshold, score }; } catch (error) { - this.logger.warn( - // read as an Error, not checked with instanceof: fetch errors come - // from another realm under Jest - `DINOv2 check failed, the run goes to review: ${(error as Error)?.message ?? error}` - ); - return { noise: false }; + return this.failed(error); } } - // an unset variable reads as empty; a trailing slash is the operator's choice + private failed(error: unknown): Dinov2Verdict { + // a cast, not instanceof: fetch errors come from another realm under Jest + this.logger.warn(`DINOv2 check failed, the run goes to review: ${(error as Error)?.message ?? error}`); + return { noise: false }; + } + private serviceUrl(): string | undefined { return this.configService.get('DINOV2_URL')?.replace(/\/$/, '') || undefined; } diff --git a/src/compare/libs/image-comparator.interface.ts b/src/compare/libs/image-comparator.interface.ts index bb28b043..8ff42a0a 100644 --- a/src/compare/libs/image-comparator.interface.ts +++ b/src/compare/libs/image-comparator.interface.ts @@ -8,12 +8,6 @@ import { VlmConfig } from './vlm/vlm.types'; export type ImageCompareConfig = PixelmatchConfig | LooksSameConfig | OdiffConfig | VlmConfig; export interface ImageComparator { - /** - * True when getDiff applies `data.blurAmplitude` itself — cheaper inside a - * pass that has the images decoded already, and before anything expensive - * (a diff written to storage, a VLM call). Otherwise CompareService measures - * the pair afterwards and may overturn a failing verdict. - */ readonly appliesBlurAmplitudeFilter?: boolean; getDiff(data: ImageCompareInput, config: ImageCompareConfig): Promise; parseConfig(configJson: string): ImageCompareConfig; diff --git a/src/compare/libs/looks-same/looks-same.core.spec.ts b/src/compare/libs/looks-same/looks-same.core.spec.ts new file mode 100644 index 00000000..08fdc754 --- /dev/null +++ b/src/compare/libs/looks-same/looks-same.core.spec.ts @@ -0,0 +1,177 @@ +import { readFileSync } from 'fs'; +import { join } from 'path'; +import { PNG } from 'pngjs'; +import { pngBytes, solidPng } from '../../test-utils'; +import { computeLooksSame, LOOKS_SAME_DEFAULT_CONFIG, LooksSameJobInput } from './looks-same.core'; + +const fixture = (name: string) => readFileSync(join(__dirname, '../../../../test', name)); + +const filled = (width: number, height: number, value: number): Buffer => + pngBytes(solidPng(width, height, [value, value, value])); + +const job = (overrides: Partial = {}): LooksSameJobInput => ({ + kind: 'looks-same', + baseline: fixture('image.png'), + image: fixture('image_edited.png'), + ignoreAreas: [], + config: LOOKS_SAME_DEFAULT_CONFIG, + saveDiff: false, + ...overrides, +}); + +describe('computeLooksSame', () => { + it('reports an unreadable baseline', async () => { + await expect(computeLooksSame(job({ baseline: Buffer.from('nope') }))).resolves.toStrictEqual({ + outcome: 'baseline-unreadable', + }); + }); + + it('reports an unreadable image', async () => { + await expect(computeLooksSame(job({ image: Buffer.from('nope') }))).resolves.toStrictEqual({ + outcome: 'image-unreadable', + }); + }); + + it('reports a baseline that is unreadable before an unreadable image', async () => { + const output = await computeLooksSame(job({ baseline: Buffer.from('a'), image: Buffer.from('b') })); + + expect(output).toStrictEqual({ outcome: 'baseline-unreadable' }); + }); + + it('does not call a 60x30 and a 30x60 image of the same pixel bytes equal', async () => { + const output = await computeLooksSame(job({ baseline: filled(60, 30, 128), image: filled(30, 60, 128) })); + + expect(output).toStrictEqual({ outcome: 'diff-dimension' }); + }); + + it('compares them when other sizes are allowed', async () => { + const output = await computeLooksSame( + job({ + baseline: filled(60, 30, 128), + image: filled(30, 60, 128), + config: { ...LOOKS_SAME_DEFAULT_CONFIG, allowDiffDimensions: true }, + }) + ); + + expect(output).toMatchObject({ outcome: 'compared', isSameDimension: false }); + }); + + it('answers equal for the same pixels', async () => { + const output = await computeLooksSame(job({ baseline: filled(10, 10, 7), image: filled(10, 10, 7) })); + + expect(output).toStrictEqual({ outcome: 'equal' }); + }); + + it('counts the different pixels of the fixtures', async () => { + const output = await computeLooksSame(job()); + + expect(output).toStrictEqual({ + outcome: 'compared', + isSameDimension: true, + result: { equal: false, differentPixels: 31068, totalPixels: 129600 }, + errors: [], + }); + }); + + it('returns the diff image when asked to', async () => { + const output = await computeLooksSame(job({ saveDiff: true })); + + expect(output.outcome).toBe('compared'); + const diffPng = (output as { diffPng?: Uint8Array }).diffPng!; + expect(diffPng).toBeInstanceOf(Uint8Array); + const diff = PNG.sync.read(Buffer.from(diffPng)); + expect([diff.width, diff.height]).toEqual([360, 360]); + }); + + it('returns no diff image for an equal result even when asked to', async () => { + const baseline = filled(10, 10, 255); + const changed = PNG.sync.read(baseline); + changed.data[0] = 0; + + const output = await computeLooksSame( + job({ + baseline, + image: PNG.sync.write(changed), + ignoreAreas: [{ x: 0, y: 0, width: 1, height: 1 }], + saveDiff: true, + }) + ); + + expect(output).toStrictEqual({ + outcome: 'compared', + isSameDimension: true, + result: { equal: true, differentPixels: undefined, totalPixels: undefined }, + errors: [], + }); + }); + + it('falls back to the defaults when looks-same refuses the project config', async () => { + const output = await computeLooksSame( + job({ config: { ...LOOKS_SAME_DEFAULT_CONFIG, strict: true, tolerance: 5 } }) + ); + + expect(output).toMatchObject({ + outcome: 'compared', + result: { equal: false, differentPixels: 31068, totalPixels: 129600 }, + errors: [ + 'Unable to use "strict" and "tolerance" options together', + 'Looks-Same cannot compare with the project config, fallback to default one', + ], + }); + }); + + it('masks the ignore areas of both images before comparing', async () => { + const baseline = filled(10, 10, 255); + const changed = PNG.sync.read(baseline); + changed.data[0] = 0; + changed.data[4 * 55] = 0; + + const output = await computeLooksSame( + job({ + baseline, + image: PNG.sync.write(changed), + ignoreAreas: [{ x: 0, y: 0, width: 1, height: 1 }], + config: { strict: true, ignoreAntialiasing: false, ignoreCaret: false }, + }) + ); + + expect(output).toMatchObject({ + outcome: 'compared', + result: { equal: false, differentPixels: 1, totalPixels: 100 }, + }); + }); + + it('produces an output that survives structured cloning', async () => { + const output = await computeLooksSame(job({ saveDiff: true })); + + const clone = structuredClone(output) as { diffPng: Uint8Array }; + + expect(clone).toStrictEqual(output); + expect(clone.diffPng.byteLength).toBeGreaterThan(0); + }); +}); + +describe('computeLooksSame when looks-same fails', () => { + afterEach(() => { + jest.resetModules(); + jest.dontMock('looks-same'); + }); + + it('gives no result and all three messages when the defaults fail too', async () => { + jest.resetModules(); + jest.doMock('looks-same', () => ({ + __esModule: true, + default: jest.fn().mockRejectedValueOnce(new Error('first')).mockRejectedValueOnce(new Error('second')), + })); + const core: typeof import('./looks-same.core') = require('./looks-same.core'); + + const output = await core.computeLooksSame(job()); + + expect(output).toStrictEqual({ + outcome: 'compared', + isSameDimension: true, + result: null, + errors: ['first', 'Looks-Same cannot compare with the project config, fallback to default one', 'second'], + }); + }); +}); diff --git a/src/compare/libs/looks-same/looks-same.core.ts b/src/compare/libs/looks-same/looks-same.core.ts new file mode 100644 index 00000000..09c4ce81 --- /dev/null +++ b/src/compare/libs/looks-same/looks-same.core.ts @@ -0,0 +1,92 @@ +import looksSame from 'looks-same'; +import { PNG } from 'pngjs'; +import { IgnoreAreaDto } from '../../../test-runs/dto/ignore-area.dto'; +import { applyIgnoreAreas, readPng } from '../../utils'; +import { LooksSameConfig } from './looks-same.types'; + +export const LOOKS_SAME_DEFAULT_CONFIG: LooksSameConfig = { + strict: false, + ignoreAntialiasing: true, + ignoreCaret: true, + allowDiffDimensions: false, +}; + +export interface LooksSameJobInput { + kind: 'looks-same'; + baseline: Buffer | Uint8Array; + image: Buffer | Uint8Array; + ignoreAreas: IgnoreAreaDto[]; + config: LooksSameConfig; + saveDiff: boolean; +} + +export type LooksSameJobOutput = + | { outcome: 'baseline-unreadable' } + | { outcome: 'image-unreadable' } + | { outcome: 'equal' } + | { outcome: 'diff-dimension' } + | { + outcome: 'compared'; + isSameDimension: boolean; + result: { equal: boolean; differentPixels?: number; totalPixels?: number } | null; + diffPng?: Uint8Array; + errors: string[]; + }; + +const FAST_PNG = { deflateLevel: 1, filterType: 0 } as const; + +export async function computeLooksSame(input: LooksSameJobInput): Promise { + let baseline: PNG; + try { + baseline = readPng(input.baseline); + } catch { + return { outcome: 'baseline-unreadable' }; + } + let image: PNG; + try { + image = readPng(input.image); + } catch { + return { outcome: 'image-unreadable' }; + } + + const isSameDimension = baseline.width === image.width && baseline.height === image.height; + if (!isSameDimension && !input.config.allowDiffDimensions) { + return { outcome: 'diff-dimension' }; + } + if (isSameDimension && baseline.data.equals(image.data)) { + return { outcome: 'equal' }; + } + + applyIgnoreAreas(baseline, input.ignoreAreas); + applyIgnoreAreas(image, input.ignoreAreas); + const baselinePng = PNG.sync.write(baseline, FAST_PNG); + const imagePng = PNG.sync.write(image, FAST_PNG); + + const errors: string[] = []; + const run = (options: LooksSameConfig) => looksSame(baselinePng, imagePng, { ...options, createDiffImage: true }); + let result: looksSame.LooksSameResult | undefined; + try { + result = await run(input.config); + } catch (error) { + errors.push((error as Error).message, 'Looks-Same cannot compare with the project config, fallback to default one'); + result = await run(LOOKS_SAME_DEFAULT_CONFIG).catch((fallbackError: Error) => { + errors.push(fallbackError.message); + return undefined; + }); + } + if (!result) { + return { outcome: 'compared', isSameDimension, result: null, errors }; + } + + const diffPng = + input.saveDiff && !result.equal && result.diffImage + ? new Uint8Array(await result.diffImage.createBuffer('png')) + : undefined; + return { + outcome: 'compared', + isSameDimension, + result: { equal: result.equal, differentPixels: result.differentPixels, totalPixels: result.totalPixels }, + ...(diffPng ? { diffPng } : {}), + errors, + }; +} diff --git a/src/compare/libs/looks-same/looks-same.service.boundaries.spec.ts b/src/compare/libs/looks-same/looks-same.service.boundaries.spec.ts deleted file mode 100644 index e7889ac1..00000000 --- a/src/compare/libs/looks-same/looks-same.service.boundaries.spec.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { Test } from '@nestjs/testing'; -import { TestStatus } from '@prisma/client'; -import { PNG } from 'pngjs'; -import { StaticService } from '../../../static/static.service'; -import { DIFF_DIMENSION_RESULT } from '../consts'; -import { DEFAULT_CONFIG, LookSameService } from './looks-same.service'; - -const filled = (width: number, height: number, value: number): PNG => { - const image = new PNG({ width, height }); - image.data.fill(value); - for (let i = 3; i < image.data.length; i += 4) { - image.data[i] = 255; - } - return image; -}; - -const initService = async (baseline: PNG, image: PNG) => { - const saveImage = jest.fn().mockResolvedValue('diff.png'); - const module = await Test.createTestingModule({ - providers: [ - LookSameService, - { - provide: StaticService, - useValue: { getImage: jest.fn().mockResolvedValueOnce(baseline).mockResolvedValueOnce(image), saveImage }, - }, - ], - }).compile(); - return { service: module.get(LookSameService), saveImage }; -}; - -const input = { baseline: 'baseline', image: 'image', diffTollerancePercent: 0, ignoreAreas: [], saveDiffAsFile: true }; - -describe('LookSameService.getDiff dimensions', () => { - it('reports a different height as a dimension change', async () => { - const { service } = await initService(filled(10, 10, 0), filled(10, 5, 0)); - - await expect(service.getDiff(input, DEFAULT_CONFIG)).resolves.toBe(DIFF_DIMENSION_RESULT); - }); - - it('reports a different width as a dimension change', async () => { - const { service } = await initService(filled(10, 10, 0), filled(5, 10, 0)); - - await expect(service.getDiff(input, DEFAULT_CONFIG)).resolves.toBe(DIFF_DIMENSION_RESULT); - }); -}); - -describe('LookSameService.getDiff saving the diff', () => { - it('stores the diff as a "diff" image', async () => { - const { service, saveImage } = await initService(filled(10, 10, 0), filled(10, 10, 255)); - - const result = await service.getDiff(input, DEFAULT_CONFIG); - - expect(result.status).toBe(TestStatus.unresolved); - expect(result.diffName).toBe('diff.png'); - expect(saveImage).toHaveBeenCalledWith('diff', expect.any(Buffer)); - expect(PNG.sync.read(saveImage.mock.calls[0][1]).width).toBe(10); - }); -}); diff --git a/src/compare/libs/looks-same/looks-same.service.real-images.spec.ts b/src/compare/libs/looks-same/looks-same.service.real-images.spec.ts new file mode 100644 index 00000000..980c92ad --- /dev/null +++ b/src/compare/libs/looks-same/looks-same.service.real-images.spec.ts @@ -0,0 +1,169 @@ +import { readFileSync } from 'fs'; +import { join } from 'path'; +import { Test } from '@nestjs/testing'; +import { TestStatus } from '@prisma/client'; +import { PNG } from 'pngjs'; +import { StaticService } from '../../../static/static.service'; +import { DiffWorkerPool } from '../../diff-worker-pool'; +import { DIFF_DIMENSION_RESULT, EQUAL_RESULT } from '../consts'; +import { solidPng } from '../../test-utils'; +import { DEFAULT_CONFIG, LookSameService } from './looks-same.service'; + +const filled = (width: number, height: number, value: number): PNG => solidPng(width, height, [value, value, value]); + +const encode = (png: PNG) => PNG.sync.write(png, { filterType: 0, deflateLevel: 1 }); + +const initService = async (baseline: PNG, image: PNG) => { + const saveImage = jest.fn().mockResolvedValue('diff.png'); + const getImageBuffer = jest + .fn() + .mockImplementationOnce(async () => encode(baseline)) + .mockImplementationOnce(async () => encode(image)); + const module = await Test.createTestingModule({ + providers: [ + LookSameService, + DiffWorkerPool, + { + provide: StaticService, + useValue: { + getImageBuffer, + saveImage, + }, + }, + ], + }).compile(); + return { service: module.get(LookSameService), saveImage }; +}; + +const input = { baseline: 'baseline', image: 'image', diffTollerancePercent: 0, ignoreAreas: [], saveDiffAsFile: true }; + +describe('LookSameService.getDiff dimensions', () => { + it.each([ + ['a different height', filled(10, 10, 0), filled(10, 5, 0)], + ['a different width', filled(10, 10, 0), filled(5, 10, 0)], + ['a 60x30 and a 30x60 image of the same pixel bytes', filled(60, 30, 128), filled(30, 60, 128)], + ])('reports %s as a dimension change', async (_, baseline, image) => { + const { service } = await initService(baseline, image); + + await expect(service.getDiff(input, DEFAULT_CONFIG)).resolves.toBe(DIFF_DIMENSION_RESULT); + }); + + it('compares them when other sizes are allowed', async () => { + const { service } = await initService(filled(60, 30, 128), filled(30, 60, 128)); + + const result = await service.getDiff(input, { ...DEFAULT_CONFIG, allowDiffDimensions: true }); + + expect(result).not.toBe(EQUAL_RESULT); + expect(result.status).not.toBe(TestStatus.ok); + expect(result.isSameDimension).toBe(false); + }); +}); + +describe('LookSameService.getDiff saving the diff', () => { + it('stores the diff as a "diff" image', async () => { + const { service, saveImage } = await initService(filled(10, 10, 0), filled(10, 10, 255)); + + const result = await service.getDiff(input, DEFAULT_CONFIG); + + expect(result.status).toBe(TestStatus.unresolved); + expect(result.diffName).toBe('diff.png'); + expect(saveImage).toHaveBeenCalledWith('diff', expect.any(Buffer)); + expect(PNG.sync.read(saveImage.mock.calls[0][1]).width).toBe(10); + }); +}); + +describe('characterization on the real looks-same', () => { + const decoded = new Map(); + const fixture = (name: string) => { + if (!decoded.has(name)) { + decoded.set(name, PNG.sync.read(readFileSync(join(__dirname, '../../../../test', name)))); + } + return decoded.get(name) as PNG; + }; + const fixtures = () => initService(fixture('image.png'), fixture('image_edited.png')); + const ignoreFirstPixel = { ...input, ignoreAreas: [{ x: 0, y: 0, width: 1, height: 1 }] }; + + it('passes a pair that is equal only after masking, without counting pixels', async () => { + const changed = filled(10, 10, 255); + changed.data[0] = 0; + const { service } = await initService(filled(10, 10, 255), changed); + + const result = await service.getDiff(ignoreFirstPixel, DEFAULT_CONFIG); + + expect(result).toStrictEqual({ + status: TestStatus.ok, + diffName: null, + pixelMisMatchCount: undefined, + diffPercent: 0, + isSameDimension: true, + }); + }); + + it('counts zero pixels when only the gamma differs after masking', async () => { + const baseline = filled(10, 10, 255); + baseline.gamma = 0.45455; + const changed = filled(10, 10, 255); + changed.data[0] = 0; + const { service } = await initService(baseline, changed); + + const result = await service.getDiff(ignoreFirstPixel, DEFAULT_CONFIG); + + expect(result).toStrictEqual({ + status: TestStatus.ok, + diffName: null, + pixelMisMatchCount: 0, + diffPercent: 0, + isSameDimension: true, + }); + }); + + it('reports the pixels looks-same counts on the fixtures', async () => { + const { service } = await fixtures(); + + const result = await service.getDiff({ ...input, saveDiffAsFile: false }, DEFAULT_CONFIG); + + expect(result.status).toBe(TestStatus.unresolved); + expect(result.pixelMisMatchCount).toBe(31068); + expect(result.diffPercent).toBeCloseTo((31068 * 100) / 129600, 10); + expect(result.isSameDimension).toBe(true); + }); + + it('counts more pixels in the strict mode without antialiasing and caret handling', async () => { + const { service } = await fixtures(); + + const result = await service.getDiff( + { ...input, saveDiffAsFile: false }, + { strict: true, ignoreAntialiasing: false, ignoreCaret: false } + ); + + expect(result.pixelMisMatchCount).toBe(33915); + }); + + it('saves the looks-same diff of the fixtures', async () => { + const { service, saveImage } = await fixtures(); + + const result = await service.getDiff(input, DEFAULT_CONFIG); + + expect(result.diffName).toBe('diff.png'); + expect(saveImage).toHaveBeenCalledWith('diff', expect.any(Buffer)); + const diff = PNG.sync.read(saveImage.mock.calls[0][1]); + expect([diff.width, diff.height]).toEqual([360, 360]); + }); + + it('falls back to the defaults when looks-same refuses the project config', async () => { + const { service } = await fixtures(); + + const result = await service.getDiff( + { ...input, saveDiffAsFile: false }, + { ...DEFAULT_CONFIG, strict: true, tolerance: 5 } + ); + + expect(result).toStrictEqual({ + status: TestStatus.unresolved, + diffName: null, + pixelMisMatchCount: 31068, + diffPercent: (31068 * 100) / 129600, + isSameDimension: true, + }); + }); +}); diff --git a/src/compare/libs/looks-same/looks-same.service.spec.ts b/src/compare/libs/looks-same/looks-same.service.spec.ts index 01e4d5af..530cd0db 100644 --- a/src/compare/libs/looks-same/looks-same.service.spec.ts +++ b/src/compare/libs/looks-same/looks-same.service.spec.ts @@ -1,31 +1,86 @@ -import { TestingModule, Test } from '@nestjs/testing'; +import { Test } from '@nestjs/testing'; +import { Logger } from '@nestjs/common'; import { TestStatus } from '@prisma/client'; import { PNG } from 'pngjs'; import { StaticService } from '../../../static/static.service'; +import { DiffWorkerPool } from '../../diff-worker-pool'; import { DIFF_DIMENSION_RESULT, EQUAL_RESULT, NO_BASELINE_RESULT } from '../consts'; import { DEFAULT_CONFIG, LookSameService } from './looks-same.service'; import { LooksSameConfig } from './looks-same.types'; import { ImageCompareInput } from '../ImageCompareInput'; -const initService = async ({ getImageMock = jest.fn(), saveImageMock = jest.fn(), deleteImageMock = jest.fn() }) => { - const module: TestingModule = await Test.createTestingModule({ +const baselineBuffer = PNG.sync.write(new PNG({ width: 20, height: 20 })); +const another = new PNG({ width: 20, height: 20 }); +another.data[0] = 1; +const imageBuffer = PNG.sync.write(another); + +interface Deps { + getImageBuffer?: jest.Mock; + saveImage?: jest.Mock; + run?: jest.Mock; +} + +const setup = async ({ + getImageBuffer = jest.fn().mockResolvedValueOnce(baselineBuffer).mockResolvedValueOnce(imageBuffer), + saveImage = jest.fn(), + run = jest.fn(), +}: Deps = {}) => { + const module = await Test.createTestingModule({ providers: [ LookSameService, - { - provide: StaticService, - useValue: { - getImage: getImageMock, - saveImage: saveImageMock, - deleteImage: deleteImageMock, - }, - }, + { provide: DiffWorkerPool, useValue: { run } }, + { provide: StaticService, useValue: { getImageBuffer, saveImage } }, ], }).compile(); + return { service: module.get(LookSameService), getImageBuffer, saveImage, run }; +}; - return module.get(LookSameService); +const input: ImageCompareInput = { + baseline: 'baseline', + image: 'image', + diffTollerancePercent: 0.1, + ignoreAreas: [], + saveDiffAsFile: true, }; -let service: LookSameService; +const diff = ( + service: LookSameService, + extra: Partial = {}, + config: LooksSameConfig = DEFAULT_CONFIG +) => service.getDiff({ ...input, ...extra }, config); + +const outcome = (name: string) => jest.fn().mockResolvedValue({ outcome: name }); + +const compared = ( + result: { equal: boolean; differentPixels?: number; totalPixels?: number } | null, + extra: { diffPng?: Uint8Array; errors?: string[] } = {} +) => jest.fn().mockResolvedValue({ outcome: 'compared', isSameDimension: true, result, errors: [], ...extra }); + +const verdict = ( + status: TestStatus, + diffName: string | null, + diffPercent: number | undefined, + pixelMisMatchCount?: number +) => ({ + status, + diffName, + diffPercent, + pixelMisMatchCount, + isSameDimension: true, +}); + +const twentyOfFourHundred = { equal: false, differentPixels: 20, totalPixels: 400 }; +const diffPng = new Uint8Array(Buffer.from('png')); + +let loggerError: jest.SpyInstance; + +beforeEach(() => { + loggerError = jest.spyOn(Logger.prototype, 'error').mockImplementation(); +}); + +afterEach(() => { + jest.restoreAllMocks(); +}); describe('parseConfig', () => { it.each<[string, LooksSameConfig]>([ @@ -43,264 +98,202 @@ describe('parseConfig', () => { ['', DEFAULT_CONFIG], ['invalid', DEFAULT_CONFIG], ])('should parse config', async (json, expected) => { - service = await initService({}); + const { service } = await setup(); - const config = service.parseConfig(json); - - expect(config).toStrictEqual(expected); + expect(service.parseConfig(json)).toStrictEqual(expected); }); }); describe('getDiff', () => { - const image = new PNG({ - width: 20, - height: 20, - }); - const anotherImage = new PNG({ - width: 20, - height: 20, + it('no baseline', async () => { + const { service, run } = await setup({ + getImageBuffer: jest.fn().mockResolvedValueOnce(null).mockResolvedValueOnce(imageBuffer), + }); + + expect(await diff(service, { baseline: null })).toStrictEqual(NO_BASELINE_RESULT); + expect(run).not.toHaveBeenCalled(); }); - anotherImage.data[0] = 1; // alterate pixel to have it different from 0 - it('no baseline', async () => { - const getImageMock = jest.fn().mockReturnValueOnce(undefined).mockReturnValueOnce(image); - service = await initService({ getImageMock }); + it('compares the image it is given without reading it again', async () => { + const { service, getImageBuffer, run } = await setup({ + getImageBuffer: jest.fn().mockResolvedValue(baselineBuffer), + run: outcome('equal'), + }); - const result = await service.getDiff( - { - baseline: null, - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: true, - }, - DEFAULT_CONFIG - ); + await diff(service, { imageBuffer }); - expect(result).toStrictEqual(NO_BASELINE_RESULT); + expect(getImageBuffer).toHaveBeenCalledTimes(1); + expect(getImageBuffer).toHaveBeenCalledWith('baseline'); + expect(run).toHaveBeenCalledWith(expect.objectContaining({ baseline: baselineBuffer, image: imageBuffer })); }); - it('equal images', async () => { - const getImageMock = jest.fn().mockReturnValueOnce(image).mockReturnValueOnce(image); - service = await initService({ getImageMock }); + it('reports no baseline when the baseline cannot be read', async () => { + const { service, run } = await setup({ + getImageBuffer: jest.fn().mockRejectedValueOnce(new Error('disk')).mockResolvedValueOnce(imageBuffer), + }); - const result = await service.getDiff( - { - baseline: 'image', - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: true, - }, - DEFAULT_CONFIG - ); + expect(await diff(service)).toStrictEqual(NO_BASELINE_RESULT); + expect(run).not.toHaveBeenCalled(); + expect(loggerError).toHaveBeenCalledWith('disk'); + }); + + it('reports no baseline when the job finds it unreadable', async () => { + const { service } = await setup({ run: outcome('baseline-unreadable') }); - expect(result).toStrictEqual(EQUAL_RESULT); + await expect(diff(service)).resolves.toStrictEqual(NO_BASELINE_RESULT); }); - it('diff image dimensions mismatch', async () => { - const baseline = new PNG({ - width: 10, - height: 10, + it('answers EQUAL for byte-equal well-formed PNGs without a job', async () => { + const { service, run } = await setup({ getImageBuffer: jest.fn().mockResolvedValue(baselineBuffer) }); + + expect(await diff(service)).toStrictEqual(EQUAL_RESULT); + expect(run).not.toHaveBeenCalled(); + }); + + it('hands byte-equal files that are not well-formed PNGs to the job', async () => { + const { service, run } = await setup({ + getImageBuffer: jest.fn().mockResolvedValue(Buffer.from('not a png')), + run: outcome('baseline-unreadable'), }); - const getImageMock = jest.fn().mockReturnValueOnce(image).mockReturnValueOnce(baseline); - service = await initService({ getImageMock }); - const result = await service.getDiff( - { - baseline: 'image', - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: true, - }, - DEFAULT_CONFIG - ); + await diff(service); - expect(result).toStrictEqual(DIFF_DIMENSION_RESULT); + expect(run).toHaveBeenCalledTimes(1); }); - const input: Omit = { - baseline: 'image', - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - }; + it.each([ + ['equal images', 'equal', EQUAL_RESULT], + ['diff image dimensions mismatch', 'diff-dimension', DIFF_DIMENSION_RESULT], + ])('%s', async (_, name, expected) => { + const { service } = await setup({ run: outcome(name) }); - it('diff not found', async () => { - const getImageMock = jest.fn().mockReturnValueOnce(image).mockReturnValueOnce(anotherImage); - service = await initService({ getImageMock }); - service.compare = jest.fn().mockReturnValueOnce({ equal: true, differentPixels: 0, totalPixels: 400 }); - - const result = await service.getDiff({ ...input, saveDiffAsFile: true }, DEFAULT_CONFIG); + expect(await diff(service)).toStrictEqual(expected); + }); - expect(result).toStrictEqual({ - status: TestStatus.ok, - diffName: null, - diffPercent: 0, - pixelMisMatchCount: 0, - isSameDimension: true, + it('throws a clear error when the image cannot be read', async () => { + const { service, run } = await setup({ + getImageBuffer: jest.fn().mockResolvedValueOnce(baselineBuffer).mockResolvedValueOnce(null), }); + + await expect(diff(service, { saveDiffAsFile: false })).rejects.toThrow('Cannot get image: image'); + expect(run).not.toHaveBeenCalled(); + }); + + it('throws a clear error when the job finds the image unreadable', async () => { + const { service } = await setup({ run: outcome('image-unreadable') }); + + await expect(diff(service, { saveDiffAsFile: false })).rejects.toThrow('Cannot get image: image'); + }); + + it('diff not found', async () => { + const { service } = await setup({ run: compared({ equal: true, differentPixels: 0, totalPixels: 400 }) }); + + expect(await diff(service)).toStrictEqual(verdict(TestStatus.ok, null, 0, 0)); }); it('diff found', async () => { - const getImageMock = jest.fn().mockReturnValueOnce(image).mockReturnValueOnce(anotherImage); - service = await initService({ getImageMock }); - service.compare = jest.fn().mockReturnValueOnce({ equal: false, differentPixels: 20, totalPixels: 400 }); - service.saveDiff = jest.fn(); + const { service, saveImage } = await setup({ run: compared(twentyOfFourHundred) }); - const result = await service.getDiff({ ...input, saveDiffAsFile: false }, DEFAULT_CONFIG); + const result = await diff(service, { saveDiffAsFile: false }); - expect(service.compare).toHaveBeenCalledWith(image, anotherImage, DEFAULT_CONFIG); - expect(service.saveDiff).not.toHaveBeenCalled(); - expect(result).toStrictEqual({ - status: TestStatus.unresolved, - diffName: null, - diffPercent: 5, - pixelMisMatchCount: 20, - isSameDimension: true, - }); + expect(saveImage).not.toHaveBeenCalled(); + expect(result).toStrictEqual(verdict(TestStatus.unresolved, null, 5, 20)); }); it('diff found and save diff', async () => { - const diffImage = { createBuffer: jest.fn() }; - const getImageMock = jest.fn().mockReturnValueOnce(image).mockReturnValueOnce(anotherImage); - service = await initService({ getImageMock }); - service.compare = jest.fn().mockReturnValueOnce({ equal: false, differentPixels: 20, totalPixels: 400, diffImage }); - service.saveDiff = jest.fn().mockReturnValueOnce('diff name'); + const { service, saveImage } = await setup({ + run: compared(twentyOfFourHundred, { diffPng }), + saveImage: jest.fn().mockResolvedValueOnce('diff name'), + }); - const result = await service.getDiff({ ...input, saveDiffAsFile: true }, DEFAULT_CONFIG); + const result = await diff(service); - expect(service.compare).toHaveBeenCalledWith(image, anotherImage, DEFAULT_CONFIG); - expect(service.saveDiff).toHaveBeenCalledWith(diffImage); - expect(result).toStrictEqual({ - status: TestStatus.unresolved, - diffName: 'diff name', - diffPercent: 5, - pixelMisMatchCount: 20, - isSameDimension: true, - }); + expect(saveImage).toHaveBeenCalledWith('diff', Buffer.from('png')); + expect(saveImage.mock.calls[0][1]).toBeInstanceOf(Buffer); + expect(result).toStrictEqual(verdict(TestStatus.unresolved, 'diff name', 5, 20)); }); it('ok, without a diff, when the change is within the tolerance', async () => { - const getImageMock = jest.fn().mockReturnValueOnce(image).mockReturnValueOnce(anotherImage); - service = await initService({ getImageMock }); - service.compare = jest.fn().mockReturnValueOnce({ equal: false, differentPixels: 20, totalPixels: 400 }); - service.saveDiff = jest.fn(); + const { service, saveImage } = await setup({ run: compared(twentyOfFourHundred) }); - const result = await service.getDiff({ ...input, diffTollerancePercent: 5, saveDiffAsFile: true }, DEFAULT_CONFIG); + const result = await diff(service, { diffTollerancePercent: 5 }); - expect(service.saveDiff).not.toHaveBeenCalled(); - expect(result).toStrictEqual({ - status: TestStatus.ok, - diffName: null, - diffPercent: 5, - pixelMisMatchCount: 20, - isSameDimension: true, - }); + expect(saveImage).not.toHaveBeenCalled(); + expect(result).toStrictEqual(verdict(TestStatus.ok, null, 5, 20)); }); it('comparison failed', async () => { - const getImageMock = jest.fn().mockReturnValueOnce(image).mockReturnValueOnce(anotherImage); - service = await initService({ getImageMock }); - service.compare = jest.fn().mockReturnValueOnce(undefined); - - const result = await service.getDiff({ ...input, saveDiffAsFile: true }, DEFAULT_CONFIG); + const { service } = await setup({ run: compared(null) }); - expect(result).toStrictEqual({ - status: TestStatus.unresolved, - diffName: null, - diffPercent: undefined, - pixelMisMatchCount: undefined, - isSameDimension: true, - }); + expect(await diff(service)).toStrictEqual(verdict(TestStatus.unresolved, null, undefined)); }); it('reports a 0% diff when Looks-Same counts no pixels at all', async () => { - const getImageMock = jest.fn().mockReturnValueOnce(image).mockReturnValueOnce(anotherImage); - service = await initService({ getImageMock }); - service.compare = jest.fn().mockReturnValueOnce({ equal: false, differentPixels: 0, totalPixels: 0 }); + const { service } = await setup({ run: compared({ equal: false, differentPixels: 0, totalPixels: 0 }) }); - const result = await service.getDiff({ ...input, saveDiffAsFile: false }, DEFAULT_CONFIG); + expect(await diff(service, { saveDiffAsFile: false })).toStrictEqual(verdict(TestStatus.unresolved, null, 0, 0)); + }); - expect(result).toStrictEqual({ - status: TestStatus.unresolved, - diffName: null, - diffPercent: 0, - pixelMisMatchCount: 0, - isSameDimension: true, - }); + it('logs what looks-same refused', async () => { + const { service } = await setup({ run: compared(null, { errors: ['a', 'b'] }) }); + + await diff(service, { saveDiffAsFile: false }); + + expect(loggerError.mock.calls).toEqual([['a'], ['b']]); }); - it('throws when the baseline exists but the image cannot be read', async () => { - const getImageMock = jest.fn().mockReturnValueOnce(image).mockReturnValueOnce(undefined); - service = await initService({ getImageMock }); + it('passes the project config, the ignore areas and saveDiffAsFile to the job', async () => { + const { service, run } = await setup({ run: outcome('equal') }); + const config = { ...DEFAULT_CONFIG, tolerance: 3, strict: undefined }; + const ignoreAreas = [{ x: 1, y: 2, width: 3, height: 4 }]; - // NOTE: current behavior — unlike pixelmatch there is no explicit check, so a - // missing image surfaces as a TypeError rather than a descriptive error. - await expect(service.getDiff({ ...input, saveDiffAsFile: false }, DEFAULT_CONFIG)).rejects.toThrow(TypeError); + await diff(service, { ignoreAreas }, config); + + expect(run).toHaveBeenCalledWith({ + kind: 'looks-same', + baseline: baselineBuffer, + image: imageBuffer, + ignoreAreas, + config, + saveDiff: true, + }); }); - it('saves the Looks-Same diff with its metrics', async () => { - const baseline = new PNG({ width: 2, height: 1 }); - const changed = new PNG({ width: 2, height: 1 }); - baseline.data.fill(255); - changed.data.fill(255); - changed.data[0] = 0; - const getImageMock = jest.fn().mockReturnValueOnce(baseline).mockReturnValueOnce(changed); - const saveImageMock = jest.fn().mockReturnValueOnce('diff name'); - service = await initService({ getImageMock, saveImageMock }); - - const result = await service.getDiff( - { ...input, saveDiffAsFile: true }, - { ...DEFAULT_CONFIG, strict: true, ignoreAntialiasing: false } - ); - - expect(result).toStrictEqual({ + it('keeps the compared outcome when the diff cannot be saved', async () => { + const { service } = await setup({ + run: compared(twentyOfFourHundred, { diffPng }), + saveImage: jest.fn().mockRejectedValueOnce(new Error('disk full')), + }); + + expect(await diff(service)).toMatchObject({ status: TestStatus.unresolved, - diffName: 'diff name', - diffPercent: 50, - pixelMisMatchCount: 1, - isSameDimension: true, + diffName: null, + pixelMisMatchCount: 20, }); - expect(saveImageMock).toHaveBeenCalledWith('diff', expect.any(Buffer)); }); }); -describe('compare', () => { - it('returns undefined when Looks-Same fails', async () => { - service = await initService({}); - const broken = new PNG({ width: 1, height: 1 }); - jest.spyOn(PNG.sync, 'write').mockImplementationOnce(() => Buffer.from('not a png')); - - const result = await service.compare(broken, broken, DEFAULT_CONFIG); +describe('saveDiff', () => { + it('returns null when there is no diff image', async () => { + const { service, saveImage } = await setup(); - expect(result).toBeUndefined(); + await expect(service.saveDiff(undefined)).resolves.toBeNull(); + expect(saveImage).not.toHaveBeenCalled(); }); - it('compares with the defaults when Looks-Same refuses the project config', async () => { - service = await initService({}); - const baseline = new PNG({ width: 2, height: 1 }); - const changed = new PNG({ width: 2, height: 1 }); - baseline.data.fill(255); - changed.data.fill(255); - changed.data[0] = 0; - - // looks-same refuses strict together with tolerance - const result = await service.compare(baseline, changed, { ...DEFAULT_CONFIG, strict: true, tolerance: 5 }); + it('returns null when the diff cannot be saved', async () => { + const { service } = await setup({ saveImage: jest.fn().mockRejectedValueOnce(new Error('disk full')) }); - expect(result).toMatchObject({ equal: false, differentPixels: 1, totalPixels: 2 }); + expect(await service.saveDiff(diffPng)).toBeNull(); }); -}); -describe('saveDiff', () => { - it('returns null when the diff cannot be saved', async () => { - const saveImageMock = jest.fn().mockRejectedValueOnce(new Error('disk full')); - service = await initService({ saveImageMock }); - const diffImage = { createBuffer: jest.fn().mockResolvedValueOnce(Buffer.from('png')) }; + it('saves only the bytes of a view into a larger buffer', async () => { + const { service, saveImage } = await setup({ saveImage: jest.fn().mockResolvedValueOnce('diff name') }); + const whole = Buffer.from('xxpngxx'); - const result = await service.saveDiff(diffImage as never); + const result = await service.saveDiff(new Uint8Array(whole.buffer, whole.byteOffset + 2, 3)); - expect(result).toBeNull(); + expect(result).toBe('diff name'); + expect(saveImage.mock.calls[0][1].toString()).toBe('png'); }); }); diff --git a/src/compare/libs/looks-same/looks-same.service.ts b/src/compare/libs/looks-same/looks-same.service.ts index 015a8d88..fc356996 100644 --- a/src/compare/libs/looks-same/looks-same.service.ts +++ b/src/compare/libs/looks-same/looks-same.service.ts @@ -1,113 +1,104 @@ import { Injectable, Logger } from '@nestjs/common'; import { TestStatus } from '@prisma/client'; -import { PNG } from 'pngjs'; import { StaticService } from '../../../static/static.service'; +import { asBuffer, isByteEqualWellFormedPng } from '../../../static/utils'; import { DiffResult } from '../../../test-runs/diffResult'; -import { applyIgnoreAreas, parseConfig } from '../../utils'; +import { parseConfig } from '../../utils'; +import { DiffWorkerPool } from '../../diff-worker-pool'; import { ImageComparator } from '../image-comparator.interface'; import { ImageCompareInput } from '../ImageCompareInput'; import { LooksSameConfig } from './looks-same.types'; -import looksSame from 'looks-same'; +import { LOOKS_SAME_DEFAULT_CONFIG } from './looks-same.core'; import { DIFF_DIMENSION_RESULT, EQUAL_RESULT, NO_BASELINE_RESULT } from '../consts'; -export const DEFAULT_CONFIG: LooksSameConfig = { - strict: false, - ignoreAntialiasing: true, - ignoreCaret: true, - allowDiffDimensions: false, -}; +export const DEFAULT_CONFIG = LOOKS_SAME_DEFAULT_CONFIG; @Injectable() export class LookSameService implements ImageComparator { private readonly logger: Logger = new Logger(LookSameService.name); - constructor(private readonly staticService: StaticService) {} + constructor( + private readonly staticService: StaticService, + private readonly diffWorkerPool: DiffWorkerPool + ) {} parseConfig(configJson: string): LooksSameConfig { return parseConfig(configJson, DEFAULT_CONFIG, this.logger); } async getDiff(data: ImageCompareInput, config: LooksSameConfig): Promise { - const result: DiffResult = { - ...NO_BASELINE_RESULT, - }; - - const baseline = await this.staticService.getImage(data.baseline); - // a missing checkpoint image is not expected here, and fails on first use - const image = (await this.staticService.getImage(data.image))!; - - if (!baseline) { + const [baselineBuffer, imageBuffer] = await Promise.all([ + this.readBaseline(data.baseline), + data.imageBuffer ?? this.staticService.getImageBuffer(data.image), + ]); + if (!baselineBuffer) { return NO_BASELINE_RESULT; } + if (!imageBuffer) { + throw new Error(`Cannot get image: ${data.image}`); + } - if (baseline.data.equals(new Uint8Array(image.data.buffer, image.data.byteOffset, image.data.byteLength))) { + if (isByteEqualWellFormedPng(baselineBuffer, imageBuffer)) { return EQUAL_RESULT; } - result.isSameDimension = baseline.width === image.width && baseline.height === image.height; - if (!result.isSameDimension && !config.allowDiffDimensions) { - return DIFF_DIMENSION_RESULT; + const output = await this.diffWorkerPool.run({ + kind: 'looks-same', + baseline: baselineBuffer, + image: imageBuffer, + ignoreAreas: data.ignoreAreas, + config, + saveDiff: data.saveDiffAsFile, + }); + switch (output.outcome) { + case 'baseline-unreadable': + return NO_BASELINE_RESULT; + case 'image-unreadable': + throw new Error(`Cannot get image: ${data.image}`); + case 'equal': + return EQUAL_RESULT; + case 'diff-dimension': + return DIFF_DIMENSION_RESULT; } - // apply ignore areas - const baselineIgnored = applyIgnoreAreas(baseline, data.ignoreAreas); - const imageIgnored = applyIgnoreAreas(image, data.ignoreAreas); - - // compare - const compareResult = await this.compare(baselineIgnored, imageIgnored, config); - if (!compareResult) { - // looks-same failed: keep the run for review instead of failing it + output.errors.forEach((message) => this.logger.error(message)); + const result: DiffResult = { ...NO_BASELINE_RESULT, isSameDimension: output.isSameDimension }; + if (!output.result) { result.status = TestStatus.unresolved; return result; } - result.pixelMisMatchCount = compareResult.differentPixels; - result.diffPercent = - compareResult.totalPixels > 0 ? (compareResult.differentPixels * 100) / compareResult.totalPixels : 0; + const { equal, differentPixels, totalPixels } = output.result; + result.pixelMisMatchCount = differentPixels; + result.diffPercent = totalPixels && totalPixels > 0 ? ((differentPixels as number) * 100) / totalPixels : 0; - // The run's tolerance forgives a change as small as it, as pixelmatch and - // odiff do. Only a counted change: one looks-same reports but cannot - // measure (no pixels counted) is left for review whatever the tolerance. const withinTolerance = result.diffPercent > 0 && result.diffPercent <= data.diffTollerancePercent; - if (compareResult.equal || withinTolerance) { + if (equal || withinTolerance) { result.status = TestStatus.ok; } else { result.status = TestStatus.unresolved; - if (data.saveDiffAsFile) { - result.diffName = await this.saveDiff(compareResult.diffImage); - } + result.diffName = await this.saveDiff(output.diffPng); } return result; } - // one pass gives the verdict, the pixel counts and the diff image together - async compare( - baseline: PNG, - image: PNG, - config: LooksSameConfig - ): Promise | undefined> { - const [baselinePng, imagePng] = [PNG.sync.write(baseline), PNG.sync.write(image)]; - const run = (options: LooksSameConfig) => looksSame(baselinePng, imagePng, { ...options, createDiffImage: true }); + private async readBaseline(name: string | null): Promise { try { - return await run(config); + return await this.staticService.getImageBuffer(name); } catch (error) { this.logger.error((error as Error).message); + return null; } - // Most likely a project config looks-same refuses ("strict" together with - // "tolerance", say). That is the project's problem, not this screenshot's: - // compare with the defaults, as for a config that is not JSON, rather than - // leave every changed screenshot without numbers. - this.logger.error('Looks-Same cannot compare with the project config, fallback to default one'); - return run(DEFAULT_CONFIG).catch((error): undefined => { - this.logger.error(error.message); - return undefined; - }); } - async saveDiff(diffImage: looksSame.DiffImage): Promise { + async saveDiff(diffPng?: Uint8Array): Promise { + if (!diffPng) { + return null; + } try { - return await this.staticService.saveImage('diff', await diffImage.createBuffer('png')); + // saveImage checks the PNG signature with Buffer#equals, which a Uint8Array from a worker lacks + return await this.staticService.saveImage('diff', asBuffer(diffPng)); } catch (error) { this.logger.error((error as Error).message); return null; diff --git a/src/compare/libs/looks-same/looks-same.types.ts b/src/compare/libs/looks-same/looks-same.types.ts index 41084b1b..5e0d8818 100644 --- a/src/compare/libs/looks-same/looks-same.types.ts +++ b/src/compare/libs/looks-same/looks-same.types.ts @@ -1,47 +1,10 @@ import { BlurAmplitudeConfig } from '../blur-amplitude.config'; export interface LooksSameConfig extends BlurAmplitudeConfig { - /** - * strict comparsion - */ strict?: boolean; - /** - * ΔE value that will be treated as error in non-strict mode - */ tolerance?: number; - /** - * makes the search algorithm of the antialiasing less strict - */ antialiasingTolerance?: number; - /** - * Ability to ignore antialiasing - */ ignoreAntialiasing?: boolean; - /** - * Ability to ignore text caret - */ ignoreCaret?: boolean; allowDiffDimensions?: boolean; } - -/** - * coordinate bounds - */ -export interface CoordBounds { - /** - * X-coordinate of upper left corner - */ - left: number; - /** - * Y-coordinate of upper left corner - */ - top: number; - /** - * X-coordinate of bottom right corner - */ - right: number; - /** - * Y-coordinate of bottom right corner - */ - bottom: number; -} diff --git a/src/compare/libs/odiff/odiff.service.spec.ts b/src/compare/libs/odiff/odiff.service.spec.ts index e7d9c89a..394092ca 100644 --- a/src/compare/libs/odiff/odiff.service.spec.ts +++ b/src/compare/libs/odiff/odiff.service.spec.ts @@ -1,9 +1,10 @@ -import { TestingModule, Test } from '@nestjs/testing'; +import { Test } from '@nestjs/testing'; import { TestStatus } from '@prisma/client'; import { IgnoreAreaDto } from '../../../test-runs/dto/ignore-area.dto'; import { StaticService } from '../../../static/static.service'; import { HddService } from '../../../static/hdd/hdd.service'; import { DIFF_DIMENSION_RESULT, EQUAL_RESULT, NO_BASELINE_RESULT } from '../consts'; +import { ImageCompareInput } from '../ImageCompareInput'; import { OdiffService, DEFAULT_CONFIG, ignoreAreaToRegionMapper } from './odiff.service'; import { OdiffConfig, OdiffIgnoreRegions } from './odiff.types'; import { compare } from 'odiff-bin'; @@ -12,225 +13,112 @@ jest.mock('odiff-bin', () => ({ compare: jest.fn(), })); +type CompareResult = Awaited>; + const compareMock = compare as jest.MockedFunction; -const initService = async ({ - generateNewImageMock = jest.fn(), - getImagePathMock = jest.fn(), - deleteImageMock = jest.fn(), -}) => { - const module: TestingModule = await Test.createTestingModule({ +const setup = async (compareResult?: CompareResult) => { + if (compareResult) { + compareMock.mockResolvedValueOnce(compareResult); + } + const deleteImage = jest.fn(); + const module = await Test.createTestingModule({ providers: [ OdiffService, - { - provide: StaticService, - useValue: { - deleteImage: deleteImageMock, - }, - }, + { provide: StaticService, useValue: { deleteImage } }, { provide: HddService, useValue: { - generateNewImage: generateNewImageMock, - getImagePath: getImagePathMock, + generateNewImage: jest.fn().mockReturnValueOnce({ imagePath: './diff.png', imageName: 'diff.png' }), + getImagePath: jest.fn().mockReturnValueOnce('./baseline.png').mockReturnValueOnce('./image.png'), }, }, ], }).compile(); + return { service: module.get(OdiffService), deleteImage }; +}; - return module.get(OdiffService); +const input: ImageCompareInput = { + baseline: 'baseline', + image: 'image', + diffTollerancePercent: 0.1, + ignoreAreas: [], + saveDiffAsFile: true, }; -let service: OdiffService; +const pixelDiff = (diffPercentage: number, diffCount = 33.33) => + ({ match: false, reason: 'pixel-diff', diffPercentage, diffCount }) as CompareResult; + +const verdict = (status: TestStatus, diffName: string | null, diffPercent: number | undefined, count = 33.33) => ({ + diffName, + diffPercent, + isSameDimension: true, + pixelMisMatchCount: diffPercent === undefined ? undefined : count, + status, +}); describe('parseConfig', () => { it.each<[string, OdiffConfig]>([ [ '{"threshold":0,"antialiasing":true,"failOnLayoutDiff":true,"outputDiffMask":true}', - { - threshold: 0, - antialiasing: true, - failOnLayoutDiff: true, - outputDiffMask: true, - }, + { threshold: 0, antialiasing: true, failOnLayoutDiff: true, outputDiffMask: true }, ], ['', DEFAULT_CONFIG], ['invalid', DEFAULT_CONFIG], ])('should parse config', async (json, expected) => { - service = await initService({}); - - const config = service.parseConfig(json); + const { service } = await setup(); - expect(config).toStrictEqual(expected); + expect(service.parseConfig(json)).toStrictEqual(expected); }); }); describe('getDiff', () => { it('no baseline', async () => { - service = await initService({}); - - const result = await service.getDiff( - { - baseline: null, - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: true, - }, - DEFAULT_CONFIG - ); + const { service } = await setup(); - expect(result).toStrictEqual(NO_BASELINE_RESULT); + expect(await service.getDiff({ ...input, baseline: null }, DEFAULT_CONFIG)).toStrictEqual(NO_BASELINE_RESULT); }); it('diff layout', async () => { - const generateNewImageMock = jest.fn().mockReturnValueOnce({ imagePath: './diff.png', imageName: 'diff.png' }); - const getImagePathMock = jest.fn().mockReturnValueOnce('./baseline.png').mockReturnValueOnce('./image.png'); - const deleteImageMock = jest.fn(); - compareMock.mockResolvedValueOnce({ match: false, reason: 'layout-diff' }); - service = await initService({ - getImagePathMock, - generateNewImageMock, - deleteImageMock, - }); + const { service, deleteImage } = await setup({ match: false, reason: 'layout-diff' }); - const result = await service.getDiff( - { - baseline: 'baseline', - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: true, - }, - { - ...DEFAULT_CONFIG, - failOnLayoutDiff: true, - } - ); + const result = await service.getDiff(input, { ...DEFAULT_CONFIG, failOnLayoutDiff: true }); expect(result).toStrictEqual(DIFF_DIMENSION_RESULT); - expect(deleteImageMock).not.toHaveBeenCalled(); + expect(deleteImage).not.toHaveBeenCalled(); }); it('diff found', async () => { - const generateNewImageMock = jest.fn().mockReturnValueOnce({ imagePath: './diff.png', imageName: 'diff.png' }); - const getImagePathMock = jest.fn().mockReturnValueOnce('./baseline.png').mockReturnValueOnce('./image.png'); - const deleteImageMock = jest.fn(); - compareMock.mockResolvedValueOnce({ match: false, reason: 'pixel-diff', diffPercentage: 12.34, diffCount: 33.33 }); - service = await initService({ - getImagePathMock, - generateNewImageMock, - deleteImageMock, - }); + const { service, deleteImage } = await setup(pixelDiff(12.34)); - const result = await service.getDiff( - { - baseline: 'baseline', - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: true, - }, - { - ...DEFAULT_CONFIG, - failOnLayoutDiff: true, - } - ); - - expect(deleteImageMock).not.toHaveBeenCalled(); - expect(result).toStrictEqual({ - diffName: 'diff.png', - diffPercent: 12.34, - isSameDimension: true, - pixelMisMatchCount: 33.33, - status: TestStatus.unresolved, - }); + const result = await service.getDiff(input, { ...DEFAULT_CONFIG, failOnLayoutDiff: true }); + + expect(deleteImage).not.toHaveBeenCalled(); + expect(result).toStrictEqual(verdict(TestStatus.unresolved, 'diff.png', 12.34)); }); it('diff found < tollerance', async () => { - const generateNewImageMock = jest.fn().mockReturnValueOnce({ imagePath: './diff.png', imageName: 'diff.png' }); - const getImagePathMock = jest.fn().mockReturnValueOnce('./baseline.png').mockReturnValueOnce('./image.png'); - const deleteImageMock = jest.fn(); - compareMock.mockResolvedValueOnce({ match: false, reason: 'pixel-diff', diffPercentage: 0.05, diffCount: 33.33 }); - service = await initService({ - getImagePathMock, - generateNewImageMock, - deleteImageMock, - }); + const { service, deleteImage } = await setup(pixelDiff(0.05)); - const result = await service.getDiff( - { - baseline: 'baseline', - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: true, - }, - DEFAULT_CONFIG - ); - - // odiff writes the diff image for every pixel diff, so one below the tolerance is not kept - expect(deleteImageMock).toHaveBeenCalledWith('diff.png'); - expect(result).toStrictEqual({ - diffName: null, - diffPercent: 0.05, - isSameDimension: true, - pixelMisMatchCount: 33.33, - status: TestStatus.ok, - }); + const result = await service.getDiff(input, DEFAULT_CONFIG); + + expect(deleteImage).toHaveBeenCalledWith('diff.png'); + expect(result).toStrictEqual(verdict(TestStatus.ok, null, 0.05)); }); it('diff found no save', async () => { - const generateNewImageMock = jest.fn().mockReturnValueOnce({ imagePath: './diff.png', imageName: 'diff.png' }); - const getImagePathMock = jest.fn().mockReturnValueOnce('./baseline.png').mockReturnValueOnce('./image.png'); - const deleteImageMock = jest.fn(); - compareMock.mockResolvedValueOnce({ match: false, reason: 'pixel-diff', diffPercentage: 3, diffCount: 33.33 }); - service = await initService({ - getImagePathMock, - generateNewImageMock, - deleteImageMock, - }); + const { service, deleteImage } = await setup(pixelDiff(3)); - const result = await service.getDiff( - { - baseline: 'baseline', - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: false, - }, - DEFAULT_CONFIG - ); - - expect(deleteImageMock).toHaveBeenCalledWith('diff.png'); - expect(result).toStrictEqual({ - diffName: null, - diffPercent: 3, - isSameDimension: true, - pixelMisMatchCount: 33.33, - status: TestStatus.unresolved, - }); + const result = await service.getDiff({ ...input, saveDiffAsFile: false }, DEFAULT_CONFIG); + + expect(deleteImage).toHaveBeenCalledWith('diff.png'); + expect(result).toStrictEqual(verdict(TestStatus.unresolved, null, 3)); }); it('passes ignore areas to odiff as regions', async () => { - const generateNewImageMock = jest.fn().mockReturnValueOnce({ imagePath: './diff.png', imageName: 'diff.png' }); - const getImagePathMock = jest.fn().mockReturnValueOnce('./baseline.png').mockReturnValueOnce('./image.png'); - compareMock.mockResolvedValueOnce({ match: true }); - service = await initService({ - getImagePathMock, - generateNewImageMock, - }); + const { service } = await setup({ match: true }); - await service.getDiff( - { - baseline: 'baseline', - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [{ x: 1, y: 2, width: 3, height: 4 }], - saveDiffAsFile: true, - }, - DEFAULT_CONFIG - ); + await service.getDiff({ ...input, ignoreAreas: [{ x: 1, y: 2, width: 3, height: 4 }] }, DEFAULT_CONFIG); expect(compareMock).toHaveBeenLastCalledWith('./baseline.png', './image.png', './diff.png', { ...DEFAULT_CONFIG, @@ -240,179 +128,61 @@ describe('getDiff', () => { }); }); - it('does not pass empty ignore regions to odiff', async () => { - // odiff 4 rejects an empty --ignore= argument with "Invalid ignore regions format" - const generateNewImageMock = jest.fn().mockReturnValueOnce({ imagePath: './diff.png', imageName: 'diff.png' }); - const getImagePathMock = jest.fn().mockReturnValueOnce('./baseline.png').mockReturnValueOnce('./image.png'); - compareMock.mockResolvedValueOnce({ match: true }); - service = await initService({ - getImagePathMock, - generateNewImageMock, - }); + it.each([ + ['there are none', []], + ['every area lies outside the image', [{ x: -20, y: 0, width: 10, height: 5 }]], + ])('does not pass ignore regions when %s', async (_, ignoreAreas) => { + const { service } = await setup({ match: true }); - await service.getDiff( - { - baseline: 'baseline', - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: true, - }, - DEFAULT_CONFIG - ); + await service.getDiff({ ...input, ignoreAreas }, DEFAULT_CONFIG); expect(compareMock.mock.lastCall![3]).not.toHaveProperty('ignoreRegions'); }); it('ok', async () => { - const generateNewImageMock = jest.fn().mockReturnValueOnce({ imagePath: './diff.png', imageName: 'diff.png' }); - const getImagePathMock = jest.fn().mockReturnValueOnce('./baseline.png').mockReturnValueOnce('./image.png'); - const deleteImageMock = jest.fn(); - compareMock.mockResolvedValueOnce({ match: true }); - service = await initService({ - getImagePathMock, - generateNewImageMock, - deleteImageMock, - }); + const { service, deleteImage } = await setup({ match: true }); - const result = await service.getDiff( - { - baseline: 'baseline', - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: false, - }, - DEFAULT_CONFIG - ); + const result = await service.getDiff({ ...input, saveDiffAsFile: false }, DEFAULT_CONFIG); - // odiff writes no diff image when the images match - expect(deleteImageMock).not.toHaveBeenCalled(); + expect(deleteImage).not.toHaveBeenCalled(); expect(result).toStrictEqual(EQUAL_RESULT); }); it('diff found without parsable numbers', async () => { - // odiff-bin resolves a pixel diff without numbers when it cannot parse the binary output - const generateNewImageMock = jest.fn().mockReturnValueOnce({ imagePath: './diff.png', imageName: 'diff.png' }); - const getImagePathMock = jest.fn().mockReturnValueOnce('./baseline.png').mockReturnValueOnce('./image.png'); - compareMock.mockResolvedValueOnce({ match: false, reason: 'pixel-diff' } as Awaited>); - service = await initService({ - getImagePathMock, - generateNewImageMock, - }); + const { service } = await setup({ match: false, reason: 'pixel-diff' } as CompareResult); - const result = await service.getDiff( - { - baseline: 'baseline', - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: true, - }, - DEFAULT_CONFIG - ); - - expect(result).toStrictEqual({ - diffName: 'diff.png', - diffPercent: undefined, - isSameDimension: true, - pixelMisMatchCount: undefined, - status: TestStatus.unresolved, - }); + const result = await service.getDiff(input, DEFAULT_CONFIG); + + expect(result).toStrictEqual(verdict(TestStatus.unresolved, 'diff.png', undefined)); }); it('baseline file missing', async () => { - const generateNewImageMock = jest.fn().mockReturnValueOnce({ imagePath: './diff.png', imageName: 'diff.png' }); - const getImagePathMock = jest.fn().mockReturnValueOnce('./baseline.png').mockReturnValueOnce('./image.png'); - compareMock.mockResolvedValueOnce({ match: false, reason: 'file-not-exists', file: './baseline.png' }); - service = await initService({ - getImagePathMock, - generateNewImageMock, - }); + const { service } = await setup({ match: false, reason: 'file-not-exists', file: './baseline.png' }); - const result = await service.getDiff( - { - baseline: 'baseline', - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: true, - }, - DEFAULT_CONFIG - ); - - expect(result).toStrictEqual(NO_BASELINE_RESULT); + expect(await service.getDiff(input, DEFAULT_CONFIG)).toStrictEqual(NO_BASELINE_RESULT); }); it('image file missing', async () => { - const generateNewImageMock = jest.fn().mockReturnValueOnce({ imagePath: './diff.png', imageName: 'diff.png' }); - const getImagePathMock = jest.fn().mockReturnValueOnce('./baseline.png').mockReturnValueOnce('./image.png'); - compareMock.mockResolvedValueOnce({ match: false, reason: 'file-not-exists', file: './image.png' }); - service = await initService({ - getImagePathMock, - generateNewImageMock, - }); + const { service } = await setup({ match: false, reason: 'file-not-exists', file: './image.png' }); - await expect( - service.getDiff( - { - baseline: 'baseline', - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: true, - }, - DEFAULT_CONFIG - ) - ).rejects.toThrow('Cannot get image: image'); + await expect(service.getDiff(input, DEFAULT_CONFIG)).rejects.toThrow('Cannot get image: image'); }); it('odiff error', async () => { - const generateNewImageMock = jest.fn().mockReturnValueOnce({ imagePath: './diff.png', imageName: 'diff.png' }); - const getImagePathMock = jest.fn().mockReturnValueOnce('./baseline.png').mockReturnValueOnce('./image.png'); + const { service } = await setup(); compareMock.mockRejectedValueOnce(new TypeError('Error: Could not load comparison image: ./image.png')); - service = await initService({ - getImagePathMock, - generateNewImageMock, - }); - await expect( - service.getDiff( - { - baseline: 'baseline', - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: true, - }, - DEFAULT_CONFIG - ) - ).rejects.toThrow('Could not load comparison image'); + await expect(service.getDiff(input, DEFAULT_CONFIG)).rejects.toThrow('Could not load comparison image'); }); }); it.each<[IgnoreAreaDto[], OdiffIgnoreRegions]>([ [[], []], - [ - [ - { - x: 1, - y: 2, - width: 3, - height: 4, - }, - ], - [ - { - x1: 1, - y1: 2, - x2: 4, - y2: 6, - }, - ], - ], + [[{ x: 1, y: 2, width: 3, height: 4 }], [{ x1: 1, y1: 2, x2: 4, y2: 6 }]], + [[{ x: 1.5, y: 2.2, width: 3, height: 4 }], [{ x1: 1, y1: 2, x2: 5, y2: 7 }]], + [[{ x: -5, y: -1, width: 10, height: 3 }], [{ x1: 0, y1: 0, x2: 5, y2: 2 }]], + [[{ x: -20, y: 0, width: 10, height: 5 }], []], + [[{ x: 10, y: 10, width: 0, height: 5 }], [{ x1: 10, y1: 10, x2: 10, y2: 15 }]], ])('ignoreAreaToRegionMapper', async (ignoreAreas, ignoreRegions) => { - const result = ignoreAreaToRegionMapper(ignoreAreas); - - expect(result).toStrictEqual(ignoreRegions); + expect(ignoreAreaToRegionMapper(ignoreAreas)).toStrictEqual(ignoreRegions); }); diff --git a/src/compare/libs/odiff/odiff.service.ts b/src/compare/libs/odiff/odiff.service.ts index ee11ee5a..c2ebdad2 100644 --- a/src/compare/libs/odiff/odiff.service.ts +++ b/src/compare/libs/odiff/odiff.service.ts @@ -32,31 +32,25 @@ export class OdiffService implements ImageComparator { } async getDiff(data: ImageCompareInput, config: OdiffConfig): Promise { - const result: DiffResult = { - ...NO_BASELINE_RESULT, - }; - if (!data.baseline) { return NO_BASELINE_RESULT; } - // compare const baselinePath = this.hddService.getImagePath(data.baseline); const diff = this.hddService.generateNewImage('diff'); + const ignoreRegions = ignoreAreaToRegionMapper(data.ignoreAreas); const compareResult = await compare(baselinePath, this.hddService.getImagePath(data.image), diff.imagePath, { ...config, diffColor: '#cd2cc9', - // a missing file comes back as a result instead of an error naming the file noFailOnFsErrors: true, // odiff rejects an empty --ignore= argument, so only pass regions when there are any - ...(data.ignoreAreas.length > 0 && { ignoreRegions: ignoreAreaToRegionMapper(data.ignoreAreas) }), + ...(ignoreRegions.length > 0 && { ignoreRegions }), }); if (compareResult.match === true) { return EQUAL_RESULT; } if (compareResult.reason === 'file-not-exists') { - // same as the other comparators: a missing baseline file means there is nothing to compare with if (compareResult.file === baselinePath) { return NO_BASELINE_RESULT; } @@ -66,34 +60,31 @@ export class OdiffService implements ImageComparator { return DIFF_DIMENSION_RESULT; } - // pixel diff: odiff has written the diff image - result.isSameDimension = true; - result.diffPercent = compareResult.diffPercentage; - result.pixelMisMatchCount = compareResult.diffCount; - - // process result - // odiff-bin leaves the numbers out when it cannot parse the binary output: keep the run for review - if (!(result.diffPercent <= data.diffTollerancePercent)) { - // save diff - if (data.saveDiffAsFile) { - result.diffName = diff.imageName; - } else { - await this.staticService.deleteImage(diff.imageName); - } - result.status = TestStatus.unresolved; + const withinTolerance = compareResult.diffPercentage <= data.diffTollerancePercent; + const result: DiffResult = { + ...NO_BASELINE_RESULT, + isSameDimension: true, + diffPercent: compareResult.diffPercentage, + pixelMisMatchCount: compareResult.diffCount, + status: withinTolerance ? TestStatus.ok : TestStatus.unresolved, + }; + if (!withinTolerance && data.saveDiffAsFile) { + result.diffName = diff.imageName; } else { await this.staticService.deleteImage(diff.imageName); - result.status = TestStatus.ok; } return result; } } -export const ignoreAreaToRegionMapper = (ignoreAreas: IgnoreAreaDto[]): OdiffIgnoreRegions => { - return ignoreAreas.map((item) => ({ - x1: item.x, - y1: item.y, - x2: item.x + item.width, - y2: item.y + item.height, - })); -}; +export const ignoreAreaToRegionMapper = (ignoreAreas: IgnoreAreaDto[]): OdiffIgnoreRegions => + ignoreAreas.flatMap((item) => { + const region = { + x1: Math.max(0, Math.floor(item.x)), + y1: Math.max(0, Math.floor(item.y)), + x2: Math.ceil(item.x + item.width), + y2: Math.ceil(item.y + item.height), + }; + const usable = Object.values(region).every(Number.isFinite) && region.x2 >= region.x1 && region.y2 >= region.y1; + return usable ? [region] : []; + }); diff --git a/src/compare/libs/odiff/odiff.types.ts b/src/compare/libs/odiff/odiff.types.ts index 5d19d181..97b0ab96 100644 --- a/src/compare/libs/odiff/odiff.types.ts +++ b/src/compare/libs/odiff/odiff.types.ts @@ -1,13 +1,9 @@ import { BlurAmplitudeConfig } from '../blur-amplitude.config'; export type OdiffConfig = BlurAmplitudeConfig & { - /** Output full diff image. */ outputDiffMask: boolean; - /** Do not compare images and produce output if images layout is different. */ failOnLayoutDiff: boolean; - /** Color difference threshold (from 0 to 1). Less more precise. */ threshold: number; - /** If this is true, antialiased pixels are not counted to the diff of an image */ antialiasing: boolean; }; diff --git a/src/compare/libs/pixelmatch/blur-amplitude.core.spec.ts b/src/compare/libs/pixelmatch/blur-amplitude.core.spec.ts index c7b1964d..467ebd96 100644 --- a/src/compare/libs/pixelmatch/blur-amplitude.core.spec.ts +++ b/src/compare/libs/pixelmatch/blur-amplitude.core.spec.ts @@ -10,6 +10,7 @@ import { MAX_SOLID_SHARE, measureBlurredDifference, } from './blur-amplitude.core'; +import { DINOV2_PATCH, gateOfDecoded } from './patch-gate.core'; import { runWorkerJob } from './worker-job'; const SIZE = 20; @@ -28,104 +29,70 @@ const image = (paint: (x: number, y: number) => Rgba | undefined = () => undefin }; const blank = image(); +const pixel = (px: number, py: number, rgba: Rgba = BLACK) => + image((x, y) => (x === px && y === py ? rgba : undefined)); +const column = (col: number) => image((x) => (x === col ? BLACK : undefined)); +const box = (rgba: Rgba = BLACK) => image((x, y) => (x >= 4 && x < 16 && y >= 4 && y < 16 ? rgba : undefined)); -// the measure of a pair that is known to be measurable -const measured = (a: RawImage, b: RawImage, sigma: number): BlurredDifference => { - const measure = measureBlurredDifference(a, b, sigma); +const measured = (a: RawImage, b: RawImage): BlurredDifference => { + const measure = measureBlurredDifference(a, b); if (!measure) { throw new Error('expected a measure'); } return measure; }; -const maxBlurredDifference = (a: RawImage, b: RawImage, sigma: number): number => measured(a, b, sigma).amplitude; +const maxBlurredDifference = (a: RawImage, b: RawImage): number => measured(a, b).amplitude; -// Expected values are scipy.ndimage.gaussian_filter (sigma=1, truncate=4, -// mode='reflect') on the same black-and-white images, where every channel and -// the luma agree. +// expected: scipy.ndimage.gaussian_filter(sigma=1, truncate=4, mode='reflect') describe('measureBlurredDifference', () => { it('is zero for identical images', () => { - expect(maxBlurredDifference(blank, image(), 1)).toBe(0); + expect(maxBlurredDifference(blank, image())).toBe(0); }); - // what anti-aliasing mostly is: single pixels flipping along edges it('keeps an isolated pixel well under the default threshold', () => { - const amplitude = maxBlurredDifference( - blank, - image((x, y) => (x === 10 && y === 10 ? BLACK : undefined)), - 1 - ); + const amplitude = maxBlurredDifference(blank, pixel(10, 10)); expect(amplitude).toBeCloseTo(40.585, 2); expect(amplitude).toBeLessThan(DEFAULT_BLUR_AMPLITUDE_THRESHOLD); }); - // the other thing it is: an edge landing one pixel further over it('keeps a hairline moved by one pixel under the default threshold', () => { - const before = image((x) => (x === 10 ? BLACK : undefined)); - const after = image((x) => (x === 11 ? BLACK : undefined)); - - const amplitude = maxBlurredDifference(before, after, 1); + const amplitude = maxBlurredDifference(column(10), column(11)); expect(amplitude).toBeCloseTo(47.935, 2); expect(amplitude).toBeLessThan(DEFAULT_BLUR_AMPLITUDE_THRESHOLD); }); it('reports the full contrast of a solid element that appeared', () => { - const block = image((x, y) => (x >= 4 && x < 16 && y >= 4 && y < 16 ? BLACK : undefined)); - - expect(maxBlurredDifference(blank, block, 1)).toBeCloseTo(255, 2); + expect(maxBlurredDifference(blank, box())).toBeCloseTo(255, 2); }); - // the image is mirrored at its border, not padded with nothing, so a change - // in a corner is not underrated it('mirrors the image at its border', () => { - const corner = image((x, y) => (x === 0 && y === 0 ? BLACK : undefined)); - - expect(maxBlurredDifference(blank, corner, 1)).toBeCloseTo(104.747, 2); + expect(maxBlurredDifference(blank, pixel(0, 0))).toBeCloseTo(104.747, 2); }); - it('spreads a change thinner under a wider blur', () => { - const dot = image((x, y) => (x === 10 && y === 10 ? BLACK : undefined)); - - expect(maxBlurredDifference(blank, dot, 2)).toBeCloseTo(10.146, 2); - }); - - // ignore areas are blanked to transparent black, and pixelmatch reads - // transparent pixels as white; reading them the same way keeps a masked area - // from looking like a change against a white background it('reads transparent pixels as white', () => { - const transparent = image((x, y) => (x === 10 && y === 10 ? [0, 0, 0, 0] : undefined)); - - expect(maxBlurredDifference(blank, transparent, 1)).toBe(0); + expect(maxBlurredDifference(blank, pixel(10, 10, [0, 0, 0, 0]))).toBe(0); }); - // luma alone would call this nothing: the grey has the red's brightness it('sees a change of hue at equal brightness', () => { - const red = image((x, y) => (x >= 4 && x < 16 && y >= 4 && y < 16 ? [255, 0, 0, 255] : undefined)); - const grey = image((x, y) => (x >= 4 && x < 16 && y >= 4 && y < 16 ? [76, 76, 76, 255] : undefined)); - - expect(maxBlurredDifference(red, grey, 1)).toBeGreaterThan(DEFAULT_BLUR_AMPLITUDE_THRESHOLD); + expect(maxBlurredDifference(box([255, 0, 0, 255]), box([76, 76, 76, 255]))).toBeGreaterThan( + DEFAULT_BLUR_AMPLITUDE_THRESHOLD + ); }); it('refuses images of different sizes', () => { const smaller: RawImage = { data: Buffer.alloc(4 * 4 * 4), width: 4, height: 4 }; - expect(() => maxBlurredDifference(blank, smaller, 1)).toThrow(); + expect(() => maxBlurredDifference(blank, smaller)).toThrow(); }); - // anti-aliasing lives on edges, which the blur flattens; a filled shape's - // inside it leaves as it was it('counts the pixels of a solid change, not those of an edge', () => { - const block = image((x, y) => (x >= 4 && x < 16 && y >= 4 && y < 16 ? [200, 200, 200, 255] : undefined)); - const hairline = image((x) => (x === 10 ? BLACK : undefined)); - - expect(measured(blank, block, 1).solidPixels).toBeGreaterThan(50); - expect(measured(blank, hairline, 1).solidPixels).toBe(0); + expect(measured(blank, box([200, 200, 200, 255])).solidPixels).toBeGreaterThan(50); + expect(measured(blank, column(10)).solidPixels).toBe(0); }); - // measuring a change spread over a huge screenshot would cost seconds and - // hundreds of megabytes in a worker it('gives no measure for a changed area beyond the size limit', () => { const width = 4000; const height = Math.ceil(MAX_MEASURED_PIXELS / width) + 1; @@ -134,7 +101,7 @@ describe('measureBlurredDifference', () => { b.data[0] = 0; b.data[b.data.length - 4] = 0; - expect(measureBlurredDifference(a, b, 1)).toBeNull(); + expect(measureBlurredDifference(a, b)).toBeNull(); }); }); @@ -147,7 +114,6 @@ describe('isRenderingNoise', () => { expect(isRenderingNoise({ amplitude: 81, solidPixels: 0, solidShare: 0 }, 80)).toBe(false); }); - // a panel turned light grey: faint, so under any threshold, but solid it('is not noise when too much of the change is solid, however faint', () => { expect(isRenderingNoise({ amplitude: 31, solidPixels: 1, solidShare: MAX_SOLID_SHARE + 0.001 }, 80)).toBe(false); }); @@ -156,7 +122,6 @@ describe('isRenderingNoise', () => { expect(isRenderingNoise(null, 80)).toBe(false); }); - // measured on a real-size picture rather than on numbers it('sees a faint panel shade as a change', () => { const size = 200; const flat = (shade: number): RawImage => { @@ -169,7 +134,7 @@ describe('isRenderingNoise', () => { return { data, width: size, height: size }; }; - const measure = measured(flat(255), flat(224), 1); + const measure = measured(flat(255), flat(224)); expect(measure.amplitude).toBeLessThan(DEFAULT_BLUR_AMPLITUDE_THRESHOLD); expect(isRenderingNoise(measure, DEFAULT_BLUR_AMPLITUDE_THRESHOLD)).toBe(false); @@ -177,19 +142,17 @@ describe('isRenderingNoise', () => { }); describe('computeBlurAmplitude', () => { - // the worker-pool job: PNG bytes in, so the decoding happens off the event loop const encode = (raw: RawImage): Buffer => { const png = new PNG({ width: raw.width, height: raw.height }); raw.data.copy(png.data); return PNG.sync.write(png); }; - const block = image((x, y) => (x >= 4 && x < 16 && y >= 4 && y < 16 ? BLACK : undefined)); + const block = box(); const job = (overrides: Partial = {}): BlurAmplitudeJobInput => ({ kind: 'blur-amplitude', baseline: encode(blank), image: encode(block), ignoreAreas: [], - sigma: 1, ...overrides, }); @@ -197,7 +160,6 @@ describe('computeBlurAmplitude', () => { expect(computeBlurAmplitude(job()).measure?.amplitude).toBeCloseTo(255, 2); }); - // what the project has masked out must not decide its verdict it('leaves out what the ignore areas cover', () => { expect(computeBlurAmplitude(job({ ignoreAreas: [{ x: 0, y: 0, width: SIZE, height: SIZE }] })).measure).toEqual({ amplitude: 0, @@ -206,25 +168,67 @@ describe('computeBlurAmplitude', () => { }); }); - // a comparator allowed to compare different sizes has nothing to line up it('has no answer for screenshots of different sizes', () => { const smaller = encode({ data: Buffer.alloc(4 * 4 * 4, 255), width: 4, height: 4 }); expect(computeBlurAmplitude(job({ image: smaller })).measure).toBeNull(); }); - // the caller keeps the comparator's verdict rather than failing the upload it('has no answer for bytes that are not a PNG', () => { expect(computeBlurAmplitude(job({ image: Buffer.from('not a png') })).measure).toBeNull(); }); - // what a Buffer arrives as on the far side of postMessage it('reads the Uint8Array views a worker message turns Buffers into', () => { const viaMessage = job({ baseline: new Uint8Array(encode(blank)), image: new Uint8Array(encode(block)) }); expect(computeBlurAmplitude(viaMessage).measure?.amplitude).toBeCloseTo(255, 2); }); + describe('with the gate asked for', () => { + const dot = pixel(10, 10); + const gated = (overrides: Partial = {}) => + job({ image: encode(dot), gateIfNoise: { threshold: DEFAULT_BLUR_AMPLITUDE_THRESHOLD }, ...overrides }); + + it('returns the DINOv2 patch gate of a pair it takes for rendering noise', () => { + const output = computeBlurAmplitude(gated()); + + expect(isRenderingNoise(output.measure, DEFAULT_BLUR_AMPLITUDE_THRESHOLD)).toBe(true); + expect(output.gate).toEqual(gateOfDecoded(blank, dot, DINOV2_PATCH)); + expect(output.gate?.cells).toEqual([0]); + }); + + it('leaves out what the ignore areas cover', () => { + const output = computeBlurAmplitude(gated({ ignoreAreas: [{ x: 8, y: 8, width: 4, height: 4 }] })); + + expect(output.gate?.cells).toEqual([]); + }); + + it('returns no gate for a change stronger than rendering noise', () => { + const output = computeBlurAmplitude(gated({ image: encode(block) })); + + expect(output.measure?.amplitude).toBeGreaterThan(DEFAULT_BLUR_AMPLITUDE_THRESHOLD); + expect(output.gate).toBeUndefined(); + }); + + it('judges the noise by the threshold it is given', () => { + expect(computeBlurAmplitude(gated({ gateIfNoise: { threshold: 30 } })).gate).toBeUndefined(); + }); + + it('returns no gate when it was not asked for', () => { + expect(computeBlurAmplitude(job({ image: encode(dot) })).gate).toBeUndefined(); + }); + + it('returns no gate for screenshots it cannot line up', () => { + expect(computeBlurAmplitude(gated({ image: Buffer.from('not a png') }))).toEqual({ measure: null }); + }); + + it('returns a gate that survives a structured clone', () => { + const output = computeBlurAmplitude(gated()); + + expect(structuredClone(output)).toEqual(output); + }); + }); + it('is what the worker runs for a blur-amplitude job', () => { expect(runWorkerJob(job())).toEqual(computeBlurAmplitude(job())); }); diff --git a/src/compare/libs/pixelmatch/blur-amplitude.core.ts b/src/compare/libs/pixelmatch/blur-amplitude.core.ts index 24c8a178..87cfbb5e 100644 --- a/src/compare/libs/pixelmatch/blur-amplitude.core.ts +++ b/src/compare/libs/pixelmatch/blur-amplitude.core.ts @@ -1,109 +1,56 @@ -import { PNG } from 'pngjs'; import { IgnoreAreaDto } from '../../../test-runs/dto/ignore-area.dto'; -import { applyIgnoreAreas, RawImage } from '../../utils'; +import { RawImage, readSameSizePair } from '../../utils'; +import { DINOV2_PATCH, gateOfDecoded, PatchGate } from './patch-gate.core'; -/** - * Filter for rendering noise, calibrated in research/jepa-screenshot-diff: - * blurring the difference between two screenshots by a pixel or so makes - * anti-aliasing and sub-pixel rasterisation differences — single pixels - * flipping, edges landing a fraction of a pixel over — average out to a weak - * change, while anything a person would notice (a letter, an icon, a moved - * block) keeps most of its contrast. - * - * The blur cannot tell a faint change that is *solid* — a panel turned light - * grey — from noise by its strength alone: blurring a uniform patch leaves it - * as it is. So the pixels the blur does not attenuate are counted too, and a - * change with many of them is never noise. - */ export interface BlurAmplitudeOptions { - // Gaussian sigma in pixels - sigma: number; - // strongest blurred colour-channel difference (0-255) still treated as - // rendering noise threshold: number; - // a second look at what the blur would pass (see dinov2/dinov2.service.ts): - // largest content distance between the two screenshots still treated as noise dinov2?: { threshold: number }; } export interface BlurredDifference { - // strongest blurred colour-channel difference, 0-255 amplitude: number; - // pixels whose change the blur leaves (nearly) intact: a solid change solidPixels: number; - // the same as a share of the screenshot, 0-1 solidShare: number; } -/** - * The worker-pool job CompareService runs for comparators that do not apply the - * filter in their own pass (looks-same, odiff). PNG bytes in, because decoding - * is the expensive part and belongs off the event loop; all input/output must - * stay structured-clone serializable. - */ export interface BlurAmplitudeJobInput { kind: 'blur-amplitude'; baseline: Buffer | Uint8Array; image: Buffer | Uint8Array; ignoreAreas: IgnoreAreaDto[]; - sigma: number; + gateIfNoise?: { threshold: number }; } export interface BlurAmplitudeJobOutput { - // null when there is nothing to measure: unreadable bytes, different sizes, - // or a change too large to measure within bounds measure: BlurredDifference | null; + gate?: PatchGate; } export const DEFAULT_BLUR_SIGMA = 1; -// Above the strongest of 57 anti-aliasing-only pairs of real VRT pages (75.1) -// in the research dataset; at this value 72% of the injected bugs still fail. +// strongest of 57 anti-aliasing-only real pairs: 75.1; 72% of injected bugs still fail export const DEFAULT_BLUR_AMPLITUDE_THRESHOLD = 80; -// A pixel counts as solid when its change is at least this many levels and the -// blur keeps at least this share of it. Sub-pixel shifts of large flat shapes -// make up to 0.7% of a page solid in the research dataset (6,300 of 1280x720); -// a panel turning #e0e0e0 makes 19%. Taken as a share of the screenshot, so it -// holds for a small component and a tall page alike. +// solid share: sub-pixel shifts up to 0.7%, #e0e0e0 panel 19% const SOLID_MIN_CHANGE = 6; const SOLID_KEEP = 0.9; export const MAX_SOLID_SHARE = 0.01; -// Beyond this many pixels in the changed area, measuring would cost seconds -// and hundreds of megabytes in a worker; the comparator's verdict stands. export const MAX_MEASURED_PIXELS = 16_000_000; -/** The filter's verdict: true only for a measured change no stronger than noise. */ export function isRenderingNoise(measure: BlurredDifference | null, threshold: number): boolean { return measure !== null && measure.amplitude <= threshold && measure.solidShare <= MAX_SOLID_SHARE; } -// postMessage turns Buffers into Uint8Array views over their own ArrayBuffer. -function toBuffer(data: Buffer | Uint8Array): Buffer { - return Buffer.isBuffer(data) ? data : Buffer.from(data.buffer, data.byteOffset, data.byteLength); -} - export function computeBlurAmplitude(input: BlurAmplitudeJobInput): BlurAmplitudeJobOutput { - // The caller keeps the comparator's verdict when this has no answer, so a - // corrupt screenshot costs the filter, not the upload. - let baseline: PNG; - let image: PNG; - try { - baseline = PNG.sync.read(toBuffer(input.baseline)); - image = PNG.sync.read(toBuffer(input.image)); - } catch { - return { measure: null }; - } - if (baseline.width !== image.width || baseline.height !== image.height) { - return { measure: null }; - } - applyIgnoreAreas(baseline, input.ignoreAreas); - applyIgnoreAreas(image, input.ignoreAreas); - return { measure: measureBlurredDifference(baseline, image, input.sigma) }; + const pair = readSameSizePair(input.baseline, input.image, input.ignoreAreas); + const measure = pair && measureBlurredDifference(pair.baseline, pair.image); + const gate = + pair && input.gateIfNoise && isRenderingNoise(measure, input.gateIfNoise.threshold) + ? gateOfDecoded(pair.baseline, pair.image, DINOV2_PATCH) + : undefined; + return { measure, ...(gate ? { gate } : {}) }; } -// Same truncation as scipy.ndimage.gaussian_filter, which the research -// prototype used: the kernel reaches out four sigmas. function gaussianKernel(sigma: number): Float32Array { const radius = Math.floor(4 * sigma + 0.5); const kernel = new Float64Array(2 * radius + 1); @@ -115,17 +62,12 @@ function gaussianKernel(sigma: number): Float32Array { return Float32Array.from(kernel, (weight) => weight / sum); } -// Mirrors an index at the image border (scipy's 'reflect': -1 -> 0, n -> n-1), -// so a change in a corner is not diluted by an imaginary blank margin. function reflect(index: number, size: number): number { const period = 2 * size; const wrapped = ((index % period) + period) % period; return wrapped < size ? wrapped : period - 1 - wrapped; } -// A colour channel of a pixel over white, the way pixelmatch reads -// transparency: the ignore areas are blanked to transparent black and must look -// like nothing. function channel(data: Buffer | Uint8Array, offset: number, rgb: number): number { return 255 + (data[offset + rgb] - 255) * (data[offset + 3] / 255); } @@ -137,7 +79,6 @@ interface Box { maxY: number; } -// the bounding box of every pixel that differs at all, or null when none does function changedBox({ data: a, width, height }: RawImage, { data: b }: RawImage): Box | null { let minX = width; let minY = height; @@ -146,7 +87,6 @@ function changedBox({ data: a, width, height }: RawImage, { data: b }: RawImage) for (let y = 0; y < height; y++) { for (let x = 0; x < width; x++) { const o = (y * width + x) * 4; - // equal bytes are the overwhelming case and need no blending to compare if (a[o] === b[o] && a[o + 1] === b[o + 1] && a[o + 2] === b[o + 2] && a[o + 3] === b[o + 3]) { continue; } @@ -166,9 +106,6 @@ function changedBox({ data: a, width, height }: RawImage, { data: b }: RawImage) return maxX < 0 ? null : { minX, minY, maxX, maxY }; } -// For each position from `from - radius` to `to + radius`, the index into the -// box it reads once mirrored at the image border, or -1 outside the box (where -// the difference is zero). function sourceIndex(from: number, to: number, radius: number, size: number, boxMin: number, boxMax: number) { const index = new Int32Array(to - from + 1 + 2 * radius); for (let i = 0; i < index.length; i++) { @@ -178,14 +115,7 @@ function sourceIndex(from: number, to: number, radius: number, size: number, box return index; } -/** - * The strongest Gaussian-blurred difference between two images of equal size, - * per colour channel — so a change of hue at equal brightness is not mistaken - * for noise — and how many pixels the blur leaves intact. Blurring is linear, - * so the difference is blurred rather than each image, and only around the box - * where it is non-zero. Null when that box is too large to measure. - */ -export function measureBlurredDifference(baseline: RawImage, image: RawImage, sigma: number): BlurredDifference | null { +export function measureBlurredDifference(baseline: RawImage, image: RawImage): BlurredDifference | null { const { width, height } = baseline; if (image.width !== width || image.height !== height) { throw new Error(`Cannot blur-compare ${width}x${height} with ${image.width}x${image.height}`); @@ -201,9 +131,8 @@ export function measureBlurredDifference(baseline: RawImage, image: RawImage, si return null; } - const kernel = gaussianKernel(sigma); + const kernel = gaussianKernel(DEFAULT_BLUR_SIGMA); const radius = (kernel.length - 1) / 2; - // where the blurred difference can be non-zero const left = Math.max(0, minX - radius); const right = Math.min(width - 1, maxX + radius); const top = Math.max(0, minY - radius); @@ -212,7 +141,6 @@ export function measureBlurredDifference(baseline: RawImage, image: RawImage, si const columns = sourceIndex(left, right, radius, width, minX, maxX); const rowsOf = sourceIndex(top, bottom, radius, height, minY, maxY); - // shared by the three channels: one set of buffers, not three const difference = new Float32Array(boxWidth * boxHeight); const rows = new Float32Array(regionWidth * boxHeight); const solid = new Uint8Array(boxWidth * boxHeight); @@ -226,7 +154,6 @@ export function measureBlurredDifference(baseline: RawImage, image: RawImage, si } } - // horizontal pass, over the box's rows only: the others blur to zero for (let by = 0; by < boxHeight; by++) { const rowStart = by * boxWidth; for (let rx = 0; rx < regionWidth; rx++) { @@ -239,7 +166,6 @@ export function measureBlurredDifference(baseline: RawImage, image: RawImage, si } } - // vertical pass, keeping the extreme and marking what the blur left intact for (let ry = 0; ry <= bottom - top; ry++) { const y = top + ry; const insideBox = y >= minY && y <= maxY; diff --git a/src/compare/libs/pixelmatch/patch-gate.core.spec.ts b/src/compare/libs/pixelmatch/patch-gate.core.spec.ts index 306a5735..00712fcc 100644 --- a/src/compare/libs/pixelmatch/patch-gate.core.spec.ts +++ b/src/compare/libs/pixelmatch/patch-gate.core.spec.ts @@ -1,67 +1,43 @@ import { PNG } from 'pngjs'; -import { computePatchGate, PatchGateJobInput } from './patch-gate.core'; -import { runWorkerJob } from './worker-job'; +import { DINOV2_PATCH, gateOfDecoded } from './patch-gate.core'; +import { applyIgnoreAreas } from '../../utils'; +import { IgnoreAreaDto } from '../../../test-runs/dto/ignore-area.dto'; +import { paintedPng, pngBytes, solidPng, square } from '../../test-utils'; const WIDTH = 60; const HEIGHT = 40; -const png = (paint: (x: number, y: number) => [number, number, number] | undefined = () => undefined): Buffer => { - const image = new PNG({ width: WIDTH, height: HEIGHT }); - for (let y = 0; y < HEIGHT; y++) { - for (let x = 0; x < WIDTH; x++) { - const [r, g, b] = paint(x, y) ?? [255, 255, 255]; - image.data.set([r, g, b, 255], (y * WIDTH + x) * 4); - } - } - return PNG.sync.write(image); -}; - -const blank = png(); -// a solid block covering patch (1, 2) only: rows 14-27, columns 28-41 -const block = png((x, y) => (x >= 30 && x < 40 && y >= 16 && y < 26 ? [200, 40, 40] : undefined)); +const blank = pngBytes(solidPng(WIDTH, HEIGHT)); +const block = pngBytes(paintedPng(WIDTH, HEIGHT, square(30, 16, 10, [200, 40, 40]))); -const job = (overrides: Partial = {}): PatchGateJobInput => ({ - kind: 'patch-gate', - baseline: blank, - image: block, - ignoreAreas: [], - patch: 14, - ...overrides, -}); +const gateOf = (baseline: Buffer, image: Buffer, ignoreAreas: IgnoreAreaDto[] = []) => { + const decodedBaseline = PNG.sync.read(baseline); + const decodedImage = PNG.sync.read(image); + applyIgnoreAreas(decodedBaseline, ignoreAreas); + applyIgnoreAreas(decodedImage, ignoreAreas); + return gateOfDecoded(decodedBaseline, decodedImage, DINOV2_PATCH); +}; -describe('computePatchGate', () => { +describe('gateOfDecoded', () => { it('marks the patches where pixelmatch sees a change', () => { - expect(computePatchGate(job()).gate).toEqual({ width: 5, height: 3, cells: [1 * 5 + 2] }); + expect(gateOf(blank, block)).toEqual({ width: 5, height: 3, cells: [1 * 5 + 2] }); }); it('marks nothing for identical screenshots', () => { - expect(computePatchGate(job({ image: blank })).gate).toEqual({ width: 5, height: 3, cells: [] }); + expect(gateOf(blank, blank)).toEqual({ width: 5, height: 3, cells: [] }); }); - // what the project has masked out must not decide the verdict it('leaves out what the ignore areas cover', () => { - expect(computePatchGate(job({ ignoreAreas: [{ x: 28, y: 14, width: 14, height: 14 }] })).gate?.cells).toEqual([]); + expect(gateOf(blank, block, [{ x: 28, y: 14, width: 14, height: 14 }]).cells).toEqual([]); }); - it('has no answer for screenshots of different sizes', () => { - const taller = new PNG({ width: WIDTH, height: HEIGHT + 4 }); - taller.data.fill(255); - - expect(computePatchGate(job({ image: PNG.sync.write(taller) })).gate).toBeNull(); - }); - - it('has no answer for bytes that are not a PNG', () => { - expect(computePatchGate(job({ image: Buffer.from('not a png') })).gate).toBeNull(); - }); + it('gives a gate that survives a structured clone', () => { + const gate = gateOf(blank, block); - // what a Buffer arrives as on the far side of postMessage - it('reads the Uint8Array views a worker message turns Buffers into', () => { - expect( - computePatchGate(job({ baseline: new Uint8Array(blank), image: new Uint8Array(block) })).gate?.cells - ).toEqual([7]); + expect(structuredClone(gate)).toEqual(gate); }); - it('is what the worker runs for a patch-gate job', () => { - expect(runWorkerJob(job())).toEqual(computePatchGate(job())); + it('uses the 14 px patch of DINOv2', () => { + expect(DINOV2_PATCH).toBe(14); }); }); diff --git a/src/compare/libs/pixelmatch/patch-gate.core.ts b/src/compare/libs/pixelmatch/patch-gate.core.ts index 0f8e7160..9b7451f4 100644 --- a/src/compare/libs/pixelmatch/patch-gate.core.ts +++ b/src/compare/libs/pixelmatch/patch-gate.core.ts @@ -1,76 +1,32 @@ import { PNG } from 'pngjs'; import Pixelmatch from 'pixelmatch'; -import { IgnoreAreaDto } from '../../../test-runs/dto/ignore-area.dto'; -import { applyIgnoreAreas } from '../../utils'; +import { RawImage } from '../../utils'; -/** - * Where a DINOv2 look at two screenshots should count (see - * dinov2/dinov2.service.ts): the patches in which pixelmatch, with its - * anti-aliasing detector on, sees a change. Elsewhere a content distance is - * the model's context bleeding across a tile, not a change — measured in - * research/jepa-screenshot-diff, where this gate took DINOv2 from 14% to 73% - * of bugs caught. Runs on the diff worker pool; all input/output must stay - * structured-clone serializable. - */ -export interface PatchGateJobInput { - kind: 'patch-gate'; - baseline: Buffer | Uint8Array; - image: Buffer | Uint8Array; - ignoreAreas: IgnoreAreaDto[]; - // the model's patch size in pixels - patch: number; -} +export const DINOV2_PATCH = 14; export interface PatchGate { - // patches across and down, covering the screenshot width: number; height: number; - // indices (row * width + col) of the patches holding a change cells: number[]; } -export interface PatchGateJobOutput { - // null when there is nothing to line up: unreadable bytes or different sizes - gate: PatchGate | null; -} - -// postMessage turns Buffers into Uint8Array views over their own ArrayBuffer. -function toBuffer(data: Buffer | Uint8Array): Buffer { - return Buffer.isBuffer(data) ? data : Buffer.from(data.buffer, data.byteOffset, data.byteLength); -} - -export function computePatchGate(input: PatchGateJobInput): PatchGateJobOutput { - let baseline: PNG; - let image: PNG; - try { - baseline = PNG.sync.read(toBuffer(input.baseline)); - image = PNG.sync.read(toBuffer(input.image)); - } catch { - return { gate: null }; - } +export function gateOfDecoded(baseline: RawImage, image: RawImage, patch: number): PatchGate { const { width, height } = baseline; - if (image.width !== width || image.height !== height) { - return { gate: null }; - } - applyIgnoreAreas(baseline, input.ignoreAreas); - applyIgnoreAreas(image, input.ignoreAreas); - // the research's settings, whatever the project's: this asks where the - // content may have changed, not whether the run fails const mask = new PNG({ width, height }); Pixelmatch(baseline.data, image.data, mask.data, width, height, { threshold: 0.1, includeAA: false, diffMask: true }); - const columns = Math.ceil(width / input.patch); - const rows = Math.ceil(height / input.patch); + const columns = Math.ceil(width / patch); + const rows = Math.ceil(height / patch); const marked = new Uint8Array(columns * rows); for (let y = 0; y < height; y++) { for (let x = 0; x < width; x++) { if (mask.data[(y * width + x) * 4 + 3] > 0) { - marked[Math.floor(y / input.patch) * columns + Math.floor(x / input.patch)] = 1; + marked[Math.floor(y / patch) * columns + Math.floor(x / patch)] = 1; } } } const cells: number[] = []; marked.forEach((value, index) => value && cells.push(index)); - return { gate: { width: columns, height: rows, cells } }; + return { width: columns, height: rows, cells }; } diff --git a/src/compare/libs/pixelmatch/pixelmatch.core.boundaries.spec.ts b/src/compare/libs/pixelmatch/pixelmatch.core.real-images.spec.ts similarity index 63% rename from src/compare/libs/pixelmatch/pixelmatch.core.boundaries.spec.ts rename to src/compare/libs/pixelmatch/pixelmatch.core.real-images.spec.ts index 2abae522..b0ea534c 100644 --- a/src/compare/libs/pixelmatch/pixelmatch.core.boundaries.spec.ts +++ b/src/compare/libs/pixelmatch/pixelmatch.core.real-images.spec.ts @@ -1,17 +1,9 @@ import { PNG } from 'pngjs'; +import { paintedPng, pngBytes } from '../../test-utils'; import { computePixelmatchDiff, PixelmatchJobInput } from './pixelmatch.core'; -// white image with the first `redPixels` pixels (row by row) painted red -const png = (width: number, height: number, redPixels = 0): Buffer => { - const image = new PNG({ width, height }); - for (let i = 0; i < width * height; i++) { - image.data[i * 4] = 255; - image.data[i * 4 + 1] = i < redPixels ? 0 : 255; - image.data[i * 4 + 2] = i < redPixels ? 0 : 255; - image.data[i * 4 + 3] = 255; - } - return PNG.sync.write(image); -}; +const png = (width: number, height: number, redPixels = 0): Buffer => + pngBytes(paintedPng(width, height, (x, y) => (y * width + x < redPixels ? [255, 0, 0] : undefined))); const job = (baseline: Buffer, image: Buffer, overrides: Partial = {}) => computePixelmatchDiff({ @@ -20,7 +12,6 @@ const job = (baseline: Buffer, image: Buffer, overrides: Partial { - // 36 of 3600 pixels differ: exactly 1% const baseline = png(60, 60); const image = png(60, 60, 36); - it('reports the mismatch count and percent', () => { + it('reports the mismatch count and percent, and does not keep the diff when the percent equals the tolerance', () => { const result = job(baseline, image, { diffTolerancePercent: 1 }); expect(result.equal).toBe(false); expect(result.isSameDimension).toBe(true); expect(result.pixelMisMatchCount).toBe(36); expect(result.diffPercent).toBe(1); - }); - - it('does not keep the diff when the percent equals the tolerance', () => { - const result = job(baseline, image, { diffTolerancePercent: 1, withThumbnails: true }); - expect(result.diffBuffer).toBeUndefined(); expect(result.imageThumbnail).toBeUndefined(); expect(result.diffThumbnail).toBeUndefined(); }); it('keeps the diff and thumbnails when the percent is just over the tolerance', () => { - const result = job(baseline, image, { diffTolerancePercent: 0.99, withThumbnails: true }); + const result = job(baseline, image, { diffTolerancePercent: 0.99 }); expect(PNG.sync.read(Buffer.from(result.diffBuffer!)).width).toBe(60); expect(result.imageThumbnail).toBeDefined(); @@ -59,7 +44,7 @@ describe('computePixelmatchDiff tolerance boundary', () => { }); it('does not keep the diff over the tolerance when saving is off', () => { - const result = job(baseline, image, { diffTolerancePercent: 0, saveDiff: false, withThumbnails: true }); + const result = job(baseline, image, { diffTolerancePercent: 0, saveDiff: false }); expect(result.diffBuffer).toBeUndefined(); expect(result.imageThumbnail).toBeUndefined(); @@ -67,16 +52,13 @@ describe('computePixelmatchDiff tolerance boundary', () => { }); describe('computePixelmatchDiff dimensions', () => { - it('stops at a different height when different dimensions are not allowed', () => { - expect(job(png(60, 60), png(60, 30, 1))).toStrictEqual({ equal: false, isSameDimension: false }); - }); - - it('stops at a different width when different dimensions are not allowed', () => { - expect(job(png(60, 60), png(30, 60, 1))).toStrictEqual({ equal: false, isSameDimension: false }); + it.each([ + ['height', png(60, 30, 1)], + ['width', png(30, 60, 1)], + ])('stops at a different %s when different dimensions are not allowed', (_, image) => { + expect(job(png(60, 60), image)).toStrictEqual({ equal: false, isSameDimension: false }); }); - // 60x30 and 30x60 with the first 60 pixels red hold the very same bytes: - // one red row against two half-width red rows const wide = png(60, 30, 60); const tall = png(30, 60, 60); @@ -93,16 +75,13 @@ describe('computePixelmatchDiff dimensions', () => { }); it('compares on the larger canvas when different dimensions are allowed', () => { - // the checkpoint is padded with transparent pixels, which pixelmatch blends to white: - // only the baseline's red first row differs - const result = job(png(10, 10, 10), png(10, 5), { allowDiffDimensions: true, withSignature: true }); + const result = job(png(10, 10, 10), png(10, 5), { allowDiffDimensions: true }); expect(result.equal).toBe(false); expect(result.isSameDimension).toBe(false); expect(result.pixelMisMatchCount).toBe(10); expect(result.diffPercent).toBe(10); expect(PNG.sync.read(Buffer.from(result.diffBuffer!)).height).toBe(10); - // no signature for screenshots of different sizes expect(result.signature).toBeUndefined(); }); }); diff --git a/src/compare/libs/pixelmatch/pixelmatch.core.spec.ts b/src/compare/libs/pixelmatch/pixelmatch.core.spec.ts index c237a668..fb76e170 100644 --- a/src/compare/libs/pixelmatch/pixelmatch.core.spec.ts +++ b/src/compare/libs/pixelmatch/pixelmatch.core.spec.ts @@ -1,77 +1,55 @@ +import Pixelmatch from 'pixelmatch'; import { PNG } from 'pngjs'; -import { DEFAULT_BLUR_AMPLITUDE_THRESHOLD, DEFAULT_BLUR_SIGMA } from './blur-amplitude.core'; +import { DEFAULT_BLUR_AMPLITUDE_THRESHOLD } from './blur-amplitude.core'; +import { readSameSizePair } from '../../utils'; +import { DINOV2_PATCH, gateOfDecoded } from './patch-gate.core'; import { computePixelmatchDiff, PixelmatchJobInput } from './pixelmatch.core'; import { computeChangeSignature } from './signature.core'; import { THUMBNAIL_MAX_DIMENSION } from './thumbnail.core'; +import { paintedPng, pngBytes, Rgb, solidPng, square } from '../../test-utils'; const WIDTH = 60; const HEIGHT = 60; -const png = (paint: (set: (x: number, y: number, rgb: [number, number, number]) => void) => void): Buffer => { - const image = new PNG({ width: WIDTH, height: HEIGHT }); - for (let i = 0; i < WIDTH * HEIGHT; i++) { - image.data[i * 4] = 255; - image.data[i * 4 + 1] = 255; - image.data[i * 4 + 2] = 255; - image.data[i * 4 + 3] = 255; - } - paint((x, y, [r, g, b]) => { - const index = (y * WIDTH + x) * 4; - image.data[index] = r; - image.data[index + 1] = g; - image.data[index + 2] = b; - }); - return PNG.sync.write(image); -}; - -const block = (left: number, top: number, rgb: [number, number, number]): Buffer => - png((set) => { - for (let y = top; y < top + 12; y++) { - for (let x = left; x < left + 12; x++) { - set(x, y, rgb); - } - } - }); +const png = (paint?: (x: number, y: number) => Rgb | undefined): Buffer => pngBytes(paintedPng(WIDTH, HEIGHT, paint)); + +const block = (left: number, top: number, rgb: Rgb): Buffer => png(square(left, top, 12, rgb)); -const blank = png(() => undefined); -const RED: [number, number, number] = [255, 0, 0]; +const blank = png(); +const RED: Rgb = [255, 0, 0]; const diffJob = ( baseline: Buffer, image: Buffer, - withSignature: boolean, - withThumbnails = false, { saveDiff = true, diffTolerancePercent = 0, includeAA = false, + allowDiffDimensions = false, blurAmplitude = undefined as PixelmatchJobInput['blurAmplitude'], + ignoreAreas = [] as PixelmatchJobInput['ignoreAreas'], } = {} ) => computePixelmatchDiff({ kind: 'diff', baseline, image, - ignoreAreas: [], + ignoreAreas, threshold: 0.1, includeAA, - allowDiffDimensions: false, + allowDiffDimensions, diffTolerancePercent, saveDiff, - withSignature, - withThumbnails, blurAmplitude, }); describe('computePixelmatchDiff with a signature', () => { - // The whole point of computing it here is to reuse the decode the diff - // already paid for. If the fused version answered differently from the - // standalone one, stored signatures would stop matching computed ones and - // variations would silently stop grouping. + afterEach(() => jest.restoreAllMocks()); + it('gives the same signature the standalone job would', () => { const image = block(10, 10, RED); - const fused = diffJob(blank, image, true).signature; + const fused = diffJob(blank, image).signature; const standalone = computeChangeSignature({ kind: 'signature', baseline: blank, @@ -85,27 +63,42 @@ describe('computePixelmatchDiff with a signature', () => { expect(fused).not.toBeNull(); }); - it('leaves the signature out when it was not asked for', () => { - expect(diffJob(blank, block(10, 10, RED), false).signature).toBeUndefined(); + it('leaves the signature out when the diff is not kept', () => { + expect(diffJob(blank, block(10, 10, RED), { saveDiff: false }).signature).toBeUndefined(); }); it('has no signature when the screenshots are identical', () => { - expect(diffJob(blank, blank, true).signature).toBeUndefined(); + expect(diffJob(blank, blank).signature).toBeUndefined(); }); it('still reports the diff it was asked for', () => { - const result = diffJob(blank, block(10, 10, RED), true); + const result = diffJob(blank, block(10, 10, RED)); expect(result.equal).toBe(false); expect(result.pixelMisMatchCount).toBe(144); }); + + it('encodes the diff with the Sub filter without changing a pixel of it', () => { + const image = block(10, 10, RED); + const write = jest.spyOn(PNG.sync, 'write'); + const expected = new PNG({ width: WIDTH, height: HEIGHT }); + Pixelmatch(PNG.sync.read(blank).data, PNG.sync.read(image).data, expected.data, WIDTH, HEIGHT, { + includeAA: false, + threshold: 0.1, + }); + + const result = diffJob(blank, image); + const decoded = PNG.sync.read(Buffer.from(result.diffBuffer!)); + + expect(write).toHaveBeenCalledWith(expect.any(PNG), expect.objectContaining({ filterType: 1 })); + expect([decoded.width, decoded.height]).toStrictEqual([WIDTH, HEIGHT]); + expect(decoded.data.equals(expected.data)).toBe(true); + }); }); describe('computePixelmatchDiff with thumbnails', () => { - // Made here from the pixels the diff has already decoded and already drawn, - // so a thumbnail costs a resize rather than a second read of the screenshot. it('returns a small picture of both the checkpoint and the diff', () => { - const result = diffJob(blank, block(10, 10, RED), false, true); + const result = diffJob(blank, block(10, 10, RED)); const image = PNG.sync.read(Buffer.from(result.imageThumbnail!)); const diff = PNG.sync.read(Buffer.from(result.diffThumbnail!)); @@ -113,64 +106,36 @@ describe('computePixelmatchDiff with thumbnails', () => { expect(Math.max(diff.width, diff.height)).toBeLessThanOrEqual(THUMBNAIL_MAX_DIMENSION); }); - it('makes none when they were not asked for', () => { - const result = diffJob(blank, block(10, 10, RED), false, false); - - expect(result.imageThumbnail).toBeUndefined(); - expect(result.diffThumbnail).toBeUndefined(); - }); - - // A build is mostly runs that pass. Resizing two images for each of those, - // when the diff they belong to is thrown away seconds later, is work nobody - // ever sees the result of. it('makes none when the change is under tolerance', () => { - const result = diffJob(blank, block(10, 10, RED), false, true, { diffTolerancePercent: 90 }); + const result = diffJob(blank, block(10, 10, RED), { diffTolerancePercent: 90 }); expect(result.imageThumbnail).toBeUndefined(); expect(result.diffThumbnail).toBeUndefined(); }); it('makes none when the caller is not keeping the diff', () => { - const result = diffJob(blank, block(10, 10, RED), false, true, { saveDiff: false }); + const result = diffJob(blank, block(10, 10, RED), { saveDiff: false }); expect(result.imageThumbnail).toBeUndefined(); }); - // nothing to show and nothing to shrink it('makes none when the screenshots are identical', () => { - expect(diffJob(blank, blank, false, true).imageThumbnail).toBeUndefined(); + expect(diffJob(blank, blank).imageThumbnail).toBeUndefined(); }); }); describe('computePixelmatchDiff with the blur-amplitude filter', () => { - const BLACK: [number, number, number] = [0, 0, 0]; - const hairline = (column: number): Buffer => - png((set) => { - for (let y = 0; y < HEIGHT; y++) { - set(column, y, BLACK); - } - }); - const filter = { sigma: DEFAULT_BLUR_SIGMA, threshold: DEFAULT_BLUR_AMPLITUDE_THRESHOLD }; - // AA counted, as the project default `ignoreAntialiasing: true` actually - // configures pixelmatch: the moved hairline is then a real mismatch for it - const withFilter = (baseline: Buffer, image: Buffer, options: Parameters[4] = {}) => - diffJob(baseline, image, false, true, { includeAA: true, blurAmplitude: filter, ...options }); - - // an edge rasterised one pixel further over: pixelmatch sees two columns of - // changed pixels, a person sees the same line - it('marks a change no stronger than rendering noise', () => { + const hairline = (column: number): Buffer => png((x) => (x === column ? [0, 0, 0] : undefined)); + const filter = { threshold: DEFAULT_BLUR_AMPLITUDE_THRESHOLD }; + const withFilter = (baseline: Buffer, image: Buffer, options: Parameters[2] = {}) => + diffJob(baseline, image, { includeAA: true, blurAmplitude: filter, ...options }); + + it('marks a change no stronger than rendering noise, and produces no diff for it', () => { const result = withFilter(hairline(10), hairline(11)); expect(result.pixelMisMatchCount).toBeGreaterThan(0); expect(result.blurAmplitude).toBeLessThan(DEFAULT_BLUR_AMPLITUDE_THRESHOLD); expect(result.renderingNoiseOnly).toBe(true); - }); - - // the run is about to pass, so a diff and thumbnails for it would be thrown - // away unseen - it('produces no diff for a change it passes', () => { - const result = withFilter(hairline(10), hairline(11)); - expect(result.diffBuffer).toBeUndefined(); expect(result.imageThumbnail).toBeUndefined(); expect(result.diffThumbnail).toBeUndefined(); @@ -185,50 +150,23 @@ describe('computePixelmatchDiff with the blur-amplitude filter', () => { expect(result.imageThumbnail).toBeDefined(); }); - // faint, so under the threshold, but a whole panel of it: not noise it('lets a faint but solid change through', () => { - const panel = png((set) => { - for (let y = 0; y < HEIGHT; y++) { - for (let x = 0; x < WIDTH; x++) { - set(x, y, [224, 224, 224]); - } - } - }); - - const result = withFilter(blank, panel); + const result = withFilter(blank, pngBytes(solidPng(WIDTH, HEIGHT, [224, 224, 224]))); expect(result.blurAmplitude).toBeLessThan(DEFAULT_BLUR_AMPLITUDE_THRESHOLD); expect(result.renderingNoiseOnly).toBe(false); }); - // as CompareService does for the other comparators: a size change is a - // finding of its own, whatever the padded pixels look like it('does not judge screenshots of different sizes', () => { - const taller = (() => { - const image = new PNG({ width: WIDTH, height: HEIGHT + 5 }); - image.data.fill(255); - return PNG.sync.write(image); - })(); - - const result = computePixelmatchDiff({ - kind: 'diff', - baseline: blank, - image: taller, - ignoreAreas: [], - threshold: 0.1, - includeAA: true, - allowDiffDimensions: true, - diffTolerancePercent: 0, - saveDiff: true, - blurAmplitude: filter, - }); + const taller = pngBytes(solidPng(WIDTH, HEIGHT + 5)); + + const result = withFilter(blank, taller, { allowDiffDimensions: true }); expect(result.isSameDimension).toBe(false); expect(result.renderingNoiseOnly).toBeUndefined(); expect(result.blurAmplitude).toBeUndefined(); }); - // a second check may yet send it to review, and then it needs its diff it('keeps the diff of a noise-like change when a DINOv2 check will follow', () => { const result = withFilter(hairline(10), hairline(11), { blurAmplitude: { ...filter, dinov2: { threshold: 0.15 } }, @@ -239,15 +177,57 @@ describe('computePixelmatchDiff with the blur-amplitude filter', () => { expect(result.imageThumbnail).toBeDefined(); }); - it('measures nothing when the filter is off', () => { - const result = diffJob(hairline(10), hairline(11), false, false, { includeAA: true }); + describe('with the DINOv2 check', () => { + const checked = { ...filter, dinov2: { threshold: 0.15 } }; + const gated = (baseline: Buffer, image: Buffer, options: Parameters[2] = {}) => + withFilter(baseline, image, { blurAmplitude: checked, ...options }); + const gateOf = (baseline: Buffer, image: Buffer, ignoreAreas: PixelmatchJobInput['ignoreAreas']) => { + const pair = readSameSizePair(baseline, image, ignoreAreas)!; + return gateOfDecoded(pair.baseline, pair.image, DINOV2_PATCH); + }; + + it('computes the DINOv2 patch gate of a noise-like change', () => { + const ignoreAreas = [{ x: 40, y: 40, width: 5, height: 5 }]; + const result = gated(hairline(10), hairline(11), { ignoreAreas }); + + expect(result.renderingNoiseOnly).toBe(true); + expect(result.gate).toEqual(gateOf(hairline(10), hairline(11), ignoreAreas)); + expect(result.gate?.cells.length).toBeGreaterThan(0); + }); + + it('leaves out what the ignore areas cover', () => { + const ignoreAreas = [{ x: 0, y: 0, width: 30, height: 28 }]; + const result = gated(hairline(10), hairline(11), { ignoreAreas }); + + expect(result.gate?.cells.length).toBeGreaterThan(0); + expect(result.gate?.cells.every((cell) => Math.floor(cell / 5) >= 2)).toBe(true); + expect(result.gate).toEqual(gateOf(hairline(10), hairline(11), ignoreAreas)); + }); + + it('computes no gate for a change that is not noise', () => { + const result = gated(blank, block(10, 10, RED)); + + expect(result.renderingNoiseOnly).toBe(false); + expect(result.gate).toBeUndefined(); + }); + + it('computes no gate when the check is off', () => { + const result = withFilter(hairline(10), hairline(11), { blurAmplitude: filter }); + + expect(result.renderingNoiseOnly).toBe(true); + expect(result.gate).toBeUndefined(); + }); + }); + + it('measures nothing and computes no gate when the filter is off', () => { + const result = diffJob(hairline(10), hairline(11), { includeAA: true }); + expect(result.gate).toBeUndefined(); expect(result.blurAmplitude).toBeUndefined(); expect(result.renderingNoiseOnly).toBeUndefined(); expect(result.diffBuffer).toBeDefined(); }); - // a run under tolerance passes anyway; blurring it would be work for nothing it('measures nothing when the change is under tolerance', () => { const result = withFilter(hairline(10), hairline(11), { diffTolerancePercent: 90 }); diff --git a/src/compare/libs/pixelmatch/pixelmatch.core.ts b/src/compare/libs/pixelmatch/pixelmatch.core.ts index 5cf732e2..6d7be66c 100644 --- a/src/compare/libs/pixelmatch/pixelmatch.core.ts +++ b/src/compare/libs/pixelmatch/pixelmatch.core.ts @@ -1,23 +1,17 @@ import { PNG } from 'pngjs'; import Pixelmatch from 'pixelmatch'; import { IgnoreAreaDto } from '../../../test-runs/dto/ignore-area.dto'; -import { applyIgnoreAreas, scaleImageToSize } from '../../utils'; +import { applyIgnoreAreas, readPng, scaleImageToSize } from '../../utils'; import { BlurAmplitudeOptions, BlurredDifference, isRenderingNoise, measureBlurredDifference, } from './blur-amplitude.core'; +import { DINOV2_PATCH, gateOfDecoded, PatchGate } from './patch-gate.core'; import { signatureOfDecoded } from './signature.core'; import { encodeThumbnail } from './thumbnail.core'; -/** - * CPU-bound part of the pixelmatch comparison, extracted so it can run inside - * a worker thread (see pixelmatch.worker.ts / DiffWorkerPool): PNG decode, - * scaling, pixelmatch and diff encode of full-size screenshots block the event - * loop for ~a second each, which makes the whole API unresponsive during - * build ingestion. All input/output must stay structured-clone serializable. - */ export interface PixelmatchJobInput { kind: 'diff'; baseline: Buffer | Uint8Array; @@ -28,23 +22,6 @@ export interface PixelmatchJobInput { allowDiffDimensions?: boolean; diffTolerancePercent: number; saveDiff: boolean; - /** - * Also produce the change signature the variations dialog matches on. Free - * here next to the diff — the images are decoded already — and computed once - * at ingest so reviewing never has to fetch and decode a build's screenshots - * again just to group them. - */ - withSignature?: boolean; - /** - * Also return small PNGs of the checkpoint and of the diff. The grids draw - * these a hundred-odd pixels wide; sending the full-size files and letting - * CSS shrink them is what made opening the variations dialog cost megabytes. - */ - withThumbnails?: boolean; - /** - * Prototype: pass a change that is over tolerance but, once blurred, no - * stronger than rendering noise (see blur-amplitude.core.ts). Off when absent. - */ blurAmplitude?: BlurAmplitudeOptions; } @@ -54,59 +31,37 @@ export interface PixelmatchJobOutput { pixelMisMatchCount?: number; diffPercent?: number; diffBuffer?: Buffer | Uint8Array; - // absent when not asked for, when the screenshots are identical, or when - // their dimensions differ and there is nothing meaningful to sign signature?: number[]; - // absent when not asked for, or when the screenshots matched and there is - // nothing to show imageThumbnail?: Buffer | Uint8Array; diffThumbnail?: Buffer | Uint8Array; - // absent unless the blur-amplitude filter was asked for and the change was - // over tolerance between screenshots of one size — the only case where its - // verdict could matter; the two numbers are absent when it could not measure blurAmplitude?: number; solidPixels?: number; renderingNoiseOnly?: boolean; -} - -// postMessage turns Buffers into Uint8Array views over their own ArrayBuffer. -function toBuffer(data: Buffer | Uint8Array): Buffer { - return Buffer.isBuffer(data) ? data : Buffer.from(data.buffer, data.byteOffset, data.byteLength); + gate?: PatchGate; } export function computePixelmatchDiff(input: PixelmatchJobInput): PixelmatchJobOutput { - const baseline = PNG.sync.read(toBuffer(input.baseline)); - const image = PNG.sync.read(toBuffer(input.image)); + const baseline = readPng(input.baseline); + const image = readPng(input.image); const isSameDimension = baseline.width === image.width && baseline.height === image.height; if (!isSameDimension && !input.allowDiffDimensions) { return { equal: false, isSameDimension }; } - // Identical bytes only mean identical screenshots at the same size: a 60x30 - // and a 30x60 image of the same pixels decode to the very same buffer. - if ( - isSameDimension && - baseline.data.equals(new Uint8Array(image.data.buffer, image.data.byteOffset, image.data.byteLength)) - ) { + if (isSameDimension && baseline.data.equals(image.data)) { return { equal: true }; } - // scale image to max size const maxWidth = Math.max(baseline.width, image.width); const maxHeight = Math.max(baseline.height, image.height); const scaledBaseline = scaleImageToSize(baseline, maxWidth, maxHeight); const scaledImage = scaleImageToSize(image, maxWidth, maxHeight); - // apply ignore areas const baselineIgnored = applyIgnoreAreas(scaledBaseline, input.ignoreAreas); const imageIgnored = applyIgnoreAreas(scaledImage, input.ignoreAreas); - // compare - const diff = new PNG({ - width: maxWidth, - height: maxHeight, - }); + const diff = new PNG({ width: maxWidth, height: maxHeight }); const pixelMisMatchCount = Pixelmatch(baselineIgnored.data, imageIgnored.data, diff.data, maxWidth, maxHeight, { includeAA: input.includeAA, threshold: input.threshold, @@ -114,42 +69,33 @@ export function computePixelmatchDiff(input: PixelmatchJobInput): PixelmatchJobO const diffPercent = (pixelMisMatchCount * 100) / (scaledImage.width * scaledImage.height); const overTolerance = diffPercent > input.diffTolerancePercent; - // A size change is a finding of its own, as CompareService treats it for the - // other comparators: the filter only judges screenshots of one size. const blurAmplitude = overTolerance && isSameDimension ? input.blurAmplitude : undefined; let measure: BlurredDifference | null = null; - let renderingNoiseOnly: boolean | undefined; + let renderingNoiseOnly = false; if (blurAmplitude) { - measure = measureBlurredDifference(baselineIgnored, imageIgnored, blurAmplitude.sigma); + measure = measureBlurredDifference(baselineIgnored, imageIgnored); renderingNoiseOnly = isRenderingNoise(measure, blurAmplitude.threshold); } - // A build is mostly runs that pass, so anything produced for a diff that is - // about to be thrown away is work nobody ever sees the result of. - // ...unless a DINOv2 check is still to come, which may send the run to - // review after all — and then it needs its diff. - const keepingTheDiff = overTolerance && input.saveDiff && (!renderingNoiseOnly || !!input.blurAmplitude?.dinov2); + const gate = + renderingNoiseOnly && blurAmplitude?.dinov2 + ? gateOfDecoded(baselineIgnored, imageIgnored, DINOV2_PATCH) + : undefined; - let diffBuffer: Buffer | undefined; - if (keepingTheDiff) { - diffBuffer = PNG.sync.write(diff); - } + const keepingTheDiff = overTolerance && input.saveDiff && (!renderingNoiseOnly || !!blurAmplitude?.dinov2); + const diffBuffer = keepingTheDiff ? PNG.sync.write(diff, { filterType: 1 }) : undefined; - // Same source images, same ignore areas, same threshold as the standalone - // signature job — sharing signatureOfDecoded is what keeps the two answers - // identical, so a stored signature still matches one computed on the fly. const signature = - input.withSignature && isSameDimension + input.saveDiff && isSameDimension ? signatureOfDecoded(baselineIgnored, imageIgnored, { threshold: input.threshold, includeAA: input.includeAA, }) : null; - const thumbnails = - input.withThumbnails && keepingTheDiff - ? { imageThumbnail: encodeThumbnail(imageIgnored), diffThumbnail: encodeThumbnail(diff) } - : {}; + const thumbnails = keepingTheDiff + ? { imageThumbnail: encodeThumbnail(imageIgnored), diffThumbnail: encodeThumbnail(diff) } + : {}; return { equal: false, @@ -159,6 +105,7 @@ export function computePixelmatchDiff(input: PixelmatchJobInput): PixelmatchJobO diffBuffer, ...(signature ? { signature } : {}), ...thumbnails, + ...(gate ? { gate } : {}), ...(blurAmplitude ? { renderingNoiseOnly, diff --git a/src/compare/libs/pixelmatch/pixelmatch.service.boundaries.spec.ts b/src/compare/libs/pixelmatch/pixelmatch.service.real-images.spec.ts similarity index 100% rename from src/compare/libs/pixelmatch/pixelmatch.service.boundaries.spec.ts rename to src/compare/libs/pixelmatch/pixelmatch.service.real-images.spec.ts diff --git a/src/compare/libs/pixelmatch/pixelmatch.service.spec.ts b/src/compare/libs/pixelmatch/pixelmatch.service.spec.ts index b820a721..72c0c682 100644 --- a/src/compare/libs/pixelmatch/pixelmatch.service.spec.ts +++ b/src/compare/libs/pixelmatch/pixelmatch.service.spec.ts @@ -6,39 +6,71 @@ import { mocked } from 'jest-mock'; import { StaticService } from '../../../static/static.service'; import { DiffWorkerPool } from '../../diff-worker-pool'; import { DIFF_DIMENSION_RESULT, EQUAL_RESULT, NO_BASELINE_RESULT } from '../consts'; +import { ImageCompareInput } from '../ImageCompareInput'; import { DEFAULT_CONFIG, PixelmatchService } from './pixelmatch.service'; import { PixelmatchConfig } from './pixelmatch.types'; +import { computePixelmatchDiff } from './pixelmatch.core'; import { Dinov2Service } from '../dinov2/dinov2.service'; jest.mock('pixelmatch'); -const toBuffer = (png: PNG | undefined): Buffer | undefined => (png ? PNG.sync.write(png) : undefined); - -const initService = async ({ - getImageBufferMock = jest.fn(), - saveImageMock = jest.fn(), - deleteImageMock = jest.fn(), -}) => { +const pngOf = (width: number, height: number) => PNG.sync.write(new PNG({ width, height })); +const toBuffer = (png: PNG): Buffer => PNG.sync.write(png); +const sequence = (...results: unknown[]) => results.reduce((m, r) => m.mockResolvedValueOnce(r), jest.fn()); +const pngReader = () => jest.fn().mockResolvedValue(Buffer.from('png')); +const poolOf = (output: Record) => ({ run: jest.fn().mockResolvedValue(output) }); + +interface Deps { + getImageBuffer?: jest.Mock; + saveImage?: jest.Mock; + deleteImage?: jest.Mock; + pool?: { run: unknown }; + confirmsNoiseForGate?: jest.Mock; +} + +const setup = async ({ + getImageBuffer = jest.fn(), + saveImage = jest.fn(), + deleteImage = jest.fn(), + pool, + confirmsNoiseForGate = jest.fn(), +}: Deps = {}) => { const module: TestingModule = await Test.createTestingModule({ providers: [ PixelmatchService, - DiffWorkerPool, - { provide: Dinov2Service, useValue: { confirmsNoise: jest.fn() } }, - { - provide: StaticService, - useValue: { - getImageBuffer: getImageBufferMock, - saveImage: saveImageMock, - deleteImage: deleteImageMock, - }, - }, + pool ? { provide: DiffWorkerPool, useValue: pool } : DiffWorkerPool, + { provide: Dinov2Service, useValue: { confirmsNoiseForGate } }, + { provide: StaticService, useValue: { getImageBuffer, saveImage, deleteImage } }, ], }).compile(); - - return module.get(PixelmatchService); + return { + service: module.get(PixelmatchService), + run: pool?.run as jest.Mock, + getImageBuffer, + saveImage, + deleteImage, + confirmsNoiseForGate, + }; }; -let service: PixelmatchService; +const compare = ( + service: PixelmatchService, + data: Partial = {}, + config: PixelmatchConfig = DEFAULT_CONFIG +) => + service.getDiff( + { + baseline: 'baseline.png', + image: 'image.png', + ignoreAreas: [], + diffTollerancePercent: 0, + saveDiffAsFile: false, + ...data, + }, + config + ); + +const mismatch = { equal: false, isSameDimension: true, pixelMisMatchCount: 10, diffPercent: 5 }; describe('parseConfig', () => { it.each<[string, PixelmatchConfig]>([ @@ -50,150 +82,65 @@ describe('parseConfig', () => { ['invalid', DEFAULT_CONFIG], ['null', DEFAULT_CONFIG], ])('should parse config', async (json, expected) => { - service = await initService({}); - - const config = service.parseConfig(json); + const { service } = await setup(); - expect(config).toStrictEqual(expected); + expect(service.parseConfig(json)).toStrictEqual(expected); }); }); describe('getDiff', () => { - const image = new PNG({ - width: 20, - height: 20, - }); + const image = new PNG({ width: 20, height: 20 }); + const flagged = (baseline: number | undefined = undefined) => { + const png = new PNG({ width: 100, height: 100 }); + if (baseline !== undefined) png.data[0] = baseline; + return toBuffer(png); + }; it('no baseline', async () => { - const getImageBufferMock = jest.fn().mockReturnValueOnce(undefined).mockReturnValueOnce(toBuffer(image)); - service = await initService({ getImageBufferMock }); - - const result = await service.getDiff( - { - baseline: null, - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: true, - }, - DEFAULT_CONFIG - ); + const { service } = await setup({ getImageBuffer: sequence(undefined, toBuffer(image)) }); - expect(result).toStrictEqual(NO_BASELINE_RESULT); + expect(await compare(service, { baseline: null, saveDiffAsFile: true })).toStrictEqual(NO_BASELINE_RESULT); }); it('throws when the image cannot be read', async () => { - const getImageBufferMock = jest.fn().mockResolvedValueOnce(toBuffer(image)).mockResolvedValueOnce(undefined); - service = await initService({ getImageBufferMock }); - - await expect( - service.getDiff( - { - baseline: 'baseline', - image: 'missing.png', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: true, - }, - DEFAULT_CONFIG - ) - ).rejects.toThrow('Cannot get image: missing.png'); - expect(getImageBufferMock).toHaveBeenNthCalledWith(2, 'missing.png'); + const { service, getImageBuffer } = await setup({ getImageBuffer: sequence(toBuffer(image), undefined) }); + + await expect(compare(service, { image: 'missing.png', saveDiffAsFile: true })).rejects.toThrow( + 'Cannot get image: missing.png' + ); + expect(getImageBuffer).toHaveBeenNthCalledWith(2, 'missing.png'); }); it('diff not found', async () => { - const getImageBufferMock = jest.fn().mockReturnValueOnce(toBuffer(image)).mockReturnValueOnce(toBuffer(image)); - service = await initService({ getImageBufferMock }); - - const result = await service.getDiff( - { - baseline: 'image', - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: true, - }, - DEFAULT_CONFIG - ); + const { service } = await setup({ getImageBuffer: sequence(toBuffer(image), toBuffer(image)) }); - expect(result).toStrictEqual(EQUAL_RESULT); + expect(await compare(service, { saveDiffAsFile: true })).toStrictEqual(EQUAL_RESULT); }); it('diff image dimensions mismatch', async () => { - const baseline = new PNG({ - width: 10, - height: 10, - }); - const getImageBufferMock = jest.fn().mockReturnValueOnce(toBuffer(baseline)).mockReturnValueOnce(toBuffer(image)); - service = await initService({ getImageBufferMock }); - - const result = await service.getDiff( - { - baseline: 'image', - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: true, - }, - DEFAULT_CONFIG - ); + const { service } = await setup({ getImageBuffer: sequence(pngOf(10, 10), toBuffer(image)) }); - expect(result).toStrictEqual(DIFF_DIMENSION_RESULT); + expect(await compare(service, { saveDiffAsFile: true })).toStrictEqual(DIFF_DIMENSION_RESULT); }); it('diff image dimensions mismatch ALLOWED', async () => { - const baseline = new PNG({ - width: 1, - height: 5, - }); - const image = new PNG({ - width: 2, - height: 4, - }); - const getImageBufferMock = jest.fn().mockReturnValueOnce(toBuffer(baseline)).mockReturnValueOnce(toBuffer(image)); const diffName = 'diff name'; - const saveImageMock = jest.fn().mockReturnValueOnce(diffName); mocked(Pixelmatch).mockReturnValueOnce(5); - service = await initService({ saveImageMock, getImageBufferMock }); - - const result = await service.getDiff( - { - baseline: 'image', - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: true, - }, - { - allowDiffDimensions: true, - ignoreAntialiasing: true, - threshold: 0.1, - } - ); + const { service, saveImage } = await setup({ + getImageBuffer: sequence(pngOf(1, 5), pngOf(2, 4)), + saveImage: sequence(diffName), + }); - expect(mocked(Pixelmatch)).toHaveBeenCalledWith( - new PNG({ - width: 2, - height: 5, - }).data, - new PNG({ - width: 2, - height: 5, - }).data, - new PNG({ - width: 2, - height: 5, - }).data, - 2, - 5, - { - includeAA: true, - threshold: 0.1, - } + const result = await compare( + service, + { diffTollerancePercent: 0.1, saveDiffAsFile: true }, + { allowDiffDimensions: true, ignoreAntialiasing: true, threshold: 0.1 } ); - // the full-size diff first, then the two thumbnails made beside it - expect(saveImageMock.mock.calls[0][0]).toBe('diff'); - expect(saveImageMock).toHaveBeenCalledTimes(3); + + const padded = new PNG({ width: 2, height: 5 }).data; + expect(mocked(Pixelmatch)).toHaveBeenCalledWith(padded, padded, padded, 2, 5, { includeAA: true, threshold: 0.1 }); + expect(saveImage.mock.calls[0][0]).toBe('diff'); + expect(saveImage).toHaveBeenCalledTimes(3); expect(result).toStrictEqual({ status: TestStatus.unresolved, diffName, @@ -203,141 +150,150 @@ describe('getDiff', () => { }); }); - it('diff found < tollerance', async () => { - const baseline = new PNG({ - width: 100, - height: 100, - }); - baseline.data[0] = 1; - const image = new PNG({ - width: 100, - height: 100, - }); - const getImageBufferMock = jest.fn().mockReturnValueOnce(toBuffer(baseline)).mockReturnValueOnce(toBuffer(image)); - const saveImageMock = jest.fn(); - service = await initService({ saveImageMock, getImageBufferMock }); - const pixelMisMatchCount = 150; - mocked(Pixelmatch).mockReturnValueOnce(pixelMisMatchCount); - - const result = await service.getDiff( - { - baseline: 'image', - image: 'image', - diffTollerancePercent: 2, - ignoreAreas: [], - saveDiffAsFile: false, - }, - DEFAULT_CONFIG - ); + it.each([ + ['below', 2, false, 150, 1.5, TestStatus.ok, null, 0], + ['above', 0.5, true, 200, 2, TestStatus.unresolved, 'diff name', 3], + ])( + 'diff found %s tollerance', + async (_, diffTollerancePercent, saveDiffAsFile, pixelMisMatchCount, diffPercent, status, diffName, saves) => { + mocked(Pixelmatch).mockReturnValueOnce(pixelMisMatchCount); + const { service, saveImage } = await setup({ + getImageBuffer: sequence(flagged(1), flagged()), + saveImage: sequence('diff name'), + }); + + const result = await compare(service, { diffTollerancePercent, saveDiffAsFile }); + + if (saves) expect(saveImage.mock.calls[0][0]).toBe('diff'); + expect(saveImage).toHaveBeenCalledTimes(saves); + expect(result).toStrictEqual({ status, diffName, pixelMisMatchCount, diffPercent, isSameDimension: true }); + } + ); +}); - expect(saveImageMock).toHaveBeenCalledTimes(0); - expect(result).toStrictEqual({ - status: TestStatus.ok, - diffName: null, - pixelMisMatchCount, - diffPercent: 1.5, - isSameDimension: true, - }); +describe('byte-equal fast path', () => { + const valid = pngOf(4, 4); + + it('answers EQUAL for byte-equal well-formed PNGs without queueing a job', async () => { + const pool = { run: jest.fn() }; + const { service } = await setup({ pool, getImageBuffer: sequence(Buffer.from(valid), Buffer.from(valid)) }); + + await expect(compare(service, { saveDiffAsFile: true })).resolves.toStrictEqual(EQUAL_RESULT); + expect(pool.run).not.toHaveBeenCalled(); }); - it('diff found > tollerance', async () => { - const baseline = new PNG({ - width: 100, - height: 100, - }); - baseline.data[0] = 1; - const image = new PNG({ - width: 100, - height: 100, - }); - const getImageBufferMock = jest.fn().mockReturnValueOnce(toBuffer(baseline)).mockReturnValueOnce(toBuffer(image)); - const pixelMisMatchCount = 200; - mocked(Pixelmatch).mockReturnValueOnce(pixelMisMatchCount); - const diffName = 'diff name'; - const saveImageMock = jest.fn().mockReturnValueOnce(diffName); - service = await initService({ - saveImageMock, - getImageBufferMock, + it('still sends a byte-equal pair to the worker when the PNG is not well formed', async () => { + const truncated = valid.subarray(0, valid.length - 20); + const pool = new DiffWorkerPool(); + const run = jest.spyOn(pool, 'run'); + const { service } = await setup({ pool, getImageBuffer: sequence(Buffer.from(truncated), Buffer.from(truncated)) }); + + await expect(compare(service, { saveDiffAsFile: true })).rejects.toThrow(); + expect(run).toHaveBeenCalledTimes(1); + }); + + it('sends different buffers to the worker', async () => { + const other = new PNG({ width: 4, height: 4 }); + other.data.fill(255); + const pool = poolOf({ equal: true }); + const { service } = await setup({ pool, getImageBuffer: sequence(Buffer.from(valid), toBuffer(other)) }); + + await compare(service, { saveDiffAsFile: true }); + + expect(pool.run).toHaveBeenCalledTimes(1); + }); +}); + +describe('image already in memory', () => { + it('uses the image it is given and reads only the baseline', async () => { + const pool = poolOf({ equal: true }); + const { service, getImageBuffer } = await setup({ + pool, + getImageBuffer: jest.fn().mockResolvedValue(Buffer.from('baseline bytes')), }); + const imageBuffer = Buffer.from('image bytes'); - const result = await service.getDiff( - { - baseline: 'image', - image: 'image', - diffTollerancePercent: 0.5, - ignoreAreas: [], - saveDiffAsFile: true, - }, - DEFAULT_CONFIG - ); + await compare(service, { imageBuffer }); - // the full-size diff first, then the two thumbnails made beside it - expect(saveImageMock.mock.calls[0][0]).toBe('diff'); - expect(saveImageMock).toHaveBeenCalledTimes(3); - expect(result).toStrictEqual({ - status: TestStatus.unresolved, - diffName, - pixelMisMatchCount, - diffPercent: 2, - isSameDimension: true, + expect(getImageBuffer).toHaveBeenCalledTimes(1); + expect(getImageBuffer).toHaveBeenCalledWith('baseline.png'); + expect(pool.run.mock.calls[0][0].image).toBe(imageBuffer); + }); + + it('reads the image itself when none is given and fails as before when it is missing', async () => { + const { service, getImageBuffer } = await setup({ + pool: { run: jest.fn() }, + getImageBuffer: sequence(Buffer.from('baseline bytes'), undefined), }); + + await expect(compare(service)).rejects.toThrow('Cannot get image: image.png'); + expect(getImageBuffer).toHaveBeenNthCalledWith(2, 'image.png'); }); }); -describe('change signature', () => { - // Correctness of the signature itself is pinned in pixelmatch.core.spec - // against the real pixelmatch; Pixelmatch is mocked in this file, so what is - // worth testing here is only the plumbing — that the service asks for one and - // carries what comes back into the result. - const initWithPool = async (output: Record) => { - const run = jest.fn().mockResolvedValue(output); - const module: TestingModule = await Test.createTestingModule({ - providers: [ - PixelmatchService, - { provide: DiffWorkerPool, useValue: { run } }, - { provide: Dinov2Service, useValue: { confirmsNoise: jest.fn() } }, - { - provide: StaticService, - useValue: { - getImageBuffer: jest.fn().mockResolvedValue(Buffer.from('png')), - saveImage: jest.fn(), - deleteImage: jest.fn(), - }, - }, - ], - }).compile(); - return { service: module.get(PixelmatchService), run }; - }; +describe('screenshots of another size', () => { + const setupSizes = (baseline: Buffer, image: Buffer, run: jest.Mock) => + setup({ pool: { run }, getImageBuffer: sequence(baseline, image) }); - const compare = (service: PixelmatchService, saveDiffAsFile = false) => - service.getDiff( - { - baseline: 'baseline.png', - image: 'image.png', - ignoreAreas: [], - diffTollerancePercent: 0, - saveDiffAsFile, - }, - DEFAULT_CONFIG - ); + it('answers different sizes from the headers without the worker', async () => { + const run = jest.fn(); + const { service } = await setupSizes(pngOf(2, 3), pngOf(3, 2), run); + + expect(await compare(service)).toBe(DIFF_DIMENSION_RESULT); + expect(run).not.toHaveBeenCalled(); + }); + + it('still sends different sizes to the worker when they are allowed', async () => { + const run = jest + .fn() + .mockResolvedValue({ equal: false, isSameDimension: false, pixelMisMatchCount: 1, diffPercent: 1 }); + const { service } = await setupSizes(pngOf(2, 3), pngOf(3, 2), run); + + await compare(service, {}, { ...DEFAULT_CONFIG, allowDiffDimensions: true }); + + expect(run).toHaveBeenCalledTimes(1); + }); + + it('sends buffers without a readable header to the worker', async () => { + const run = jest.fn().mockResolvedValue({ equal: false, isSameDimension: false }); + const { service } = await setupSizes(Buffer.from('a'), Buffer.from('b'), run); + + expect(await compare(service)).toBe(DIFF_DIMENSION_RESULT); + expect(run).toHaveBeenCalledTimes(1); + }); - // Asked for on every comparison, not only when the project has bulk approve - // switched on: computing it lazily would leave every run ingested before the - // flag was turned on without one. + it('gives the result the worker gave for screenshots of different sizes', async () => { + const baseline = pngOf(2, 3); + const image = pngOf(3, 2); + const output = computePixelmatchDiff({ + kind: 'diff', + baseline, + image, + ignoreAreas: [], + threshold: DEFAULT_CONFIG.threshold, + includeAA: DEFAULT_CONFIG.ignoreAntialiasing, + diffTolerancePercent: 0, + saveDiff: false, + }); + expect(output.isSameDimension).toBe(false); + expect(output.equal).toBeFalsy(); + + const run = jest.fn().mockImplementation(async (input) => computePixelmatchDiff(input)); + const { service } = await setupSizes(baseline, image, run); + expect(await compare(service)).toStrictEqual(DIFF_DIMENSION_RESULT); + }); +}); + +describe('change signature', () => { it('asks for a signature and carries it into the result', async () => { - const { service, run } = await initWithPool({ - equal: false, - isSameDimension: true, - pixelMisMatchCount: 10, - diffPercent: 5, - signature: [0.25, 0.75], + const { service, run } = await setup({ + pool: poolOf({ ...mismatch, signature: [0.25, 0.75] }), + getImageBuffer: pngReader(), }); - const result = await compare(service); + const result = await compare(service, { saveDiffAsFile: true }); - expect(run).toHaveBeenCalledWith(expect.objectContaining({ kind: 'diff', withSignature: true })); - // stamped with the settings it was computed under, so a stored one can be - // discarded when the project's config moves on + expect(run).toHaveBeenCalledWith(expect.objectContaining({ kind: 'diff', saveDiff: true })); expect(result.changeSignature).toEqual({ threshold: DEFAULT_CONFIG.threshold, includeAA: DEFAULT_CONFIG.ignoreAntialiasing, @@ -345,144 +301,113 @@ describe('change signature', () => { }); }); - const initSaving = async (output: Record) => { - const saveImage = jest - .fn() - .mockResolvedValueOnce('diff.png') - .mockResolvedValueOnce('image.thumb.png') - .mockResolvedValueOnce('diff.thumb.png'); - const module: TestingModule = await Test.createTestingModule({ - providers: [ - PixelmatchService, - { provide: DiffWorkerPool, useValue: { run: jest.fn().mockResolvedValue(output) } }, - { provide: Dinov2Service, useValue: { confirmsNoise: jest.fn() } }, - { - provide: StaticService, - useValue: { - getImageBuffer: jest.fn().mockResolvedValue(Buffer.from('png')), - saveImage, - deleteImage: jest.fn(), - }, - }, - ], - }).compile(); - return { service: module.get(PixelmatchService), saveImage }; - }; + it('does not ask for a signature for a comparison whose diff is thrown away', async () => { + const { service, run } = await setup({ pool: poolOf(mismatch), getImageBuffer: pngReader() }); - const overTolerance = { - equal: false, - isSameDimension: true, - pixelMisMatchCount: 10, - diffPercent: 5, - imageThumbnail: Buffer.from('small image'), - diffThumbnail: Buffer.from('small diff'), - }; + const result = await compare(service); - it('saves the thumbnails and reports the names they went under', async () => { - const { service } = await initSaving({ ...overTolerance, diffBuffer: Buffer.from('the diff') }); + expect(run).toHaveBeenCalledWith(expect.objectContaining({ saveDiff: false })); + expect(result.changeSignature).toBeUndefined(); + }); - const result = await compare(service, true); + describe('thumbnails', () => { + const overTolerance = { + ...mismatch, + imageThumbnail: Buffer.from('small image'), + diffThumbnail: Buffer.from('small diff'), + diffBuffer: Buffer.from('the diff'), + }; + const setupSaving = (saveImage: jest.Mock, output: Record = overTolerance) => + setup({ pool: poolOf(output), getImageBuffer: pngReader(), saveImage }); + const savedAs = (...names: (string | Error)[]) => + names.reduce( + (m, name) => (name instanceof Error ? m.mockRejectedValueOnce(name) : m.mockResolvedValueOnce(name)), + jest.fn() + ); - expect(result.imageThumbnailName).toBe('image.thumb.png'); - expect(result.diffThumbnailName).toBe('diff.thumb.png'); - }); + it('saves the thumbnails and reports the names they went under', async () => { + const { service } = await setupSaving(savedAs('diff.png', 'image.thumb.png', 'diff.thumb.png')); - // shouldAutoApprove compares against past baselines with saveDiffAsFile off - // and throws the result away. Storing thumbnails for those comparisons would - // leave two objects per attempt that nothing ever references or deletes. - it('stores nothing when the caller is not keeping the diff', async () => { - const { service, saveImage } = await initSaving(overTolerance); + const result = await compare(service, { saveDiffAsFile: true }); - const result = await compare(service, false); + expect(result.imageThumbnailName).toBe('image.thumb.png'); + expect(result.diffThumbnailName).toBe('diff.thumb.png'); + }); - expect(saveImage).not.toHaveBeenCalled(); - expect(result.imageThumbnailName).toBeUndefined(); - expect(result.diffThumbnailName).toBeUndefined(); - }); + it('keeps the diff and the verdict when a thumbnail cannot be saved, and deletes the one that was', async () => { + const { service, deleteImage } = await setupSaving(savedAs('diff.png', new Error('disk full'), 'diff.thumb.png')); + + const result = await compare(service, { saveDiffAsFile: true }); + + expect(result).toMatchObject({ status: TestStatus.unresolved, diffName: 'diff.png' }); + expect(result.imageThumbnailName).toBeUndefined(); + expect(result.diffThumbnailName).toBeUndefined(); + expect(deleteImage).toHaveBeenCalledTimes(1); + expect(deleteImage).toHaveBeenCalledWith('diff.thumb.png'); + }); + + it('drops a thumbnail pair when one of the names comes back empty', async () => { + const { service, deleteImage } = await setupSaving(savedAs('diff.png', '', 'diff.thumb.png')); + + const result = await compare(service, { saveDiffAsFile: true }); - it('leaves it out when the comparison produced none', async () => { - const { service } = await initWithPool({ - equal: false, - isSameDimension: true, - pixelMisMatchCount: 10, - diffPercent: 5, + expect(result.diffName).toBe('diff.png'); + expect(result.diffThumbnailName).toBeUndefined(); + expect(deleteImage).toHaveBeenCalledWith('diff.thumb.png'); }); - expect((await compare(service)).changeSignature).toBeUndefined(); + it('stores nothing when the caller is not keeping the diff', async () => { + const saveImage = savedAs('diff.png', 'image.thumb.png', 'diff.thumb.png'); + const { service } = await setupSaving(saveImage, { ...overTolerance, diffBuffer: undefined }); + + const result = await compare(service); + + expect(saveImage).not.toHaveBeenCalled(); + expect(result.imageThumbnailName).toBeUndefined(); + expect(result.diffThumbnailName).toBeUndefined(); + }); }); }); describe('blur-amplitude filter', () => { - // The filter itself is pinned in pixelmatch.core.spec and - // blur-amplitude.core.spec, and whether a project has it on is - // CompareService's business; here only the plumbing: what the service asks - // the worker for, and what it makes of the answer. - const init = async (output: Record, confirmsNoise = jest.fn()) => { - const run = jest.fn().mockResolvedValue(output); - const saveImage = jest.fn().mockResolvedValue('diff.png'); - const module: TestingModule = await Test.createTestingModule({ - providers: [ - PixelmatchService, - { provide: DiffWorkerPool, useValue: { run } }, - { provide: Dinov2Service, useValue: { confirmsNoise } }, - { - provide: StaticService, - useValue: { - getImageBuffer: jest.fn().mockResolvedValue(Buffer.from('png')), - saveImage, - deleteImage: jest.fn(), - }, - }, - ], - }).compile(); - return { service: module.get(PixelmatchService), run, saveImage, confirmsNoise }; - }; - - const filter: { sigma: number; threshold: number; dinov2?: { threshold: number } } = { sigma: 1, threshold: 80 }; - const compare = (service: PixelmatchService, blurAmplitude?: typeof filter) => - service.getDiff( - { - baseline: 'baseline.png', - image: 'image.png', - ignoreAreas: [], - diffTollerancePercent: 0, - saveDiffAsFile: true, - ...(blurAmplitude ? { blurAmplitude } : {}), - }, - DEFAULT_CONFIG - ); + const init = (output: Record, confirmsNoiseForGate = jest.fn()) => + setup({ + pool: poolOf(output), + getImageBuffer: pngReader(), + saveImage: jest.fn().mockResolvedValue('diff.png'), + confirmsNoiseForGate, + }); - const overTolerance = { equal: false, isSameDimension: true, pixelMisMatchCount: 10, diffPercent: 5 }; + const filter: { threshold: number; dinov2?: { threshold: number } } = { threshold: 80 }; + const compareWith = (service: PixelmatchService, blurAmplitude?: typeof filter) => + compare(service, { saveDiffAsFile: true, ...(blurAmplitude ? { blurAmplitude } : {}) }); - // so CompareService does not measure the same pair a second time it('applies the filter itself', async () => { - const { service } = await init(overTolerance); + const { service } = await init(mismatch); expect(service.appliesBlurAmplitudeFilter).toBe(true); }); it('asks for no measurement when the comparison does not carry the filter', async () => { - const { service, run } = await init(overTolerance); + const { service, run } = await init(mismatch); - await compare(service); + await compareWith(service); expect(run.mock.calls[0][0].blurAmplitude).toBeUndefined(); }); it('measures with the settings the comparison carries', async () => { - const { service, run } = await init(overTolerance); + const { service, run } = await init(mismatch); - await compare(service, { sigma: 2, threshold: 50 }); + await compareWith(service, { threshold: 50 }); - expect(run).toHaveBeenCalledWith(expect.objectContaining({ blurAmplitude: { sigma: 2, threshold: 50 } })); + expect(run).toHaveBeenCalledWith(expect.objectContaining({ blurAmplitude: { threshold: 50 } })); }); - // the measured mismatch stays on the run, so a reviewer can still see the - // pixels moved; only the verdict changes it('passes a run whose change is only rendering noise', async () => { - const { service, saveImage } = await init({ ...overTolerance, blurAmplitude: 40, renderingNoiseOnly: true }); + const { service, saveImage } = await init({ ...mismatch, blurAmplitude: 40, renderingNoiseOnly: true }); - const result = await compare(service, filter); + const result = await compareWith(service, filter); expect(result.status).toBe(TestStatus.ok); expect(result.diffPercent).toBe(5); @@ -492,13 +417,13 @@ describe('blur-amplitude filter', () => { it('fails a run whose change is stronger than rendering noise', async () => { const { service } = await init({ - ...overTolerance, + ...mismatch, blurAmplitude: 200, renderingNoiseOnly: false, diffBuffer: Buffer.from('the diff'), }); - const result = await compare(service, filter); + const result = await compareWith(service, filter); expect(result.status).toBe(TestStatus.unresolved); expect(result.diffName).toBe('diff.png'); @@ -506,71 +431,80 @@ describe('blur-amplitude filter', () => { describe('with the DINOv2 check', () => { const checked = { ...filter, dinov2: { threshold: 0.15 } }; + const gate = { width: 5, height: 3, cells: [7] }; const noise = { - ...overTolerance, + ...mismatch, blurAmplitude: 40, renderingNoiseOnly: true, diffBuffer: Buffer.from('the diff'), + gate, }; - // what the blur would pass, DINOv2 looks at once more - it('passes a run DINOv2 confirms is only noise', async () => { - const confirmsNoise = jest.fn().mockResolvedValue({ noise: true, score: 0.05 }); - const { service, saveImage } = await init(noise, confirmsNoise); + it('passes a run DINOv2 confirms is only noise, with the gate the diff job computed', async () => { + const confirmsNoiseForGate = jest.fn().mockResolvedValue({ noise: true, score: 0.05 }); + const { service, saveImage } = await init(noise, confirmsNoiseForGate); - const result = await compare(service, checked); + const result = await compareWith(service, checked); - expect(confirmsNoise).toHaveBeenCalledWith(Buffer.from('png'), Buffer.from('png'), [], 0.15); + expect(confirmsNoiseForGate).toHaveBeenCalledWith(gate, Buffer.from('png'), Buffer.from('png'), 0.15); expect(result.status).toBe(TestStatus.ok); expect(saveImage).not.toHaveBeenCalled(); }); it('fails a run in which DINOv2 sees a change of content, with its diff', async () => { - const confirmsNoise = jest.fn().mockResolvedValue({ noise: false, score: 0.56 }); - const { service, saveImage } = await init(noise, confirmsNoise); + const confirmsNoiseForGate = jest.fn().mockResolvedValue({ noise: false, score: 0.56 }); + const { service, saveImage } = await init(noise, confirmsNoiseForGate); - const result = await compare(service, checked); + const result = await compareWith(service, checked); expect(result.status).toBe(TestStatus.unresolved); expect(result.diffName).toBe('diff.png'); expect(saveImage).toHaveBeenCalledWith('diff', Buffer.from('the diff')); }); - // the worker must keep the diff of a noise-like change, or a run DINOv2 - // sends to review would arrive with nothing to show + it('fails a run whose diff job returned no gate, without asking DINOv2', async () => { + const confirmsNoiseForGate = jest.fn(); + const { service, saveImage } = await init({ ...noise, gate: undefined }, confirmsNoiseForGate); + + const result = await compareWith(service, checked); + + expect(confirmsNoiseForGate).not.toHaveBeenCalled(); + expect(result.status).toBe(TestStatus.unresolved); + expect(saveImage).toHaveBeenCalledWith('diff', Buffer.from('the diff')); + }); + it('asks the worker to keep the diff when a check will follow', async () => { const { service, run } = await init(noise, jest.fn().mockResolvedValue({ noise: true })); - await compare(service, checked); + await compareWith(service, checked); expect(run).toHaveBeenCalledWith(expect.objectContaining({ blurAmplitude: checked })); }); - // a check that could not look answers without a score: still to review it('fails a run DINOv2 could not confirm', async () => { const { service } = await init(noise, jest.fn().mockResolvedValue({ noise: false })); - expect((await compare(service, checked)).status).toBe(TestStatus.unresolved); + expect((await compareWith(service, checked)).status).toBe(TestStatus.unresolved); }); it('does not ask DINOv2 about a change the blur already fails', async () => { - const confirmsNoise = jest.fn(); + const confirmsNoiseForGate = jest.fn(); const { service } = await init( - { ...overTolerance, blurAmplitude: 200, renderingNoiseOnly: false }, - confirmsNoise + { ...mismatch, blurAmplitude: 200, renderingNoiseOnly: false }, + confirmsNoiseForGate ); - await compare(service, checked); + await compareWith(service, checked); - expect(confirmsNoise).not.toHaveBeenCalled(); + expect(confirmsNoiseForGate).not.toHaveBeenCalled(); }); it('does not ask DINOv2 when the project did not turn the check on', async () => { - const confirmsNoise = jest.fn(); - const { service } = await init(noise, confirmsNoise); + const confirmsNoiseForGate = jest.fn(); + const { service } = await init(noise, confirmsNoiseForGate); - expect((await compare(service, filter)).status).toBe(TestStatus.ok); - expect(confirmsNoise).not.toHaveBeenCalled(); + expect((await compareWith(service, filter)).status).toBe(TestStatus.ok); + expect(confirmsNoiseForGate).not.toHaveBeenCalled(); }); }); }); diff --git a/src/compare/libs/pixelmatch/pixelmatch.service.ts b/src/compare/libs/pixelmatch/pixelmatch.service.ts index f7136caf..128e53cc 100644 --- a/src/compare/libs/pixelmatch/pixelmatch.service.ts +++ b/src/compare/libs/pixelmatch/pixelmatch.service.ts @@ -1,17 +1,25 @@ import { Injectable, Logger } from '@nestjs/common'; import { TestStatus } from '@prisma/client'; import { StaticService } from '../../../static/static.service'; +import { isByteEqualWellFormedPng, readPngDimensions } from '../../../static/utils'; import { DiffResult } from '../../../test-runs/diffResult'; import { parseConfig } from '../../utils'; import { DIFF_DIMENSION_RESULT, EQUAL_RESULT, NO_BASELINE_RESULT } from '../consts'; import { ImageComparator } from '../image-comparator.interface'; import { ImageCompareInput } from '../ImageCompareInput'; import { PixelmatchConfig } from './pixelmatch.types'; +import { PixelmatchJobOutput } from './pixelmatch.core'; import { DiffWorkerPool } from '../../diff-worker-pool'; import { Dinov2Service } from '../dinov2/dinov2.service'; export const DEFAULT_CONFIG: PixelmatchConfig = { threshold: 0.1, ignoreAntialiasing: true }; +function sameDimensions(a: Buffer, b: Buffer): boolean { + const x = readPngDimensions(a); + const y = readPngDimensions(b); + return !x || !y || (x.width === y.width && x.height === y.height); +} + @Injectable() export class PixelmatchService implements ImageComparator { private readonly logger: Logger = new Logger(PixelmatchService.name); @@ -32,12 +40,19 @@ export class PixelmatchService implements ImageComparator { if (!baselineBuffer) { return NO_BASELINE_RESULT; } - const imageBuffer = await this.staticService.getImageBuffer(data.image); + const imageBuffer = data.imageBuffer ?? (await this.staticService.getImageBuffer(data.image)); if (!imageBuffer) { throw new Error(`Cannot get image: ${data.image}`); } - // decode + pixelmatch + diff encode run off the event loop + if (isByteEqualWellFormedPng(baselineBuffer, imageBuffer)) { + return EQUAL_RESULT; + } + + if (!config.allowDiffDimensions && !sameDimensions(baselineBuffer, imageBuffer)) { + return DIFF_DIMENSION_RESULT; + } + const output = await this.diffWorkerPool.run({ kind: 'diff', baseline: baselineBuffer, @@ -48,18 +63,6 @@ export class PixelmatchService implements ImageComparator { allowDiffDimensions: config.allowDiffDimensions, diffTolerancePercent: data.diffTollerancePercent, saveDiff: data.saveDiffAsFile, - // Asked for on every comparison, not only when the project has bulk - // approve of variations switched on: the pass costs little beside the - // full-size diff that has already been paid for, and computing it lazily - // would leave every run ingested before the flag was turned on without - // one — which is exactly the build someone then tries to review. - withSignature: true, - // Only when the caller is keeping the diff. shouldAutoApprove compares - // against past baselines with this off and throws the result away — - // making thumbnails for those would burn the CPU and, worse, store two - // objects per attempt that nothing ever references or deletes. - withThumbnails: data.saveDiffAsFile, - // measured here, next to the diff, while the images are decoded anyway ...(data.blurAmplitude ? { blurAmplitude: data.blurAmplitude } : {}), }); @@ -86,9 +89,7 @@ export class PixelmatchService implements ImageComparator { : {}), }; - // The mismatch stays on the run as measured, so a reviewer can still see - // the pixels moved; only the verdict is the filter's. - if (output.renderingNoiseOnly && (await this.confirmedByDinov2(data, baselineBuffer, imageBuffer))) { + if (output.renderingNoiseOnly && (await this.confirmedByDinov2(data, baselineBuffer, imageBuffer, output))) { this.logger.debug( `Diff of ${result.diffPercent}% passed as rendering noise (blur amplitude ${output.blurAmplitude?.toFixed(1)}, ${output.solidPixels} solid pixels)` ); @@ -100,23 +101,8 @@ export class PixelmatchService implements ImageComparator { if (output.diffBuffer) { result.diffName = await this.staticService.saveImage('diff', Buffer.from(output.diffBuffer)); } - // Kept to the same condition as the diff itself: a thumbnail of a diff - // that was never saved would point at a file that does not exist. Stored - // as ordinary images so they follow the same storage, the same deletion - // and the same URLs as everything else, with no naming convention for - // the UI to guess at. - // tied to the diff actually landing, not merely to being over tolerance - if (result.diffName && output.imageThumbnail && output.diffThumbnail) { - const [imageThumbnailName, diffThumbnailName] = await Promise.all([ - this.staticService.saveImage('screenshot', Buffer.from(output.imageThumbnail)), - this.staticService.saveImage('diff', Buffer.from(output.diffThumbnail)), - ]); - // only when both came back: a result carrying one half of a pair, or a - // key set to undefined, is worse than none at all - if (imageThumbnailName && diffThumbnailName) { - result.imageThumbnailName = imageThumbnailName; - result.diffThumbnailName = diffThumbnailName; - } + if (result.diffName) { + Object.assign(result, await this.saveThumbnails(output)); } result.status = TestStatus.unresolved; } else { @@ -125,17 +111,44 @@ export class PixelmatchService implements ImageComparator { return result; } - // The project's DINOv2 check, when it has one, gets the last word on what the - // blur would pass; without one the blur's verdict stands. - private async confirmedByDinov2(data: ImageCompareInput, baseline: Buffer, image: Buffer): Promise { + private async confirmedByDinov2( + data: ImageCompareInput, + baseline: Buffer, + image: Buffer, + output: PixelmatchJobOutput + ): Promise { const dinov2 = data.blurAmplitude?.dinov2; if (!dinov2) { return true; } - const verdict = await this.dinov2Service.confirmsNoise(baseline, image, data.ignoreAreas, dinov2.threshold); + if (!output.gate) { + return false; + } + const verdict = await this.dinov2Service.confirmsNoiseForGate(output.gate, baseline, image, dinov2.threshold); if (!verdict.noise) { this.logger.debug(`DINOv2 sees a change of content (score ${verdict.score?.toFixed(3) ?? 'n/a'}): to review`); } return verdict.noise; } + + private async saveThumbnails( + output: PixelmatchJobOutput + ): Promise<{ imageThumbnailName?: string; diffThumbnailName?: string }> { + if (!output.imageThumbnail || !output.diffThumbnail) { + return {}; + } + const [image, diff] = await Promise.allSettled([ + this.staticService.saveImage('screenshot', Buffer.from(output.imageThumbnail)), + this.staticService.saveImage('diff', Buffer.from(output.diffThumbnail)), + ]); + if (image.status === 'fulfilled' && diff.status === 'fulfilled' && image.value && diff.value) { + return { imageThumbnailName: image.value, diffThumbnailName: diff.value }; + } + for (const saved of [image, diff]) { + if (saved.status === 'fulfilled' && saved.value) void this.staticService.deleteImage(saved.value); + } + const failure = [image, diff].find((saved) => saved.status === 'rejected'); + this.logger.warn(`Thumbnails not saved: ${failure?.reason?.message ?? 'empty name'}`); + return {}; + } } diff --git a/src/compare/libs/pixelmatch/pixelmatch.worker.spec.ts b/src/compare/libs/pixelmatch/pixelmatch.worker.spec.ts index 9b40be0a..9e1254ca 100644 --- a/src/compare/libs/pixelmatch/pixelmatch.worker.spec.ts +++ b/src/compare/libs/pixelmatch/pixelmatch.worker.spec.ts @@ -27,10 +27,11 @@ const input: SignatureJobInput = { }; describe('pixelmatch.worker', () => { + let postedOnLoad: unknown[][]; + beforeAll(() => { - // a plain require: under `module: nodenext` an import() stays a real ESM import - // eslint-disable-next-line @typescript-eslint/no-require-imports require('./pixelmatch.worker'); + postedOnLoad = [...port.postMessage.mock.calls]; }); beforeEach(() => { @@ -38,31 +39,40 @@ describe('pixelmatch.worker', () => { runWorkerJobMock.mockReset(); }); + it('announces that it has loaded', () => { + expect(postedOnLoad).toEqual([[{ ready: true }]]); + }); + it('listens for jobs on the parent port', () => { expect(port.listenerCount('message')).toBe(1); }); - it('runs the job it receives and posts the result back', () => { + const flush = () => new Promise(setImmediate); + + it('runs the job it receives and posts the result back', async () => { runWorkerJobMock.mockReturnValue({ signature: [1] }); port.emit('message', input); + await flush(); expect(runWorkerJobMock).toHaveBeenCalledWith(input); expect(port.postMessage).toHaveBeenCalledWith({ signature: [1] }); }); - it('posts the error message back when the job throws an Error', () => { + it('posts the error message back when the job throws an Error', async () => { runWorkerJobMock.mockImplementation(() => { throw new Error('Invalid PNG'); }); port.emit('message', input); + await flush(); expect(port.postMessage).toHaveBeenCalledWith({ error: 'Invalid PNG' }); }); it('posts the stringified value back when the job throws something that is not an Error', () => { runWorkerJobMock.mockImplementation(() => { + // eslint-disable-next-line @typescript-eslint/only-throw-error -- worker must stringify non-Error throws throw 42; }); @@ -70,4 +80,24 @@ describe('pixelmatch.worker', () => { expect(port.postMessage).toHaveBeenCalledWith({ error: '42' }); }); + + it('awaits an async job before posting its result', async () => { + runWorkerJobMock.mockResolvedValue({ outcome: 'equal' }); + + port.emit('message', input); + await flush(); + + expect(port.postMessage).toHaveBeenCalledTimes(1); + expect(port.postMessage).toHaveBeenCalledWith({ outcome: 'equal' }); + }); + + it('posts the message of an async job that rejects', async () => { + runWorkerJobMock.mockRejectedValue(new Error('boom')); + + port.emit('message', input); + await flush(); + + expect(port.postMessage).toHaveBeenCalledTimes(1); + expect(port.postMessage).toHaveBeenCalledWith({ error: 'boom' }); + }); }); diff --git a/src/compare/libs/pixelmatch/pixelmatch.worker.ts b/src/compare/libs/pixelmatch/pixelmatch.worker.ts index 67520b56..40c6de79 100644 --- a/src/compare/libs/pixelmatch/pixelmatch.worker.ts +++ b/src/compare/libs/pixelmatch/pixelmatch.worker.ts @@ -1,13 +1,15 @@ import { parentPort } from 'worker_threads'; import { runWorkerJob, WorkerJobInput } from './worker-job'; -// this file is only ever loaded as a worker thread, where parentPort is set const port = parentPort!; -port.on('message', (input: WorkerJobInput) => { +// eslint-disable-next-line @typescript-eslint/no-misused-promises -- handler reports errors via postMessage +port.on('message', async (input: WorkerJobInput) => { try { - port.postMessage(runWorkerJob(input)); + port.postMessage(await runWorkerJob(input)); } catch (error) { port.postMessage({ error: error instanceof Error ? error.message : String(error) }); } }); + +port.postMessage({ ready: true }); diff --git a/src/compare/libs/pixelmatch/signature.core.spec.ts b/src/compare/libs/pixelmatch/signature.core.spec.ts index 3c48e99c..e8584fff 100644 --- a/src/compare/libs/pixelmatch/signature.core.spec.ts +++ b/src/compare/libs/pixelmatch/signature.core.spec.ts @@ -1,4 +1,4 @@ -import { PNG } from 'pngjs'; +import { paintedPng, pngBytes, Rgb, solidPng, square } from '../../test-utils'; import { computeChangeSignature, signaturesMatch, @@ -10,32 +10,13 @@ import { IgnoreAreaDto } from '../../../test-runs/dto/ignore-area.dto'; const WIDTH = 40; const HEIGHT = 40; -const png = (paint: (set: (x: number, y: number, rgb: [number, number, number]) => void) => void): Buffer => { - const image = new PNG({ width: WIDTH, height: HEIGHT }); - image.data.fill(255); - paint((x, y, [r, g, b]) => { - const index = (y * WIDTH + x) * 4; - image.data[index] = r; - image.data[index + 1] = g; - image.data[index + 2] = b; - image.data[index + 3] = 255; - }); - return PNG.sync.write(image); -}; - -const block = (left: number, top: number, rgb: [number, number, number]): Buffer => - png((set) => { - for (let y = top; y < top + 8; y++) { - for (let x = left; x < left + 8; x++) { - set(x, y, rgb); - } - } - }); +const block = (left: number, top: number, rgb: Rgb): Buffer => + pngBytes(paintedPng(WIDTH, HEIGHT, square(left, top, 8, rgb))); -const blank = png(() => undefined); +const blank = pngBytes(solidPng(WIDTH, HEIGHT)); -const RED: [number, number, number] = [255, 0, 0]; -const BLUE: [number, number, number] = [0, 0, 255]; +const RED: Rgb = [255, 0, 0]; +const BLUE: Rgb = [0, 0, 255]; const signatureOf = (baseline: Buffer, image: Buffer, ignoreAreas: IgnoreAreaDto[] = []): number[] | null => computeChangeSignature({ @@ -48,34 +29,18 @@ const signatureOf = (baseline: Buffer, image: Buffer, ignoreAreas: IgnoreAreaDto }).signature; describe('computeChangeSignature', () => { - it('has no signature when the images have different dimensions', () => { - const taller = new PNG({ width: WIDTH, height: HEIGHT * 2 }); - taller.data.fill(255); - - expect(signatureOf(blank, PNG.sync.write(taller))).toBeNull(); - }); - - it('has no signature for a checkpoint with the same pixel count but another shape', () => { - const wide = new PNG({ width: WIDTH * 2, height: HEIGHT / 2 }); - wide.data.fill(0); - - expect(signatureOf(blank, PNG.sync.write(wide))).toBeNull(); - }); - - it('has no signature for a checkpoint of another width only', () => { - const wider = new PNG({ width: WIDTH * 2, height: HEIGHT }); - wider.data.fill(0); - - expect(signatureOf(blank, PNG.sync.write(wider))).toBeNull(); + it.each([ + ['different dimensions', WIDTH, HEIGHT * 2], + ['the same pixel count but another shape', WIDTH * 2, HEIGHT / 2], + ['another width only', WIDTH * 2, HEIGHT], + ])('has no signature for a checkpoint with %s', (_, width, height) => { + expect(signatureOf(blank, pngBytes(solidPng(width, height)))).toBeNull(); }); it('has no signature when nothing changed', () => { expect(signatureOf(blank, blank)).toBeNull(); }); - // One unreadable screenshot must not reject the whole matching request: the - // siblings are signed in a single fan-out, so a throw here would take the - // variations dialog down with it rather than skipping that one candidate. it('has no signature when the checkpoint cannot be decoded', () => { expect(signatureOf(blank, Buffer.from('not a png'))).toBeNull(); }); @@ -88,7 +53,6 @@ describe('computeChangeSignature', () => { const signature = signatureOf(blank, block(0, 0, RED))!; const brightest = signature.indexOf(Math.max(...signature)); - // 4 buckets per channel: pure red is the last red bucket, first green/blue expect(brightest).toBe(3 * 16); expect(signature.reduce((sum, value) => sum + value, 0)).toBeCloseTo(1); }); @@ -109,7 +73,6 @@ describe('computeChangeSignature', () => { }); describe('computeChangeSignature with structured-clone input', () => { - // postMessage hands the worker Uint8Array views, possibly over a larger buffer it('reads Uint8Array views the same as Buffers', () => { const changed = block(0, 0, RED); const padded = new Uint8Array(changed.length + 16); @@ -147,9 +110,7 @@ describe('signaturesMatch', () => { it('matches at a cosine similarity of at least 0.9', () => { expect(SIGNATURE_SIMILARITY_THRESHOLD).toBe(0.9); - // cos = 0.95 / sqrt(0.95^2 + 0.3^2) ≈ 0.954 expect(signaturesMatch([1, 0], [0.95, 0.3])).toBe(true); - // cos = 0.8 / sqrt(0.8^2 + 0.6^2) = 0.8 expect(signaturesMatch([1, 0], [0.8, 0.6])).toBe(false); }); @@ -158,10 +119,8 @@ describe('signaturesMatch', () => { expect(signaturesMatch([1, 0], [0, 0])).toBe(false); }); - it('compares only as many buckets as the first signature has', () => { + it('compares only the buckets of the first signature (signatures are always SIGNATURE_LENGTH long)', () => { expect(SIGNATURE_LENGTH).toBe(64); - // NOTE: current behavior — a shorter second signature yields NaN (no match), - // while extra buckets in the second one are silently ignored (a match). expect(signaturesMatch([1, 0], [1])).toBe(false); expect(signaturesMatch([1], [1, 5])).toBe(true); }); diff --git a/src/compare/libs/pixelmatch/signature.core.ts b/src/compare/libs/pixelmatch/signature.core.ts index 020d37a7..4df8181f 100644 --- a/src/compare/libs/pixelmatch/signature.core.ts +++ b/src/compare/libs/pixelmatch/signature.core.ts @@ -1,21 +1,10 @@ import { PNG } from 'pngjs'; import Pixelmatch from 'pixelmatch'; import { IgnoreAreaDto } from '../../../test-runs/dto/ignore-area.dto'; -import { RawImage } from '../../utils'; -import { applyIgnoreAreas, downscale } from '../../utils'; +import { downscale, RawImage, readSameSizePair } from '../../utils'; -/** - * CPU-bound part of "do these two screens carry the same change?": PNG decode, - * downscale, pixelmatch and histogram. Extracted so it can run inside a worker - * thread (see pixelmatch.worker.ts / DiffWorkerPool) — the variations dialog - * asks for one of these per sibling, and decoding full-size screenshots on the - * main thread froze the whole API for as long as it took. All input/output must - * stay structured-clone serializable. - */ export interface SignatureJobInput { kind: 'signature'; - // PNG bytes, not decoded pixels: decoding is the expensive part and belongs - // in the worker. baseline: Buffer | Uint8Array; image: Buffer | Uint8Array; ignoreAreas: IgnoreAreaDto[]; @@ -27,70 +16,19 @@ export interface SignatureJobOutput { signature: number[] | null; } -// Colors are quantized to this many levels per RGB channel, giving -// COLOR_BUCKETS_PER_CHANNEL^3 histogram buckets. const COLOR_BUCKETS_PER_CHANNEL = 4; -// How many buckets a signature has. Exported so a stored one can be checked -// against the shape this build produces: comparing signatures of two different -// lengths yields a similarity score computed over undefined entries — a number, -// and a meaningless one. export const SIGNATURE_LENGTH = COLOR_BUCKETS_PER_CHANNEL ** 3; -// Longest side (px) images are downscaled to before computing the color -// signature — keeps the histogram representative while cutting pixelmatch cost. const SIGNATURE_MAX_DIMENSION = 500; -// Two changes are considered the same pattern when their color signatures' -// cosine similarity is at least this value. export const SIGNATURE_SIMILARITY_THRESHOLD = 0.9; -// postMessage turns Buffers into Uint8Array views over their own ArrayBuffer. -function toBuffer(data: Buffer | Uint8Array): Buffer { - return Buffer.isBuffer(data) ? data : Buffer.from(data.buffer, data.byteOffset, data.byteLength); -} - -/** - * Position-independent signature of a change: a normalized histogram of the - * colors that the changed pixels take in the new image. Because it ignores - * *where* the change is, it is robust to per-locale layout shifts (a title - * wrapping to a different number of lines, options moving down, etc.) while - * still capturing *what* changed (a selection highlight, a recolored button). - * Null when there is nothing to compare: unreadable bytes, differing - * dimensions, or no change outside the ignore areas. - */ export function computeChangeSignature(input: SignatureJobInput): SignatureJobOutput { - // A corrupt or truncated screenshot is one candidate the reviewer cannot be - // offered, not a failed request: the siblings are signed in a single - // fan-out, so throwing here would take the whole variations dialog down. - let baselineImage: PNG; - let checkpointImage: PNG; - try { - baselineImage = PNG.sync.read(toBuffer(input.baseline)); - checkpointImage = PNG.sync.read(toBuffer(input.image)); - } catch { - return { signature: null }; - } - - if (baselineImage.width !== checkpointImage.width || baselineImage.height !== checkpointImage.height) { - return { signature: null }; - } - - // Masked regions must not count toward the signature, so they are blanked at - // full resolution — the areas are given in full-resolution coordinates — - // before the downscale makes both images cheap to compare. - applyIgnoreAreas(baselineImage, input.ignoreAreas); - applyIgnoreAreas(checkpointImage, input.ignoreAreas); - - return { signature: signatureOfDecoded(baselineImage, checkpointImage, input) }; + const pair = readSameSizePair(input.baseline, input.image, input.ignoreAreas); + return { signature: pair && signatureOfDecoded(pair.baseline, pair.image, input) }; } -/** - * The signature of a pair that is already decoded, of equal size, and with its - * ignore areas blanked. Split out so the diff can produce a signature from the - * images it has just decoded rather than paying for a second decode — the two - * must not drift apart, so there is only ever one implementation. - */ export function signatureOfDecoded( baselineImage: RawImage, checkpointImage: RawImage, @@ -110,7 +48,7 @@ export function signatureOfDecoded( } const bucketSize = 256 / COLOR_BUCKETS_PER_CHANNEL; - const histogram = new Array(COLOR_BUCKETS_PER_CHANNEL ** 3).fill(0); + const histogram = new Array(SIGNATURE_LENGTH).fill(0); for (let i = 0; i < width * height; i++) { if (mask.data[i * 4 + 3] === 0) { continue; diff --git a/src/compare/libs/pixelmatch/thumbnail.core.spec.ts b/src/compare/libs/pixelmatch/thumbnail.core.spec.ts index 4dcfa7a8..3ec7665a 100644 --- a/src/compare/libs/pixelmatch/thumbnail.core.spec.ts +++ b/src/compare/libs/pixelmatch/thumbnail.core.spec.ts @@ -1,42 +1,28 @@ import { PNG } from 'pngjs'; +import { solidPng } from '../../test-utils'; import { encodeThumbnail, THUMBNAIL_MAX_DIMENSION } from './thumbnail.core'; -const solid = (width: number, height: number): PNG => { - const png = new PNG({ width, height }); - for (let i = 0; i < width * height; i++) { - png.data[i * 4] = 200; - png.data[i * 4 + 1] = 100; - png.data[i * 4 + 2] = 50; - png.data[i * 4 + 3] = 255; - } - return png; -}; +const solid = (width: number, height: number): PNG => solidPng(width, height, [200, 100, 50]); describe('encodeThumbnail', () => { - // The grids draw these a hundred-odd pixels wide. Sending the full - // screenshot and letting CSS shrink it cost ~7 MB and 68 requests every time - // the variations dialog opened. + const source = solid(1284, 2778); + it('brings a screenshot down to the thumbnail size', () => { - const thumbnail = PNG.sync.read(encodeThumbnail(solid(1284, 2778))); + const thumbnail = PNG.sync.read(encodeThumbnail(source)); expect(Math.max(thumbnail.width, thumbnail.height)).toBe(THUMBNAIL_MAX_DIMENSION); }); it('keeps the proportions of the screenshot it came from', () => { - const source = solid(1284, 2778); - const thumbnail = PNG.sync.read(encodeThumbnail(source)); expect(thumbnail.width / thumbnail.height).toBeCloseTo(source.width / source.height, 2); }); it('is a fraction of the bytes of the original', () => { - const source = solid(1284, 2778); - - expect(encodeThumbnail(source).length).toBeLessThan(PNG.sync.write(source).length / 10); + expect(encodeThumbnail(source).length).toBeLessThan(PNG.sync.write(source, { filterType: 1 }).length / 10); }); - // a small screenshot re-encoded larger would be worse than leaving it alone it('never enlarges an image that is already small', () => { const thumbnail = PNG.sync.read(encodeThumbnail(solid(100, 80))); diff --git a/src/compare/libs/pixelmatch/thumbnail.core.ts b/src/compare/libs/pixelmatch/thumbnail.core.ts index 1fef0cd6..901cef66 100644 --- a/src/compare/libs/pixelmatch/thumbnail.core.ts +++ b/src/compare/libs/pixelmatch/thumbnail.core.ts @@ -1,22 +1,8 @@ import { PNG } from 'pngjs'; import { downscale, RawImage } from '../../utils'; -/** - * Longest side, in pixels, of the picture the grids draw. They lay these out a - * hundred-odd pixels wide, so this leaves room for a retina screen and nothing - * more. Sending the full screenshot and letting CSS shrink it cost roughly 7 MB - * and 68 requests every time the variations dialog opened. - */ export const THUMBNAIL_MAX_DIMENSION = 400; -/** - * A small PNG of an already-decoded image, for the card grids. Produced beside - * the diff, from the pixels it has already decoded, so a thumbnail costs a - * resize rather than a second read of the screenshot. - * - * An image already within the bound is re-encoded as it is: enlarging it would - * make it bigger on the wire for no gain. - */ export function encodeThumbnail(source: RawImage): Buffer { const small = downscale(source, THUMBNAIL_MAX_DIMENSION); const png = new PNG({ width: small.width, height: small.height }); diff --git a/src/compare/libs/pixelmatch/worker-job.spec.ts b/src/compare/libs/pixelmatch/worker-job.spec.ts new file mode 100644 index 00000000..fdbc4872 --- /dev/null +++ b/src/compare/libs/pixelmatch/worker-job.spec.ts @@ -0,0 +1,41 @@ +import { computeLooksSame, LooksSameJobInput } from '../looks-same/looks-same.core'; +import { computeVlmInputs, VlmInputsJobInput } from '../vlm/vlm-inputs.core'; +import { runWorkerJob } from './worker-job'; + +jest.mock('../looks-same/looks-same.core', () => ({ + computeLooksSame: jest.fn(), +})); +jest.mock('../vlm/vlm-inputs.core', () => ({ + computeVlmInputs: jest.fn(), +})); + +describe('runWorkerJob', () => { + it('hands a looks-same job to computeLooksSame and returns its promise', async () => { + const input: LooksSameJobInput = { + kind: 'looks-same', + baseline: Buffer.from('baseline'), + image: Buffer.from('image'), + ignoreAreas: [], + config: {}, + saveDiff: false, + }; + (computeLooksSame as jest.Mock).mockResolvedValue({ outcome: 'equal' }); + + await expect(runWorkerJob(input)).resolves.toEqual({ outcome: 'equal' }); + expect(computeLooksSame).toHaveBeenCalledWith(input); + }); + + it('hands a vlm-inputs job to computeVlmInputs', () => { + const input: VlmInputsJobInput = { + kind: 'vlm-inputs', + baseline: Buffer.from('baseline'), + image: Buffer.from('image'), + ignoreAreas: [], + }; + const output = { baseline: null, image: null }; + (computeVlmInputs as jest.Mock).mockReturnValue(output); + + expect(runWorkerJob(input)).toBe(output); + expect(computeVlmInputs).toHaveBeenCalledWith(input); + }); +}); diff --git a/src/compare/libs/pixelmatch/worker-job.ts b/src/compare/libs/pixelmatch/worker-job.ts index 729f4727..9f574dc8 100644 --- a/src/compare/libs/pixelmatch/worker-job.ts +++ b/src/compare/libs/pixelmatch/worker-job.ts @@ -1,25 +1,26 @@ -import { BlurAmplitudeJobInput, BlurAmplitudeJobOutput, computeBlurAmplitude } from './blur-amplitude.core'; -import { computePatchGate, PatchGateJobInput, PatchGateJobOutput } from './patch-gate.core'; -import { computePixelmatchDiff, PixelmatchJobInput, PixelmatchJobOutput } from './pixelmatch.core'; -import { computeChangeSignature, SignatureJobInput, SignatureJobOutput } from './signature.core'; +import { computeBlurAmplitude } from './blur-amplitude.core'; +import { computeLooksSame } from '../looks-same/looks-same.core'; +import { computePixelmatchDiff } from './pixelmatch.core'; +import { computeVlmInputs } from '../vlm/vlm-inputs.core'; +import { computeChangeSignature } from './signature.core'; -/** - * The jobs {@link DiffWorkerPool} runs off the event loop, discriminated by - * `kind`. Both the worker thread and the pool's inline fallback go through - * {@link runWorkerJob}, so the two paths cannot answer a job differently. - */ -export type WorkerJobInput = PixelmatchJobInput | SignatureJobInput | BlurAmplitudeJobInput | PatchGateJobInput; -export type WorkerJobOutput = PixelmatchJobOutput | SignatureJobOutput | BlurAmplitudeJobOutput | PatchGateJobOutput; +const JOBS = { + diff: computePixelmatchDiff, + signature: computeChangeSignature, + 'blur-amplitude': computeBlurAmplitude, + 'looks-same': computeLooksSame, + 'vlm-inputs': computeVlmInputs, +} as const; -export function runWorkerJob(input: WorkerJobInput): WorkerJobOutput { - switch (input.kind) { - case 'signature': - return computeChangeSignature(input); - case 'blur-amplitude': - return computeBlurAmplitude(input); - case 'patch-gate': - return computePatchGate(input); - default: - return computePixelmatchDiff(input); - } +type Jobs = typeof JOBS; + +export type WorkerJobKind = keyof Jobs; +export type WorkerJobInput = { [K in WorkerJobKind]: Parameters[0] }[WorkerJobKind]; +export type WorkerJobOutput = { [K in WorkerJobKind]: Awaited> }[WorkerJobKind]; + +export type WorkerJobResult = Awaited>; +export type WorkerJobInputOf = Parameters[0] & { kind: K }; + +export function runWorkerJob(input: WorkerJobInput): WorkerJobOutput | Promise { + return (JOBS[input.kind] as (input: WorkerJobInput) => WorkerJobOutput | Promise)(input); } diff --git a/src/compare/libs/vlm/concurrency-limiter.spec.ts b/src/compare/libs/vlm/concurrency-limiter.spec.ts new file mode 100644 index 00000000..39fb966e --- /dev/null +++ b/src/compare/libs/vlm/concurrency-limiter.spec.ts @@ -0,0 +1,115 @@ +import { ConcurrencyLimiter } from './concurrency-limiter'; + +interface Deferred { + promise: Promise; + resolve: (value: T) => void; + reject: (error: Error) => void; +} + +const deferred = (): Deferred => { + let resolve!: (value: T) => void; + let reject!: (error: Error) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +}; + +const flush = async () => { + for (let i = 0; i < 5; i++) await Promise.resolve(); +}; + +describe('ConcurrencyLimiter', () => { + it('runs at most limit tasks at once', async () => { + const limiter = new ConcurrencyLimiter(2); + const gates = Array.from({ length: 5 }, () => deferred()); + let active = 0; + let peak = 0; + const started: number[] = []; + const results = gates.map((gate, index) => + limiter.run(async () => { + started.push(index); + active += 1; + peak = Math.max(peak, active); + try { + return await gate.promise; + } finally { + active -= 1; + } + }) + ); + + await flush(); + expect(started).toEqual([0, 1]); + + gates[0].resolve(10); + await flush(); + expect(started).toEqual([0, 1, 2]); + + gates[2].resolve(12); + gates[1].resolve(11); + await flush(); + expect(started).toEqual([0, 1, 2, 3, 4]); + + gates[3].resolve(13); + gates[4].resolve(14); + await expect(Promise.all(results)).resolves.toEqual([10, 11, 12, 13, 14]); + expect(peak).toBe(2); + }); + + it('frees the slot when a task rejects', async () => { + const limiter = new ConcurrencyLimiter(1); + const first = deferred(); + const failure = new Error('model is down'); + + const failing = limiter.run(() => first.promise); + const next = limiter.run(async () => 'second'); + first.reject(failure); + + await expect(failing).rejects.toBe(failure); + await expect(next).resolves.toBe('second'); + }); + + it('never exceeds the limit when a new task arrives while a waiting one is being woken', async () => { + const limiter = new ConcurrencyLimiter(1); + const gates = { a: deferred(), b: deferred(), c: deferred() }; + let active = 0; + let peak = 0; + const started: string[] = []; + const track = (name: keyof typeof gates) => + limiter.run(async () => { + started.push(name); + active += 1; + peak = Math.max(peak, active); + await gates[name].promise; + active -= 1; + }); + + const a = track('a'); + const b = track('b'); + await flush(); + expect(started).toEqual(['a']); + + gates.a.resolve(); + const c = track('c'); + await flush(); + expect(started).toEqual(['a', 'b']); + + gates.b.resolve(); + await flush(); + expect(started).toEqual(['a', 'b', 'c']); + gates.c.resolve(); + + await Promise.all([a, b, c]); + expect(peak).toBe(1); + }); + + it('lets a task start at once when a slot is free again', async () => { + const limiter = new ConcurrencyLimiter(1); + + await limiter.run(async () => 1); + + await expect(limiter.run(async () => 2)).resolves.toBe(2); + }); +}); diff --git a/src/compare/libs/vlm/concurrency-limiter.ts b/src/compare/libs/vlm/concurrency-limiter.ts new file mode 100644 index 00000000..bfad3caa --- /dev/null +++ b/src/compare/libs/vlm/concurrency-limiter.ts @@ -0,0 +1,28 @@ +export class ConcurrencyLimiter { + private active = 0; + private readonly waiting: (() => void)[] = []; + + constructor(private readonly limit: number) {} + + async run(task: () => Promise): Promise { + if (this.active < this.limit) { + this.active += 1; + } else { + await new Promise((resolve) => this.waiting.push(resolve)); + } + try { + return await task(); + } finally { + this.release(); + } + } + + private release(): void { + const next = this.waiting.shift(); + if (next) { + next(); + } else { + this.active -= 1; + } + } +} diff --git a/src/compare/libs/vlm/providers/gemini/gemini.service.spec.ts b/src/compare/libs/vlm/providers/gemini/gemini.service.spec.ts index fc1bdb2f..ef7abe64 100644 --- a/src/compare/libs/vlm/providers/gemini/gemini.service.spec.ts +++ b/src/compare/libs/vlm/providers/gemini/gemini.service.spec.ts @@ -1,12 +1,8 @@ import { Test, TestingModule } from '@nestjs/testing'; +import { GoogleGenAI } from '@google/genai'; import { GeminiService } from './gemini.service'; import { GeminiVlmConfig } from '../../vlm.types'; -jest.mock('zod/v3', () => { - const actualZod = jest.requireActual('zod'); - return actualZod; -}); - const mockGenerateContent = jest.fn(); jest.mock('@google/genai', () => { @@ -104,8 +100,21 @@ describe('GeminiService', () => { const callArgs = mockGenerateContent.mock.calls[0][0]; expect(callArgs.config.responseMimeType).toBe('application/json'); const schema = callArgs.config.responseJsonSchema; - expect(schema).toBeDefined(); - expect(schema).toBeTruthy(); + expect(schema).toEqual({ + $schema: 'http://json-schema.org/draft-07/schema#', + type: 'object', + properties: { identical: { type: 'boolean' }, description: { type: 'string' } }, + required: ['identical', 'description'], + additionalProperties: false, + }); + }); + + it('gives the SDK a five-minute request timeout', async () => { + mockGenerateContent.mockResolvedValue(createMockResponse('{}')); + + await service.generate(createConfig(), []); + + expect(GoogleGenAI).toHaveBeenCalledWith({ apiKey: 'test-api-key', httpOptions: { timeout: 300000 } }); }); it.each([ diff --git a/src/compare/libs/vlm/providers/gemini/gemini.service.ts b/src/compare/libs/vlm/providers/gemini/gemini.service.ts index e3c0ff55..c96919cc 100644 --- a/src/compare/libs/vlm/providers/gemini/gemini.service.ts +++ b/src/compare/libs/vlm/providers/gemini/gemini.service.ts @@ -2,9 +2,11 @@ import { Injectable, Logger } from '@nestjs/common'; import { GoogleGenAI } from '@google/genai'; import { zodToJsonSchema } from 'zod-to-json-schema'; import { VlmProvider, VlmProviderResponse } from '../../vlm-provider.interface'; -import { VlmConfig, GeminiVlmConfig } from '../../vlm.types'; +import { VlmConfig } from '../../vlm.types'; import { VlmComparisonResultSchema } from '../../vlm-result.schema'; +const TIMEOUT_MS = 300_000; + @Injectable() export class GeminiService implements VlmProvider { private readonly logger: Logger = new Logger(GeminiService.name); @@ -13,47 +15,32 @@ export class GeminiService implements VlmProvider { if (!apiKey) { throw new Error('Gemini API key is required'); } - return new GoogleGenAI({ apiKey }); - } - - private imageToBase64(imageBytes: Uint8Array): string { - const base64 = Buffer.from(imageBytes).toString('base64'); - return base64; + return new GoogleGenAI({ + apiKey, + httpOptions: { timeout: TIMEOUT_MS }, + }); } async generate(config: VlmConfig, images: Uint8Array[]): Promise { - // Type guard: ensure this is Gemini config if (config.provider !== 'gemini') { throw new Error(`GeminiService requires GeminiVlmConfig, got ${config.provider}`); } - - const geminiConfig: GeminiVlmConfig = config; - - const genAI = this.getGenAI(geminiConfig.apiKey); + const genAI = this.getGenAI(config.apiKey); try { const imageParts = images.map((img) => ({ - inlineData: { - data: this.imageToBase64(img), - mimeType: 'image/png', - }, + inlineData: { data: Buffer.from(img).toString('base64'), mimeType: 'image/png' }, })); - - const parts = [{ text: geminiConfig.prompt }, ...imageParts]; - const result = await genAI.models.generateContent({ - model: geminiConfig.model, - contents: parts, + model: config.model, + contents: [{ text: config.prompt }, ...imageParts], config: { - temperature: geminiConfig.temperature, - responseMimeType: 'application/json' as const, + temperature: config.temperature, + responseMimeType: 'application/json', responseJsonSchema: zodToJsonSchema(VlmComparisonResultSchema), }, }); - - return { - content: result.text, - }; + return { content: result.text }; } catch (error) { this.logger.error(`Gemini generate request failed: ${(error as Error).message}`, (error as Error).stack); throw error; diff --git a/src/compare/libs/vlm/providers/ollama/ollama.controller.spec.ts b/src/compare/libs/vlm/providers/ollama/ollama.controller.spec.ts index 0ce2ac1c..15f0e3ff 100644 --- a/src/compare/libs/vlm/providers/ollama/ollama.controller.spec.ts +++ b/src/compare/libs/vlm/providers/ollama/ollama.controller.spec.ts @@ -87,8 +87,6 @@ describe('OllamaController', () => { }); it('passes a missing temperature on as NaN', async () => { - // NOTE: current behavior — the query value is not validated, so leaving it - // out reaches the provider as NaN rather than its default temperature. ollamaService.generate.mockResolvedValue({}); await controller.compareImages([image('a'), image('b')], 'gemma3:12b', 'prompt', undefined as unknown as string); diff --git a/src/compare/libs/vlm/providers/ollama/ollama.service.spec.ts b/src/compare/libs/vlm/providers/ollama/ollama.service.spec.ts index c7b8df86..da0bd7ce 100644 --- a/src/compare/libs/vlm/providers/ollama/ollama.service.spec.ts +++ b/src/compare/libs/vlm/providers/ollama/ollama.service.spec.ts @@ -1,13 +1,9 @@ import { Test, TestingModule } from '@nestjs/testing'; import { ConfigService } from '@nestjs/config'; +import { Ollama } from 'ollama'; import { OllamaService } from './ollama.service'; import { OllamaVlmConfig } from '../../vlm.types'; -jest.mock('zod/v3', () => { - const actualZod = jest.requireActual('zod'); - return actualZod; -}); - const mockChat = jest.fn(); const mockList = jest.fn(); @@ -25,11 +21,13 @@ describe('OllamaService', () => { let service: OllamaService; const mockConfigService = { getOrThrow: jest.fn().mockReturnValue('http://localhost:11434'), + get: jest.fn(), }; beforeEach(async () => { jest.clearAllMocks(); mockConfigService.getOrThrow.mockReturnValue('http://localhost:11434'); + mockConfigService.get.mockReset(); const module: TestingModule = await Test.createTestingModule({ providers: [ @@ -85,7 +83,13 @@ describe('OllamaService', () => { }, ], stream: false, - format: expect.any(Object), + format: { + $schema: 'http://json-schema.org/draft-07/schema#', + type: 'object', + properties: { identical: { type: 'boolean' }, description: { type: 'string' } }, + required: ['identical', 'description'], + additionalProperties: false, + }, options: { temperature: config.temperature, }, @@ -150,53 +154,108 @@ describe('OllamaService', () => { }); }); + describe('request timeout', () => { + type Fetch = (url: string, init?: RequestInit) => Promise; + const realFetch = global.fetch; + + afterEach(() => { + global.fetch = realFetch; + }); + + const sdkFetch = async (): Promise => { + mockChat.mockResolvedValue(createMockResponse('{}')); + const fresh = new OllamaService(mockConfigService as any); + await fresh.generate(createConfig(), []); + const [options] = (Ollama as unknown as jest.Mock).mock.calls.at(-1)!; + expect(options.host).toBe('http://localhost:11434'); + return options.fetch; + }; + + const stalledFetch = () => + jest.fn().mockImplementation( + (_url: string, init: RequestInit) => + new Promise((_, reject) => { + init.signal!.addEventListener('abort', () => reject(init.signal!.reason)); + }) + ); + + it('gives the SDK its own fetch and signals every request', async () => { + const fetchMock = jest.fn().mockResolvedValue({ ok: true }); + global.fetch = fetchMock; + const fetchForSdk = await sdkFetch(); + + await fetchForSdk('http://localhost:11434/api/chat', { method: 'POST' }); + + const [url, init] = fetchMock.mock.calls[0]; + expect(url).toBe('http://localhost:11434/api/chat'); + expect(init.method).toBe('POST'); + expect(init.signal).toBeInstanceOf(AbortSignal); + expect(init.signal.aborted).toBe(false); + }); + + it('aborts a request that outlasts its timeout', async () => { + const realTimeout = AbortSignal.timeout.bind(AbortSignal); + const timeout = jest.spyOn(AbortSignal, 'timeout').mockImplementation(() => realTimeout(1)); + global.fetch = stalledFetch(); + const fetchForSdk = await sdkFetch(); + + await expect(fetchForSdk('http://localhost:11434/api/chat')).rejects.toMatchObject({ name: 'TimeoutError' }); + timeout.mockRestore(); + }); + + it('defaults to five minutes', async () => { + const timeout = jest.spyOn(AbortSignal, 'timeout'); + global.fetch = jest.fn().mockResolvedValue({}); + const fetchForSdk = await sdkFetch(); + + await fetchForSdk('http://localhost:11434/api/chat'); + + expect(timeout).toHaveBeenCalledWith(300_000); + timeout.mockRestore(); + }); + + it('still honours the signal of the caller', async () => { + global.fetch = stalledFetch(); + const fetchForSdk = await sdkFetch(); + const controller = new AbortController(); + + const request = fetchForSdk('http://localhost:11434/api/chat', { signal: controller.signal }); + controller.abort(); + + await expect(request).rejects.toMatchObject({ name: 'AbortError' }); + }); + }); + describe('listModels', () => { it('should return list of models', async () => { - const mockDate = new Date('2024-01-01'); - const mockResponse = { - models: [ - { - name: 'llava:7b', - model: 'llava:7b', - size: 1000, - digest: 'abc123', - modified_at: mockDate, - expires_at: mockDate, - size_vram: 500, - details: { - parent_model: '', - format: 'gguf', - family: 'llama', - families: ['llama'], - parameter_size: '7B', - quantization_level: 'Q4_0', - }, - }, - { - name: 'moondream', - model: 'moondream', - size: 2000, - digest: 'def456', - modified_at: mockDate, - expires_at: mockDate, - size_vram: 1000, - details: { - parent_model: '', - format: 'gguf', - family: 'moondream', - families: ['moondream'], - parameter_size: '1.6B', - quantization_level: 'Q4_0', - }, - }, - ], - }; - mockList.mockResolvedValue(mockResponse); + const date = new Date('2024-01-01'); + const modelEntry = (name: string, size: number, digest: string, family: string, parameterSize: string) => ({ + name, + model: name, + size, + digest, + modified_at: date, + expires_at: date, + size_vram: size / 2, + details: { + parent_model: '', + format: 'gguf', + family, + families: [family], + parameter_size: parameterSize, + quantization_level: 'Q4_0', + }, + }); + const models = [ + modelEntry('llava:7b', 1000, 'abc123', 'llama', '7B'), + modelEntry('moondream', 2000, 'def456', 'moondream', '1.6B'), + ]; + mockList.mockResolvedValue({ models }); const result = await service.listModels(); expect(mockList).toHaveBeenCalled(); - expect(result).toEqual(mockResponse.models); + expect(result).toEqual(models); }); it('should throw error when API fails', async () => { diff --git a/src/compare/libs/vlm/providers/ollama/ollama.service.ts b/src/compare/libs/vlm/providers/ollama/ollama.service.ts index e4d4ae78..180b3458 100644 --- a/src/compare/libs/vlm/providers/ollama/ollama.service.ts +++ b/src/compare/libs/vlm/providers/ollama/ollama.service.ts @@ -1,11 +1,13 @@ import { Injectable, Logger } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; -import { Ollama, ChatRequest, ChatResponse, ListResponse, ModelResponse } from 'ollama'; +import { Ollama, ModelResponse } from 'ollama'; import { zodToJsonSchema } from 'zod-to-json-schema'; import { VlmProvider, VlmProviderResponse } from '../../vlm-provider.interface'; -import { VlmConfig, OllamaVlmConfig } from '../../vlm.types'; +import { VlmConfig } from '../../vlm.types'; import { VlmComparisonResultSchema } from '../../vlm-result.schema'; +const TIMEOUT_MS = 300_000; + @Injectable() export class OllamaService implements VlmProvider { private readonly logger: Logger = new Logger(OllamaService.name); @@ -16,44 +18,35 @@ export class OllamaService implements VlmProvider { private getOllamaClient(): Ollama { if (!this.ollamaClient) { const baseUrl = this.configService.getOrThrow('OLLAMA_BASE_URL'); - this.ollamaClient = new Ollama({ host: baseUrl }); + this.ollamaClient = new Ollama({ + host: baseUrl, + // non-streaming chat sends headers only when done: without this a stalled model holds the upload for good + fetch: (url, init) => + fetch(url, { + ...init, + signal: init?.signal + ? AbortSignal.any([init.signal, AbortSignal.timeout(TIMEOUT_MS)]) + : AbortSignal.timeout(TIMEOUT_MS), + }), + }); } return this.ollamaClient; } async generate(config: VlmConfig, images: Uint8Array[]): Promise { - // Type guard: ensure this is Ollama config if (config.provider && config.provider !== 'ollama') { throw new Error(`OllamaService requires OllamaVlmConfig, got ${config.provider}`); } - - const ollamaConfig = config as OllamaVlmConfig; const client = this.getOllamaClient(); try { - const format = zodToJsonSchema(VlmComparisonResultSchema); - - const chatRequest: ChatRequest = { - model: ollamaConfig.model, - messages: [ - { - role: 'user', - content: ollamaConfig.prompt, - images, - }, - ], - stream: false, - format, - options: { - temperature: ollamaConfig.temperature, - }, - }; - - const response: ChatResponse = await client.chat({ - ...chatRequest, + const response = await client.chat({ + model: config.model, + messages: [{ role: 'user', content: config.prompt, images }], stream: false, + format: zodToJsonSchema(VlmComparisonResultSchema), + options: { temperature: config.temperature }, }); - return { content: response.message.content, thinking: response.message.thinking, @@ -67,8 +60,7 @@ export class OllamaService implements VlmProvider { async listModels(): Promise { const client = this.getOllamaClient(); try { - const response: ListResponse = await client.list(); - return response.models; + return (await client.list()).models; } catch (error) { this.logger.error(`Failed to list models: ${(error as Error).message}`); throw error; diff --git a/src/compare/libs/vlm/vlm-inputs.core.spec.ts b/src/compare/libs/vlm/vlm-inputs.core.spec.ts new file mode 100644 index 00000000..ce3d0242 --- /dev/null +++ b/src/compare/libs/vlm/vlm-inputs.core.spec.ts @@ -0,0 +1,94 @@ +import { PNG } from 'pngjs'; +import { pngBytes, solidPng } from '../../test-utils'; +import { computeVlmInputs } from './vlm-inputs.core'; + +const redPng = (width = 4, height = 4, gamma?: number): Buffer => { + const png = solidPng(width, height, [255, 0, 0]); + if (gamma !== undefined) { + png.gamma = gamma; + } + return pngBytes(png); +}; + +const pixelAt = (png: PNG, x: number, y: number) => + Array.from(png.data.subarray(4 * (png.width * y + x), 4 * (png.width * y + x) + 4)); + +describe('computeVlmInputs', () => { + afterEach(() => jest.restoreAllMocks()); + + it('masks the ignore areas of both pictures', () => { + const output = computeVlmInputs({ + kind: 'vlm-inputs', + baseline: redPng(), + image: redPng(), + ignoreAreas: [{ x: 1, y: 1, width: 2, height: 2 }], + }); + + for (const bytes of [output.baseline, output.image]) { + const png = PNG.sync.read(Buffer.from(bytes!)); + expect([png.width, png.height]).toEqual([4, 4]); + for (let y = 0; y < 4; y++) { + for (let x = 0; x < 4; x++) { + const ignored = x >= 1 && x < 3 && y >= 1 && y < 3; + expect(pixelAt(png, x, y)).toEqual(ignored ? [0, 0, 0, 0] : [255, 0, 0, 255]); + } + } + } + }); + + it('keeps the gamma of the source', () => { + const output = computeVlmInputs({ + kind: 'vlm-inputs', + baseline: redPng(4, 4, 0.45455), + image: redPng(), + ignoreAreas: [], + }); + + expect(PNG.sync.read(Buffer.from(output.baseline!)).gamma).toBe(0.45455); + }); + + it('encodes with the Sub filter', () => { + const [baseline, image] = [redPng(), redPng()]; + const write = jest.spyOn(PNG.sync, 'write'); + + computeVlmInputs({ kind: 'vlm-inputs', baseline, image, ignoreAreas: [] }); + + expect(write).toHaveBeenCalledTimes(2); + for (const [, options] of write.mock.calls) { + expect(options).toEqual(expect.objectContaining({ filterType: 1 })); + } + }); + + it('returns null for a picture that does not decode', () => { + const output = computeVlmInputs({ + kind: 'vlm-inputs', + baseline: Buffer.from('not a png'), + image: redPng(), + ignoreAreas: [], + }); + + expect(output.baseline).toBeNull(); + expect(output.image).toBeInstanceOf(Uint8Array); + }); + + it('accepts the Uint8Array views a worker receives', () => { + const source = redPng(); + const view = new Uint8Array(source.buffer.slice(source.byteOffset, source.byteOffset + source.byteLength)); + + const output = computeVlmInputs({ kind: 'vlm-inputs', baseline: view, image: view, ignoreAreas: [] }); + + expect(PNG.sync.read(Buffer.from(output.image!)).width).toBe(4); + }); + + it('returns a structured-clone serializable result', () => { + const output = computeVlmInputs({ + kind: 'vlm-inputs', + baseline: redPng(), + image: Buffer.from('not a png'), + ignoreAreas: [], + }); + + expect(() => structuredClone(output)).not.toThrow(); + expect(structuredClone(output).image).toBeNull(); + }); +}); diff --git a/src/compare/libs/vlm/vlm-inputs.core.ts b/src/compare/libs/vlm/vlm-inputs.core.ts new file mode 100644 index 00000000..d77d0753 --- /dev/null +++ b/src/compare/libs/vlm/vlm-inputs.core.ts @@ -0,0 +1,29 @@ +import { PNG } from 'pngjs'; +import { IgnoreAreaDto } from '../../../test-runs/dto/ignore-area.dto'; +import { applyIgnoreAreas, readPng } from '../../utils'; + +export interface VlmInputsJobInput { + kind: 'vlm-inputs'; + baseline: Buffer | Uint8Array; + image: Buffer | Uint8Array; + ignoreAreas: IgnoreAreaDto[]; +} + +export interface VlmInputsJobOutput { + baseline: Uint8Array | null; + image: Uint8Array | null; +} + +const masked = (bytes: Buffer | Uint8Array, ignoreAreas: IgnoreAreaDto[]): Uint8Array | null => { + let png: PNG; + try { + png = readPng(bytes); + } catch { + return null; + } + return new Uint8Array(PNG.sync.write(applyIgnoreAreas(png, ignoreAreas), { filterType: 1 })); +}; + +export function computeVlmInputs(input: VlmInputsJobInput): VlmInputsJobOutput { + return { baseline: masked(input.baseline, input.ignoreAreas), image: masked(input.image, input.ignoreAreas) }; +} diff --git a/src/compare/libs/vlm/vlm-provider.interface.ts b/src/compare/libs/vlm/vlm-provider.interface.ts index 7c6f2ed9..a086814c 100644 --- a/src/compare/libs/vlm/vlm-provider.interface.ts +++ b/src/compare/libs/vlm/vlm-provider.interface.ts @@ -1,29 +1,10 @@ import { VlmConfig } from './vlm.types'; -/** - * Interface for VLM (Vision Language Model) providers. - * This abstraction allows easy addition of future providers (OpenAI, Anthropic, etc.) - * without modifying VlmService logic. - * Each provider is responsible for converting its own config format to internal API calls. - */ export interface VlmProvider { - /** - * Generate a response from the VLM provider. - * @param config - Provider-specific configuration - * @param images - Array of images as Uint8Array (provider implementations handle format conversion) - * @returns Response containing content and optional thinking field - */ generate(config: VlmConfig, images: Uint8Array[]): Promise; } export interface VlmProviderResponse { - /** - * Main response content from the model - */ content?: string; - - /** - * Optional thinking/reasoning field (some models use this instead of content) - */ thinking?: string; } diff --git a/src/compare/libs/vlm/vlm-result.schema.ts b/src/compare/libs/vlm/vlm-result.schema.ts index a6ec8555..5b9e7fa1 100644 --- a/src/compare/libs/vlm/vlm-result.schema.ts +++ b/src/compare/libs/vlm/vlm-result.schema.ts @@ -1,8 +1,5 @@ import z from 'zod/v3'; -// The answer a VLM provider must give. Its own module because the providers -// validate with it and VlmService uses the providers: kept in vlm.service it -// made an import cycle, which only resolves when vlm.service loads first. export const VlmComparisonResultSchema = z.object({ identical: z.boolean(), description: z.string(), diff --git a/src/compare/libs/vlm/vlm.service.spec.ts b/src/compare/libs/vlm/vlm.service.spec.ts index 5629797c..3faa481d 100644 --- a/src/compare/libs/vlm/vlm.service.spec.ts +++ b/src/compare/libs/vlm/vlm.service.spec.ts @@ -1,4 +1,4 @@ -import { Test, TestingModule } from '@nestjs/testing'; +import { Test } from '@nestjs/testing'; import { TestStatus } from '@prisma/client'; import { PNG } from 'pngjs'; import { StaticService } from '../../../static/static.service'; @@ -9,44 +9,17 @@ import { GeminiService } from './providers/gemini/gemini.service'; import { PixelmatchService } from '../pixelmatch/pixelmatch.service'; import { DiffResult } from '../../../test-runs/diffResult'; import { ImageCompareInput } from '../ImageCompareInput'; +import { VlmConfig } from './vlm.types'; +import { DiffWorkerPool } from '../../diff-worker-pool'; +import { pngBytes, solidPng } from '../../test-utils'; -jest.mock('zod/v3', () => { - const actualZod = jest.requireActual('zod'); - return actualZod; -}); - -const initService = async ({ - getImageMock = jest.fn(), - ollamaGenerateMock = jest.fn(), - geminiGenerateMock = jest.fn(), - pixelmatchGetDiffMock = jest.fn(), -}) => { - const module: TestingModule = await Test.createTestingModule({ - providers: [ - VlmService, - { - provide: StaticService, - useValue: { getImage: getImageMock }, - }, - { - provide: OllamaService, - useValue: { generate: ollamaGenerateMock }, - }, - { - provide: GeminiService, - useValue: { generate: geminiGenerateMock }, - }, - { - provide: PixelmatchService, - useValue: { getDiff: pixelmatchGetDiffMock }, - }, - ], - }).compile(); +const pngBuffer = (width: number, height: number): Buffer => PNG.sync.write(new PNG({ width, height })); +const sequence = (...results: unknown[]) => results.reduce((m, r) => m.mockResolvedValueOnce(r), jest.fn()); +const reply = (identical: boolean, description: string) => ({ content: JSON.stringify({ identical, description }) }); +const modelSaying = (identical: boolean, description = 'ok') => + jest.fn().mockResolvedValue(reply(identical, description)); - return module.get(VlmService); -}; - -const createPixelmatchResult = (overrides: Partial): DiffResult => ({ +const createPixelmatchResult = (overrides: Partial = {}): DiffResult => ({ status: TestStatus.unresolved, diffName: 'diff.png', pixelMisMatchCount: 100, @@ -55,197 +28,154 @@ const createPixelmatchResult = (overrides: Partial): DiffResult => ( ...overrides, }); -const createImageMocks = () => { - const image = new PNG({ width: 20, height: 20 }); - const diffImage = new PNG({ width: 20, height: 20 }); - return { - image, - diffImage, - getImageMock: jest.fn().mockReturnValueOnce(image).mockReturnValueOnce(image).mockReturnValueOnce(diffImage), - }; +interface Deps { + pixelmatch?: DiffResult; + buffers?: unknown[]; + getImageBuffer?: jest.Mock; + ollama?: jest.Mock; + gemini?: jest.Mock; +} + +const setup = async ({ + pixelmatch = createPixelmatchResult(), + buffers = [pngBuffer(20, 20), pngBuffer(20, 20), pngBuffer(20, 20)], + getImageBuffer = sequence(...buffers), + ollama = jest.fn(), + gemini = jest.fn(), +}: Deps = {}) => { + const pixelmatchGetDiff = jest.fn().mockResolvedValue(pixelmatch); + const module = await Test.createTestingModule({ + providers: [ + VlmService, + { provide: StaticService, useValue: { getImageBuffer } }, + { provide: OllamaService, useValue: { generate: ollama } }, + { provide: GeminiService, useValue: { generate: gemini } }, + { provide: PixelmatchService, useValue: { getDiff: pixelmatchGetDiff } }, + DiffWorkerPool, + ], + }).compile(); + return { service: module.get(VlmService), pixelmatchGetDiff, getImageBuffer, ollama, gemini }; }; -describe('VlmService', () => { - it('should return NO_BASELINE_RESULT when pixelmatch returns no baseline', async () => { - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(NO_BASELINE_RESULT); - const service = await initService({ pixelmatchGetDiffMock }); +const input: ImageCompareInput = { + baseline: 'baseline', + image: 'image', + diffTollerancePercent: 0.1, + ignoreAreas: [], + saveDiffAsFile: true, +}; - const result = await service.getDiff( - { baseline: null, image: 'image', diffTollerancePercent: 0.1, ignoreAreas: [], saveDiffAsFile: true }, - DEFAULT_CONFIG - ); +const geminiConfig = { + provider: 'gemini' as const, + model: 'gemini-1.5-pro', + prompt: DEFAULT_CONFIG.prompt, + temperature: 0.1, + apiKey: 'test-api-key', +}; - expect(result).toStrictEqual(NO_BASELINE_RESULT); - expect(pixelmatchGetDiffMock).toHaveBeenCalled(); - }); +const threeImages = expect.arrayContaining([expect.any(Uint8Array), expect.any(Uint8Array), expect.any(Uint8Array)]); +describe('VlmService', () => { it('should return OK immediately when pixelmatch finds no differences (VLM not called)', async () => { - const pixelmatchResult: DiffResult = { - ...EQUAL_RESULT, - status: TestStatus.ok, - pixelMisMatchCount: 0, - diffPercent: 0, - }; - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(pixelmatchResult); - const ollamaGenerateMock = jest.fn(); - const service = await initService({ pixelmatchGetDiffMock, ollamaGenerateMock }); - - const result = await service.getDiff( - { baseline: 'baseline', image: 'image', diffTollerancePercent: 0.1, ignoreAreas: [], saveDiffAsFile: false }, - DEFAULT_CONFIG - ); + const { service, ollama } = await setup({ + pixelmatch: { ...EQUAL_RESULT, status: TestStatus.ok, pixelMisMatchCount: 0, diffPercent: 0 }, + }); + + const result = await service.getDiff(input, DEFAULT_CONFIG); expect(result.status).toBe(TestStatus.ok); expect(result.pixelMisMatchCount).toBe(0); expect(result.diffPercent).toBe(0); - expect(ollamaGenerateMock).not.toHaveBeenCalled(); // VLM should not be called + expect(ollama).not.toHaveBeenCalled(); }); - // CompareService leaves the rendering-noise filter to VLM, which hands it to - // the pixelmatch pass it runs first: noise then passes before any model sees it it('applies the rendering-noise filter through pixelmatch, before the model', async () => { - const blurAmplitude = { sigma: 1, threshold: 80 }; - const pixelmatchGetDiffMock = jest - .fn() - .mockResolvedValue(createPixelmatchResult({ status: TestStatus.ok, diffName: null })); - const ollamaGenerateMock = jest.fn(); - const service = await initService({ pixelmatchGetDiffMock, ollamaGenerateMock }); - - const result = await service.getDiff( - { - baseline: 'baseline', - image: 'image', - diffTollerancePercent: 0, - ignoreAreas: [], - saveDiffAsFile: true, - blurAmplitude, - }, - DEFAULT_CONFIG - ); + const blurAmplitude = { threshold: 80 }; + const { service, pixelmatchGetDiff, ollama } = await setup({ + pixelmatch: createPixelmatchResult({ status: TestStatus.ok, diffName: null }), + }); + + const result = await service.getDiff({ ...input, diffTollerancePercent: 0, blurAmplitude }, DEFAULT_CONFIG); expect(service.appliesBlurAmplitudeFilter).toBe(true); - expect(pixelmatchGetDiffMock).toHaveBeenCalledWith(expect.objectContaining({ blurAmplitude }), expect.anything()); + expect(pixelmatchGetDiff).toHaveBeenCalledWith(expect.objectContaining({ blurAmplitude }), expect.anything()); expect(result.status).toBe(TestStatus.ok); - expect(ollamaGenerateMock).not.toHaveBeenCalled(); + expect(ollama).not.toHaveBeenCalled(); }); it.each([ [ 'override to OK when VLM says not noticeable', - { identical: true, description: 'Differences are minor rendering artifacts, not noticeable to humans.' }, + reply(true, 'Differences are minor rendering artifacts, not noticeable to humans.'), TestStatus.ok, { pixelMisMatchCount: 100, diffPercent: 2.5 }, ], [ 'keep unresolved when VLM confirms noticeable', - { - identical: false, - description: 'Button text changed from Submit to Send, and user count changed from 12 to 15.', - }, + reply(false, 'Button text changed from Submit to Send, and user count changed from 12 to 15.'), TestStatus.unresolved, { pixelMisMatchCount: 500, diffPercent: 12.5 }, ], ])('should %s', async (_, vlmResponse, expectedStatus, pixelmatchOverrides) => { - const pixelmatchResult = createPixelmatchResult(pixelmatchOverrides); - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(pixelmatchResult); - const { getImageMock } = createImageMocks(); - const ollamaGenerateMock = jest.fn().mockResolvedValue({ - content: JSON.stringify(vlmResponse), + const { service, ollama } = await setup({ + pixelmatch: createPixelmatchResult(pixelmatchOverrides), + ollama: jest.fn().mockResolvedValue(vlmResponse), }); - const service = await initService({ getImageMock, ollamaGenerateMock, pixelmatchGetDiffMock }); - const result = await service.getDiff( - { baseline: 'baseline', image: 'image', diffTollerancePercent: 0.1, ignoreAreas: [], saveDiffAsFile: false }, - DEFAULT_CONFIG - ); + const result = await service.getDiff(input, DEFAULT_CONFIG); expect(result.status).toBe(expectedStatus); - expect(result.vlmDescription).toBe(vlmResponse.description); + expect(result.vlmDescription).toBe(JSON.parse(vlmResponse.content).description); expect(result.pixelMisMatchCount).toBe(pixelmatchOverrides.pixelMisMatchCount); expect(result.diffPercent).toBe(pixelmatchOverrides.diffPercent); - expect(ollamaGenerateMock).toHaveBeenCalledWith( - DEFAULT_CONFIG, - expect.arrayContaining([expect.any(Uint8Array), expect.any(Uint8Array), expect.any(Uint8Array)]) - ); + expect(ollama).toHaveBeenCalledWith(DEFAULT_CONFIG, threeImages); + }); + + it('sends the image it is given to the model without reading it again', async () => { + const image = pngBuffer(20, 20); + const { service, getImageBuffer, ollama } = await setup({ + buffers: [image, pngBuffer(20, 20)], + ollama: modelSaying(true), + }); + + await service.getDiff({ ...input, imageBuffer: image }, DEFAULT_CONFIG); + + expect(getImageBuffer.mock.calls).toEqual([['baseline'], ['diff.png']]); + expect(ollama).toHaveBeenCalled(); }); it.each([ - [ - 'invalid JSON response', - { content: 'Invalid JSON response from model' }, - { pixelMisMatchCount: 200, diffPercent: 5 }, - ], - ['API error', null, { pixelMisMatchCount: 300, diffPercent: 7.5 }], - ])('should handle %s gracefully and return pixelmatch result', async (_, vlmResponse, pixelmatchOverrides) => { - const pixelmatchResult = createPixelmatchResult(pixelmatchOverrides); - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(pixelmatchResult); - const { getImageMock } = createImageMocks(); - const ollamaGenerateMock = vlmResponse - ? jest.fn().mockResolvedValue(vlmResponse) - : jest.fn().mockRejectedValue(new Error('Connection refused')); - const service = await initService({ getImageMock, ollamaGenerateMock, pixelmatchGetDiffMock }); - - const result = await service.getDiff( - { baseline: 'baseline', image: 'image', diffTollerancePercent: 0.1, ignoreAreas: [], saveDiffAsFile: false }, - DEFAULT_CONFIG - ); + ['invalid JSON response', jest.fn().mockResolvedValue({ content: 'Invalid JSON response from model' }), 200, 5], + ['API error', jest.fn().mockRejectedValue(new Error('Connection refused')), 300, 7.5], + ])('should handle %s gracefully and return pixelmatch result', async (_, ollama, pixelMisMatchCount, diffPercent) => { + const { service } = await setup({ + pixelmatch: createPixelmatchResult({ pixelMisMatchCount, diffPercent }), + ollama, + }); + + const result = await service.getDiff(input, DEFAULT_CONFIG); expect(result.status).toBe(TestStatus.unresolved); expect(result.vlmDescription).toContain('VLM analysis failed'); - expect(result.pixelMisMatchCount).toBe(pixelmatchOverrides.pixelMisMatchCount); - expect(result.diffPercent).toBe(pixelmatchOverrides.diffPercent); + expect(result.pixelMisMatchCount).toBe(pixelMisMatchCount); + expect(result.diffPercent).toBe(diffPercent); }); it.each([ ['ollama', DEFAULT_CONFIG], - [ - 'gemini', - { - provider: 'gemini' as const, - model: 'gemini-1.5-pro', - prompt: DEFAULT_CONFIG.prompt, - temperature: 0.1, - apiKey: 'test-api-key', - }, - ], + ['gemini', geminiConfig], ])('should send baseline and image with ignore areas masked to %s', async (_, config) => { - const createFilledImage = (r: number, g: number, b: number) => { - const png = new PNG({ width: 20, height: 20 }); - for (let k = 0; k < png.data.length; k += 4) { - png.data[k] = r; - png.data[k + 1] = g; - png.data[k + 2] = b; - png.data[k + 3] = 255; - } - return png; - }; - const baseline = createFilledImage(255, 255, 255); - const image = createFilledImage(255, 0, 0); - const diffImage = new PNG({ width: 20, height: 20 }); - const getImageMock = jest - .fn() - .mockReturnValueOnce(baseline) - .mockReturnValueOnce(image) - .mockReturnValueOnce(diffImage); - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(createPixelmatchResult({})); - const generateMock = jest.fn().mockResolvedValue({ - content: '{"identical": false, "description": "Changed."}', - }); - const service = await initService({ - getImageMock, - pixelmatchGetDiffMock, - ollamaGenerateMock: generateMock, - geminiGenerateMock: generateMock, + const generate = modelSaying(false, 'Changed.'); + const { service } = await setup({ + buffers: [pngBytes(solidPng(20, 20)), pngBytes(solidPng(20, 20, [255, 0, 0])), pngBuffer(20, 20)], + ollama: generate, + gemini: generate, }); - const ignoreAreas = [{ x: 2, y: 3, width: 5, height: 4 }]; - await service.getDiff( - { baseline: 'baseline', image: 'image', diffTollerancePercent: 0.1, ignoreAreas, saveDiffAsFile: false }, - config - ); + await service.getDiff({ ...input, ignoreAreas: [{ x: 2, y: 3, width: 5, height: 4 }] }, config); - expect(generateMock).toHaveBeenCalledTimes(1); - const [sentBaseline, sentImage] = (generateMock.mock.calls[0][1] as Uint8Array[]).map((bytes) => + expect(generate).toHaveBeenCalledTimes(1); + const [sentBaseline, sentImage] = (generate.mock.calls[0][1] as Uint8Array[]).map((bytes) => PNG.sync.read(Buffer.from(bytes)) ); const pixelAt = (png: PNG, x: number, y: number) => @@ -260,60 +190,35 @@ describe('VlmService', () => { }); it('should use custom model and temperature from config', async () => { - const pixelmatchResult = createPixelmatchResult({ pixelMisMatchCount: 150, diffPercent: 3.75 }); - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(pixelmatchResult); - const { getImageMock } = createImageMocks(); - const ollamaGenerateMock = jest.fn().mockResolvedValue({ - content: '{"identical": true, "description": "No differences."}', - }); - const service = await initService({ getImageMock, ollamaGenerateMock, pixelmatchGetDiffMock }); - + const { service, ollama } = await setup({ ollama: modelSaying(true, 'No differences.') }); const customConfig = { ...DEFAULT_CONFIG, model: 'llava:13b', prompt: 'Custom context', temperature: 0.2 }; - await service.getDiff( - { baseline: 'baseline', image: 'image', diffTollerancePercent: 0.1, ignoreAreas: [], saveDiffAsFile: false }, - customConfig - ); - expect(ollamaGenerateMock).toHaveBeenCalledWith( - customConfig, - expect.arrayContaining([expect.any(Uint8Array), expect.any(Uint8Array), expect.any(Uint8Array)]) - ); + await service.getDiff(input, customConfig); + + expect(ollama).toHaveBeenCalledWith(customConfig, threeImages); }); it('should use thinking field when useThinking is true', async () => { - const pixelmatchResult = createPixelmatchResult({ pixelMisMatchCount: 80, diffPercent: 2 }); - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(pixelmatchResult); - const { getImageMock } = createImageMocks(); - const ollamaGenerateMock = jest.fn().mockResolvedValue({ - content: '{"identical": false, "description": "Content field"}', - thinking: '{"identical": true, "description": "Thinking field"}', + const { service } = await setup({ + ollama: jest.fn().mockResolvedValue({ + content: JSON.stringify({ identical: false, description: 'Content field' }), + thinking: JSON.stringify({ identical: true, description: 'Thinking field' }), + }), }); - const service = await initService({ getImageMock, ollamaGenerateMock, pixelmatchGetDiffMock }); - const result = await service.getDiff( - { baseline: 'baseline', image: 'image', diffTollerancePercent: 0.1, ignoreAreas: [], saveDiffAsFile: false }, - { ...DEFAULT_CONFIG, useThinking: true } - ); + const result = await service.getDiff(input, { ...DEFAULT_CONFIG, useThinking: true }); expect(result.status).toBe(TestStatus.ok); expect(result.vlmDescription).toBe('Thinking field'); }); it('should handle missing diff image gracefully', async () => { - const pixelmatchResult = createPixelmatchResult({ diffName: null }); - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(pixelmatchResult); - const { image } = createImageMocks(); - const getImageMock = jest.fn().mockReturnValueOnce(image).mockReturnValueOnce(image).mockReturnValueOnce(null); - const ollamaGenerateMock = jest.fn(); - const service = await initService({ getImageMock, ollamaGenerateMock, pixelmatchGetDiffMock }); - - const result = await service.getDiff( - { baseline: 'baseline', image: 'image', diffTollerancePercent: 0.1, ignoreAreas: [], saveDiffAsFile: false }, - DEFAULT_CONFIG - ); + const pixelmatch = createPixelmatchResult({ diffName: null }); + const image = pngBuffer(20, 20); + const { service, ollama } = await setup({ pixelmatch, buffers: [image, image, null] }); - expect(result).toEqual(pixelmatchResult); - expect(ollamaGenerateMock).not.toHaveBeenCalled(); + expect(await service.getDiff(input, DEFAULT_CONFIG)).toEqual(pixelmatch); + expect(ollama).not.toHaveBeenCalled(); }); it.each([ @@ -321,7 +226,6 @@ describe('VlmService', () => { ['invalid JSON', 'invalid', DEFAULT_CONFIG], ['partial config, over the defaults', '{"model":"llava:7b"}', { ...DEFAULT_CONFIG, model: 'llava:7b' }], [ - // what a project created without a config holds: the column default is pixelmatch's 'the pixelmatch column default, over the defaults', '{ "threshold": 0.1, "ignoreAntialiasing": true, "allowDiffDimensions": false }', { ...DEFAULT_CONFIG, threshold: 0.1, ignoreAntialiasing: true, allowDiffDimensions: false }, @@ -329,105 +233,113 @@ describe('VlmService', () => { [ 'full config', '{"model":"llava:13b","prompt":"Custom prompt","temperature":0.2,"useThinking":true}', - { - provider: 'ollama', - model: 'llava:13b', - prompt: 'Custom prompt', - temperature: 0.2, - useThinking: true, - }, + { provider: 'ollama', model: 'llava:13b', prompt: 'Custom prompt', temperature: 0.2, useThinking: true }, ], ])('should parse config: %s', async (_, configJson, expected) => { - const service = await initService({}); + const { service } = await setup(); + expect(service.parseConfig(configJson)).toEqual(expected); }); describe('characterization', () => { - const input: ImageCompareInput = { - baseline: 'baseline', - image: 'image', - diffTollerancePercent: 0.1, - ignoreAreas: [], - saveDiffAsFile: false, - }; - - it('always asks pixelmatch for a saved diff with the pixelmatch defaults', async () => { - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(EQUAL_RESULT); - const service = await initService({ pixelmatchGetDiffMock }); + it('asks pixelmatch for a saved diff with the pixelmatch defaults when the diff is saved', async () => { + const { service, pixelmatchGetDiff } = await setup({ pixelmatch: EQUAL_RESULT }); await service.getDiff(input, { ...DEFAULT_CONFIG, temperature: 0.9 }); - expect(pixelmatchGetDiffMock).toHaveBeenCalledWith( - { ...input, saveDiffAsFile: true }, - { threshold: 0.1, ignoreAntialiasing: true } - ); + expect(pixelmatchGetDiff).toHaveBeenCalledWith(input, { threshold: 0.1, ignoreAntialiasing: true }); }); - it('returns a pixelmatch result with status new without asking the model', async () => { - const pixelmatchResult = createPixelmatchResult({ status: TestStatus.new }); - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(pixelmatchResult); - const getImageMock = jest.fn(); - const ollamaGenerateMock = jest.fn(); - const service = await initService({ pixelmatchGetDiffMock, getImageMock, ollamaGenerateMock }); + it.each([ + ['a pixelmatch result with status new', createPixelmatchResult({ status: TestStatus.new }), input], + ['the no-baseline result', NO_BASELINE_RESULT, { ...input, baseline: null }], + ])('returns %s without loading images or asking the model', async (_, pixelmatch, data) => { + const { service, getImageBuffer, ollama, pixelmatchGetDiff } = await setup({ + pixelmatch, + getImageBuffer: jest.fn(), + }); - const result = await service.getDiff(input, DEFAULT_CONFIG); + const result = await service.getDiff(data, DEFAULT_CONFIG); - expect(result).toBe(pixelmatchResult); - expect(getImageMock).not.toHaveBeenCalled(); - expect(ollamaGenerateMock).not.toHaveBeenCalled(); + expect(result).toBe(pixelmatch); + expect(pixelmatchGetDiff).toHaveBeenCalled(); + expect(getImageBuffer).not.toHaveBeenCalled(); + expect(ollama).not.toHaveBeenCalled(); }); - it('still loads images when pixelmatch reports no baseline', async () => { - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(NO_BASELINE_RESULT); - const getImageMock = jest.fn().mockResolvedValue(undefined); - const service = await initService({ pixelmatchGetDiffMock, getImageMock }); + describe('auto-approve probe (saveDiffAsFile: false)', () => { + const probe: ImageCompareInput = { ...input, saveDiffAsFile: false }; + + it('answers an auto-approve probe with the pixelmatch verdict, saving nothing and asking no model', async () => { + const pixelmatch = createPixelmatchResult({ status: TestStatus.unresolved }); + const { service, pixelmatchGetDiff, getImageBuffer, ollama, gemini } = await setup({ + pixelmatch, + getImageBuffer: jest.fn(), + }); + + const result = await service.getDiff(probe, DEFAULT_CONFIG); + + expect(result).toStrictEqual(pixelmatch); + expect(pixelmatchGetDiff).toHaveBeenCalledWith( + expect.objectContaining({ saveDiffAsFile: false }), + expect.anything() + ); + expect(ollama).not.toHaveBeenCalled(); + expect(gemini).not.toHaveBeenCalled(); + expect(getImageBuffer).not.toHaveBeenCalled(); + }); - const result = await service.getDiff({ ...input, baseline: null }, DEFAULT_CONFIG); + it('passes an ok probe through', async () => { + const pixelmatch = createPixelmatchResult({ status: TestStatus.ok, diffName: null }); + const { service, ollama } = await setup({ pixelmatch }); - // NOTE: current behavior — NO_BASELINE_RESULT has status undefined, not `new`, - // so the early return above never applies and the VLM path runs until it - // finds the images missing. - expect(getImageMock).toHaveBeenCalledWith(null); - expect(result).toBe(NO_BASELINE_RESULT); + expect(await service.getDiff(probe, DEFAULT_CONFIG)).toStrictEqual(pixelmatch); + expect(ollama).not.toHaveBeenCalled(); + }); }); it('reports an empty model response as a failed analysis', async () => { - const pixelmatchResult = createPixelmatchResult({}); - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(pixelmatchResult); - const { getImageMock } = createImageMocks(); - const ollamaGenerateMock = jest.fn().mockResolvedValue({ content: '', thinking: undefined }); - const service = await initService({ getImageMock, ollamaGenerateMock, pixelmatchGetDiffMock }); - - const result = await service.getDiff(input, DEFAULT_CONFIG); + const pixelmatch = createPixelmatchResult(); + const { service } = await setup({ + pixelmatch, + ollama: jest.fn().mockResolvedValue({ content: '', thinking: undefined }), + }); - expect(result).toStrictEqual({ - ...pixelmatchResult, + expect(await service.getDiff(input, DEFAULT_CONFIG)).toStrictEqual({ + ...pixelmatch, vlmDescription: 'VLM analysis failed: Empty response from model', }); }); - it('falls back to the thinking field when the content is empty', async () => { - const { getImageMock } = createImageMocks(); - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(createPixelmatchResult({})); - const ollamaGenerateMock = jest.fn().mockResolvedValue({ - content: '', - thinking: '{"identical": true, "description": "From thinking"}', - }); - const service = await initService({ getImageMock, ollamaGenerateMock, pixelmatchGetDiffMock }); - - const result = await service.getDiff(input, DEFAULT_CONFIG); - - expect(result.status).toBe(TestStatus.ok); - expect(result.vlmDescription).toBe('From thinking'); + it.each([ + [ + 'the thinking field when the content is empty', + { content: '', thinking: JSON.stringify({ identical: true, description: 'From thinking' }) }, + DEFAULT_CONFIG, + TestStatus.ok, + 'From thinking', + ], + [ + 'the content field when thinking is preferred but empty', + reply(false, 'From content'), + { ...DEFAULT_CONFIG, useThinking: true }, + TestStatus.unresolved, + 'From content', + ], + ])('falls back to %s', async (_, response, config, status, description) => { + const { service } = await setup({ ollama: jest.fn().mockResolvedValue(response) }); + + const result = await service.getDiff(input, config); + + expect(result.status).toBe(status); + expect(result.vlmDescription).toBe(description); }); it('marks the run unresolved when the model sees a difference, whatever status pixelmatch gave', async () => { - const { getImageMock } = createImageMocks(); - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(createPixelmatchResult({ status: TestStatus.failed })); - const ollamaGenerateMock = jest.fn().mockResolvedValue({ - content: '{"identical": false, "description": "Button moved"}', + const { service } = await setup({ + pixelmatch: createPixelmatchResult({ status: TestStatus.failed }), + ollama: modelSaying(false, 'Button moved'), }); - const service = await initService({ getImageMock, ollamaGenerateMock, pixelmatchGetDiffMock }); const result = await service.getDiff(input, DEFAULT_CONFIG); @@ -435,25 +347,8 @@ describe('VlmService', () => { expect(result.vlmDescription).toBe('Button moved'); }); - it('falls back to the content field when thinking is preferred but empty', async () => { - const { getImageMock } = createImageMocks(); - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(createPixelmatchResult({})); - const ollamaGenerateMock = jest.fn().mockResolvedValue({ - content: '{"identical": false, "description": "From content"}', - }); - const service = await initService({ getImageMock, ollamaGenerateMock, pixelmatchGetDiffMock }); - - const result = await service.getDiff(input, { ...DEFAULT_CONFIG, useThinking: true }); - - expect(result.status).toBe(TestStatus.unresolved); - expect(result.vlmDescription).toBe('From content'); - }); - it('fills in a description when the model returns an empty one', async () => { - const { getImageMock } = createImageMocks(); - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(createPixelmatchResult({})); - const ollamaGenerateMock = jest.fn().mockResolvedValue({ content: '{"identical": true, "description": ""}' }); - const service = await initService({ getImageMock, ollamaGenerateMock, pixelmatchGetDiffMock }); + const { service } = await setup({ ollama: modelSaying(true, '') }); const result = await service.getDiff(input, DEFAULT_CONFIG); @@ -461,11 +356,7 @@ describe('VlmService', () => { }); it('reports a response that does not match the schema as a failed analysis', async () => { - const { getImageMock } = createImageMocks(); - const pixelmatchResult = createPixelmatchResult({}); - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(pixelmatchResult); - const ollamaGenerateMock = jest.fn().mockResolvedValue({ content: '{"identical": "yes"}' }); - const service = await initService({ getImageMock, ollamaGenerateMock, pixelmatchGetDiffMock }); + const { service } = await setup({ ollama: jest.fn().mockResolvedValue({ content: '{"identical": "yes"}' }) }); const result = await service.getDiff(input, DEFAULT_CONFIG); @@ -477,98 +368,109 @@ describe('VlmService', () => { ['no provider', undefined], ['an unknown provider', 'openai'], ])('uses Ollama when the config names %s', async (_, provider) => { - const { getImageMock } = createImageMocks(); - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(createPixelmatchResult({})); - const ollamaGenerateMock = jest.fn().mockResolvedValue({ content: '{"identical": true, "description": "ok"}' }); - const geminiGenerateMock = jest.fn(); - const service = await initService({ - getImageMock, - ollamaGenerateMock, - geminiGenerateMock, - pixelmatchGetDiffMock, - }); - const config = { ...DEFAULT_CONFIG, provider } as any; + const { service, ollama, gemini } = await setup({ ollama: modelSaying(true) }); + const config = { ...DEFAULT_CONFIG, provider } as unknown as VlmConfig; await service.getDiff(input, config); - expect(ollamaGenerateMock).toHaveBeenCalledWith(config, expect.any(Array)); - expect(geminiGenerateMock).not.toHaveBeenCalled(); + expect(ollama).toHaveBeenCalledWith(config, expect.any(Array)); + expect(gemini).not.toHaveBeenCalled(); }); it('sends baseline, image and diff to the model as PNG bytes in that order', async () => { - const baseline = new PNG({ width: 1, height: 1 }); - const image = new PNG({ width: 2, height: 1 }); - const diff = new PNG({ width: 3, height: 1 }); - const getImageMock = jest - .fn() - .mockResolvedValueOnce(baseline) - .mockResolvedValueOnce(image) - .mockResolvedValueOnce(diff); - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(createPixelmatchResult({ diffName: 'diff.png' })); - const ollamaGenerateMock = jest.fn().mockResolvedValue({ content: '{"identical": true, "description": "ok"}' }); - const service = await initService({ getImageMock, ollamaGenerateMock, pixelmatchGetDiffMock }); + const diff = pngBuffer(3, 1); + const { service, getImageBuffer, ollama } = await setup({ + buffers: [pngBuffer(1, 1), pngBuffer(2, 1), diff], + ollama: modelSaying(true), + }); await service.getDiff(input, DEFAULT_CONFIG); - expect(getImageMock.mock.calls).toEqual([['baseline'], ['image'], ['diff.png']]); - const [[, images]] = ollamaGenerateMock.mock.calls; + expect(getImageBuffer.mock.calls).toEqual([['baseline'], ['image'], ['diff.png']]); + const [[, images]] = ollama.mock.calls; expect(images.map((bytes: Uint8Array) => PNG.sync.read(Buffer.from(bytes)).width)).toEqual([1, 2, 3]); + expect(Buffer.from(images[2]).equals(diff)).toBe(true); + }); + + it('masks ignore areas in a worker job', async () => { + const run = jest.spyOn(DiffWorkerPool.prototype, 'run'); + const opaque = new PNG({ width: 4, height: 4 }); + opaque.data.fill(255); + const picture = PNG.sync.write(opaque); + const { service, ollama } = await setup({ + buffers: [picture, picture, pngBuffer(4, 4)], + ollama: modelSaying(true), + }); + const ignoreAreas = [{ x: 1, y: 1, width: 2, height: 2 }]; + + await service.getDiff({ ...input, ignoreAreas }, DEFAULT_CONFIG); + + expect(run).toHaveBeenCalledWith(expect.objectContaining({ kind: 'vlm-inputs', ignoreAreas })); + const [[, [sentBaseline]]] = ollama.mock.calls; + const png = PNG.sync.read(Buffer.from(sentBaseline)); + const alpha = (x: number, y: number) => png.data[4 * (png.width * y + x) + 3]; + expect([alpha(0, 0), alpha(1, 1), alpha(2, 2), alpha(3, 3)]).toEqual([255, 0, 0, 255]); + run.mockRestore(); + }); + + it.each([ + ['a picture does not decode', [Buffer.from('not a png'), pngBuffer(2, 2), pngBuffer(2, 2)]], + ['the image does not decode', [pngBuffer(2, 2), Buffer.from('not a png'), pngBuffer(2, 2)]], + ['a stored picture is missing', [null, pngBuffer(2, 2), pngBuffer(2, 2)]], + ])('returns the pixelmatch result when %s', async (_, buffers) => { + const pixelmatch = createPixelmatchResult(); + const { service, ollama } = await setup({ pixelmatch, buffers }); + + expect(await service.getDiff(input, DEFAULT_CONFIG)).toStrictEqual(pixelmatch); + expect(ollama).not.toHaveBeenCalled(); + }); + }); + + describe('model concurrency', () => { + it('asks the model at most four times at once', async () => { + const gates: { resolve: (value: { content: string }) => void }[] = []; + const ollama = jest.fn().mockImplementation( + () => + new Promise((resolve) => { + gates.push({ resolve }); + }) + ); + const { service } = await setup({ + getImageBuffer: jest.fn().mockImplementation(async () => pngBuffer(2, 2)), + ollama, + }); + const answer = reply(true, 'ok'); + + const diffs = [1, 2, 3, 4, 5].map(() => service.getDiff(input, DEFAULT_CONFIG)); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(ollama).toHaveBeenCalledTimes(4); + + gates[0].resolve(answer); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(ollama).toHaveBeenCalledTimes(5); + + gates.slice(1).forEach((gate) => gate.resolve(answer)); + await Promise.all(diffs); }); }); describe('Gemini provider', () => { it('should use GeminiService when provider is gemini', async () => { - const pixelmatchResult = createPixelmatchResult({}); - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(pixelmatchResult); - const { getImageMock } = createImageMocks(); - const geminiGenerateMock = jest.fn().mockResolvedValue({ - content: '{"identical": true, "description": "No noticeable differences."}', - }); - const ollamaGenerateMock = jest.fn(); - const service = await initService({ - getImageMock, - geminiGenerateMock, - ollamaGenerateMock, - pixelmatchGetDiffMock, - }); + const { service, gemini, ollama } = await setup({ gemini: modelSaying(true, 'No noticeable differences.') }); - const geminiConfig = { - provider: 'gemini' as const, - model: 'gemini-1.5-pro', - prompt: DEFAULT_CONFIG.prompt, - temperature: 0.1, - apiKey: 'test-api-key', - }; - - const result = await service.getDiff( - { baseline: 'baseline', image: 'image', diffTollerancePercent: 0.1, ignoreAreas: [], saveDiffAsFile: false }, - geminiConfig - ); + const result = await service.getDiff(input, geminiConfig); expect(result.status).toBe(TestStatus.ok); expect(result.vlmDescription).toBe('No noticeable differences.'); - expect(geminiGenerateMock).toHaveBeenCalledWith( - geminiConfig, - expect.arrayContaining([expect.any(Uint8Array), expect.any(Uint8Array), expect.any(Uint8Array)]) - ); - expect(ollamaGenerateMock).not.toHaveBeenCalled(); + expect(gemini).toHaveBeenCalledWith(geminiConfig, threeImages); + expect(ollama).not.toHaveBeenCalled(); }); it('should handle error when Gemini API key is missing', async () => { - const pixelmatchResult = createPixelmatchResult({}); - const pixelmatchGetDiffMock = jest.fn().mockResolvedValue(pixelmatchResult); - const { getImageMock } = createImageMocks(); - const service = await initService({ getImageMock, pixelmatchGetDiffMock }); - - const result = await service.getDiff( - { baseline: 'baseline', image: 'image', diffTollerancePercent: 0.1, ignoreAreas: [], saveDiffAsFile: false }, - { - provider: 'gemini' as const, - model: 'gemini-1.5-pro', - prompt: DEFAULT_CONFIG.prompt, - temperature: 0.1, - } as any - ); + const { service } = await setup(); + const { apiKey: _apiKey, ...withoutKey } = geminiConfig; + + const result = await service.getDiff(input, withoutKey as unknown as VlmConfig); expect(result.vlmDescription).toContain('VLM analysis failed'); expect(result.status).toBe(TestStatus.unresolved); diff --git a/src/compare/libs/vlm/vlm.service.ts b/src/compare/libs/vlm/vlm.service.ts index 7ecadd50..67a4db3a 100644 --- a/src/compare/libs/vlm/vlm.service.ts +++ b/src/compare/libs/vlm/vlm.service.ts @@ -2,16 +2,17 @@ import { Injectable, Logger } from '@nestjs/common'; import { TestStatus } from '@prisma/client'; import { StaticService } from '../../../static/static.service'; import { DiffResult } from '../../../test-runs/diffResult'; -import { applyIgnoreAreas, parseConfig } from '../../utils'; +import { parseConfig } from '../../utils'; +import { ConcurrencyLimiter } from './concurrency-limiter'; import { ImageComparator } from '../image-comparator.interface'; import { ImageCompareInput } from '../ImageCompareInput'; import { VlmConfig, OllamaVlmConfig } from './vlm.types'; -import { VlmProvider } from './vlm-provider.interface'; +import { VlmProvider, VlmProviderResponse } from './vlm-provider.interface'; import { VlmComparisonResultSchema } from './vlm-result.schema'; import { OllamaService } from './providers/ollama/ollama.service'; import { GeminiService } from './providers/gemini/gemini.service'; +import { DiffWorkerPool } from '../../diff-worker-pool'; import { PixelmatchService, DEFAULT_CONFIG as PIXELMATCH_DEFAULT_CONFIG } from '../pixelmatch/pixelmatch.service'; -import { PNG } from 'pngjs'; export const DEFAULT_PROMPT = `You are provided with three images: 1. First image: baseline screenshot @@ -22,8 +23,6 @@ Spot any difference in text, color, shape and position of elements - treat as di Ignore minor rendering artifacts that are imperceptible to users like antialiasing. Describe the difference in about 100 words.`; -export { VlmComparisonResultSchema } from './vlm-result.schema'; - export const DEFAULT_CONFIG: OllamaVlmConfig = { provider: 'ollama', model: 'gemma3:12b', @@ -35,15 +34,15 @@ export const DEFAULT_CONFIG: OllamaVlmConfig = { @Injectable() export class VlmService implements ImageComparator { private readonly logger: Logger = new Logger(VlmService.name); - // through the pixelmatch pass below, which passes rendering noise before the - // model is ever asked about it readonly appliesBlurAmplitudeFilter = true; + private readonly limiter = new ConcurrencyLimiter(4); constructor( private readonly staticService: StaticService, private readonly ollamaService: OllamaService, private readonly geminiService: GeminiService, - private readonly pixelmatchService: PixelmatchService + private readonly pixelmatchService: PixelmatchService, + private readonly diffWorkerPool: DiffWorkerPool ) {} parseConfig(configJson: string): VlmConfig { @@ -51,55 +50,54 @@ export class VlmService implements ImageComparator { } async getDiff(data: ImageCompareInput, config: VlmConfig): Promise { - const pixelmatchResult = await this.pixelmatchService.getDiff( - { - ...data, - saveDiffAsFile: true, - }, - PIXELMATCH_DEFAULT_CONFIG - ); - - if (pixelmatchResult.status === TestStatus.new) { - return pixelmatchResult; - } - - if (pixelmatchResult.status === TestStatus.ok) { + const pixelmatchResult = await this.pixelmatchService.getDiff(data, PIXELMATCH_DEFAULT_CONFIG); + + if ( + !data.saveDiffAsFile || + !pixelmatchResult.status || + pixelmatchResult.status === TestStatus.new || + pixelmatchResult.status === TestStatus.ok + ) { return pixelmatchResult; } this.logger.debug('Pixel diff is being sent to VLM'); try { - const baseline = await this.staticService.getImage(data.baseline); - const image = await this.staticService.getImage(data.image); - const diffImage = pixelmatchResult.diffName ? await this.staticService.getImage(pixelmatchResult.diffName) : null; - - if (!baseline || !image || !diffImage) { + const [baselineBuffer, imageBuffer, diffBuffer] = await Promise.all([ + this.staticService.getImageBuffer(data.baseline), + data.imageBuffer ?? this.staticService.getImageBuffer(data.image), + pixelmatchResult.diffName ? this.staticService.getImageBuffer(pixelmatchResult.diffName) : null, + ]); + if (!baselineBuffer || !imageBuffer || !diffBuffer) { this.logger.warn('Missing images for VLM analysis, returning pixelmatch result'); return pixelmatchResult; } - // mask ignore areas the same way pixelmatch did, so the VLM does not flag changes inside them - const baselineBytes = new Uint8Array(PNG.sync.write(applyIgnoreAreas(baseline, data.ignoreAreas))); - const imageBytes = new Uint8Array(PNG.sync.write(applyIgnoreAreas(image, data.ignoreAreas))); - const diffBytes = new Uint8Array(PNG.sync.write(diffImage)); + const { baseline, image } = await this.diffWorkerPool.run({ + kind: 'vlm-inputs', + baseline: baselineBuffer, + image: imageBuffer, + ignoreAreas: data.ignoreAreas, + }); + if (!baseline || !image) { + this.logger.warn('Missing images for VLM analysis, returning pixelmatch result'); + return pixelmatchResult; + } - const { pass, description } = await this.compareImagesWithVLM(baselineBytes, imageBytes, diffBytes, config); + const { pass, description } = await this.compareImagesWithVLM( + baseline, + image, + new Uint8Array(diffBuffer), + config + ); - // Build result from pixelmatch, but override status based on VLM analysis - const result: DiffResult = { + return { ...pixelmatchResult, + status: pass ? TestStatus.ok : TestStatus.unresolved, vlmDescription: description, }; - - if (pass) { - result.status = TestStatus.ok; - } else { - result.status = TestStatus.unresolved; - } - - return result; } catch (error) { - // a cast, not instanceof: errors from Node's own fetch can come from another realm + // a cast, not instanceof: fetch errors come from another realm const { message, stack } = error as Error; this.logger.error(`VLM comparison failed: ${message}`, stack); return { @@ -110,18 +108,10 @@ export class VlmService implements ImageComparator { } private getProvider(config: VlmConfig): VlmProvider { - const provider = config.provider || 'ollama'; // Default to ollama for backward compatibility - - switch (provider) { - case 'gemini': - return this.geminiService; - case 'ollama': - default: - return this.ollamaService; - } + return config.provider === 'gemini' ? this.geminiService : this.ollamaService; } - private extractResponseContent(config: VlmConfig, response: { content?: string; thinking?: string }): string { + private extractResponseContent(config: VlmConfig, response: VlmProviderResponse): string { const preferred = config.useThinking ? response.thinking : response.content; const fallback = config.useThinking ? response.content : response.thinking; const content = preferred || fallback; @@ -135,9 +125,7 @@ export class VlmService implements ImageComparator { } private parseVlmResponse(responseContent: string): { pass: boolean; description: string } { - const parsed = JSON.parse(responseContent); - const validated = VlmComparisonResultSchema.parse(parsed); - + const validated = VlmComparisonResultSchema.parse(JSON.parse(responseContent)); return { pass: validated.identical, description: validated.description || 'No description provided', @@ -150,9 +138,9 @@ export class VlmService implements ImageComparator { diffBytes: Uint8Array, config: VlmConfig ): Promise<{ pass: boolean; description: string }> { - const provider = this.getProvider(config); - const images = [baselineBytes, imageBytes, diffBytes]; - const response = await provider.generate(config, images); + const response = await this.limiter.run(() => + this.getProvider(config).generate(config, [baselineBytes, imageBytes, diffBytes]) + ); const content = this.extractResponseContent(config, response); return this.parseVlmResponse(content); } diff --git a/src/compare/libs/vlm/vlm.types.ts b/src/compare/libs/vlm/vlm.types.ts index 1cf03060..36fa226a 100644 --- a/src/compare/libs/vlm/vlm.types.ts +++ b/src/compare/libs/vlm/vlm.types.ts @@ -1,63 +1,20 @@ import { BlurAmplitudeConfig } from '../blur-amplitude.config'; -/** - * Base configuration shared by all VLM providers - */ export interface BaseVlmConfig extends BlurAmplitudeConfig { - /** - * Custom prompt for image comparison. - */ prompt: string; - - /** - * Temperature parameter controlling response randomness (0.0-1.0). - * Lower values = more consistent results. - * @default 0.1 - */ temperature: number; - - /** - * Whether to prefer thinking field over content field for response. - * Some models return result in thinking field instead of response. - * @default false - */ useThinking?: boolean; } -/** - * Configuration for Ollama provider - */ export interface OllamaVlmConfig extends BaseVlmConfig { provider?: 'ollama'; - - /** - * Ollama vision model name. - * Examples: "gemma3:12b", "llava:13b" - * @default "gemma3:12b" - */ model: string; } -/** - * Configuration for Gemini provider - */ export interface GeminiVlmConfig extends BaseVlmConfig { provider: 'gemini'; - - /** - * Gemini model name. - * Examples: "gemini-1.5-pro", "gemini-1.5-flash" - */ model: string; - - /** - * Google Gemini API key (required). - * Stored in project settings for per-project configuration. - */ apiKey: string; } -/** - * Union type for all VLM provider configurations - */ export type VlmConfig = OllamaVlmConfig | GeminiVlmConfig; diff --git a/src/compare/test-utils.ts b/src/compare/test-utils.ts new file mode 100644 index 00000000..418140a3 --- /dev/null +++ b/src/compare/test-utils.ts @@ -0,0 +1,33 @@ +import { PNG } from 'pngjs'; + +export type Rgb = [number, number, number]; + +export const WHITE: Rgb = [255, 255, 255]; + +export const paintedPng = ( + width: number, + height: number, + paint: (x: number, y: number) => Rgb | undefined = () => undefined +): PNG => { + const png = new PNG({ width, height }); + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + const [r, g, b] = paint(x, y) ?? WHITE; + const index = (y * width + x) * 4; + png.data[index] = r; + png.data[index + 1] = g; + png.data[index + 2] = b; + png.data[index + 3] = 255; + } + } + return png; +}; + +export const solidPng = (width: number, height: number, rgb: Rgb = WHITE): PNG => paintedPng(width, height, () => rgb); + +export const pngBytes = (png: PNG): Buffer => PNG.sync.write(png); + +export const square = + (left: number, top: number, size: number, rgb: Rgb) => + (x: number, y: number): Rgb | undefined => + x >= left && x < left + size && y >= top && y < top + size ? rgb : undefined; diff --git a/src/compare/utils/index.spec.ts b/src/compare/utils/index.spec.ts index b514f537..ba397371 100644 --- a/src/compare/utils/index.spec.ts +++ b/src/compare/utils/index.spec.ts @@ -37,7 +37,6 @@ describe('parseConfig', () => { }); }); -// pixel i of a w x h image gets channels [i, 100 + i, 200 - i, 50 + i] so every channel is distinguishable const distinctPixels = (width: number, height: number): Buffer => { const data = Buffer.alloc(width * height * 4); for (let i = 0; i < width * height; i++) { @@ -59,9 +58,7 @@ describe('applyIgnoreAreas edges', () => { expect(pixelOf(image.data, 0)).toEqual([0, 0, 0, 0]); expect(pixelOf(image.data, 1)).toEqual([0, 0, 0, 0]); - // column right after the area expect(pixelOf(image.data, 2)).toEqual([255, 255, 255, 255]); - // row right after the area expect(pixelOf(image.data, 3)).toEqual([255, 255, 255, 255]); expect(pixelOf(image.data, 4)).toEqual([255, 255, 255, 255]); }); @@ -78,40 +75,122 @@ describe('applyIgnoreAreas edges', () => { }); }); -describe('scaleImageToSize', () => { - it('returns the same image when the requested size equals the image size', () => { - const image = generatePng(3, 2); +const masked = (image: PNG): number[] => { + const indexes: number[] = []; + for (let i = 0; i < image.width * image.height; i++) { + if (image.data[i * 4 + 3] === 0) { + indexes.push(i); + } + } + return indexes; +}; + +const legacyApply = (image: PNG, ignoreAreas: IgnoreAreaDto[]): PNG => { + ignoreAreas.forEach((area) => { + for (let y = area.y; y < Math.min(area.y + area.height, image.height); y++) { + for (let x = area.x; x < Math.min(area.x + area.width, image.width); x++) { + const k = 4 * (image.width * y + x); + image.data[k + 0] = 0; + image.data[k + 1] = 0; + image.data[k + 2] = 0; + image.data[k + 3] = 0; + } + } + }); + return image; +}; + +describe('applyIgnoreAreas rounding and clamping', () => { + it('masks every pixel a fractional area touches', () => { + const image = generatePng(4, 4); + + applyIgnoreAreas(image, [{ x: 0.5, y: 0.5, width: 1, height: 1 }]); + + expect(masked(image)).toEqual([0, 1, 4, 5]); + for (const index of masked(image)) { + expect(pixelOf(image.data, index)).toEqual([0, 0, 0, 0]); + } + }); + + it.each([ + ['does not wrap a negative x into the previous row', 4, { x: -2, y: 1, width: 3, height: 1 }, [4]], + ['clips a negative y to the first row', 4, { x: 1, y: -2, width: 1, height: 3 }, [1]], + [ + 'keeps the old ceil end for a fractional size at an integer start', + 4, + { x: 0, y: 0, width: 1.5, height: 1.2 }, + [0, 1, 4, 5], + ], + [ + 'returns at once for absurd coordinates', + 3, + { x: -1e12, y: -1e12, width: 2e12, height: 2e12 }, + [0, 1, 2, 3, 4, 5, 6, 7, 8], + ], + ])('%s', (_, size, area, expected) => { + const image = generatePng(size, size); + + applyIgnoreAreas(image, [area]); - expect(scaleImageToSize(image, 3, 2)).toBe(image); + expect(masked(image)).toEqual(expected); }); - it('returns the same image when only the height is requested smaller', () => { + it('skips null, empty and non-numeric areas', () => { const image = generatePng(3, 3); - expect(scaleImageToSize(image, 3, 2)).toBe(image); + applyIgnoreAreas(image, [ + null, + { x: 1, y: 1, width: 0, height: 2 }, + { x: NaN, y: 0, width: 2, height: 2 }, + ] as unknown as IgnoreAreaDto[]); + + expect(masked(image)).toEqual([]); }); - it('pads a wider canvas with transparent black, keeping the original pixels top-left', () => { - const image = new PNG({ width: 2, height: 1 }); - distinctPixels(2, 1).copy(image.data); + it('masks byte for byte what the old loop masked for integer areas', () => { + let seed = 12345; + const next = (min: number, max: number): number => { + seed = (seed * 1664525 + 1013904223) % 4294967296; + return min + Math.floor((seed / 4294967296) * (max - min)); + }; - const result = scaleImageToSize(image, 3, 1); + for (let i = 0; i < 300; i++) { + const area = { x: next(0, 12), y: next(-5, 12), width: next(-3, 12), height: next(-3, 12) }; + const legacy = new PNG({ width: 9, height: 7 }); + const current = new PNG({ width: 9, height: 7 }); + distinctPixels(9, 7).copy(legacy.data); + distinctPixels(9, 7).copy(current.data); - expect(result).not.toBe(image); - expect(result.width).toBe(3); - expect(result.height).toBe(1); - expect([...result.data]).toEqual([0, 100, 200, 50, 1, 101, 199, 51, 0, 0, 0, 0]); + legacyApply(legacy, [area]); + applyIgnoreAreas(current, [area]); + + expect(current.data.equals(legacy.data)).toBe(true); + } + }); +}); + +describe('scaleImageToSize', () => { + it.each([ + ['the requested size equals the image size', 3, 2, 3, 2], + ['only the height is requested smaller', 3, 3, 3, 2], + ])('returns the same image when %s', (_, imageWidth, imageHeight, width, height) => { + const image = generatePng(imageWidth, imageHeight); + + expect(scaleImageToSize(image, width, height)).toBe(image); }); - it('pads a taller canvas with transparent black, keeping the original pixels top-left', () => { - const image = new PNG({ width: 1, height: 1 }); - distinctPixels(1, 1).copy(image.data); + it.each([ + ['wider', [2, 1], [3, 1], [0, 100, 200, 50, 1, 101, 199, 51, 0, 0, 0, 0]], + ['taller', [1, 1], [1, 2], [0, 100, 200, 50, 0, 0, 0, 0]], + ])('pads a %s canvas with transparent black, keeping the original pixels top-left', (_, from, to, expected) => { + const image = new PNG({ width: from[0], height: from[1] }); + distinctPixels(from[0], from[1]).copy(image.data); - const result = scaleImageToSize(image, 1, 2); + const result = scaleImageToSize(image, to[0], to[1]); - expect(result.width).toBe(1); - expect(result.height).toBe(2); - expect([...result.data]).toEqual([0, 100, 200, 50, 0, 0, 0, 0]); + expect(result).not.toBe(image); + expect([result.width, result.height]).toEqual(to); + expect([...result.data]).toEqual(expected); }); }); @@ -130,7 +209,6 @@ describe('downscale', () => { expect(result.width).toBe(2); expect(result.height).toBe(2); expect(result.data.length).toBe(2 * 2 * 4); - // target (x, y) <- source (2x, 2y): source indexes 0, 2, 8, 10 expect([...result.data]).toEqual([ ...pixelOf(source.data, 0), ...pixelOf(source.data, 2), @@ -146,7 +224,6 @@ describe('downscale', () => { expect(result.width).toBe(2); expect(result.height).toBe(1); - // target x 0 <- source x 0, target x 1 <- source x 3 (scale 1/3) expect([...result.data]).toEqual([...pixelOf(source.data, 0), ...pixelOf(source.data, 3)]); }); }); diff --git a/src/compare/utils/index.ts b/src/compare/utils/index.ts index cd944b9e..4855b181 100644 --- a/src/compare/utils/index.ts +++ b/src/compare/utils/index.ts @@ -1,6 +1,7 @@ import { Logger } from '@nestjs/common'; import { PNG } from 'pngjs'; import { IgnoreAreaDto } from '../../test-runs/dto/ignore-area.dto'; +import { asBuffer } from '../../static/utils'; export function scaleImageToSize(image: PNG, width: number, height: number): PNG { if (width > image.width || height > image.height) { @@ -11,24 +12,50 @@ export function scaleImageToSize(image: PNG, width: number, height: number): PNG return image; } +export function readPng(data: Buffer | Uint8Array): PNG { + return PNG.sync.read(asBuffer(data)); +} + +export function readSameSizePair( + baselineBytes: Buffer | Uint8Array, + imageBytes: Buffer | Uint8Array, + ignoreAreas: IgnoreAreaDto[] +): { baseline: PNG; image: PNG } | null { + let baseline: PNG; + let image: PNG; + try { + baseline = readPng(baselineBytes); + image = readPng(imageBytes); + } catch { + return null; + } + if (baseline.width !== image.width || baseline.height !== image.height) { + return null; + } + applyIgnoreAreas(baseline, ignoreAreas); + applyIgnoreAreas(image, ignoreAreas); + return { baseline, image }; +} + export function applyIgnoreAreas(image: PNG, ignoreAreas: IgnoreAreaDto[]): PNG { - ignoreAreas.forEach((area) => { - for (let y = area.y; y < Math.min(area.y + area.height, image.height); y++) { - for (let x = area.x; x < Math.min(area.x + area.width, image.width); x++) { - const k = 4 * (image.width * y + x); - image.data[k + 0] = 0; - image.data[k + 1] = 0; - image.data[k + 2] = 0; - image.data[k + 3] = 0; - } + for (const area of ignoreAreas) { + if (!area) { + continue; + } + const x0 = Math.max(0, Math.floor(area.x)); + const y0 = Math.max(0, Math.floor(area.y)); + const x1 = Math.min(image.width, Math.ceil(area.x + area.width)); + const y1 = Math.min(image.height, Math.ceil(area.y + area.height)); + if (!(x1 > x0 && y1 > y0)) { + continue; } - }); + for (let y = y0; y < y1; y++) { + image.data.fill(0, 4 * (image.width * y + x0), 4 * (image.width * y + x1)); + } + } return image; } -// The stored config is laid over the comparator's defaults, not used in their -// place: a project that sets one option keeps the others, and a VLM project -// still holding the column's pixelmatch default still gets a model and prompt. export const parseConfig = (configJson: string, defaultConfig: T, logger: Logger): T => { try { const config = JSON.parse(configJson); @@ -50,8 +77,6 @@ export interface RawImage { height: number; } -// Nearest-neighbour downscale so the longest side is at most maxDimension. -// Returns the original when already small enough. export function downscale(source: RawImage, maxDimension: number): RawImage { const scale = maxDimension / Math.max(source.width, source.height); if (scale >= 1) { diff --git a/src/config/env.validation.spec.ts b/src/config/env.validation.spec.ts new file mode 100644 index 00000000..5040c52e --- /dev/null +++ b/src/config/env.validation.spec.ts @@ -0,0 +1,96 @@ +import { Logger } from '@nestjs/common'; +import { DEFAULT_JWT_LIFE_TIME, DEVELOPMENT_JWT_SECRET, validateEnv } from './env.validation'; + +describe('validateEnv', () => { + const GOOD = { JWT_SECRET: 'a-random-secret', JWT_LIFE_TIME: '12h', APP_FRONTEND_URL: 'http://localhost:8080' }; + let logger: { warn: jest.Mock }; + + beforeEach(() => { + logger = { warn: jest.fn() }; + }); + + it('returns a valid config unchanged and silent', () => { + expect(validateEnv({ ...GOOD, OTHER: 'x' }, logger)).toEqual({ ...GOOD, OTHER: 'x' }); + expect(logger.warn).not.toHaveBeenCalled(); + }); + + it('does not mutate the config it was given', () => { + const config = { JWT_SECRET: 'a-random-secret' }; + + validateEnv(config, logger); + + expect(config).toEqual({ JWT_SECRET: 'a-random-secret' }); + }); + + describe('JWT_SECRET', () => { + it.each([undefined, '', ' '])('refuses to start without it (%p)', (value) => { + expect(() => validateEnv({ ...GOOD, JWT_SECRET: value }, logger)).toThrow('JWT_SECRET is not set'); + }); + + it.each([ + ['the public sample value when NODE_ENV=production', 'jwtPrivateKey', 'production'], + ['the public sample value when NODE_ENV=development', 'jwtPrivateKey', 'development'], + ['the public sample value when NODE_ENV is unset', 'jwtPrivateKey', undefined], + ['the committed development secret in production', DEVELOPMENT_JWT_SECRET, 'production'], + ])('only warns about %s', (_, secret, nodeEnv) => { + expect(() => validateEnv({ ...GOOD, JWT_SECRET: secret, NODE_ENV: nodeEnv }, logger)).not.toThrow(); + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('publicly known')); + }); + }); + + describe('JWT_LIFE_TIME', () => { + it.each([undefined, '', ' ', 'abc', '1 fortnight', 'x'.repeat(101)])( + 'falls back to 1d and warns for %p', + (value) => { + const env = validateEnv({ ...GOOD, JWT_LIFE_TIME: value }, logger); + + expect(env.JWT_LIFE_TIME).toBe(DEFAULT_JWT_LIFE_TIME); + expect(DEFAULT_JWT_LIFE_TIME).toBe('1d'); + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('JWT_LIFE_TIME')); + } + ); + + it.each(['12h', '1d', '30 minutes', '2 days', '500ms', '1.5h'])('accepts %p without a warning', (value) => { + const env = validateEnv({ ...GOOD, JWT_LIFE_TIME: value }, logger); + + expect(env.JWT_LIFE_TIME).toBe(value); + expect(logger.warn).not.toHaveBeenCalled(); + }); + + it('keeps a number without a unit but warns that it counts milliseconds', () => { + const env = validateEnv({ ...GOOD, JWT_LIFE_TIME: '3600' }, logger); + + expect(env.JWT_LIFE_TIME).toBe('3600'); + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('milliseconds')); + }); + }); + + describe('other settings only warn', () => { + it('warns when APP_FRONTEND_URL is not set', () => { + validateEnv({ JWT_SECRET: 'a-random-secret', JWT_LIFE_TIME: '1d' }, logger); + + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('APP_FRONTEND_URL')); + }); + + it('warns when STATIC_SERVICE=s3 has no bucket', () => { + validateEnv({ ...GOOD, STATIC_SERVICE: 's3' }, logger); + + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('AWS_S3_BUCKET_NAME')); + }); + + it('is silent when STATIC_SERVICE=s3 has a bucket', () => { + validateEnv({ ...GOOD, STATIC_SERVICE: 's3', AWS_S3_BUCKET_NAME: 'vrt' }, logger); + + expect(logger.warn).not.toHaveBeenCalled(); + }); + }); + + it('logs through a Nest Logger by default', () => { + const warn = jest.spyOn(Logger.prototype, 'warn').mockImplementation(); + + validateEnv({ JWT_SECRET: 'a-random-secret', JWT_LIFE_TIME: 'abc' }); + + expect(warn).toHaveBeenCalledWith(expect.stringContaining('JWT_LIFE_TIME')); + warn.mockRestore(); + }); +}); diff --git a/src/config/env.validation.ts b/src/config/env.validation.ts new file mode 100644 index 00000000..68cea789 --- /dev/null +++ b/src/config/env.validation.ts @@ -0,0 +1,49 @@ +import { Logger } from '@nestjs/common'; + +export const DEFAULT_JWT_LIFE_TIME = '1d'; +export const DEVELOPMENT_JWT_SECRET = 'vrt-local-development-only-jwt-secret'; +const PUBLIC_JWT_SECRETS: readonly string[] = ['jwtPrivateKey', DEVELOPMENT_JWT_SECRET]; + +const TIMESPAN = + /^(-?(?:\d+)?\.?\d+) *(milliseconds?|msecs?|ms|seconds?|secs?|s|minutes?|mins?|m|hours?|hrs?|h|days?|d|weeks?|w|years?|yrs?|y)?$/i; + +const text = (value: unknown): string => (value === undefined || value === null ? '' : String(value).trim()); + +type Warn = Pick; + +export function validateEnv( + config: Record, + logger: Warn = new Logger('Config') +): Record { + const env = { ...config }; + + const secret = text(env.JWT_SECRET); + if (!secret) { + throw new Error('JWT_SECRET is not set. Generate one, e.g. `openssl rand -hex 32`.'); + } + if (PUBLIC_JWT_SECRETS.includes(secret)) { + logger.warn( + 'JWT_SECRET is a publicly known sample value: anyone can sign tokens for any user. Set a random one, e.g. `openssl rand -hex 32`.' + ); + } + + const lifeTime = text(env.JWT_LIFE_TIME); + if (!lifeTime || lifeTime.length > 100 || !TIMESPAN.test(lifeTime)) { + logger.warn( + `JWT_LIFE_TIME ${lifeTime ? `"${lifeTime}" is not a timespan` : 'is not set'}; tokens expire after ${DEFAULT_JWT_LIFE_TIME}. Use e.g. "1d" or "12h".` + ); + env.JWT_LIFE_TIME = DEFAULT_JWT_LIFE_TIME; + } else if (/^\d+(\.\d+)?$/.test(lifeTime)) { + logger.warn( + `JWT_LIFE_TIME "${lifeTime}" has no unit and counts milliseconds; add "s", "h" or "d" if that is not meant.` + ); + } + + if (!text(env.APP_FRONTEND_URL)) { + logger.warn('APP_FRONTEND_URL is not set: links in test run results start with "undefined".'); + } + if (text(env.STATIC_SERVICE) === 's3' && !text(env.AWS_S3_BUCKET_NAME)) { + logger.warn('STATIC_SERVICE=s3 but AWS_S3_BUCKET_NAME is not set: storing images will fail.'); + } + return env; +} diff --git a/src/health/health.controller.spec.ts b/src/health/health.controller.spec.ts index af45bb2e..578072aa 100644 --- a/src/health/health.controller.spec.ts +++ b/src/health/health.controller.spec.ts @@ -1,34 +1,20 @@ import { ServiceUnavailableException } from '@nestjs/common'; -import { Test, TestingModule } from '@nestjs/testing'; -import { HealthCheckService, TerminusModule } from '@nestjs/terminus'; +import { Test } from '@nestjs/testing'; +import { HealthCheckService, PrismaHealthIndicator, TerminusModule } from '@nestjs/terminus'; import { PrismaService } from '../prisma/prisma.service'; import { HealthController } from './health.controller'; describe('Health Controller', () => { let controller: HealthController; - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - controllers: [HealthController], - providers: [ - { provide: HealthCheckService, useValue: {} }, - { provide: PrismaService, useValue: {} }, - ], - }).compile(); - - controller = module.get(HealthController); - }); - - it('should be defined', () => { - expect(controller).toBeDefined(); - }); - describe('check', () => { - it('runs a single indicator that queries SELECT 1', async () => { + it('runs a single indicator that pings the database with a timeout', async () => { const health = { check: jest.fn().mockResolvedValue('result') }; - const prisma = { $queryRaw: jest.fn().mockResolvedValue([{ '?column?': 1 }]) }; + const prismaHealth = { pingCheck: jest.fn().mockResolvedValue({ database: { status: 'up' } }) }; + const prisma = {}; const healthController = new HealthController( health as unknown as HealthCheckService, + prismaHealth as unknown as PrismaHealthIndicator, prisma as unknown as PrismaService ); @@ -36,17 +22,23 @@ describe('Health Controller', () => { const [indicators] = health.check.mock.calls[0]; expect(indicators).toHaveLength(1); - await indicators[0](); - const [strings] = prisma.$queryRaw.mock.calls[0]; - expect([...strings]).toEqual(['SELECT 1']); + await expect(indicators[0]()).resolves.toEqual({ database: { status: 'up' } }); + expect(prismaHealth.pingCheck).toHaveBeenCalledWith('database', prisma, { timeout: 3000 }); }); }); describe('check with the real terminus HealthCheckService', () => { - let prisma: { $queryRaw: jest.Mock }; + let prisma: { $runCommandRaw: jest.Mock; $queryRawUnsafe: jest.Mock }; beforeEach(async () => { - prisma = { $queryRaw: jest.fn() }; + prisma = { + $runCommandRaw: jest + .fn() + .mockRejectedValue( + new Error('The postgresql provider does not support $runCommandRaw. Use the mongodb provider.') + ), + $queryRawUnsafe: jest.fn(), + }; const module = await Test.createTestingModule({ imports: [TerminusModule.forRoot({ logger: false })], controllers: [HealthController], @@ -55,25 +47,40 @@ describe('Health Controller', () => { controller = module.get(HealthController); }); - // NOTE: current behavior — the raw query result is not a terminus - // HealthIndicatorResult, so the report names no "database" indicator at all. - it('reports ok with empty details when the database answers', async () => { - prisma.$queryRaw.mockResolvedValue([{ '?column?': 1 }]); + afterEach(() => { + jest.useRealTimers(); + }); - await expect(controller.check()).resolves.toEqual({ status: 'ok', info: {}, error: {}, details: {} }); + it('reports the database as up', async () => { + prisma.$queryRawUnsafe.mockResolvedValue([{ '?column?': 1 }]); + + await expect(controller.check()).resolves.toEqual({ + status: 'ok', + info: { database: { status: 'up', responseTime: expect.any(Number) } }, + error: {}, + details: { database: { status: 'up', responseTime: expect.any(Number) } }, + }); + expect(prisma.$queryRawUnsafe).toHaveBeenCalledWith('SELECT 1'); }); - // NOTE: current behavior — the indicator does not throw a terminus - // HealthCheckError, so terminus rethrows the raw error instead of - // answering 503; through the global filter /health then answers 500. - it('rethrows the raw database error instead of a 503', async () => { - const error = new Error('Can not reach database server'); - prisma.$queryRaw.mockRejectedValue(error); + it('answers 503 when the database fails', async () => { + prisma.$queryRawUnsafe.mockRejectedValue(new Error('Can not reach database server')); const result = controller.check(); - await expect(result).rejects.toBe(error); - await expect(result).rejects.not.toBeInstanceOf(ServiceUnavailableException); + await expect(result).rejects.toBeInstanceOf(ServiceUnavailableException); + const error = (await result.catch((e: ServiceUnavailableException) => e)) as ServiceUnavailableException; + expect((error.getResponse() as { error: Record }).error.database.status).toBe('down'); + }); + + it('answers 503 when the database does not answer within 3 s', async () => { + jest.useFakeTimers(); + prisma.$queryRawUnsafe.mockReturnValue(new Promise(() => undefined)); + + const result = controller.check().catch((e: unknown) => e); + await jest.advanceTimersByTimeAsync(3000); + + await expect(result).resolves.toBeInstanceOf(ServiceUnavailableException); }); }); }); diff --git a/src/health/health.controller.ts b/src/health/health.controller.ts index 22410d27..44965c0d 100644 --- a/src/health/health.controller.ts +++ b/src/health/health.controller.ts @@ -1,13 +1,16 @@ import { Controller, Get } from '@nestjs/common'; -import { HealthCheckService, HealthCheck, HealthIndicatorResult } from '@nestjs/terminus'; +import { HealthCheckService, HealthCheck, PrismaHealthIndicator } from '@nestjs/terminus'; import { PrismaService } from '../prisma/prisma.service'; import { ApiOkResponse, ApiTags } from '@nestjs/swagger'; +const DATABASE_TIMEOUT_MS = 3_000; + @ApiTags('health') @Controller('health') export class HealthController { constructor( private health: HealthCheckService, + private readonly prismaHealth: PrismaHealthIndicator, private readonly prismaService: PrismaService ) {} @@ -15,7 +18,8 @@ export class HealthController { @ApiOkResponse({ type: Object }) @HealthCheck() check() { - // the raw query result has always been passed through as the indicator result - return this.health.check([() => this.prismaService.$queryRaw`SELECT 1`]); + return this.health.check([ + () => this.prismaHealth.pingCheck('database', this.prismaService, { timeout: DATABASE_TIMEOUT_MS }), + ]); } } diff --git a/src/http-exception.filter.spec.ts b/src/http-exception.filter.spec.ts index feebef95..113a58d4 100644 --- a/src/http-exception.filter.spec.ts +++ b/src/http-exception.filter.spec.ts @@ -6,12 +6,13 @@ import { HttpStatus, Logger, NotFoundException, + ServiceUnavailableException, } from '@nestjs/common'; import { Prisma } from '@prisma/client'; import { HttpExceptionFilter } from './http-exception.filter'; const createHost = (url = '/some/path') => { - const response = { status: jest.fn(), json: jest.fn() }; + const response = { status: jest.fn(), json: jest.fn(), setHeader: jest.fn() }; response.status.mockReturnValue(response); const request = { url }; const host = { @@ -20,150 +21,195 @@ const createHost = (url = '/some/path') => { return { host, response }; }; -// the errors Prisma 7 raises through the pg adapter, as captured from PostgreSQL -const prisma7UniqueConstraintError = (method: string, constraint: string) => - new Prisma.PrismaClientKnownRequestError( - `\nInvalid \`this.prismaService.${method}\` invocation in\n/app/dist/x.js:1:1\n\n\nUnique constraint failed on the constraint: \`${constraint}\``, - { code: 'P2002', clientVersion: Prisma.prismaVersion.client } - ); +const prismaError = (code: string, message: string) => + new Prisma.PrismaClientKnownRequestError(message, { code, clientVersion: Prisma.prismaVersion.client }); const uniqueConstraintError = (method: string) => - new Prisma.PrismaClientKnownRequestError( - `\nInvalid \`prisma.${method}\` invocation:\n\n\nUnique constraint failed on the fields: (\`name\`)`, - { code: 'P2002', clientVersion: Prisma.prismaVersion.client } + prismaError( + 'P2002', + `\nInvalid \`prisma.${method}\` invocation:\n\n\nUnique constraint failed on the fields: (\`name\`)` + ); + +const prisma7UniqueConstraintError = (method: string, constraint: string) => + prismaError( + 'P2002', + `\nInvalid \`this.prismaService.${method}\` invocation in\n/app/dist/x.js:1:1\n\n\nUnique constraint failed on the constraint: \`${constraint}\`` ); describe('HttpExceptionFilter', () => { let filter: HttpExceptionFilter; - let loggerSpy: jest.SpyInstance; + let errorSpy: jest.SpyInstance; + let warnSpy: jest.SpyInstance; + + const catchWith = (exception: unknown, url?: string) => { + const { host, response } = createHost(url); + filter.catch(exception, host); + return response; + }; beforeEach(() => { filter = new HttpExceptionFilter(); - loggerSpy = jest.spyOn(Logger, 'error').mockImplementation(); + errorSpy = jest.spyOn(Logger.prototype, 'error').mockImplementation(); + warnSpy = jest.spyOn(Logger.prototype, 'warn').mockImplementation(); }); afterEach(() => { - loggerSpy.mockRestore(); + errorSpy.mockRestore(); + warnSpy.mockRestore(); }); describe('catch', () => { - it('answers an HttpException with its own status and the full JSON shape', () => { - const { host, response } = createHost('/projects/1'); - const exception = new NotFoundException('Project not found'); - - filter.catch(exception, host); + it('answers an HttpException with its own status and a body without the exception or its stack', () => { + const response = catchWith(new NotFoundException('Project not found'), '/projects/1'); expect(response.status).toHaveBeenCalledWith(404); expect(response.json).toHaveBeenCalledWith({ + statusCode: 404, path: '/projects/1', name: 'NotFoundException', message: 'Project not found', - exception, - stack: exception.stack, }); }); - it('logs the exception with its stack and its response body', () => { - const { host } = createHost(); - const exception = new BadRequestException('Bad input'); + it('logs a 5xx at error with the stack', () => { + const exception = new Error('Something broke'); - filter.catch(exception, host); + catchWith(exception, '/boom'); - expect(loggerSpy).toHaveBeenCalledWith(exception, exception.stack, { - message: 'Bad input', - error: 'Bad Request', - statusCode: 400, - }); + expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('500: Something broke'), exception.stack); + expect(warnSpy).not.toHaveBeenCalled(); }); - it('answers 500 for an error without getStatus and logs it without a response body', () => { - const { host, response } = createHost(); - const exception = new Error('Something broke'); + it('logs a 4xx at warn without the stack', () => { + catchWith(new BadRequestException('Bad input'), '/projects/1'); - filter.catch(exception as HttpException, host); + expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('400: Bad input')); + expect(warnSpy.mock.calls[0]).toHaveLength(1); + expect(errorSpy).not.toHaveBeenCalled(); + }); + + it('answers 500 for an error without getStatus', () => { + const response = catchWith(new Error('Something broke')); expect(response.status).toHaveBeenCalledWith(HttpStatus.INTERNAL_SERVER_ERROR); - expect(response.json).toHaveBeenCalledWith( - expect.objectContaining({ name: 'Error', message: 'Something broke', exception }) - ); - expect(loggerSpy).toHaveBeenCalledWith(exception, exception.stack, undefined); + expect(response.json).toHaveBeenCalledWith({ + statusCode: 500, + path: '/some/path', + name: 'Error', + message: 'Something broke', + }); }); - it('answers a ValidationPipe rejection with the validation messages', () => { - const { host, response } = createHost(); - const exception = new BadRequestException([ - 'name should not be empty', - 'imageComparisonConfig must be a json string', - ]); - - filter.catch(exception, host); + it('answers 500 for a thrown string', () => { + const response = catchWith('plain string'); - expect(response.status).toHaveBeenCalledWith(400); + expect(response.status).toHaveBeenCalledWith(500); expect(response.json).toHaveBeenCalledWith( - expect.objectContaining({ - message: 'name should not be empty, imageComparisonConfig must be a json string', - }) + expect.objectContaining({ statusCode: 500, name: 'Error', message: 'plain string' }) ); }); - it('answers 404 when Prisma finds no record to update or delete (P2025)', () => { - const { host, response } = createHost(); - const exception = new Prisma.PrismaClientKnownRequestError('No record was found for an update.', { - code: 'P2025', - clientVersion: Prisma.prismaVersion.client, - }); + it.each([ + ['an error with a status but no parser type', Object.assign(new Error('odd'), { status: 413 })], + [ + 'a typed error whose status is not a client error', + Object.assign(new Error('odd'), { statusCode: 502, type: 'x' }), + ], + [ + 'an exception whose getStatus throws', + Object.assign(new HttpException('Weird', 418), { + getStatus: () => { + throw new Error('no status'); + }, + }), + ], + ])('answers 500 for %s', (_, exception) => { + expect(catchWith(exception).status).toHaveBeenCalledWith(500); + }); - filter.catch(exception as unknown as HttpException, host); + it.each([ + ['a foreign key violation (P2003)', prismaError('P2003', 'Foreign key constraint violated.'), 409], + ['no record to update or delete (P2025)', prismaError('P2025', 'No record was found for an update.'), 404], + ])('answers %s with %i', (_, exception, status) => { + const response = catchWith(exception); - expect(response.status).toHaveBeenCalledWith(HttpStatus.NOT_FOUND); + expect(response.status).toHaveBeenCalledWith(status); expect(response.json).toHaveBeenCalledWith( - expect.objectContaining({ message: 'No record was found for an update.', exception }) + expect.objectContaining({ statusCode: status, message: exception.message }) ); }); - it('answers 500 when getStatus itself throws', () => { - const { host, response } = createHost(); - const exception = new HttpException('Weird', 418); - jest.spyOn(exception, 'getStatus').mockImplementation(() => { - throw new Error('no status'); - }); + it('sets Retry-After from retryAfterSeconds', () => { + const response = catchWith(Object.assign(new ServiceUnavailableException('busy'), { retryAfterSeconds: 5 })); - filter.catch(exception, host); + expect(response.setHeader).toHaveBeenCalledWith('Retry-After', '5'); + expect(response.status).toHaveBeenCalledWith(503); + }); - expect(response.status).toHaveBeenCalledWith(500); + it('sends no Retry-After without retryAfterSeconds', () => { + expect(catchWith(new ServiceUnavailableException('busy')).setHeader).not.toHaveBeenCalled(); }); - it.each([ - ['build.update()', 'There is already a build with this ci build id.'], - ['project.create()', 'Project exists with this name.'], - ['user.create()', 'This user already exists.'], - ])('answers 400 with a friendly message for a unique constraint failure in %s', (method, message) => { - const { host, response } = createHost(); - const exception = uniqueConstraintError(method); + it('answers 413 for a body over the parser limit', () => { + const response = catchWith( + Object.assign(new Error('request entity too large'), { + status: 413, + statusCode: 413, + type: 'entity.too.large', + }) + ); - filter.catch(exception as unknown as HttpException, host); + expect(response.status).toHaveBeenCalledWith(413); + expect(response.json).toHaveBeenCalledWith({ + statusCode: 413, + path: '/some/path', + name: 'Error', + message: 'request entity too large', + }); + }); - expect(response.status).toHaveBeenCalledWith(HttpStatus.BAD_REQUEST); + it('answers a ValidationPipe rejection with the validation messages', () => { + const response = catchWith( + new BadRequestException(['name should not be empty', 'imageComparisonConfig must be a json string']) + ); + + expect(response.status).toHaveBeenCalledWith(400); expect(response.json).toHaveBeenCalledWith( - expect.objectContaining({ name: 'PrismaClientKnownRequestError', message, exception }) + expect.objectContaining({ + message: 'name should not be empty, imageComparisonConfig must be a json string', + }) ); }); + describe.each([ + ['Prisma 6', (method: string, _constraint: string) => uniqueConstraintError(method)], + ['Prisma 7', prisma7UniqueConstraintError], + ])('a unique constraint failure from %s', (_, build) => { + it.each([ + ['build.update()', 'Build_projectId_ciBuildId_key', 'There is already a build with this ci build id.'], + ['project.create()', 'Project_name_key', 'Project exists with this name.'], + ['user.create()', 'User_email_key', 'This user already exists.'], + ])('in %s answers 400 with a friendly message', (method, constraint, message) => { + const response = catchWith(build(method, constraint)); + + expect(response.status).toHaveBeenCalledWith(HttpStatus.BAD_REQUEST); + expect(response.json).toHaveBeenCalledWith( + expect.objectContaining({ name: 'PrismaClientKnownRequestError', message }) + ); + }); + }); + it('answers 400 with the raw Prisma message for a unique constraint failure on another model', () => { - const { host, response } = createHost(); const exception = uniqueConstraintError('testVariation.create()'); - filter.catch(exception as unknown as HttpException, host); + const response = catchWith(exception); expect(response.status).toHaveBeenCalledWith(400); expect(response.json).toHaveBeenCalledWith(expect.objectContaining({ message: exception.message })); }); - // the exception Nest throws when a guard returns false it('keeps the 403 status of a guard rejection and rewrites its message', () => { - const { host, response } = createHost(); - - filter.catch(new ForbiddenException('Forbidden resource'), host); + const response = catchWith(new ForbiddenException('Forbidden resource')); expect(response.status).toHaveBeenCalledWith(403); expect(response.json).toHaveBeenCalledWith( @@ -174,92 +220,40 @@ describe('HttpExceptionFilter', () => { ); }); - // NOTE: current behavior — the "forbidden" rewrite is keyed on the message - // text only, so any error mentioning "Forbidden resource" is reworded while - // keeping its original status. - it('rewrites the message of a non-403 error that mentions "Forbidden resource"', () => { - const { host, response } = createHost(); - - filter.catch(new Error('Forbidden resource: storage bucket') as HttpException, host); + it('keeps the message of a non-403 error mentioning "Forbidden resource"', () => { + const response = catchWith(new Error('Forbidden resource: storage bucket')); expect(response.status).toHaveBeenCalledWith(500); expect(response.json).toHaveBeenCalledWith( - expect.objectContaining({ message: 'You do not have permission to perform this operation.' }) + expect.objectContaining({ message: 'Forbidden resource: storage bucket' }) ); }); - - // NOTE: current behavior — a thrown value without a string `message` (a - // thrown string, or an object literal) crashes the filter itself with a - // TypeError instead of producing a 500 response. - it('throws a TypeError for a thrown value without a message', () => { - const { host, response } = createHost(); - - expect(() => filter.catch('plain string' as unknown as HttpException, host)).toThrow(TypeError); - expect(response.status).not.toHaveBeenCalled(); - }); }); describe('isUniqueConstraintException', () => { - it('detects the Prisma unique constraint message', () => { - expect(filter.isUniqueConstraintException(uniqueConstraintError('user.create()') as never)).toBe(true); - }); - - it('ignores other messages', () => { - expect(filter.isUniqueConstraintException(new BadRequestException('nope'))).toBe(false); - }); - - it("detects Prisma 7's wording, which names the constraint instead of the fields", () => { - expect( - filter.isUniqueConstraintException( - prisma7UniqueConstraintError('project.create()', 'Project_name_key') as never - ) - ).toBe(true); - }); - - it('detects the P2002 code whatever the message says', () => { - const error = new Prisma.PrismaClientKnownRequestError('reworded in some later version', { - code: 'P2002', - clientVersion: Prisma.prismaVersion.client, - }); - - expect(filter.isUniqueConstraintException(error as never)).toBe(true); - }); - - it('does not take another Prisma error code for a duplicate', () => { - const error = new Prisma.PrismaClientKnownRequestError('Record to update not found.', { - code: 'P2025', - clientVersion: Prisma.prismaVersion.client, - }); - - expect(filter.isUniqueConstraintException(error as never)).toBe(false); - }); - }); - - describe.each([ - ['build.update()', 'Build_projectId_ciBuildId_key', 'There is already a build with this ci build id.'], - ['project.create()', 'Project_name_key', 'Project exists with this name.'], - ['user.create()', 'User_email_key', 'This user already exists.'], - ])('a Prisma 7 duplicate from %s', (method, constraint, message) => { - it(`answers 400 "${message}"`, () => { - const { host, response } = createHost(); - - filter.catch(prisma7UniqueConstraintError(method, constraint) as never, host); - - expect(response.status).toHaveBeenCalledWith(400); - expect(response.json).toHaveBeenCalledWith(expect.objectContaining({ message })); + it.each([ + ['the Prisma unique constraint message', uniqueConstraintError('user.create()'), true], + ['other messages', new BadRequestException('nope'), false], + [ + "Prisma 7's wording, which names the constraint instead of the fields", + prisma7UniqueConstraintError('project.create()', 'Project_name_key'), + true, + ], + ['the P2002 code whatever the message says', prismaError('P2002', 'reworded in some later version'), true], + ['no other Prisma error code as a duplicate', prismaError('P2025', 'Record to update not found.'), false], + ])('handles %s', (_, error, expected) => { + expect(filter.isUniqueConstraintException(error)).toBe(expected); }); }); describe('isForbiddenException', () => { - it('detects the message of a guard rejection', () => { - expect(filter.isForbiddenException(new ForbiddenException('Forbidden resource'))).toBe(true); - }); - - // NOTE: current behavior — detection is by message text only, so a - // ForbiddenException thrown by application code ("Forbidden" by default, - // or a custom message) is not recognised and keeps its message. - it.each([undefined, 'Admins only'])('does not detect a ForbiddenException with message %p', (message) => { - expect(filter.isForbiddenException(new ForbiddenException(message))).toBe(false); + it.each([ + ['the message of a guard rejection', new ForbiddenException('Forbidden resource'), true], + ['only a ForbiddenException with the guard message', new Error('Forbidden resource'), false], + ['no ForbiddenException without a message', new ForbiddenException(undefined), false], + ['no ForbiddenException with another message', new ForbiddenException('Admins only'), false], + ])('handles %s', (_, error, expected) => { + expect(filter.isForbiddenException(error)).toBe(expected); }); }); @@ -270,14 +264,9 @@ describe('HttpExceptionFilter', () => { ['... user.create() ...', 'This user already exists.'], ['Forbidden resource', 'You do not have permission to perform this operation.'], ['anything else', 'anything else'], + ['user.create() build.update() Forbidden resource', 'There is already a build with this ci build id.'], ])('maps %p to %p', (message, expected) => { - expect(filter.getCustomMessageForException(new Error(message) as HttpException)).toBe(expected); - }); - - it('prefers the build message when several markers are present', () => { - expect( - filter.getCustomMessageForException(new Error('user.create() build.update() Forbidden resource') as never) - ).toBe('There is already a build with this ci build id.'); + expect(filter.getCustomMessageForException(new Error(message))).toBe(expected); }); }); }); diff --git a/src/http-exception.filter.ts b/src/http-exception.filter.ts index 98a93eed..e7a15e6c 100644 --- a/src/http-exception.filter.ts +++ b/src/http-exception.filter.ts @@ -1,63 +1,68 @@ -import { ExceptionFilter, Catch, ArgumentsHost, HttpException, HttpStatus, Logger } from '@nestjs/common'; +import { ExceptionFilter, Catch, ArgumentsHost, ForbiddenException, HttpStatus, Logger } from '@nestjs/common'; import { Request, Response } from 'express'; @Catch() export class HttpExceptionFilter implements ExceptionFilter { - catch(exception: HttpException, host: ArgumentsHost): any { + private readonly logger = new Logger(HttpExceptionFilter.name); + + catch(exception: unknown, host: ArgumentsHost): void { const ctx = host.switchToHttp(); const response = ctx.getResponse(); const request = ctx.getRequest(); + const error = exception instanceof Error ? exception : new Error(String(exception)); - let status: number; - const isUniqueConstaintException = this.isUniqueConstraintException(exception); - const isForbiddenException = this.isForbiddenException(exception); - try { - status = isUniqueConstaintException - ? HttpStatus.BAD_REQUEST - : this.isRecordNotFoundException(exception) - ? HttpStatus.NOT_FOUND - : exception.getStatus(); - } catch { - status = HttpStatus.INTERNAL_SERVER_ERROR; - } + const status = this.getStatus(error); const message = - isUniqueConstaintException || isForbiddenException - ? this.getCustomMessageForException(exception) - : this.getMessage(exception); + this.isUniqueConstraintException(error) || this.isForbiddenException(error) + ? this.getCustomMessageForException(error) + : this.getMessage(error); + + if (status >= 500) { + this.logger.error(`${request.method} ${request.url} ${status}: ${message}`, error.stack); + } else { + this.logger.warn(`${request.method} ${request.url} ${status}: ${message}`); + } - Logger.error(exception, exception.stack, exception.getResponse?.()); + const retryAfter = (exception as { retryAfterSeconds?: unknown } | null)?.retryAfterSeconds; + if (typeof retryAfter === 'number') { + response.setHeader('Retry-After', String(retryAfter)); + } + + response.status(status).json({ statusCode: status, path: request.url, name: error.name, message }); + } - response.status(status).json({ - path: request.url, - name: exception.name, - message: message, - exception: exception, - stack: exception.stack, - }); + getStatus(exception: Error): number { + if (this.isUniqueConstraintException(exception)) return HttpStatus.BAD_REQUEST; + const code = (exception as { code?: unknown }).code; + if (code === 'P2025') return HttpStatus.NOT_FOUND; + if (code === 'P2003') return HttpStatus.CONFLICT; + try { + const status = (exception as { getStatus?: () => unknown }).getStatus?.(); + if (typeof status === 'number') return status; + } catch { + return HttpStatus.INTERNAL_SERVER_ERROR; + } + return this.getParserStatus(exception) ?? HttpStatus.INTERNAL_SERVER_ERROR; } - isForbiddenException(exception: HttpException): boolean { - return exception.message.includes('Forbidden resource'); + private getParserStatus(exception: Error): number | undefined { + const { status, statusCode, type } = exception as { status?: unknown; statusCode?: unknown; type?: unknown }; + const clientStatus = Number(status ?? statusCode); + return typeof type === 'string' && clientStatus >= 400 && clientStatus < 500 ? clientStatus : undefined; } - // P2025: an update or delete found no row to act on — the id the client sent - // names nothing, which is a 404 like any other lookup that comes back empty - isRecordNotFoundException(exception: HttpException): boolean { - return (exception as { code?: unknown }).code === 'P2025'; + isForbiddenException(exception: Error): boolean { + return exception instanceof ForbiddenException && exception.message === 'Forbidden resource'; } - // ValidationPipe puts the reasons in the response body and leaves the - // exception's own message at the generic "Bad Request Exception"; the - // reasons are what the client needs to see - getMessage(exception: HttpException): string { - const response = exception.getResponse?.(); - const details = typeof response === 'object' ? (response as { message?: unknown })?.message : undefined; + getMessage(exception: Error): string { + const response = (exception as { getResponse?: () => unknown }).getResponse?.(); + const details = typeof response === 'object' ? (response as { message?: unknown } | null)?.message : undefined; return Array.isArray(details) ? details.join(', ') : exception.message; } - isUniqueConstraintException(exception: HttpException): boolean { - // P2002 is Prisma's code for it in every version; the wording changed in - // Prisma 7 ("on the constraint" instead of "on the fields") + isUniqueConstraintException(exception: Error): boolean { + // P2002 message says "on the fields" before Prisma 7, "on the constraint" since return ( (exception as { code?: unknown }).code === 'P2002' || exception.message.includes('Unique constraint failed on the fields') || @@ -65,17 +70,12 @@ export class HttpExceptionFilter implements ExceptionFilter { ); } - getCustomMessageForException(exception: HttpException): string { - let message = exception.message; - message = message.includes('build.update()') - ? 'There is already a build with this ci build id.' - : message.includes('project.create()') - ? 'Project exists with this name.' - : message.includes('user.create()') - ? 'This user already exists.' - : message.includes('Forbidden resource') - ? 'You do not have permission to perform this operation.' - : message; + getCustomMessageForException(exception: Error): string { + const { message } = exception; + if (message.includes('build.update()')) return 'There is already a build with this ci build id.'; + if (message.includes('project.create()')) return 'Project exists with this name.'; + if (message.includes('user.create()')) return 'This user already exists.'; + if (message.includes('Forbidden resource')) return 'You do not have permission to perform this operation.'; return message; } } diff --git a/src/jest.config.ts b/src/jest.config.ts index a544aa21..914f623e 100644 --- a/src/jest.config.ts +++ b/src/jest.config.ts @@ -1,25 +1,21 @@ -/** @returns {Promise} */ module.exports = async () => { return { displayName: 'Unit Tests', roots: ['./'], moduleFileExtensions: ['js', 'json', 'ts'], transform: { - // SWC only transpiles; type errors are caught once by `npm run typecheck` '^.+\\.(t|j)s$': '@swc/jest', }, - // count every source file, not just the ones some test happens to import - // measured on the TypeScript source through source maps: istanbul would - // instrument SWC's output and count its decorator helpers as branches + // v8 on the TypeScript source via source maps: istanbul would count SWC's decorator helpers as branches coverageProvider: 'v8', collectCoverageFrom: [ '**/*.ts', '!**/*.spec.ts', '!jest.config.ts', - // startup only + '!jest.setup.ts', + '!**/test-utils.ts', '!main.ts', '!swagger.ts', - // types only: no code runs, so v8 would report them as never executed '!**/*.interface.ts', '!**/*.types.ts', '!**/*.model.ts', @@ -29,9 +25,10 @@ module.exports = async () => { '!test-variations/dto/test-variation-update.dto.ts', ], coverageThreshold: { - global: { statements: 98, branches: 85, functions: 98, lines: 98 }, + global: { statements: 98.5, branches: 98, functions: 99, lines: 98.5 }, }, coverageDirectory: '../coverage', testEnvironment: 'node', + setupFiles: ['/jest.setup.ts'], }; }; diff --git a/src/jest.setup.ts b/src/jest.setup.ts new file mode 100644 index 00000000..8cf16cbc --- /dev/null +++ b/src/jest.setup.ts @@ -0,0 +1,4 @@ +import { Logger } from '@nestjs/common'; + +Logger.overrideLogger(false); +Logger.overrideLogger = () => undefined; diff --git a/src/main.helpers.spec.ts b/src/main.helpers.spec.ts new file mode 100644 index 00000000..35815f2d --- /dev/null +++ b/src/main.helpers.spec.ts @@ -0,0 +1,101 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'fs'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { DEFAULT_HTTPS_CERT_PATH, DEFAULT_HTTPS_KEY_PATH, getHttpsOptions, parseLogLevels } from './main.helpers'; + +describe('getHttpsOptions', () => { + let dir: string; + let logger: { log: jest.Mock; warn: jest.Mock }; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'vrt-https-')); + logger = { log: jest.fn(), warn: jest.fn() }; + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + it('reads the key and certificate from HTTPS_KEY_PATH and HTTPS_CERT_PATH', () => { + writeFileSync(join(dir, 'a.key'), 'KEY'); + writeFileSync(join(dir, 'a.cert'), 'CERT'); + + const options = getHttpsOptions( + { HTTPS_KEY_PATH: join(dir, 'a.key'), HTTPS_CERT_PATH: join(dir, 'a.cert') }, + logger + ); + + expect(options).toEqual({ key: Buffer.from('KEY'), cert: Buffer.from('CERT') }); + expect(logger.warn).not.toHaveBeenCalled(); + }); + + it('keeps the documented default paths', () => { + expect(DEFAULT_HTTPS_KEY_PATH).toBe('./secrets/ssl.key'); + expect(DEFAULT_HTTPS_CERT_PATH).toBe('./secrets/ssl.cert'); + }); + + const inEmptyDir = (fn: () => T): T => { + const original = process.cwd(); + process.chdir(dir); + try { + return fn(); + } finally { + process.chdir(original); + } + }; + + it('falls back to HTTP with a log line when no files are there and nothing is configured', () => { + expect(inEmptyDir(() => getHttpsOptions({}, logger))).toBeUndefined(); + + expect(logger.log).toHaveBeenCalledWith('HTTPS config not found. Fall back to HTTP'); + expect(logger.warn).not.toHaveBeenCalled(); + }); + + it('falls back to HTTP with a warning when the configured files are missing', () => { + const env = { HTTPS_KEY_PATH: join(dir, 'missing.key'), HTTPS_CERT_PATH: join(dir, 'missing.cert') }; + + expect(getHttpsOptions(env, logger)).toBeUndefined(); + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('missing.key')); + expect(logger.log).not.toHaveBeenCalled(); + }); + + it('warns when only one of the two paths is configured and the other default is absent', () => { + writeFileSync(join(dir, 'a.key'), 'KEY'); + + expect(getHttpsOptions({ HTTPS_KEY_PATH: join(dir, 'a.key') }, logger)).toBeUndefined(); + expect(logger.warn).toHaveBeenCalled(); + }); + + it('treats blank paths as not configured', () => { + expect(inEmptyDir(() => getHttpsOptions({ HTTPS_KEY_PATH: ' ', HTTPS_CERT_PATH: '' }, logger))).toBeUndefined(); + expect(logger.warn).not.toHaveBeenCalled(); + expect(logger.log).toHaveBeenCalled(); + }); +}); + +describe('parseLogLevels', () => { + let logger: { warn: jest.Mock }; + + beforeEach(() => { + logger = { warn: jest.fn() }; + }); + + it.each([undefined, '', ' ,, '])('keeps Nest default (every level) for %p', (value) => { + expect(parseLogLevels(value, logger)).toBeUndefined(); + expect(logger.warn).not.toHaveBeenCalled(); + }); + + it('parses a comma separated list, trimming and lower-casing', () => { + expect(parseLogLevels('warn, ERROR', logger)).toEqual(['warn', 'error']); + }); + + it('ignores unknown levels with a warning', () => { + expect(parseLogLevels('warn,loud', logger)).toEqual(['warn']); + expect(logger.warn).toHaveBeenCalledWith('LOG_LEVEL: unknown levels ignored: loud'); + }); + + it('keeps Nest default when no level is known', () => { + expect(parseLogLevels('loud', logger)).toBeUndefined(); + expect(logger.warn).toHaveBeenCalled(); + }); +}); diff --git a/src/main.helpers.ts b/src/main.helpers.ts new file mode 100644 index 00000000..e7a3d74f --- /dev/null +++ b/src/main.helpers.ts @@ -0,0 +1,40 @@ +import { Logger, LogLevel } from '@nestjs/common'; +import { HttpsOptions } from '@nestjs/common/interfaces/external/https-options.interface'; +import { existsSync, readFileSync } from 'fs'; + +export const DEFAULT_HTTPS_KEY_PATH = './secrets/ssl.key'; +export const DEFAULT_HTTPS_CERT_PATH = './secrets/ssl.cert'; +const LOG_LEVELS: readonly LogLevel[] = ['log', 'error', 'warn', 'debug', 'verbose', 'fatal']; + +export function getHttpsOptions( + env: NodeJS.ProcessEnv = process.env, + logger: Pick = new Logger('Bootstrap') +): HttpsOptions | undefined { + const keyPath = env.HTTPS_KEY_PATH?.trim() || DEFAULT_HTTPS_KEY_PATH; + const certPath = env.HTTPS_CERT_PATH?.trim() || DEFAULT_HTTPS_CERT_PATH; + if (!existsSync(keyPath) || !existsSync(certPath)) { + if (env.HTTPS_KEY_PATH?.trim() || env.HTTPS_CERT_PATH?.trim()) { + logger.warn(`HTTPS key or certificate not found (${keyPath}, ${certPath}). Fall back to HTTP`); + } else { + logger.log('HTTPS config not found. Fall back to HTTP'); + } + return undefined; + } + return { key: readFileSync(keyPath), cert: readFileSync(certPath) }; +} + +export function parseLogLevels( + value: string | undefined, + logger: Pick = new Logger('Bootstrap') +): LogLevel[] | undefined { + const requested = (value ?? '') + .split(',') + .map((level) => level.trim().toLowerCase()) + .filter(Boolean); + const levels = requested.filter((level): level is LogLevel => (LOG_LEVELS as readonly string[]).includes(level)); + const unknown = requested.filter((level) => !levels.includes(level as LogLevel)); + if (unknown.length) { + logger.warn(`LOG_LEVEL: unknown levels ignored: ${unknown.join(', ')}`); + } + return levels.length ? levels : undefined; +} diff --git a/src/main.ts b/src/main.ts index e923b014..3b6a22eb 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,60 +1,40 @@ import { NestFactory } from '@nestjs/core'; import { AppModule } from './app.module'; import { setupSwagger } from './swagger'; -import { Logger, ValidationPipe } from '@nestjs/common'; +import { Logger } from '@nestjs/common'; import { join } from 'path'; -import * as bodyParser from 'body-parser'; import compression from 'compression'; -import { readFileSync, existsSync } from 'fs'; -import { HttpsOptions } from '@nestjs/common/interfaces/external/https-options.interface'; import { NestExpressApplication } from '@nestjs/platform-express'; import { HDD_IMAGE_PATH } from './static/hdd/constants'; import { RedisIoAdapter } from './redis-io.adapter'; - -function getHttpsOptions(): HttpsOptions | null { - const keyPath = './secrets/ssl.key'; - const certPath = './secrets/ssl.cert'; - if (!existsSync(keyPath) || !existsSync(certPath)) { - Logger.log('HTTPS config not found. Fall back to HTTP'); - return null; - } - return { - key: readFileSync(keyPath), - cert: readFileSync(certPath), - }; -} +import { getHttpsOptions, parseLogLevels } from './main.helpers'; +import { GlobalValidationPipe } from './shared/validation'; async function bootstrap() { + const logger = parseLogLevels(process.env.LOG_LEVEL); const app = await NestFactory.create(AppModule, { cors: true, - // null leaves HTTPS off, as undefined would - httpsOptions: getHttpsOptions() as HttpsOptions | undefined, + httpsOptions: getHttpsOptions(), + abortOnError: false, + ...(logger && { logger }), }); - app.useGlobalPipes(new ValidationPipe()); - // On SIGTERM, close the server and run the providers' teardown (diff - // workers, database pool) so a rolling deploy does not cut requests short + app.useGlobalPipes(new GlobalValidationPipe()); app.enableShutdownHooks(); - // Large builds return multi-megabyte test-run lists; gzip shrinks them ~10x. - // Images are already-compressed PNGs — recompressing them wastes CPU. - // Must be registered before any routes (incl. swagger) to cover them. app.use( compression({ filter: (req, res) => compression.filter(req, res) && res.getHeader('Content-Type') !== 'image/png' }) ); setupSwagger(app); - // Fan out socket.io events across API instances when running multiple - // replicas. Without REDIS_URL the default in-memory adapter is used. await RedisIoAdapter.use(app, process.env.REDIS_URL); if (process.env.BODY_PARSER_JSON_LIMIT) { - app.use(bodyParser.json({ limit: process.env.BODY_PARSER_JSON_LIMIT })); + app.useBodyParser('json', { limit: process.env.BODY_PARSER_JSON_LIMIT }); } app.useStaticAssets(join(process.cwd(), HDD_IMAGE_PATH), { maxAge: 31536000, - // allow cors setHeaders: (res) => { res.set('Access-Control-Allow-Origin', '*'); }, @@ -62,4 +42,10 @@ async function bootstrap() { await app.listen(process.env.APP_PORT || 3000); } -bootstrap(); +bootstrap().catch((error: unknown) => { + Logger.error( + `Failed to start: ${error instanceof Error ? error.message : String(error)}`, + error instanceof Error ? error.stack : undefined + ); + process.exit(1); +}); diff --git a/src/prisma/prisma-errors.spec.ts b/src/prisma/prisma-errors.spec.ts new file mode 100644 index 00000000..3331a5e4 --- /dev/null +++ b/src/prisma/prisma-errors.spec.ts @@ -0,0 +1,20 @@ +import { Prisma } from '@prisma/client'; +import { isPrismaError } from './prisma-errors'; + +describe('isPrismaError', () => { + const known = (code: string) => new Prisma.PrismaClientKnownRequestError(code, { code, clientVersion: 'test' }); + + it('matches a known request error with the same code', () => { + expect(isPrismaError(known('P2002'), 'P2002')).toBe(true); + }); + + it('rejects a known request error with another code', () => { + expect(isPrismaError(known('P2003'), 'P2002')).toBe(false); + }); + + it('rejects look-alikes and non-errors', () => { + expect(isPrismaError({ code: 'P2002' }, 'P2002')).toBe(false); + expect(isPrismaError(new Error('P2002'), 'P2002')).toBe(false); + expect(isPrismaError(undefined, 'P2002')).toBe(false); + }); +}); diff --git a/src/prisma/prisma-errors.ts b/src/prisma/prisma-errors.ts new file mode 100644 index 00000000..606b4d4d --- /dev/null +++ b/src/prisma/prisma-errors.ts @@ -0,0 +1,4 @@ +import { Prisma } from '@prisma/client'; + +export const isPrismaError = (e: unknown, code: string): e is Prisma.PrismaClientKnownRequestError => + e instanceof Prisma.PrismaClientKnownRequestError && e.code === code; diff --git a/src/prisma/prisma.module.spec.ts b/src/prisma/prisma.module.spec.ts index e97b36f0..687477f6 100644 --- a/src/prisma/prisma.module.spec.ts +++ b/src/prisma/prisma.module.spec.ts @@ -18,8 +18,6 @@ describe('PrismaModule', () => { await moduleRef.close(); }); - // Every PrismaService is a separate PrismaClient with its own connection pool, - // so the whole app must share a single instance (issue #443). it('creates a single PrismaService for the whole app', () => { const instances = new Set(); const declaredIn: string[] = []; diff --git a/src/prisma/prisma.module.ts b/src/prisma/prisma.module.ts index 3c11b27a..7207426f 100644 --- a/src/prisma/prisma.module.ts +++ b/src/prisma/prisma.module.ts @@ -1,9 +1,6 @@ import { Global, Module } from '@nestjs/common'; import { PrismaService } from './prisma.service'; -// PrismaService is a PrismaClient with its own connection pool. It must be -// provided only here: listing it in another module's providers creates an -// extra client (and pool) per module. @Global() @Module({ providers: [PrismaService], diff --git a/src/prisma/prisma.service.spec.ts b/src/prisma/prisma.service.spec.ts index 36b36705..80510c70 100644 --- a/src/prisma/prisma.service.spec.ts +++ b/src/prisma/prisma.service.spec.ts @@ -3,7 +3,7 @@ import { Test, TestingModule } from '@nestjs/testing'; import { mkdtempSync, writeFileSync } from 'fs'; import { availableParallelism, tmpdir } from 'os'; import { join, relative, resolve } from 'path'; -import { Client } from 'pg'; +import { Client, ClientConfig } from 'pg'; import { getPgOptions, negotiatePreferredSsl, physicalCpuCount, PrismaService } from './prisma.service'; describe('PrismaService', () => { @@ -63,7 +63,6 @@ describe('physicalCpuCount', () => { `processor\t: ${id}\nmodel name\t: x86\nphysical id\t: ${physicalId}\ncore id\t\t: ${coreId}\n`; it('counts each physical core once, however many hyper-threads it runs', () => { - // two sockets, two cores each, two threads per core const cpuinfo = [0, 1, 2, 3, 4, 5, 6, 7] .map((id) => processor(id, Math.floor(id / 4), Math.floor(id / 2) % 2)) .join('\n'); @@ -90,6 +89,11 @@ describe('getPgOptions', () => { const url = 'postgresql://user:secret@db:5432/vrt'; const defaultMax = physicalCpuCount() * 2 + 1; const noVerify = { rejectUnauthorized: false }; + const writeCa = (content: string) => { + const ca = join(mkdtempSync(join(tmpdir(), 'vrt-ca-')), 'server-ca.pem'); + writeFileSync(ca, content); + return ca; + }; it('keeps the Prisma 6 pool defaults when the URL sets none', () => { expect(getPgOptions(url)).toEqual({ @@ -129,24 +133,21 @@ describe('getPgOptions', () => { }); describe('TLS, as Prisma 6 read the URL', () => { - it('tries TLS without checking the certificate when the URL names no sslmode (prefer)', () => { - expect(getPgOptions(url)).toMatchObject({ pool: { ssl: noVerify }, preferSsl: true }); - }); - - it('treats an explicit sslmode=prefer the same, and takes it out of the URL for pg', () => { - const { pool, preferSsl } = getPgOptions(`${url}?sslmode=prefer&schema=vrt`); - - expect(preferSsl).toBe(true); - expect(pool.ssl).toEqual(noVerify); - expect(pool.connectionString).toBe(`${url}?schema=vrt`); - }); - - it('insists on TLS under sslmode=require, still without checking the certificate', () => { - const { pool, preferSsl } = getPgOptions(`${url}?sslmode=require`); - - expect(preferSsl).toBe(false); + it.each([ + ['takes sslmode=prefer out of the URL for pg', '?sslmode=prefer&schema=vrt', true, '?schema=vrt'], + ['insists on TLS under sslmode=require, unchecked', '?sslmode=require', false, ''], + [ + 'drops sslcert without sslaccept=strict (Prisma 6 accepted any certificate)', + '?sslmode=require&sslcert=a.pem', + false, + '', + ], + ])('%s', (_, query, preferSsl, remainingQuery) => { + const { pool, preferSsl: prefer } = getPgOptions(`${url}${query}`); + + expect(prefer).toBe(preferSsl); expect(pool.ssl).toEqual(noVerify); - expect(pool.connectionString).toBe(url); + expect(pool.connectionString).toBe(`${url}${remainingQuery}`); }); it('checks the certificate against the system CAs under sslaccept=strict', () => { @@ -154,19 +155,16 @@ describe('getPgOptions', () => { }); it('checks it against sslcert, the server certificate to Prisma, under sslaccept=strict', () => { - const ca = join(mkdtempSync(join(tmpdir(), 'vrt-ca-')), 'server-ca.pem'); - writeFileSync(ca, 'ca-pem'); + const ca = writeCa('ca-pem'); const { pool } = getPgOptions(`${url}?sslmode=require&sslaccept=strict&sslcert=${encodeURIComponent(ca)}`); expect(pool.ssl).toEqual({ ca: Buffer.from('ca-pem') }); - // pg would read sslcert as a client certificate expect(pool.connectionString).toBe(url); }); it('resolves a relative sslcert from the prisma/ folder, as Prisma did', () => { - const ca = join(mkdtempSync(join(tmpdir(), 'vrt-ca-')), 'server-ca.pem'); - writeFileSync(ca, 'relative-ca'); + const ca = writeCa('relative-ca'); const fromPrismaFolder = relative(resolve('prisma'), ca); const { pool } = getPgOptions(`${url}?sslaccept=strict&sslcert=${encodeURIComponent(fromPrismaFolder)}`); @@ -174,13 +172,6 @@ describe('getPgOptions', () => { expect(pool.ssl).toEqual({ ca: Buffer.from('relative-ca') }); }); - it('drops sslcert without sslaccept=strict, where Prisma 6 accepted any certificate', () => { - const { pool } = getPgOptions(`${url}?sslmode=require&sslcert=server-ca.pem`); - - expect(pool.ssl).toEqual(noVerify); - expect(pool.connectionString).toBe(url); - }); - it('connects without TLS under sslmode=disable', () => { expect(getPgOptions(`${url}?sslmode=disable`)).toMatchObject({ pool: { ssl: false }, preferSsl: false }); }); @@ -216,14 +207,16 @@ describe('negotiatePreferredSsl', () => { afterEach(() => jest.restoreAllMocks()); const stubClient = (connect: () => Promise) => { - jest.spyOn(Client.prototype, 'connect').mockImplementation(connect as never); + jest.spyOn(Client.prototype, 'connect').mockImplementation(connect); return jest.spyOn(Client.prototype, 'end').mockResolvedValue(undefined as never); }; + const preferringConfig = () => ({ connectionString: 'postgresql://db/vrt', ssl: { rejectUnauthorized: false } }); + it('falls back to plaintext when the server has no TLS', async () => { const end = stubClient(() => Promise.reject(new Error('The server does not support SSL connections'))); const log = jest.fn(); - const config = { connectionString: 'postgresql://db/vrt', ssl: { rejectUnauthorized: false } }; + const config: ClientConfig = preferringConfig(); await negotiatePreferredSsl(config, { log } as unknown as Logger); @@ -234,7 +227,7 @@ describe('negotiatePreferredSsl', () => { it('keeps TLS when the trial connection succeeds', async () => { const end = stubClient(() => Promise.resolve()); - const config = { connectionString: 'postgresql://db/vrt', ssl: { rejectUnauthorized: false } }; + const config = preferringConfig(); await negotiatePreferredSsl(config); @@ -244,7 +237,7 @@ describe('negotiatePreferredSsl', () => { it('keeps TLS when the connection fails for another reason, left for the real connection to report', async () => { stubClient(() => Promise.reject(new Error('password authentication failed'))); - const config = { connectionString: 'postgresql://db/vrt', ssl: { rejectUnauthorized: false } }; + const config = preferringConfig(); await expect(negotiatePreferredSsl(config)).resolves.toBeUndefined(); diff --git a/src/prisma/prisma.service.ts b/src/prisma/prisma.service.ts index 5e2f2f4a..98b452d9 100644 --- a/src/prisma/prisma.service.ts +++ b/src/prisma/prisma.service.ts @@ -6,16 +6,10 @@ import { availableParallelism } from 'os'; import { isAbsolute, resolve } from 'path'; import { Client, ClientConfig, PoolConfig } from 'pg'; -// Connection URL parameters Prisma 6 understood that neither the pg driver nor -// getPgOptions carries over. const UNSUPPORTED_PRISMA_PARAMS = ['pgbouncer', 'statement_cache_size', 'socket_timeout', 'sslidentity', 'sslpassword']; -// TLS parameters with Prisma 6's meaning: getPgOptions turns them into pg's -// `ssl` option and takes them out of the URL, where pg would read them its -// own way (sslcert as a client certificate, require as verify-full). +// Translated into pg's `ssl`: pg reads sslcert as a client certificate and require as verify-full const PRISMA_SSL_PARAMS = ['sslmode', 'sslaccept', 'sslcert']; -// Parameters only pg knows. A URL carrying one was written for pg, so its -// TLS settings are left to pg as they are. const PG_ONLY_SSL_PARAMS = ['sslrootcert', 'sslkey', 'uselibpqcompat']; const SSL_NOT_SUPPORTED = 'The server does not support SSL connections'; @@ -28,14 +22,7 @@ const readCpuinfo = (): string => { } }; -/** - * Physical cores, counted the way Prisma 6 sized its default pool - * (`num_physical_cpus * 2 + 1`, Rust's num_cpus::get_physical): distinct - * (physical id, core id) pairs in /proc/cpuinfo. Hyper-threads would double a - * logical count, and with it every replica's share of max_connections. Where - * cpuinfo has no such fields (ARM, macOS) it falls back to the logical count, - * as num_cpus does. - */ +// Prisma 6 sized its pool from physical cores; hyper-threads would double connections export function physicalCpuCount(cpuinfo: string = readCpuinfo()): number { const cores = new Set(); for (const block of cpuinfo.split(/\n\s*\n/)) { @@ -48,15 +35,6 @@ export function physicalCpuCount(cpuinfo: string = readCpuinfo()): number { return cores.size || availableParallelism(); } -/** - * Pool settings for the `pg` driver Prisma 7 talks to the database through. - * Prisma 6 read its pool and TLS settings from the connection URL and had its - * own defaults; `pg` ignores those parameters, defaults to a 10-connection pool - * that waits forever for a connection, and verifies the certificate under - * `sslmode=require`. Carry the URL parameters over and keep Prisma 6's - * behaviour, so a deployment tuned through DATABASE_URL works the same after - * the upgrade. - */ export function getPgOptions(databaseUrl: string | undefined): { pool: PoolConfig; schema?: string; @@ -65,14 +43,10 @@ export function getPgOptions(databaseUrl: string | undefined): { } { let url: URL | undefined; try { - // undefined throws here like any other unparsable URL url = new URL(databaseUrl as string); - } catch { - // an unparsable URL is left for pg to report when it connects - } + } catch {} const params = url?.searchParams ?? new URLSearchParams(); - // 0 means "no timeout" to Prisma; seconds otherwise const seconds = (name: string, fallback: number) => { const value = params.has(name) ? Number(params.get(name)) : NaN; return (Number.isFinite(value) && value >= 0 ? value : fallback) * 1000; @@ -84,12 +58,8 @@ export function getPgOptions(databaseUrl: string | undefined): { const pool: PoolConfig = { connectionString: databaseUrl, max: connectionLimit > 0 ? connectionLimit : physicalCpuCount() * 2 + 1, - // Prisma 6 waited up to 5s to open a connection and 10s for a free one; - // pg has a single timeout for both, so take the longer, and none at all - // when either was switched off connectionTimeoutMillis: connectTimeout && poolTimeout ? Math.max(connectTimeout, poolTimeout) : undefined, - // Prisma 6 kept idle connections for 300s; pg drops them after 10s, - // which reconnects over and over between ingestion bursts + // Prisma 6's value; pg's 10s default reconnects over and over between ingestion bursts idleTimeoutMillis: 300_000, }; @@ -98,19 +68,15 @@ export function getPgOptions(databaseUrl: string | undefined): { url !== undefined && ['prefer', 'require', 'disable'].includes(sslmode) && !PG_ONLY_SSL_PARAMS.some((name) => params.has(name)); - if (prismaTls) { - // Prisma 6: prefer (its default) tries TLS and falls back to plaintext, - // require insists on TLS, and neither checks the certificate unless - // sslaccept=strict, which checks it against sslcert (resolved, like - // Prisma did, from the prisma/ folder) or the system CAs + if (url && prismaTls) { + // Prisma 6: no certificate check unless sslaccept=strict const sslcert = params.get('sslcert'); const ssl = params.get('sslaccept') === 'strict' ? { ...(sslcert && { ca: readFileSync(isAbsolute(sslcert) ? sslcert : resolve('prisma', sslcert)) }) } : { rejectUnauthorized: false }; - // prismaTls is only true for a parsed URL - PRISMA_SSL_PARAMS.forEach((name) => url!.searchParams.delete(name)); - pool.connectionString = url!.toString(); + PRISMA_SSL_PARAMS.forEach((name) => url.searchParams.delete(name)); + pool.connectionString = url.toString(); pool.ssl = sslmode === 'disable' ? false : ssl; } @@ -122,11 +88,6 @@ export function getPgOptions(databaseUrl: string | undefined): { }; } -/** - * sslmode=prefer, which pg cannot do per connection: one trial connection - * over TLS, and plaintext from then on if the server has no TLS at all. - * Anything else that goes wrong is left for the real connection to report. - */ export async function negotiatePreferredSsl(config: ClientConfig, logger?: Logger): Promise { const client = new Client(config); try { @@ -144,7 +105,6 @@ export async function negotiatePreferredSsl(config: ClientConfig, logger?: Logge @Injectable() export class PrismaService extends PrismaClient implements OnModuleInit, OnApplicationShutdown { private readonly logger = new Logger(PrismaService.name); - // the very object the adapter builds its pool from when Prisma first connects private readonly pgConfig: PoolConfig; private readonly preferSsl: boolean; @@ -152,7 +112,6 @@ export class PrismaService extends PrismaClient implements OnModuleInit, OnAppli const { pool, schema, unsupportedParams, preferSsl } = getPgOptions(process.env.DATABASE_URL); super({ adapter: new PrismaPg(pool, schema ? { schema } : undefined), - // log: [{ emit: 'stdout', level: 'query' }], }); this.pgConfig = pool; this.preferSsl = preferSsl; @@ -163,6 +122,7 @@ export class PrismaService extends PrismaClient implements OnModuleInit, OnAppli ); } } + async onModuleInit() { if (this.preferSsl) { await negotiatePreferredSsl(this.pgConfig, this.logger); @@ -170,18 +130,11 @@ export class PrismaService extends PrismaClient implements OnModuleInit, OnAppli await this.$connect(); } - /** - * A connection of its own, with the same settings as the pool but outside - * it, for state bound to one connection that must not tie up a pooled one - * for long — such as the advisory lock the nightly cleanup holds while it runs. - * Not connected yet; the caller connects and ends it. - */ dedicatedClient(): Client { return new Client(this.pgConfig); } - // After the HTTP server has stopped and drained its requests (Nest runs - // onModuleDestroy before that), so none of them finds the pool closed + // Not onModuleDestroy: that runs before the HTTP server has drained its requests async onApplicationShutdown() { await this.$disconnect(); } diff --git a/src/projects/dto/project.dto.spec.ts b/src/projects/dto/project.dto.spec.ts index 49fe2776..028e4fd6 100644 --- a/src/projects/dto/project.dto.spec.ts +++ b/src/projects/dto/project.dto.spec.ts @@ -39,6 +39,8 @@ describe('ProjectDto', () => { ['imageComparisonConfig', { imageComparisonConfig: '{not json' }, 'isJson'], ['autoApproveFeature', { autoApproveFeature: 'yes' }, 'isBoolean'], ['bulkApproveGroupBy', { bulkApproveGroupBy: 1 }, 'isString'], + ['maxBuildAllowed', { maxBuildAllowed: 1.5 }, 'isInt'], + ['maxBranchLifetime', { maxBranchLifetime: 0.5 }, 'isInt'], ])('rejects an invalid %s', async (property, override, constraint) => { const errors = await validate(plainToInstance(ProjectDto, { ...plainProject, ...override })); @@ -46,4 +48,10 @@ describe('ProjectDto', () => { expect(errors[0].property).toBe(property); expect(Object.keys(errors[0].constraints!)).toEqual([constraint]); }); + + it.each([0, -1, 1, 100])('still accepts the integer %p for both counters', async (value) => { + const dto = plainToInstance(ProjectDto, { ...plainProject, maxBuildAllowed: value, maxBranchLifetime: value }); + + await expect(validate(dto)).resolves.toEqual([]); + }); }); diff --git a/src/projects/dto/project.dto.ts b/src/projects/dto/project.dto.ts index 1de2bf77..778838d9 100644 --- a/src/projects/dto/project.dto.ts +++ b/src/projects/dto/project.dto.ts @@ -1,5 +1,5 @@ import { ApiProperty } from '@nestjs/swagger'; -import { IsUUID, IsString, IsNumber, IsBoolean, IsEnum, IsJSON, IsDate } from 'class-validator'; +import { IsUUID, IsString, IsNumber, IsInt, IsBoolean, IsEnum, IsJSON, IsDate } from 'class-validator'; import { ImageComparison, Project } from '@prisma/client'; import { Type } from 'class-transformer'; @@ -39,11 +39,11 @@ export class ProjectDto implements Project { imageComparison!: ImageComparison; @ApiProperty() - @IsNumber() + @IsInt() maxBuildAllowed!: number; @ApiProperty() - @IsNumber() + @IsInt() maxBranchLifetime!: number; @ApiProperty() diff --git a/src/projects/projects.controller.spec.ts b/src/projects/projects.controller.spec.ts index 761beb89..0b2393f9 100644 --- a/src/projects/projects.controller.spec.ts +++ b/src/projects/projects.controller.spec.ts @@ -37,10 +37,6 @@ describe('Projects Controller', () => { controller = module.get(ProjectsController); }); - it('should be defined', () => { - expect(controller).toBeDefined(); - }); - it('lists all projects', async () => { projectsService.findAll.mockResolvedValue([TEST_PROJECT]); @@ -96,8 +92,6 @@ describe('Projects Controller', () => { let app: INestApplication; beforeEach(async () => { - // the guard is stubbed to sign in a guest: favourites are personal, so - // no role guard may stand in front of them const module: TestingModule = await Test.createTestingModule({ controllers: [ProjectsController], providers: [ diff --git a/src/projects/projects.controller.ts b/src/projects/projects.controller.ts index 816ca69c..a7869d81 100644 --- a/src/projects/projects.controller.ts +++ b/src/projects/projects.controller.ts @@ -42,9 +42,7 @@ export class ProjectsController { return this.projectsService.findAll(); } - // Favourites are personal, not project settings, so every role (guests - // included) may use them: no RoleGuard. Declared before any `:id` route so - // "favorites" is never read as a project id. + // No RoleGuard: favourites are personal. Declared before any `:id` route so "favorites" is not read as an id @Get('favorites') @ApiBearerAuth() @UseGuards(JwtAuthGuard) diff --git a/src/projects/projects.module.ts b/src/projects/projects.module.ts index 6344c6de..d40dd7cb 100644 --- a/src/projects/projects.module.ts +++ b/src/projects/projects.module.ts @@ -3,9 +3,10 @@ import { ProjectsService } from './projects.service'; import { ProjectsController } from './projects.controller'; import { BuildsModule } from '../builds/builds.module'; import { TestVariationsModule } from '../test-variations/test-variations.module'; +import { StaticModule } from '../static/static.module'; @Module({ - imports: [forwardRef(() => BuildsModule), forwardRef(() => TestVariationsModule)], + imports: [forwardRef(() => BuildsModule), forwardRef(() => TestVariationsModule), StaticModule], providers: [ProjectsService], controllers: [ProjectsController], exports: [ProjectsService], diff --git a/src/projects/projects.service.spec.ts b/src/projects/projects.service.spec.ts index 246a83c1..a10e97af 100644 --- a/src/projects/projects.service.spec.ts +++ b/src/projects/projects.service.spec.ts @@ -1,235 +1,342 @@ -import { Test, TestingModule } from '@nestjs/testing'; +import { Test } from '@nestjs/testing'; import { ProjectsService } from './projects.service'; import { PrismaService } from '../prisma/prisma.service'; -import { TestVariationsService } from '../test-variations/test-variations.service'; -import { BuildsService } from '../builds/builds.service'; -import { HttpException, HttpStatus } from '@nestjs/common'; -import { ImageComparison } from '@prisma/client'; +import { StaticService } from '../static/static.service'; +import { EventsGateway } from '../shared/events/events.gateway'; +import { BuildDto } from '../builds/dto/build.dto'; +import { BadRequestException, ConflictException, HttpException, HttpStatus } from '@nestjs/common'; +import { ImageComparison, Prisma } from '@prisma/client'; import { TEST_PROJECT } from '../_data_'; import { CreateProjectDto } from './dto/create-project.dto'; import { UpdateProjectDto } from './dto/update-project.dto'; -const initService = async ({ - projectFindUniqueMock = jest.fn(), - projectFindManyMock = jest.fn(), - projectCreateMock = jest.fn(), - projectUpdateMock = jest.fn(), - projectDeleteMock = jest.fn(), - userUpdateMock = jest.fn(), - buildRemoveMock = jest.fn(), - testVariationDeleteMock = jest.fn(), -}) => { - const module: TestingModule = await Test.createTestingModule({ +const projectId = 'a9385fc1-884d-4f9f-915e-40da0e7773d5'; +const notFound = new HttpException('Project not found', HttpStatus.NOT_FOUND); + +const setup = async () => { + const prisma = { + project: { + findUnique: jest.fn(), + findMany: jest.fn(), + create: jest.fn(), + update: jest.fn(), + delete: jest.fn(), + }, + user: { update: jest.fn() }, + $transaction: jest.fn(), + }; + const deleteImage = jest.fn(); + const buildDeleted = jest.fn(); + const module = await Test.createTestingModule({ providers: [ ProjectsService, + { provide: PrismaService, useValue: prisma }, { - provide: PrismaService, + provide: StaticService, useValue: { - project: { - findUnique: projectFindUniqueMock, - findMany: projectFindManyMock, - create: projectCreateMock, - update: projectUpdateMock, - delete: projectDeleteMock, - }, - user: { - update: userUpdateMock, + deleteImages: async (names: Iterable) => { + for (const name of names) if (name) await deleteImage(name); }, }, }, - { provide: TestVariationsService, useValue: { delete: testVariationDeleteMock } }, - { provide: BuildsService, useValue: { remove: buildRemoveMock } }, + { provide: EventsGateway, useValue: { buildDeleted } }, ], }).compile(); - return module.get(ProjectsService); + return { service: module.get(ProjectsService), prisma, deleteImage, buildDeleted }; }; -describe('ProjectsService', () => { - let service: ProjectsService; +describe('ProjectsService', () => { describe('findOne', () => { it.each([ ['name', 'someName', { id: undefined, name: 'someName' }], - ['id', 'a9385fc1-884d-4f9f-915e-40da0e7773d5', { id: 'a9385fc1-884d-4f9f-915e-40da0e7773d5', name: undefined }], + ['id', projectId, { id: projectId, name: undefined }], ])('should find by %s', async (_, query, expected) => { - const projectFindUniqueMock = jest.fn().mockResolvedValue({}); - service = await initService({ projectFindUniqueMock }); + const { service, prisma } = await setup(); + prisma.project.findUnique.mockResolvedValue({}); await service.findOne(query); - expect(projectFindUniqueMock).toHaveBeenCalledWith({ - where: expected, - }); + expect(prisma.project.findUnique).toHaveBeenCalledWith({ where: expected }); }); it('should throw exception if not found', async () => { - const projectFindUniqueMock = jest.fn().mockResolvedValueOnce(undefined); - service = await initService({ projectFindUniqueMock }); + const { service, prisma } = await setup(); + prisma.project.findUnique.mockResolvedValueOnce(undefined); - await expect(service.findOne('foo')).rejects.toThrow( - new HttpException('Project not found', HttpStatus.NOT_FOUND) - ); + await expect(service.findOne('foo')).rejects.toThrow(notFound); }); }); - describe('findAll', () => { - it('returns every project from the database', async () => { - const projectFindManyMock = jest.fn().mockResolvedValue([TEST_PROJECT]); - service = await initService({ projectFindManyMock }); - const result = await service.findAll(); + it('findAll returns every project from the database', async () => { + const { service, prisma } = await setup(); + prisma.project.findMany.mockResolvedValue([TEST_PROJECT]); - expect(projectFindManyMock).toHaveBeenCalledWith(); - expect(result).toEqual([TEST_PROJECT]); - }); + const result = await service.findAll(); + + expect(prisma.project.findMany).toHaveBeenCalledWith(); + expect(result).toEqual([TEST_PROJECT]); }); - describe('create', () => { - it('creates the project from the dto fields', async () => { - const projectCreateMock = jest.fn().mockResolvedValue(TEST_PROJECT); - service = await initService({ projectCreateMock }); - const dto: CreateProjectDto = { - name: 'New project', - mainBranchName: 'main', - autoApproveFeature: true, - imageComparison: ImageComparison.lookSame, - imageComparisonConfig: '{"strict":true}', - maxBuildAllowed: 50, - maxBranchLifetime: 10, - bulkApproveVariations: true, - bulkApproveGroupBy: 'name', - }; + it('create creates the project from the dto fields', async () => { + const { service, prisma } = await setup(); + prisma.project.create.mockResolvedValue(TEST_PROJECT); + const dto: CreateProjectDto = { + name: 'New project', + mainBranchName: 'main', + autoApproveFeature: true, + imageComparison: ImageComparison.lookSame, + imageComparisonConfig: '{"strict":true}', + maxBuildAllowed: 50, + maxBranchLifetime: 10, + bulkApproveVariations: true, + bulkApproveGroupBy: 'name', + }; + + const result = await service.create({ ...dto, extra: 'ignored' } as CreateProjectDto); + + expect(prisma.project.create).toHaveBeenCalledWith({ data: dto }); + expect(result).toBe(TEST_PROJECT); + }); - const result = await service.create({ ...dto, extra: 'ignored' } as CreateProjectDto); + it('update updates the project by id with the dto fields', async () => { + const { service, prisma } = await setup(); + prisma.project.update.mockResolvedValue(TEST_PROJECT); + const dto: UpdateProjectDto = { + id: projectId, + name: 'Renamed', + mainBranchName: 'develop', + autoApproveFeature: false, + imageComparison: ImageComparison.pixelmatch, + imageComparisonConfig: '{}', + maxBuildAllowed: 5, + maxBranchLifetime: 7, + bulkApproveVariations: false, + bulkApproveGroupBy: 'customTags', + }; + + const result = await service.update(dto); + + const { id, ...data } = dto; + expect(prisma.project.update).toHaveBeenCalledWith({ where: { id }, data }); + expect(result).toBe(TEST_PROJECT); + }); - expect(projectCreateMock).toHaveBeenCalledWith({ data: dto }); - expect(result).toBe(TEST_PROJECT); + describe('odiff on S3 storage', () => { + const originalStaticService = process.env.STATIC_SERVICE; + const odiffDto = { ...TEST_PROJECT, id: projectId, imageComparison: ImageComparison.odiff } as UpdateProjectDto; + + afterEach(() => { + if (originalStaticService === undefined) { + delete process.env.STATIC_SERVICE; + } else { + process.env.STATIC_SERVICE = originalStaticService; + } }); - }); - describe('update', () => { - it('updates the project by id with the dto fields', async () => { - const projectUpdateMock = jest.fn().mockResolvedValue(TEST_PROJECT); - service = await initService({ projectUpdateMock }); - const dto: UpdateProjectDto = { - id: 'a9385fc1-884d-4f9f-915e-40da0e7773d5', - name: 'Renamed', - mainBranchName: 'develop', - autoApproveFeature: false, - imageComparison: ImageComparison.pixelmatch, - imageComparisonConfig: '{}', - maxBuildAllowed: 5, - maxBranchLifetime: 7, - bulkApproveVariations: false, - bulkApproveGroupBy: 'customTags', - }; + it.each(['create', 'update'] as const)( + '%s rejects odiff on S3 storage with 400 and writes nothing', + async (method) => { + process.env.STATIC_SERVICE = 's3'; + const { service, prisma } = await setup(); - const result = await service.update(dto); + await expect(service[method](odiffDto)).rejects.toThrow(BadRequestException); - const { id, ...data } = dto; - expect(projectUpdateMock).toHaveBeenCalledWith({ where: { id }, data }); - expect(result).toBe(TEST_PROJECT); + expect(prisma.project.create).not.toHaveBeenCalled(); + expect(prisma.project.update).not.toHaveBeenCalled(); + } + ); + + it.each(['create', 'update'] as const)('%s accepts odiff on HDD storage', async (method) => { + delete process.env.STATIC_SERVICE; + const { service, prisma } = await setup(); + prisma.project.create.mockResolvedValue(TEST_PROJECT); + prisma.project.update.mockResolvedValue(TEST_PROJECT); + + await expect(service[method](odiffDto)).resolves.toBe(TEST_PROJECT); + }); + + it('still accepts other comparators on S3 storage', async () => { + process.env.STATIC_SERVICE = 's3'; + const { service, prisma } = await setup(); + prisma.project.create.mockResolvedValue(TEST_PROJECT); + + await expect(service.create({ ...odiffDto, imageComparison: ImageComparison.lookSame })).resolves.toBe( + TEST_PROJECT + ); }); }); describe('remove', () => { - it('removes every build and test variation one by one before deleting the project', async () => { + const knownRequestError = (code: string) => + new Prisma.PrismaClientKnownRequestError(code, { code, clientVersion: 'test' }); + + const initRemove = async (transaction?: (fn: (tx: unknown) => unknown) => unknown) => { + const ctx = await setup(); const calls: string[] = []; - const projectFindUniqueMock = jest.fn().mockResolvedValue({ - ...TEST_PROJECT, - builds: [{ id: 'build1' }, { id: 'build2' }], - testVariations: [{ id: 'variation1' }], - }); - const buildRemoveMock = jest.fn().mockImplementation(async (id) => calls.push(`build ${id}`)); - const testVariationDeleteMock = jest.fn().mockImplementation(async (id) => calls.push(`variation ${id}`)); - const projectDeleteMock = jest.fn().mockImplementation(async () => { - calls.push('project'); - return TEST_PROJECT; - }); - service = await initService({ - projectFindUniqueMock, - buildRemoveMock, - testVariationDeleteMock, - projectDeleteMock, - }); + const record = (name: string, result?: T) => + jest.fn().mockImplementation(async () => { + calls.push(name); + return result; + }); + const builds = ['build1', 'build2'].map((id) => ({ + id, + ciBuildId: null, + number: 1, + branchName: 'master', + status: null, + projectId, + updatedAt: new Date(), + createdAt: new Date(), + userId: null, + isRunning: false, + })); + const tx = { + build: { findMany: record('build.findMany', builds), deleteMany: record('build.deleteMany') }, + testRun: { + findMany: record('testRun.findMany', [ + { imageName: 'run1.png', diffName: 'run1.diff.png', imageThumbnailName: null, diffThumbnailName: null }, + { imageName: 'run2.png', diffName: null, imageThumbnailName: 'run2.thumb.png', diffThumbnailName: '' }, + ]), + deleteMany: record('testRun.deleteMany'), + }, + baseline: { + findMany: record('baseline.findMany', [{ baselineName: 'baseline1.png' }]), + deleteMany: record('baseline.deleteMany'), + }, + testVariation: { deleteMany: record('testVariation.deleteMany') }, + project: { delete: record('project.delete', TEST_PROJECT) }, + }; + ctx.prisma.project.findUnique.mockResolvedValue(TEST_PROJECT); + ctx.prisma.$transaction.mockImplementation(transaction ?? ((fn, _options) => fn(tx))); + ctx.deleteImage.mockImplementation(async (name) => calls.push(`deleteImage ${name}`)); + ctx.buildDeleted.mockImplementation((dto) => calls.push(`buildDeleted ${dto.id}`)); + return { ...ctx, calls, tx, builds, transaction: ctx.prisma.$transaction }; + }; + + it('removes rows in FK order inside one transaction, then files, then announces builds', async () => { + const { service, calls, tx, transaction } = await initRemove(); + + const result = await service.remove(projectId); + + expect(calls).toEqual([ + 'build.findMany', + 'testRun.findMany', + 'baseline.findMany', + 'testRun.deleteMany', + 'baseline.deleteMany', + 'testVariation.deleteMany', + 'build.deleteMany', + 'project.delete', + 'deleteImage run1.png', + 'deleteImage run1.diff.png', + 'deleteImage run2.png', + 'deleteImage run2.thumb.png', + 'deleteImage baseline1.png', + 'buildDeleted build1', + 'buildDeleted build2', + ]); + expect(tx.testRun.deleteMany).toHaveBeenCalledWith({ where: { build: { projectId } } }); + expect(tx.baseline.deleteMany).toHaveBeenCalledWith({ where: { testVariation: { projectId } } }); + expect(tx.project.delete).toHaveBeenCalledWith({ where: { id: projectId } }); + expect(transaction).toHaveBeenCalledTimes(1); + expect(transaction).toHaveBeenCalledWith(expect.any(Function), { maxWait: 10000, timeout: 120000 }); + expect(result).toBe(TEST_PROJECT); + }); + + it('announces every build with the build dto', async () => { + const { service, buildDeleted, builds } = await initRemove(); + + await service.remove(projectId); + + expect(buildDeleted).toHaveBeenNthCalledWith(1, new BuildDto(builds[0])); + expect(buildDeleted).toHaveBeenNthCalledWith(2, new BuildDto(builds[1])); + }); - const result = await service.remove('projectId'); + it('deletes no file when the transaction fails', async () => { + const { service, tx, deleteImage, buildDeleted } = await initRemove(); + tx.project.delete.mockRejectedValue(new Error('db down')); - expect(projectFindUniqueMock).toHaveBeenCalledWith({ - where: { id: 'projectId' }, - include: { builds: true, testVariations: true }, + await expect(service.remove(projectId)).rejects.toThrow('db down'); + + expect(deleteImage).not.toHaveBeenCalled(); + expect(buildDeleted).not.toHaveBeenCalled(); + }); + + it('retries when an upload added rows meanwhile (P2003)', async () => { + let attempt = 0; + const { service, transaction, tx } = await initRemove((fn) => { + if (++attempt === 1) { + throw knownRequestError('P2003'); + } + return fn(tx); }); - expect(calls).toEqual(['build build1', 'build build2', 'variation variation1', 'project']); - expect(projectDeleteMock).toHaveBeenCalledWith({ where: { id: 'projectId' } }); - expect(result).toBe(TEST_PROJECT); + + await expect(service.remove(projectId)).resolves.toBe(TEST_PROJECT); + + expect(transaction).toHaveBeenCalledTimes(2); }); - it('stops at the first failing build and leaves the project in place', async () => { - const projectFindUniqueMock = jest.fn().mockResolvedValue({ - ...TEST_PROJECT, - builds: [{ id: 'build1' }, { id: 'build2' }], - testVariations: [{ id: 'variation1' }], + it('answers 409 when uploads keep arriving', async () => { + const { service, transaction, deleteImage } = await initRemove(() => { + throw knownRequestError('P2003'); }); - const buildRemoveMock = jest.fn().mockRejectedValueOnce(new Error('cannot delete')); - const testVariationDeleteMock = jest.fn(); - const projectDeleteMock = jest.fn(); - service = await initService({ - projectFindUniqueMock, - buildRemoveMock, - testVariationDeleteMock, - projectDeleteMock, + + await expect(service.remove(projectId)).rejects.toThrow(ConflictException); + + expect(transaction).toHaveBeenCalledTimes(3); + expect(deleteImage).not.toHaveBeenCalled(); + }); + + it('answers 404 when the project disappears meanwhile (P2025)', async () => { + const { service, transaction } = await initRemove(() => { + throw knownRequestError('P2025'); }); - await expect(service.remove('projectId')).rejects.toThrow('cannot delete'); + await expect(service.remove(projectId)).rejects.toThrow(notFound); - expect(buildRemoveMock).toHaveBeenCalledTimes(1); - expect(testVariationDeleteMock).not.toHaveBeenCalled(); - expect(projectDeleteMock).not.toHaveBeenCalled(); + expect(transaction).toHaveBeenCalledTimes(1); }); - it('throws a TypeError when the project does not exist', async () => { - const projectFindUniqueMock = jest.fn().mockResolvedValue(null); - const projectDeleteMock = jest.fn(); - service = await initService({ projectFindUniqueMock, projectDeleteMock }); + it('answers 404 for an unknown project and deletes nothing', async () => { + const { service, prisma } = await setup(); + prisma.project.findUnique.mockResolvedValue(null); - // NOTE: current behavior — no not-found handling (unlike findOne), the null project is dereferenced - await expect(service.remove('missing')).rejects.toThrow(TypeError); - expect(projectDeleteMock).not.toHaveBeenCalled(); + await expect(service.remove('missing')).rejects.toThrow(notFound); + expect(prisma.project.delete).not.toHaveBeenCalled(); }); }); describe('favourites', () => { const userId = 'b1e2c3d4-0000-4000-8000-000000000001'; - const projectId = 'a9385fc1-884d-4f9f-915e-40da0e7773d5'; - it('lists the ids of the projects the user marked as favourite', async () => { - const projectFindManyMock = jest.fn().mockResolvedValue([{ id: 'p1' }, { id: 'p2' }]); - service = await initService({ projectFindManyMock }); + it.each([ + [ + [{ id: 'p1' }, { id: 'p2' }], + ['p1', 'p2'], + ], + [[], []], + ])('lists the ids of the projects the user marked as favourite (%j)', async (rows, ids) => { + const { service, prisma } = await setup(); + prisma.project.findMany.mockResolvedValue(rows); const result = await service.findFavoriteIds(userId); - expect(projectFindManyMock).toHaveBeenCalledWith({ + expect(prisma.project.findMany).toHaveBeenCalledWith({ where: { favoritedBy: { some: { id: userId } } }, select: { id: true }, }); - expect(result).toEqual(['p1', 'p2']); - }); - - it('lists nothing for a user without favourites', async () => { - const projectFindManyMock = jest.fn().mockResolvedValue([]); - service = await initService({ projectFindManyMock }); - - await expect(service.findFavoriteIds(userId)).resolves.toEqual([]); + expect(result).toEqual(ids); }); it('connects the project to the user favourites', async () => { - const projectFindUniqueMock = jest.fn().mockResolvedValue(TEST_PROJECT); - const userUpdateMock = jest.fn().mockResolvedValue({}); - service = await initService({ projectFindUniqueMock, userUpdateMock }); + const { service, prisma } = await setup(); + prisma.project.findUnique.mockResolvedValue(TEST_PROJECT); await service.addFavorite(userId, projectId); - expect(projectFindUniqueMock).toHaveBeenCalledWith({ where: { id: projectId, name: undefined } }); - expect(userUpdateMock).toHaveBeenCalledWith({ + expect(prisma.project.findUnique).toHaveBeenCalledWith({ where: { id: projectId, name: undefined } }); + expect(prisma.user.update).toHaveBeenCalledWith({ where: { id: userId }, data: { favoriteProjects: { connect: { id: projectId } } }, select: { id: true }, @@ -237,49 +344,34 @@ describe('ProjectsService', () => { }); it('adds a favourite twice without an error', async () => { - const projectFindUniqueMock = jest.fn().mockResolvedValue(TEST_PROJECT); - const userUpdateMock = jest.fn().mockResolvedValue({}); - service = await initService({ projectFindUniqueMock, userUpdateMock }); + const { service, prisma } = await setup(); + prisma.project.findUnique.mockResolvedValue(TEST_PROJECT); await service.addFavorite(userId, projectId); await expect(service.addFavorite(userId, projectId)).resolves.toBeUndefined(); - expect(userUpdateMock).toHaveBeenCalledTimes(2); + expect(prisma.user.update).toHaveBeenCalledTimes(2); }); it('answers 404 and changes nothing when the project does not exist', async () => { - const projectFindUniqueMock = jest.fn().mockResolvedValue(null); - const userUpdateMock = jest.fn(); - service = await initService({ projectFindUniqueMock, userUpdateMock }); + const { service, prisma } = await setup(); + prisma.project.findUnique.mockResolvedValue(null); - await expect(service.addFavorite(userId, projectId)).rejects.toThrow( - new HttpException('Project not found', HttpStatus.NOT_FOUND) - ); - expect(userUpdateMock).not.toHaveBeenCalled(); + await expect(service.addFavorite(userId, projectId)).rejects.toThrow(notFound); + expect(prisma.user.update).not.toHaveBeenCalled(); }); - it('disconnects the project from the user favourites', async () => { - const userUpdateMock = jest.fn().mockResolvedValue({}); - service = await initService({ userUpdateMock }); + it('disconnects the project from the user favourites without looking it up', async () => { + const { service, prisma } = await setup(); await expect(service.removeFavorite(userId, projectId)).resolves.toBeUndefined(); - expect(userUpdateMock).toHaveBeenCalledWith({ + expect(prisma.user.update).toHaveBeenCalledWith({ where: { id: userId }, data: { favoriteProjects: { disconnect: { id: projectId } } }, select: { id: true }, }); - }); - - it('removes a project that is not a favourite without an error', async () => { - // Prisma treats disconnecting an unconnected record as a no-op, so the - // service does no lookup of its own first - const projectFindUniqueMock = jest.fn(); - const userUpdateMock = jest.fn().mockResolvedValue({}); - service = await initService({ projectFindUniqueMock, userUpdateMock }); - - await expect(service.removeFavorite(userId, projectId)).resolves.toBeUndefined(); - expect(projectFindUniqueMock).not.toHaveBeenCalled(); + expect(prisma.project.findUnique).not.toHaveBeenCalled(); }); }); }); diff --git a/src/projects/projects.service.ts b/src/projects/projects.service.ts index 251b08d9..5831d082 100644 --- a/src/projects/projects.service.ts +++ b/src/projects/projects.service.ts @@ -1,12 +1,30 @@ -import { forwardRef, HttpException, HttpStatus, Inject, Injectable, Logger } from '@nestjs/common'; +import { BadRequestException, ConflictException, HttpException, HttpStatus, Injectable, Logger } from '@nestjs/common'; import { CreateProjectDto } from './dto/create-project.dto'; -import { BuildsService } from '../builds/builds.service'; -import { TestVariationsService } from '../test-variations/test-variations.service'; +import { BuildDto } from '../builds/dto/build.dto'; +import { EventsGateway } from '../shared/events/events.gateway'; +import { StaticService } from '../static/static.service'; import { PrismaService } from '../prisma/prisma.service'; -import { Project } from '@prisma/client'; +import { isPrismaError } from '../prisma/prisma-errors'; +import { ImageComparison, Prisma, Project } from '@prisma/client'; import { isUuidShaped } from '../utils/random-key'; +import { testRunImageNames } from '../utils'; import { UpdateProjectDto } from './dto/update-project.dto'; -import type { WrapperType } from '../shared/wrapper-type'; +import { isHddStaticServiceConfigured } from '../static/utils'; + +const MAX_REMOVE_ATTEMPTS = 3; +const REMOVE_TRANSACTION = { maxWait: 10_000, timeout: 120_000 }; + +const projectData = (dto: CreateProjectDto | UpdateProjectDto) => ({ + name: dto.name, + mainBranchName: dto.mainBranchName, + autoApproveFeature: dto.autoApproveFeature, + imageComparison: dto.imageComparison, + imageComparisonConfig: dto.imageComparisonConfig, + maxBuildAllowed: dto.maxBuildAllowed, + maxBranchLifetime: dto.maxBranchLifetime, + bulkApproveVariations: dto.bulkApproveVariations, + bulkApproveGroupBy: dto.bulkApproveGroupBy, +}); @Injectable() export class ProjectsService { @@ -14,15 +32,13 @@ export class ProjectsService { constructor( private prismaService: PrismaService, - @Inject(forwardRef(() => BuildsService)) - private buildsService: WrapperType, - @Inject(forwardRef(() => TestVariationsService)) - private testVariationsService: WrapperType + private staticService: StaticService, + private eventsGateway: EventsGateway ) {} - async findOne(idOrName: string): Promise { + async findOne(idOrName: string): Promise { const isUUID = isUuidShaped(idOrName); - const project: Project | null = await this.prismaService.project.findUnique({ + const project = await this.prismaService.project.findUnique({ where: { id: isUUID ? idOrName : undefined, name: !isUUID ? idOrName : undefined, @@ -48,10 +64,7 @@ export class ProjectsService { } async addFavorite(userId: string, projectId: string): Promise { - // connect alone would fail with a Prisma error for an unknown project; - // findOne turns that into the API's usual 404 await this.findOne(projectId); - // connecting an already connected project is a no-op, so adding twice is fine await this.prismaService.user.update({ where: { id: userId }, data: { favoriteProjects: { connect: { id: projectId } } }, @@ -60,8 +73,6 @@ export class ProjectsService { } async removeFavorite(userId: string, projectId: string): Promise { - // disconnecting a project that is not connected (or does not exist) is a - // no-op, which makes removal idempotent without a lookup await this.prismaService.user.update({ where: { id: userId }, data: { favoriteProjects: { disconnect: { id: projectId } } }, @@ -69,58 +80,64 @@ export class ProjectsService { }); } + private assertComparisonSupported(imageComparison: ImageComparison): void { + if (imageComparison === ImageComparison.odiff && !isHddStaticServiceConfigured()) { + throw new BadRequestException('Odiff can only be used with the HDD static service (STATIC_SERVICE=hdd)'); + } + } + async create(projectDto: CreateProjectDto): Promise { - return this.prismaService.project.create({ - data: { - name: projectDto.name, - mainBranchName: projectDto.mainBranchName, - autoApproveFeature: projectDto.autoApproveFeature, - imageComparison: projectDto.imageComparison, - imageComparisonConfig: projectDto.imageComparisonConfig, - maxBuildAllowed: projectDto.maxBuildAllowed, - maxBranchLifetime: projectDto.maxBranchLifetime, - bulkApproveVariations: projectDto.bulkApproveVariations, - bulkApproveGroupBy: projectDto.bulkApproveGroupBy, - }, - }); + this.assertComparisonSupported(projectDto.imageComparison); + return this.prismaService.project.create({ data: projectData(projectDto) }); } async update(projectDto: UpdateProjectDto): Promise { - return this.prismaService.project.update({ - where: { id: projectDto.id }, - data: { - name: projectDto.name, - mainBranchName: projectDto.mainBranchName, - autoApproveFeature: projectDto.autoApproveFeature, - imageComparison: projectDto.imageComparison, - maxBuildAllowed: projectDto.maxBuildAllowed, - maxBranchLifetime: projectDto.maxBranchLifetime, - imageComparisonConfig: projectDto.imageComparisonConfig, - bulkApproveVariations: projectDto.bulkApproveVariations, - bulkApproveGroupBy: projectDto.bulkApproveGroupBy, - }, - }); + this.assertComparisonSupported(projectDto.imageComparison); + return this.prismaService.project.update({ where: { id: projectDto.id }, data: projectData(projectDto) }); } async remove(id: string): Promise { this.logger.debug(`Going to remove Project ${id}`); - const project = (await this.prismaService.project.findUnique({ - where: { id }, - include: { - builds: true, - testVariations: true, - }, - }))!; - - for (const build of project.builds) { - await this.buildsService.remove(build.id); - } - for (const testVariation of project.testVariations) { - await this.testVariationsService.delete(testVariation.id); + await this.findOne(id); + for (let attempt = 1; ; attempt++) { + try { + const { project, builds, fileNames } = await this.prismaService.$transaction( + (tx) => this.removeRows(tx, id), + REMOVE_TRANSACTION + ); + // Files only after the commit: a failure leaks bytes instead of leaving rows that point at deleted images + await this.staticService.deleteImages(fileNames); + builds.forEach((build) => this.eventsGateway.buildDeleted(new BuildDto(build))); + return project; + } catch (e) { + if (isPrismaError(e, 'P2025')) { + throw new HttpException('Project not found', HttpStatus.NOT_FOUND); + } + if (!isPrismaError(e, 'P2003')) { + throw e; + } + if (attempt === MAX_REMOVE_ATTEMPTS) { + throw new ConflictException(`Project ${id} keeps receiving uploads, try again later`); + } + } } + } - return this.prismaService.project.delete({ - where: { id }, + private async removeRows(tx: Prisma.TransactionClient, projectId: string) { + const ofProjectBuilds = { build: { projectId } }; + const ofProjectVariations = { testVariation: { projectId } }; + const builds = await tx.build.findMany({ where: { projectId } }); + const testRuns = await tx.testRun.findMany({ + where: ofProjectBuilds, + select: { imageName: true, diffName: true, imageThumbnailName: true, diffThumbnailName: true }, }); + const baselines = await tx.baseline.findMany({ where: ofProjectVariations, select: { baselineName: true } }); + await tx.testRun.deleteMany({ where: ofProjectBuilds }); + await tx.baseline.deleteMany({ where: ofProjectVariations }); + await tx.testVariation.deleteMany({ where: { projectId } }); + await tx.build.deleteMany({ where: { projectId } }); + const project = await tx.project.delete({ where: { id: projectId } }); + const fileNames = [...testRuns.flatMap(testRunImageNames), ...baselines.map((b) => b.baselineName)]; + return { project, builds, fileNames }; } } diff --git a/src/redis-io.adapter.spec.ts b/src/redis-io.adapter.spec.ts index f7b87bf5..838b5530 100644 --- a/src/redis-io.adapter.spec.ts +++ b/src/redis-io.adapter.spec.ts @@ -10,29 +10,37 @@ describe('RedisIoAdapter', () => { const createClientMock = createClient as jest.Mock; const createAdapterMock = createAdapter as jest.Mock; - let pubClient: { on: jest.Mock; connect: jest.Mock; duplicate: jest.Mock }; - let subClient: { on: jest.Mock; connect: jest.Mock }; + let pubClient: { on: jest.Mock; connect: jest.Mock; disconnect: jest.Mock; duplicate: jest.Mock }; + let subClient: { on: jest.Mock; connect: jest.Mock; disconnect: jest.Mock }; beforeEach(() => { jest.clearAllMocks(); - subClient = { on: jest.fn(), connect: jest.fn().mockResolvedValue(undefined) }; + subClient = { + on: jest.fn(), + connect: jest.fn().mockResolvedValue(undefined), + disconnect: jest.fn().mockResolvedValue(undefined), + }; pubClient = { on: jest.fn(), connect: jest.fn().mockResolvedValue(undefined), + disconnect: jest.fn().mockResolvedValue(undefined), duplicate: jest.fn(() => subClient), }; createClientMock.mockReturnValue(pubClient); createAdapterMock.mockReturnValue('ADAPTER_CONSTRUCTOR'); }); - const makeAdapter = () => new RedisIoAdapter(undefined as never); + const makeAdapter = () => new RedisIoAdapter(undefined); it('connects a pub/sub client pair and builds the adapter', async () => { const adapter = makeAdapter(); await adapter.connectToRedis('redis://localhost:6379'); - expect(createClientMock).toHaveBeenCalledWith({ url: 'redis://localhost:6379' }); + expect(createClientMock).toHaveBeenCalledWith({ + url: 'redis://localhost:6379', + socket: { reconnectStrategy: expect.any(Function) }, + }); expect(pubClient.duplicate).toHaveBeenCalledTimes(1); expect(pubClient.connect).toHaveBeenCalledTimes(1); expect(subClient.connect).toHaveBeenCalledTimes(1); @@ -58,11 +66,104 @@ describe('RedisIoAdapter', () => { expect(() => subErrorHandler(new Error('boom'))).not.toThrow(); }); - it('rejects when a client fails to connect', async () => { - pubClient.connect.mockRejectedValueOnce(new Error('redis down')); - const adapter = makeAdapter(); + describe('first connection deadline', () => { + const originalEnv = { ...process.env }; + const strategyOf = () => + createClientMock.mock.calls[0][0].socket.reconnectStrategy as (retries: number, cause: Error) => number | Error; + let now: jest.SpyInstance; + const connectPending = (url: string, timeoutMs?: number) => { + let release!: () => void; + pubClient.connect.mockReturnValue(new Promise((resolve) => (release = resolve))); + const pending = makeAdapter().connectToRedis(url, timeoutMs); + return { strategy: strategyOf(), finish: async () => (release(), pending) }; + }; + + beforeEach(() => { + now = jest.spyOn(Date, 'now').mockReturnValue(1_000_000); + }); + + afterEach(() => { + now.mockRestore(); + process.env = { ...originalEnv }; + }); + + it('gives up reconnecting once the connect timeout passes before the first connection', async () => { + const { strategy, finish } = connectPending('redis://:secret@redis-host:6379', 30_000); + + now.mockReturnValue(1_000_000 + 29_999); + expect(strategy(3, new Error('ECONNREFUSED'))).toBeLessThanOrEqual(5000); + now.mockReturnValue(1_000_000 + 30_000); + const result = strategy(4, new Error('ECONNREFUSED')); + + expect(result).toBeInstanceOf(Error); + expect((result as Error).message).toContain('redis-host:6379'); + expect((result as Error).message).toContain('ECONNREFUSED'); + expect((result as Error).message).not.toContain('secret'); + await finish(); + }); + + it('keeps reconnecting after it has connected once', async () => { + await makeAdapter().connectToRedis('redis://localhost:6379', 30_000); + const strategy = strategyOf(); + + now.mockReturnValue(1_000_000 + 10 * 60_000); + + expect(strategy(100, new Error('ECONNRESET'))).toBe(5000); + }); - await expect(adapter.connectToRedis('redis://localhost:6379')).rejects.toThrow('redis down'); + it('backs off by 200 ms per retry up to 5 s', async () => { + await makeAdapter().connectToRedis('redis://localhost:6379', 30_000); + const strategy = strategyOf(); + + expect(strategy(1, new Error('x'))).toBe(200); + expect(strategy(10, new Error('x'))).toBe(2000); + expect(strategy(1000, new Error('x'))).toBe(5000); + }); + + it('does not disable the offline queue', async () => { + await makeAdapter().connectToRedis('redis://localhost:6379'); + + expect(createClientMock.mock.calls[0][0]).not.toHaveProperty('disableOfflineQueue'); + expect(createClientMock.mock.calls[0][0].socket).not.toHaveProperty('disableOfflineQueue'); + }); + + it('disconnects both clients when the first connection fails', async () => { + pubClient.connect.mockRejectedValueOnce(new Error('redis down')); + + await expect(makeAdapter().connectToRedis('redis://localhost:6379')).rejects.toThrow('redis down'); + + expect(pubClient.disconnect).toHaveBeenCalledTimes(1); + expect(subClient.disconnect).toHaveBeenCalledTimes(1); + expect(createAdapterMock).not.toHaveBeenCalled(); + }); + + it('still reports the connect error when a disconnect fails too', async () => { + pubClient.connect.mockRejectedValueOnce(new Error('redis down')); + subClient.disconnect.mockRejectedValueOnce(new Error('already closed')); + + await expect(makeAdapter().connectToRedis('redis://localhost:6379')).rejects.toThrow('redis down'); + }); + + it('gives up after 30 seconds by default', async () => { + const { strategy, finish } = connectPending('redis://localhost:6379'); + + now.mockReturnValue(1_000_000 + 30_000 - 1); + expect(typeof strategy(1, new Error('x'))).toBe('number'); + now.mockReturnValue(1_000_000 + 30_000); + expect(strategy(2, new Error('x'))).toBeInstanceOf(Error); + await finish(); + }); + + it('does not put a malformed url into the error', async () => { + const { strategy, finish } = connectPending('not a url with secret', 1000); + + now.mockReturnValue(1_000_000 + 1000); + const result = strategy(1, new Error('x')) as Error; + + expect(result.message).toContain('REDIS_URL'); + expect(result.message).not.toContain('secret'); + await finish(); + }); }); it('attaches the redis adapter to the created server', async () => { @@ -102,7 +203,7 @@ describe('RedisIoAdapter', () => { const enabled = await RedisIoAdapter.use(app as never, 'redis://localhost:6379'); expect(enabled).toBe(true); - expect(createClientMock).toHaveBeenCalledWith({ url: 'redis://localhost:6379' }); + expect(createClientMock).toHaveBeenCalledWith(expect.objectContaining({ url: 'redis://localhost:6379' })); expect(app.useWebSocketAdapter).toHaveBeenCalledWith(expect.any(RedisIoAdapter)); }); }); diff --git a/src/redis-io.adapter.ts b/src/redis-io.adapter.ts index 888f3766..cf0ed7d4 100644 --- a/src/redis-io.adapter.ts +++ b/src/redis-io.adapter.ts @@ -1,45 +1,55 @@ -import { INestApplication, INestApplicationContext, Logger } from '@nestjs/common'; +import { INestApplication, Logger } from '@nestjs/common'; import { IoAdapter } from '@nestjs/platform-socket.io'; import { createAdapter } from '@socket.io/redis-adapter'; import { createClient } from 'redis'; import { ServerOptions } from 'socket.io'; -/** - * Socket.io adapter backed by Redis pub/sub so that events emitted from any - * API instance are delivered to clients connected to every other instance. - * Without it a multi-replica deployment only broadcasts to clients that happen - * to be connected to the same instance that emitted the event. - */ +const DEFAULT_CONNECT_TIMEOUT_MS = 30_000; +const MAX_RECONNECT_DELAY_MS = 5_000; + +const describeUrl = (url: string): string => { + try { + return new URL(url).host; + } catch { + return 'REDIS_URL'; + } +}; + export class RedisIoAdapter extends IoAdapter { private static readonly logger = new Logger(RedisIoAdapter.name); private adapterConstructor!: ReturnType; - constructor(app: INestApplicationContext) { - super(app); - } + async connectToRedis(url: string, connectTimeoutMs = DEFAULT_CONNECT_TIMEOUT_MS): Promise { + const startedAt = Date.now(); + let connected = false; + const reconnectStrategy = (retries: number, cause: Error): number | Error => + !connected && Date.now() - startedAt >= connectTimeoutMs + ? new Error(`Redis at ${describeUrl(url)} is not reachable after ${connectTimeoutMs} ms: ${cause.message}`) + : Math.min(retries * 200, MAX_RECONNECT_DELAY_MS); - async connectToRedis(url: string): Promise { - const pubClient = createClient({ url }); + // no disableOfflineQueue: @socket.io/redis-adapter publishes without catch, an offline rejection would crash the process + const pubClient = createClient({ url, socket: { reconnectStrategy } }); const subClient = pubClient.duplicate(); pubClient.on('error', (err) => RedisIoAdapter.logger.error(`Redis pub client error: ${err}`)); subClient.on('error', (err) => RedisIoAdapter.logger.error(`Redis sub client error: ${err}`)); - await Promise.all([pubClient.connect(), subClient.connect()]); + try { + await Promise.all([pubClient.connect(), subClient.connect()]); + } catch (error) { + await Promise.allSettled([pubClient.disconnect(), subClient.disconnect()]); + throw error; + } + connected = true; this.adapterConstructor = createAdapter(pubClient, subClient); } - createIOServer(port: number, options?: ServerOptions): any { + override createIOServer(port: number, options?: ServerOptions): any { const server = super.createIOServer(port, options); server.adapter(this.adapterConstructor); return server; } - /** - * Installs the Redis-backed socket.io adapter on the app when `redisUrl` is - * provided. Returns whether the Redis adapter was enabled (false = default - * in-memory adapter, i.e. single-instance mode). - */ static async use(app: INestApplication, redisUrl?: string): Promise { if (!redisUrl) { this.logger.log('REDIS_URL not set — socket.io running with the in-memory adapter (single instance only)'); diff --git a/src/shared/api-file.decorator.spec.ts b/src/shared/api-file.decorator.spec.ts index e47c76b4..770ad756 100644 --- a/src/shared/api-file.decorator.spec.ts +++ b/src/shared/api-file.decorator.spec.ts @@ -1,6 +1,5 @@ import { ApiFile } from './api-file.decorator'; -// @nestjs/swagger does not export its metadata keys const API_MODEL_PROPERTIES = 'swagger/apiModelProperties'; const API_MODEL_PROPERTIES_ARRAY = 'swagger/apiModelPropertiesArray'; @@ -39,8 +38,7 @@ describe('ApiFile', () => { }); }); - // NOTE: current behavior — every option other than isArray is dropped. - it('ignores options other than isArray', () => { + it('describes only isArray in the schema', () => { expect(propertyMetadata('explicitSingle')).toEqual({ type: 'string', format: 'binary', isArray: false }); }); }); diff --git a/src/shared/current-user.decorator.spec.ts b/src/shared/current-user.decorator.spec.ts index 4e24bf3e..aab26dae 100644 --- a/src/shared/current-user.decorator.spec.ts +++ b/src/shared/current-user.decorator.spec.ts @@ -2,7 +2,6 @@ import { ExecutionContext } from '@nestjs/common'; import { ROUTE_ARGS_METADATA } from '@nestjs/common/constants'; import { CurrentUser } from './current-user.decorator'; -// a param decorator only exposes its factory through the route metadata const getFactory = () => { class Controller { handler(@CurrentUser() _user: unknown) {} diff --git a/src/shared/dto/baseline.dto.ts b/src/shared/dto/baseline.dto.ts index 1598b90c..8c20252b 100644 --- a/src/shared/dto/baseline.dto.ts +++ b/src/shared/dto/baseline.dto.ts @@ -2,7 +2,6 @@ import { ApiProperty, OmitType } from '@nestjs/swagger'; import { TestRunDto } from '../../test-runs/dto/testRun.dto'; import { UserDto } from '../../users/dto/user.dto'; -// who approved a baseline, as anyone who can open a variation sees them export class BaselineUserDto extends OmitType(UserDto, ['apiKey'] as const) {} export class BaselineDto { diff --git a/src/shared/events/debounce.spec.ts b/src/shared/events/debounce.spec.ts new file mode 100644 index 00000000..fc820059 --- /dev/null +++ b/src/shared/events/debounce.spec.ts @@ -0,0 +1,94 @@ +import { debounce } from './debounce'; + +const setup = () => { + const fn = jest.fn(); + return { fn, debounced: debounce(fn, 1500, 3000) }; +}; + +describe('debounce', () => { + beforeEach(() => { + jest.useFakeTimers(); + }); + + afterEach(() => { + jest.clearAllTimers(); + jest.useRealTimers(); + }); + + it('runs the first call at once', () => { + const { fn, debounced } = setup(); + debounced(); + + expect(fn).toHaveBeenCalledTimes(1); + }); + + it('does not run again after the wait when it was called once', () => { + const { fn, debounced } = setup(); + debounced(); + jest.advanceTimersByTime(5000); + + expect(fn).toHaveBeenCalledTimes(1); + }); + + it('runs once more 1500ms after the last of the calls that followed the first', () => { + const { fn, debounced } = setup(); + + debounced(); + debounced(); + jest.advanceTimersByTime(1000); + debounced(); + jest.advanceTimersByTime(1499); + expect(fn).toHaveBeenCalledTimes(1); + + jest.advanceTimersByTime(1); + expect(fn).toHaveBeenCalledTimes(2); + }); + + it('runs a steady stream of calls at least every maxWait', () => { + const { fn, debounced } = setup(); + + debounced(); + for (let elapsed = 0; elapsed < 9000; elapsed += 1000) { + jest.advanceTimersByTime(1000); + debounced(); + } + + expect(fn).toHaveBeenCalledTimes(4); + }); + + it('runs at once again when called after a quiet period', () => { + const { fn, debounced } = setup(); + + debounced(); + jest.advanceTimersByTime(1500); + debounced(); + + expect(fn).toHaveBeenCalledTimes(2); + }); + + it('runs at once when maxWait has passed before its timer could fire', () => { + const { fn, debounced } = setup(); + + debounced(); + debounced(); + jest.setSystemTime(Date.now() + 3000); + debounced(); + expect(fn).toHaveBeenCalledTimes(2); + + jest.advanceTimersByTime(5000); + expect(fn).toHaveBeenCalledTimes(2); + }); + + it('drops the pending run when cancelled and starts afresh afterwards', () => { + const { fn, debounced } = setup(); + + debounced(); + debounced(); + debounced.cancel(); + jest.advanceTimersByTime(5000); + expect(fn).toHaveBeenCalledTimes(1); + + debounced(); + expect(fn).toHaveBeenCalledTimes(2); + }); +}); diff --git a/src/shared/events/debounce.ts b/src/shared/events/debounce.ts new file mode 100644 index 00000000..d0204ab9 --- /dev/null +++ b/src/shared/events/debounce.ts @@ -0,0 +1,57 @@ +export interface Debounced { + (): void; + cancel(): void; +} + +export function debounce(fn: () => void, wait: number, maxWait: number): Debounced { + let timer: NodeJS.Timeout | undefined; + let lastCallTime: number | undefined; + let lastInvokeTime = 0; + let pending = false; + + const invoke = (time: number) => { + lastInvokeTime = time; + pending = false; + fn(); + }; + + const shouldInvoke = (time: number) => { + if (lastCallTime === undefined) return true; + const sinceLastCall = time - lastCallTime; + return sinceLastCall >= wait || time - lastInvokeTime >= maxWait; + }; + + const timerExpired = () => { + const time = Date.now(); + if (shouldInvoke(time)) { + timer = undefined; + if (pending) invoke(time); + return; + } + timer = setTimeout(timerExpired, Math.min(wait - (time - lastCallTime!), maxWait - (time - lastInvokeTime))); + }; + + const debounced = () => { + const time = Date.now(); + const isInvoking = shouldInvoke(time); + pending = true; + lastCallTime = time; + if (!isInvoking) { + timer ??= setTimeout(timerExpired, wait); + return; + } + if (timer !== undefined) clearTimeout(timer); + timer = setTimeout(timerExpired, wait); + invoke(time); + }; + + debounced.cancel = () => { + if (timer !== undefined) clearTimeout(timer); + timer = undefined; + lastCallTime = undefined; + lastInvokeTime = 0; + pending = false; + }; + + return debounced; +} diff --git a/src/shared/events/events.gateway.spec.ts b/src/shared/events/events.gateway.spec.ts index 32ddae0b..6240b2ee 100644 --- a/src/shared/events/events.gateway.spec.ts +++ b/src/shared/events/events.gateway.spec.ts @@ -5,16 +5,14 @@ import { Server } from 'socket.io'; import { EventsGateway } from './events.gateway'; import { PrismaService } from '../../prisma/prisma.service'; import { BuildDto } from '../../builds/dto/build.dto'; +import { TestRunDto } from '../../test-runs/dto/testRun.dto'; import { generateTestRun, TEST_BUILD } from '../../_data_'; -// the debounced build_updated emit resolves Prisma before emitting; setImmediate -// is left real so the pending promise chain can be drained const flushPromises = () => new Promise((resolve) => setImmediate(resolve)); -const initGateway = async ({ - buildFindManyMock = jest.fn().mockResolvedValue([]), - testRunGroupByMock = jest.fn().mockResolvedValue([]), -}) => { +const initGateway = async () => { + const buildFindManyMock = jest.fn().mockResolvedValue([]); + const testRunGroupByMock = jest.fn().mockResolvedValue([]); const module: TestingModule = await Test.createTestingModule({ providers: [ EventsGateway, @@ -31,7 +29,7 @@ const initGateway = async ({ const gateway = module.get(EventsGateway); const emitMock = jest.fn(); gateway.server = { emit: emitMock } as unknown as Server; - return { gateway, emitMock }; + return { gateway, emitMock, buildFindManyMock, testRunGroupByMock }; }; const emitsOf = (emitMock: jest.Mock, event: string) => @@ -51,24 +49,17 @@ describe('EventsGateway', () => { jest.useRealTimers(); }); - it('emits build_created straight away with the build it is given', async () => { - const { gateway, emitMock } = await initGateway({}); + it.each([ + ['buildCreated', 'build_created'], + ['buildDeleted', 'build_deleted'], + ] as const)('%s emits %s straight away with the build it is given', async (method, event) => { + const { gateway, emitMock } = await initGateway(); const build = new BuildDto(TEST_BUILD); - gateway.buildCreated(build); + gateway[method](build); expect(emitMock).toHaveBeenCalledTimes(1); - expect(emitMock).toHaveBeenCalledWith('build_created', build); - }); - - it('emits build_deleted straight away with the build it is given', async () => { - const { gateway, emitMock } = await initGateway({}); - const build = new BuildDto(TEST_BUILD); - - gateway.buildDeleted(build); - - expect(emitMock).toHaveBeenCalledTimes(1); - expect(emitMock).toHaveBeenCalledWith('build_deleted', build); + expect(emitMock).toHaveBeenCalledWith(event, build); }); describe.each([ @@ -77,24 +68,24 @@ describe('EventsGateway', () => { ['testRunDeleted', 'testRun_deleted'], ] as const)('%s', (method, event) => { it('emits the first test run on the leading edge', async () => { - const { gateway, emitMock } = await initGateway({}); + const { gateway, emitMock } = await initGateway(); gateway[method](testRunA); - expect(emitsOf(emitMock, event)).toEqual([[testRunA]]); + expect(emitsOf(emitMock, event)).toEqual([[new TestRunDto(testRunA)]]); }); it('does not emit again after the wait when only one test run came in', async () => { - const { gateway, emitMock } = await initGateway({}); + const { gateway, emitMock } = await initGateway(); gateway[method](testRunA); jest.advanceTimersByTime(5000); - expect(emitsOf(emitMock, event)).toEqual([[testRunA]]); + expect(emitsOf(emitMock, event)).toEqual([[new TestRunDto(testRunA)]]); }); it('batches the test runs that follow the leading edge into one emit 1500ms after the last call', async () => { - const { gateway, emitMock } = await initGateway({}); + const { gateway, emitMock } = await initGateway(); gateway[method](testRunA); gateway[method](testRunB); @@ -102,15 +93,18 @@ describe('EventsGateway', () => { gateway[method](testRunC); jest.advanceTimersByTime(1499); - expect(emitsOf(emitMock, event)).toEqual([[testRunA]]); + expect(emitsOf(emitMock, event)).toEqual([[new TestRunDto(testRunA)]]); jest.advanceTimersByTime(1); - expect(emitsOf(emitMock, event)).toEqual([[testRunA], [testRunB, testRunC]]); + expect(emitsOf(emitMock, event)).toEqual([ + [new TestRunDto(testRunA)], + [new TestRunDto(testRunB), new TestRunDto(testRunC)], + ]); }); it('flushes a steady stream of test runs at the 3000ms max wait', async () => { - const { gateway, emitMock } = await initGateway({}); + const { gateway, emitMock } = await initGateway(); gateway[method](testRunA); jest.advanceTimersByTime(1000); @@ -119,16 +113,18 @@ describe('EventsGateway', () => { gateway[method](testRunC); jest.advanceTimersByTime(999); - expect(emitsOf(emitMock, event)).toEqual([[testRunA]]); + expect(emitsOf(emitMock, event)).toEqual([[new TestRunDto(testRunA)]]); jest.advanceTimersByTime(1); - expect(emitsOf(emitMock, event)).toEqual([[testRunA], [testRunB, testRunC]]); + expect(emitsOf(emitMock, event)).toEqual([ + [new TestRunDto(testRunA)], + [new TestRunDto(testRunB), new TestRunDto(testRunC)], + ]); }); it('queues a build update for the build of the test run', async () => { - const buildFindManyMock = jest.fn().mockResolvedValue([]); - const { gateway } = await initGateway({ buildFindManyMock }); + const { gateway, buildFindManyMock } = await initGateway(); gateway[method](testRunC); @@ -138,9 +134,8 @@ describe('EventsGateway', () => { describe('testRunDeleted', () => { it('drops the deleted test run from the created and updated queues', async () => { - const { gateway, emitMock } = await initGateway({}); + const { gateway, emitMock } = await initGateway(); - // leading edges go out at once; the second calls are queued gateway.testRunCreated(testRunA); gateway.testRunCreated(testRunB); gateway.testRunCreated(testRunC); @@ -150,12 +145,13 @@ describe('EventsGateway', () => { jest.advanceTimersByTime(1500); - expect(emitsOf(emitMock, 'testRun_created')).toEqual([[testRunA], [testRunC]]); - expect(emitsOf(emitMock, 'testRun_deleted')).toEqual([[testRunB]]); + expect(emitsOf(emitMock, 'testRun_created')).toEqual([[new TestRunDto(testRunA)], [new TestRunDto(testRunC)]]); + expect(emitsOf(emitMock, 'testRun_updated')).toEqual([[new TestRunDto(testRunA)]]); + expect(emitsOf(emitMock, 'testRun_deleted')).toEqual([[new TestRunDto(testRunB)]]); }); it('keeps the other queued updates when one updated test run is deleted', async () => { - const { gateway, emitMock } = await initGateway({}); + const { gateway, emitMock } = await initGateway(); gateway.testRunUpdated(testRunA); gateway.testRunUpdated(testRunB); @@ -164,11 +160,11 @@ describe('EventsGateway', () => { jest.advanceTimersByTime(1500); - expect(emitsOf(emitMock, 'testRun_updated')).toEqual([[testRunA], [testRunC]]); + expect(emitsOf(emitMock, 'testRun_updated')).toEqual([[new TestRunDto(testRunA)], [new TestRunDto(testRunC)]]); }); - it('still emits an empty batch once every queued entry was deleted', async () => { - const { gateway, emitMock } = await initGateway({}); + it('emits nothing once every queued entry was deleted', async () => { + const { gateway, emitMock } = await initGateway(); gateway.testRunUpdated(testRunA); gateway.testRunUpdated(testRunB); @@ -176,18 +172,89 @@ describe('EventsGateway', () => { jest.advanceTimersByTime(1500); - // NOTE: current behavior — the trailing debounce call still fires with the - // now-empty queue, so clients receive `testRun_updated` with []. - expect(emitsOf(emitMock, 'testRun_updated')).toEqual([[testRunA], []]); + expect(emitsOf(emitMock, 'testRun_updated')).toEqual([[new TestRunDto(testRunA)]]); + }); + }); + + describe('payload', () => { + it.each([ + ['testRunCreated', 'testRun_created'], + ['testRunUpdated', 'testRun_updated'], + ['testRunDeleted', 'testRun_deleted'], + ] as const)('%s emits TestRunDto, not the database row', async (method, event) => { + const { gateway, emitMock } = await initGateway(); + const row = generateTestRun({ id: 'row', changeSignature: '[0.5]', pixelMisMatchCount: 7 }); + + gateway[method](row); + + const [[payload]] = emitsOf(emitMock, event); + expect(payload).toBeInstanceOf(TestRunDto); + expect(payload).toStrictEqual(new TestRunDto(row)); + expect(payload).not.toHaveProperty('changeSignature'); + expect(payload).not.toHaveProperty('createdAt'); + }); + }); + + describe('ordering and de-duplication', () => { + it('sends an update of a run still queued as created with both the created and the updated event', async () => { + const { gateway, emitMock } = await initGateway(); + const diffed = { ...testRunB, status: TestStatus.unresolved, diffName: 'diff.png' }; + const approved = { ...diffed, status: TestStatus.approved }; + + gateway.testRunCreated(testRunA); + gateway.testRunCreated(testRunB); + gateway.testRunUpdated(diffed); + gateway.testRunUpdated(approved); + jest.advanceTimersByTime(1500); + + expect(emitsOf(emitMock, 'testRun_created')).toEqual([[new TestRunDto(testRunA)], [new TestRunDto(approved)]]); + expect(emitsOf(emitMock, 'testRun_updated')).toEqual([[new TestRunDto(diffed)], [new TestRunDto(approved)]]); + }); + + it('sends an update of a run already announced as created as an update', async () => { + const { gateway, emitMock } = await initGateway(); + const diffed = { ...testRunA, status: TestStatus.unresolved }; + + gateway.testRunCreated(testRunA); + gateway.testRunUpdated(diffed); + + expect(emitsOf(emitMock, 'testRun_created')).toEqual([[new TestRunDto(testRunA)]]); + expect(emitsOf(emitMock, 'testRun_updated')).toEqual([[new TestRunDto(diffed)]]); + }); + + it('keeps only the latest queued update of a run, in the order the runs first came in', async () => { + const { gateway, emitMock } = await initGateway(); + const first = { ...testRunB, status: TestStatus.unresolved }; + const latest = { ...testRunB, status: TestStatus.approved }; + + gateway.testRunUpdated(testRunA); + gateway.testRunUpdated(first); + gateway.testRunUpdated(testRunC); + gateway.testRunUpdated(latest); + jest.advanceTimersByTime(1500); + + expect(emitsOf(emitMock, 'testRun_updated')).toEqual([ + [new TestRunDto(testRunA)], + [new TestRunDto(latest), new TestRunDto(testRunC)], + ]); + }); + + it('announces a deleted run once even when it is deleted twice in a batch', async () => { + const { gateway, emitMock } = await initGateway(); + + gateway.testRunDeleted(testRunA); + gateway.testRunDeleted(testRunB); + gateway.testRunDeleted(testRunB); + jest.advanceTimersByTime(1500); + + expect(emitsOf(emitMock, 'testRun_deleted')).toEqual([[new TestRunDto(testRunA)], [new TestRunDto(testRunB)]]); }); }); describe('onApplicationShutdown', () => { it('drops the batches still waiting for their debounce', async () => { - const buildFindManyMock = jest.fn().mockResolvedValue([]); - const { gateway, emitMock } = await initGateway({ buildFindManyMock }); + const { gateway, emitMock, buildFindManyMock } = await initGateway(); - // the leading edge goes out at once; the second call waits for the trailing edge gateway.testRunCreated(testRunA); gateway.testRunCreated(testRunB); await flushPromises(); @@ -206,22 +273,19 @@ describe('EventsGateway', () => { describe('build_updated', () => { it('emits the build with stats aggregated by the database on the leading edge', async () => { const build: Build = { ...TEST_BUILD, id: 'buildX' }; - const buildFindManyMock = jest.fn().mockResolvedValue([build]); - const testRunGroupByMock = jest - .fn() - .mockResolvedValueOnce([ - { buildId: 'buildX', status: TestStatus.ok, _count: { _all: 3 } }, - { buildId: 'buildX', status: TestStatus.failed, _count: { _all: 1 } }, - ]) - .mockResolvedValueOnce([{ buildId: 'buildX' }]); - const { gateway, emitMock } = await initGateway({ buildFindManyMock, testRunGroupByMock }); + const { gateway, emitMock, buildFindManyMock, testRunGroupByMock } = await initGateway(); + buildFindManyMock.mockResolvedValue([build]); + testRunGroupByMock.mockResolvedValueOnce([ + { buildId: 'buildX', status: TestStatus.ok, merge: false, _count: { _all: 3 } }, + { buildId: 'buildX', status: TestStatus.failed, merge: true, _count: { _all: 1 } }, + ]); gateway.buildUpdated('buildX'); await flushPromises(); expect(buildFindManyMock).toHaveBeenCalledWith({ where: { id: { in: ['buildX'] } } }); expect(testRunGroupByMock).toHaveBeenCalledWith({ - by: ['buildId', 'status'], + by: ['buildId', 'status', 'merge'], where: { buildId: { in: ['buildX'] } }, _count: { _all: true }, }); @@ -234,8 +298,8 @@ describe('EventsGateway', () => { }); it('logs a failed read instead of leaving the rejection unhandled', async () => { - const buildFindManyMock = jest.fn().mockRejectedValue(new Error('Connection terminated')); - const { gateway, emitMock } = await initGateway({ buildFindManyMock }); + const { gateway, emitMock, buildFindManyMock } = await initGateway(); + buildFindManyMock.mockRejectedValue(new Error('Connection terminated')); const logError = jest.spyOn(Logger.prototype, 'error').mockImplementation(); const unhandled = jest.fn(); process.on('unhandledRejection', unhandled); @@ -252,9 +316,7 @@ describe('EventsGateway', () => { }); it('deduplicates the queued build ids before reading them', async () => { - const buildFindManyMock = jest.fn().mockResolvedValue([]); - const testRunGroupByMock = jest.fn().mockResolvedValue([]); - const { gateway } = await initGateway({ buildFindManyMock, testRunGroupByMock }); + const { gateway, buildFindManyMock, testRunGroupByMock } = await initGateway(); gateway.buildUpdated('first'); gateway.testRunCreated(testRunA); @@ -268,16 +330,17 @@ describe('EventsGateway', () => { expect(buildFindManyMock).toHaveBeenNthCalledWith(1, { where: { id: { in: ['first'] } } }); expect(buildFindManyMock).toHaveBeenNthCalledWith(2, { where: { id: { in: ['buildX', 'buildY'] } } }); expect(testRunGroupByMock).toHaveBeenCalledWith({ - by: ['buildId'], - where: { buildId: { in: ['buildX', 'buildY'] }, merge: true }, + by: ['buildId', 'status', 'merge'], + where: { buildId: { in: ['buildX', 'buildY'] } }, + _count: { _all: true }, }); }); it('emits the builds in the order Prisma returns them, without stats for builds it did not ask about', async () => { const known: Build = { ...TEST_BUILD, id: 'buildX' }; const unknown: Build = { ...TEST_BUILD, id: 'other' }; - const buildFindManyMock = jest.fn().mockResolvedValue([unknown, known]); - const { gateway, emitMock } = await initGateway({ buildFindManyMock }); + const { gateway, emitMock, buildFindManyMock } = await initGateway(); + buildFindManyMock.mockResolvedValue([unknown, known]); gateway.buildUpdated('buildX'); await flushPromises(); @@ -290,13 +353,13 @@ describe('EventsGateway', () => { ]); }); - it('emits an empty list when no queued build exists any more', async () => { - const { gateway, emitMock } = await initGateway({}); + it('emits nothing when no queued build exists any more', async () => { + const { gateway, emitMock } = await initGateway(); gateway.buildUpdated('gone'); await flushPromises(); - expect(emitsOf(emitMock, 'build_updated')).toEqual([[]]); + expect(emitsOf(emitMock, 'build_updated')).toEqual([]); }); }); }); diff --git a/src/shared/events/events.gateway.ts b/src/shared/events/events.gateway.ts index 87ae09e3..94f904a7 100644 --- a/src/shared/events/events.gateway.ts +++ b/src/shared/events/events.gateway.ts @@ -4,8 +4,15 @@ import { Server } from 'socket.io'; import { TestRun } from '@prisma/client'; import { BuildDto } from '../../builds/dto/build.dto'; import { getBuildsStats } from '../../builds/build-stats'; -import { debounce } from 'lodash'; +import { TestRunDto } from '../../test-runs/dto/testRun.dto'; import { PrismaService } from '../../prisma/prisma.service'; +import { Debounced, debounce } from './debounce'; + +const DEBOUNCE_WAIT = 1500; +const MAX_WAIT = 3000; + +const TEST_RUN_EVENTS = ['testRun_created', 'testRun_updated', 'testRun_deleted'] as const; +type TestRunEvent = (typeof TEST_RUN_EVENTS)[number]; @WebSocketGateway({ cors: true }) export class EventsGateway implements OnApplicationShutdown { @@ -16,39 +23,41 @@ export class EventsGateway implements OnApplicationShutdown { constructor(private prismaService: PrismaService) {} - private debounceTimeout = 1500; - private maxWait = 3000; - private testRunsCreatedQueued: Array = []; - private testRunsDeletedQueued: Array = []; - private testRunsUpdatedQueued: Array = []; - private buildsUpdatedQueued: Array = []; + private readonly testRunQueues = Object.fromEntries( + TEST_RUN_EVENTS.map((event) => [event, new Map()]) + ) as Record>; + private readonly testRunEmitters = Object.fromEntries( + TEST_RUN_EVENTS.map((event) => [event, debounce(() => this.emitQueued(event), DEBOUNCE_WAIT, MAX_WAIT)]) + ) as Record; + private readonly buildsUpdatedQueued = new Set(); buildCreated(build: BuildDto): void { this.server.emit('build_created', build); } buildUpdated(id: string): void { - this.buildsUpdatedQueued.push(id); + this.buildsUpdatedQueued.add(id); this.buildUpdatedDebounced(); } testRunCreated(testRun: TestRun): void { - this.testRunsCreatedQueued.push(testRun); - this.testRunCreatedDebounced(); + this.enqueueTestRun('testRun_created', testRun); this.buildUpdated(testRun.buildId); } testRunUpdated(testRun: TestRun): void { - this.testRunsUpdatedQueued.push(testRun); - this.testRunUpdatedDebounced(); + // Clients skip testRun_created for runs they already fetched, so the update must go out as well + if (this.testRunQueues.testRun_created.has(testRun.id)) { + this.testRunQueues.testRun_created.set(testRun.id, new TestRunDto(testRun)); + } + this.enqueueTestRun('testRun_updated', testRun); this.buildUpdated(testRun.buildId); } testRunDeleted(testRun: TestRun): void { - this.testRunsCreatedQueued = this.testRunsCreatedQueued.filter((tr) => tr.id !== testRun.id); - this.testRunsUpdatedQueued = this.testRunsUpdatedQueued.filter((tr) => tr.id !== testRun.id); - this.testRunsDeletedQueued.push(testRun); - this.testRunDeletedDebounced(); + this.testRunQueues.testRun_created.delete(testRun.id); + this.testRunQueues.testRun_updated.delete(testRun.id); + this.enqueueTestRun('testRun_deleted', testRun); this.buildUpdated(testRun.buildId); } @@ -56,77 +65,43 @@ export class EventsGateway implements OnApplicationShutdown { this.server.emit('build_deleted', buildDto); } - // A batch still waiting for its debounce would otherwise fire after shutdown - // and query a database pool that is already closed. Runs once the HTTP - // server has drained, so no request queues another batch after it. + // Runs after the HTTP server drained: a pending batch would otherwise hit a closed database pool onApplicationShutdown(): void { - this.testRunCreatedDebounced.cancel(); - this.testRunUpdatedDebounced.cancel(); - this.testRunDeletedDebounced.cancel(); + TEST_RUN_EVENTS.forEach((event) => this.testRunEmitters[event].cancel()); this.buildUpdatedDebounced.cancel(); } - private testRunUpdatedDebounced = debounce( - () => { - this.server.emit('testRun_updated', this.testRunsUpdatedQueued); - this.testRunsUpdatedQueued = []; - }, - this.debounceTimeout, - { - leading: true, - maxWait: this.maxWait, - } - ); - - private testRunCreatedDebounced = debounce( - () => { - this.server.emit('testRun_created', this.testRunsCreatedQueued); - this.testRunsCreatedQueued = []; - }, - this.debounceTimeout, - { - leading: true, - maxWait: this.maxWait, - } - ); + private enqueueTestRun(event: TestRunEvent, testRun: TestRun): void { + this.testRunQueues[event].set(testRun.id, new TestRunDto(testRun)); + this.testRunEmitters[event](); + } - private testRunDeletedDebounced = debounce( - () => { - this.server.emit('testRun_deleted', this.testRunsDeletedQueued); - this.testRunsDeletedQueued = []; - }, - this.debounceTimeout, - { - leading: true, - maxWait: this.maxWait, - } - ); + private emitQueued(event: TestRunEvent): void { + const queue = this.testRunQueues[event]; + if (queue.size === 0) return; + const testRuns = [...queue.values()]; + queue.clear(); + this.server.emit(event, testRuns); + } private buildUpdatedDebounced = debounce( () => { - // Deduplicate: during ingestion the queue holds one entry per test run. - // Stats are aggregated in the database — a build can hold thousands of - // test runs and this fires on every debounced event burst. - const buildIds = [...new Set(this.buildsUpdatedQueued)]; - this.buildsUpdatedQueued = []; + const buildIds = [...this.buildsUpdatedQueued]; + this.buildsUpdatedQueued.clear(); Promise.all([ this.prismaService.build.findMany({ where: { id: { in: buildIds } } }), getBuildsStats(this.prismaService, buildIds), ]) .then(([builds, stats]) => { + if (builds.length === 0) return; this.server.emit( 'build_updated', builds.map((build) => new BuildDto(build, stats.get(build.id))) ); }) - // nothing awaits this chain: unhandled, a failed query would take the - // whole process down .catch((error) => this.logger.error(`Cannot emit build_updated: ${error}`)); }, - this.debounceTimeout, - { - leading: true, - maxWait: this.maxWait, - } + DEBOUNCE_WAIT, + MAX_WAIT ); } diff --git a/src/shared/fite-to-body.interceptor.spec.ts b/src/shared/fite-to-body.interceptor.spec.ts index ffcdbd43..665aaf4f 100644 --- a/src/shared/fite-to-body.interceptor.spec.ts +++ b/src/shared/fite-to-body.interceptor.spec.ts @@ -10,6 +10,16 @@ const contextWith = (request: object) => const handler = (): CallHandler & { handle: jest.Mock } => ({ handle: jest.fn().mockReturnValue(of('handled')) }); +const expectUntouched = (interceptor: { intercept: FilesToBodyInterceptor['intercept'] }, request: object) => { + const before = JSON.parse(JSON.stringify(request)); + const next = handler(); + + interceptor.intercept(contextWith(request), next); + + expect(JSON.parse(JSON.stringify(request))).toEqual(before); + expect(next.handle).toHaveBeenCalledTimes(1); +}; + describe('FilesToBodyInterceptor', () => { const interceptor = new FilesToBodyInterceptor(); @@ -35,8 +45,6 @@ describe('FilesToBodyInterceptor', () => { expect(request.body.a).toEqual([existing, added]); }); - // NOTE: current behavior — a non-array body value under the same field name - // has `.push` called on it, so a plain string value makes the interceptor throw. it('throws when the body already has a non-array value for the field', () => { const request = { body: { a: 'text' }, files: [file('a')] }; @@ -49,13 +57,7 @@ describe('FilesToBodyInterceptor', () => { ['an empty files array', { body: {}, files: [] }], ['files given as a field map', { body: {}, files: { a: [file('a')] } }], ])('leaves the request untouched with %s', (_label, request) => { - const before = JSON.parse(JSON.stringify(request)); - const next = handler(); - - interceptor.intercept(contextWith(request), next); - - expect(JSON.parse(JSON.stringify(request))).toEqual(before); - expect(next.handle).toHaveBeenCalledTimes(1); + expectUntouched(interceptor, request); }); }); @@ -86,12 +88,6 @@ describe('FileToBodyInterceptor', () => { ['no file', { body: {} }], ['a file without a field name', { body: {}, file: { originalname: 'x.png' } }], ])('leaves the request untouched with %s', (_label, request) => { - const before = JSON.parse(JSON.stringify(request)); - const next = handler(); - - interceptor.intercept(contextWith(request), next); - - expect(JSON.parse(JSON.stringify(request))).toEqual(before); - expect(next.handle).toHaveBeenCalledTimes(1); + expectUntouched(interceptor, request); }); }); diff --git a/src/shared/tasks/tasks.service.spec.ts b/src/shared/tasks/tasks.service.spec.ts index bcf475da..6d91f0e2 100644 --- a/src/shared/tasks/tasks.service.spec.ts +++ b/src/shared/tasks/tasks.service.spec.ts @@ -1,10 +1,9 @@ import { CLEANUP_LOCK_KEY, TasksService } from './tasks.service'; import { generateTestVariation, TEST_PROJECT } from '../../_data_'; -import { Test, TestingModule } from '@nestjs/testing'; +import { Test } from '@nestjs/testing'; import { PrismaService } from '../../prisma/prisma.service'; import { TestVariationsService } from '../../test-variations/test-variations.service'; -// the dedicated pg connection the cleanup takes its lock on const lockClient = (locked: boolean) => ({ on: jest.fn(), connect: jest.fn().mockResolvedValue(undefined), @@ -12,88 +11,72 @@ const lockClient = (locked: boolean) => ({ end: jest.fn().mockResolvedValue(undefined), }); -const initService = async ({ - client = lockClient(true), - projectFindManyMock = jest.fn().mockResolvedValue([]), - testVariationFindManyMock = jest.fn(), - testVariationsDeleteMock = jest.fn(), -}) => { - const module: TestingModule = await Test.createTestingModule({ +const secondProject = { ...TEST_PROJECT, id: '2', name: 'Second Project' }; +const variations = (...ids: string[]) => ids.map((id) => generateTestVariation({ id })); + +const setup = async (client = lockClient(true)) => { + const projectFindMany = jest.fn().mockResolvedValue([]); + const variationFindMany = jest.fn(); + const variationFindFirst = jest.fn(async ({ where }: { where: { id: string } }): Promise<{ id: string } | null> => ({ + id: where.id, + })); + const variationDelete = jest.fn(); + const module = await Test.createTestingModule({ providers: [ TasksService, { provide: PrismaService, useValue: { dedicatedClient: () => client, - testVariation: { - findMany: testVariationFindManyMock, - }, - project: { - findMany: projectFindManyMock, - }, - }, - }, - { - provide: TestVariationsService, - useValue: { - delete: testVariationsDeleteMock, + testVariation: { findMany: variationFindMany, findFirst: variationFindFirst }, + project: { findMany: projectFindMany }, }, }, + { provide: TestVariationsService, useValue: { delete: variationDelete } }, ], }).compile(); - return module.get(TasksService); + const service = module.get(TasksService); + const spyLogger = (method: 'log' | 'warn') => jest.spyOn((service as any).logger, method).mockImplementation(); + return { service, client, projectFindMany, variationFindMany, variationFindFirst, variationDelete, spyLogger }; }; describe('cleanOldTestVariations', () => { - let service: TasksService; - afterEach(() => { jest.useRealTimers(); }); it('findMany', async () => { - // .Arrange const project = TEST_PROJECT; const testVariation = generateTestVariation(); - const projectFindManyMock = jest.fn().mockResolvedValueOnce([project]); - const testVariationFindManyMock = jest.fn().mockResolvedValueOnce([testVariation]); - const testVariationsDeleteMock = jest.fn(); - service = await initService({ - projectFindManyMock: projectFindManyMock, - testVariationFindManyMock: testVariationFindManyMock, - testVariationsDeleteMock: testVariationsDeleteMock, - }); + const { service, projectFindMany, variationFindMany, variationDelete } = await setup(); + projectFindMany.mockResolvedValueOnce([project]); + variationFindMany.mockResolvedValueOnce([testVariation]); const dateNow = new Date('2022-10-23'); jest.useFakeTimers().setSystemTime(dateNow); const dateRemoveAfter: Date = new Date(dateNow); dateRemoveAfter.setDate(dateRemoveAfter.getDate() - project.maxBranchLifetime); - // .Act await service.cleanOldTestVariations(); - // .Assert - expect(testVariationFindManyMock).toHaveBeenCalledWith({ + expect(variationFindMany).toHaveBeenCalledWith({ where: { updatedAt: { lte: dateRemoveAfter }, - // nor run since: a run that matches the baseline does not move updatedAt testRuns: { none: { createdAt: { gt: dateRemoveAfter } } }, branchName: { not: project.mainBranchName }, projectId: project.id, }, }); - expect(testVariationsDeleteMock).toHaveBeenCalledWith(testVariation.id); + expect(variationDelete).toHaveBeenCalledWith(testVariation.id); }); it('removes only while holding the advisory lock, taken in a transaction on its own connection', async () => { - const client = lockClient(true); - const projectFindManyMock = jest.fn().mockResolvedValueOnce([TEST_PROJECT]); - const testVariationFindManyMock = jest.fn().mockResolvedValueOnce([generateTestVariation()]); - const testVariationsDeleteMock = jest.fn(async () => { - // the lock is still held: the connection carrying it has not been closed + const { service, client, projectFindMany, variationFindMany, variationDelete } = await setup(); + projectFindMany.mockResolvedValueOnce([TEST_PROJECT]); + variationFindMany.mockResolvedValueOnce([generateTestVariation()]); + variationDelete.mockImplementation(async () => { expect(client.end).not.toHaveBeenCalled(); }); - service = await initService({ client, projectFindManyMock, testVariationFindManyMock, testVariationsDeleteMock }); await service.cleanOldTestVariations(); @@ -102,30 +85,25 @@ describe('cleanOldTestVariations', () => { ['BEGIN'], ['SELECT pg_try_advisory_xact_lock($1) AS locked', [CLEANUP_LOCK_KEY]], ]); - expect(testVariationsDeleteMock).toHaveBeenCalledTimes(1); - // closing the connection ends the transaction and releases the lock + expect(variationDelete).toHaveBeenCalledTimes(1); expect(client.end).toHaveBeenCalledTimes(1); }); it('skips when another instance holds the lock', async () => { - const client = lockClient(false); - const projectFindManyMock = jest.fn(); - const testVariationsDeleteMock = jest.fn(); - service = await initService({ client, projectFindManyMock, testVariationsDeleteMock }); - const logSpy = jest.spyOn((service as any).logger, 'log').mockImplementation(); + const { service, client, projectFindMany, variationDelete, spyLogger } = await setup(lockClient(false)); + const logSpy = spyLogger('log'); await service.cleanOldTestVariations(); - expect(projectFindManyMock).not.toHaveBeenCalled(); - expect(testVariationsDeleteMock).not.toHaveBeenCalled(); + expect(projectFindMany).not.toHaveBeenCalled(); + expect(variationDelete).not.toHaveBeenCalled(); expect(logSpy).toHaveBeenCalledWith('Another instance is removing old test variations, skipping'); expect(client.end).toHaveBeenCalledTimes(1); }); it('releases the lock when the cleanup fails', async () => { - const client = lockClient(true); - const projectFindManyMock = jest.fn().mockRejectedValueOnce(new Error('connection reset')); - service = await initService({ client, projectFindManyMock }); + const { service, client, projectFindMany } = await setup(); + projectFindMany.mockRejectedValueOnce(new Error('connection reset')); await expect(service.cleanOldTestVariations()).rejects.toThrow('connection reset'); @@ -133,9 +111,8 @@ describe('cleanOldTestVariations', () => { }); it('logs a lost lock connection instead of letting the error event crash the process', async () => { - const client = lockClient(true); - service = await initService({ client }); - const warnSpy = jest.spyOn((service as any).logger, 'warn').mockImplementation(); + const { service, client, spyLogger } = await setup(); + const warnSpy = spyLogger('warn'); await service.cleanOldTestVariations(); const [event, listener] = client.on.mock.calls[0]; @@ -146,4 +123,63 @@ describe('cleanOldTestVariations', () => { 'Lost the cleanup lock connection: terminating connection due to administrator command' ); }); + + it('one failing variation does not stop the cleanup of the next variation and project', async () => { + const { service, projectFindMany, variationFindMany, variationDelete, spyLogger } = await setup(); + projectFindMany.mockResolvedValueOnce([TEST_PROJECT, secondProject]); + variationFindMany.mockResolvedValueOnce(variations('v1', 'v2')).mockResolvedValueOnce(variations('v3')); + variationDelete.mockRejectedValueOnce(new Error('FK')).mockResolvedValue(undefined); + spyLogger('warn'); + + await expect(service.cleanOldTestVariations()).resolves.toBeUndefined(); + + expect(variationDelete.mock.calls).toEqual([['v1'], ['v2'], ['v3']]); + }); + + it('a failing project does not stop the next one', async () => { + const { service, projectFindMany, variationFindMany, variationDelete, spyLogger } = await setup(); + projectFindMany.mockResolvedValueOnce([TEST_PROJECT, secondProject]); + variationFindMany.mockRejectedValueOnce(new Error('db down')).mockResolvedValueOnce(variations('v3')); + variationDelete.mockResolvedValue(undefined); + const warnSpy = spyLogger('warn'); + + await service.cleanOldTestVariations(); + + expect(variationDelete.mock.calls).toEqual([['v3']]); + expect(warnSpy).toHaveBeenCalledWith('Cannot clean up TestVariations of Test Project: Error: db down'); + }); + + it('logs a summary per project', async () => { + const { service, projectFindMany, variationFindMany, variationDelete, spyLogger } = await setup(); + projectFindMany.mockResolvedValueOnce([TEST_PROJECT]); + variationFindMany.mockResolvedValueOnce(variations('v1', 'v2')); + variationDelete.mockRejectedValueOnce(new Error('FK')).mockResolvedValue(undefined); + const logSpy = spyLogger('log'); + spyLogger('warn'); + + await service.cleanOldTestVariations(); + + expect(logSpy).toHaveBeenCalledWith('Removed 1 of 2 TestVariations of Test Project, 0 in use again, 1 failed'); + }); + + it('skips a variation used after the list was read', async () => { + const { service, projectFindMany, variationFindMany, variationFindFirst, variationDelete, spyLogger } = + await setup(); + projectFindMany.mockResolvedValueOnce([TEST_PROJECT]); + variationFindMany.mockResolvedValueOnce(variations('v1', 'v2')); + variationFindFirst.mockImplementation(async ({ where }) => (where.id === 'v1' ? null : { id: where.id })); + variationDelete.mockResolvedValue(undefined); + const logSpy = spyLogger('log'); + spyLogger('warn'); + + await service.cleanOldTestVariations(); + + const { where: listWhere } = variationFindMany.mock.calls[0][0]; + expect(variationFindFirst.mock.calls).toEqual([ + [{ where: { id: 'v1', ...listWhere }, select: { id: true } }], + [{ where: { id: 'v2', ...listWhere }, select: { id: true } }], + ]); + expect(variationDelete.mock.calls).toEqual([['v2']]); + expect(logSpy).toHaveBeenCalledWith('Removed 1 of 2 TestVariations of Test Project, 1 in use again, 0 failed'); + }); }); diff --git a/src/shared/tasks/tasks.service.ts b/src/shared/tasks/tasks.service.ts index a6b67176..84e44285 100644 --- a/src/shared/tasks/tasks.service.ts +++ b/src/shared/tasks/tasks.service.ts @@ -1,11 +1,10 @@ import { forwardRef, Inject, Injectable, Logger } from '@nestjs/common'; import { Cron, CronExpression } from '@nestjs/schedule'; +import { Prisma, Project } from '@prisma/client'; import { PrismaService } from '../../prisma/prisma.service'; import { TestVariationsService } from '../../test-variations/test-variations.service'; import type { WrapperType } from '../wrapper-type'; -// Advisory lock key of the cleanup, shared by every instance against the same -// database: "VRT" and 1, a value nothing else in the database locks on export const CLEANUP_LOCK_KEY = 0x56525401; @Injectable() @@ -18,27 +17,10 @@ export class TasksService { private testVariationService: WrapperType ) {} - /** - * Every API instance schedules this for the same midnight. One of them runs - * it and the others skip, or they would race to delete the same variations - * and baseline images. - * - * The lock is a transaction-level advisory lock, taken on a connection of - * its own and held in an otherwise idle transaction until the cleanup is - * done. Taking it only to claim the run and committing before the work would - * let an instance whose timer fires a moment later run alongside; a - * session-level lock behind a PgBouncer in transaction mode would stay on - * whichever server connection took it. This one goes with the transaction, - * which ends with the connection, also when the instance dies. The - * transaction holds no rows and, between statements, no snapshot, so it does - * not hold up vacuum; the deletions go through Prisma's pool as usual. An - * instance that comes after the run has finished finds nothing left to remove. - */ + // Transaction-level lock on a dedicated connection: a session-level one would stick to a PgBouncer connection @Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT) async cleanOldTestVariations() { const client = this.prismaService.dedicatedClient(); - // an idle pg client emits a lost connection as an event, which would - // otherwise crash the process; the lock goes with it, the cleanup goes on client.on('error', (error) => this.logger.warn(`Lost the cleanup lock connection: ${error.message}`)); await client.connect(); try { @@ -52,40 +34,64 @@ export class TasksService { } await this.removeOldTestVariations(); } finally { - // closing the connection rolls the transaction back and releases the lock await client.end(); } } - /** - * Feature-branch variations nobody has used within the project's - * maxBranchLifetime. Use is an approval or edit of the variation (its - * updatedAt) or a test run compared against it: a run that matches its - * baseline leaves the variation untouched, yet the branch is plainly alive, - * and dropping its baseline would turn its next run into "new". - */ private async removeOldTestVariations() { const projects = await this.prismaService.project.findMany(); for (const project of projects) { - const dateRemoveAfter: Date = new Date(); - dateRemoveAfter.setDate(dateRemoveAfter.getDate() - project.maxBranchLifetime); + try { + await this.removeOldTestVariationsOf(project); + } catch (e) { + this.logger.warn(`Cannot clean up TestVariations of ${project.name}: ${e}`); + } + } + } - const testVariations = await this.prismaService.testVariation.findMany({ - where: { - projectId: project.id, - updatedAt: { lte: dateRemoveAfter }, - testRuns: { none: { createdAt: { gt: dateRemoveAfter } } }, - branchName: { not: project.mainBranchName }, - }, - }); - this.logger.debug( - `Removing ${testVariations.length} TestVariations for ${project.name} unused since ${dateRemoveAfter}` - ); + private async removeOldTestVariationsOf(project: Project) { + const dateRemoveAfter: Date = new Date(); + dateRemoveAfter.setDate(dateRemoveAfter.getDate() - project.maxBranchLifetime); - for (const testVariation of testVariations) { + const testVariations = await this.prismaService.testVariation.findMany({ + where: this.staleVariationsWhere(project, dateRemoveAfter), + }); + this.logger.debug( + `Removing ${testVariations.length} TestVariations for ${project.name} unused since ${dateRemoveAfter}` + ); + + let removed = 0; + let skipped = 0; + let failed = 0; + for (const testVariation of testVariations) { + try { + const stillStale = await this.prismaService.testVariation.findFirst({ + where: { id: testVariation.id, ...this.staleVariationsWhere(project, dateRemoveAfter) }, + select: { id: true }, + }); + if (!stillStale) { + skipped++; + continue; + } await this.testVariationService.delete(testVariation.id); + removed++; + } catch (e) { + failed++; + this.logger.warn(`Cannot remove TestVariation ${testVariation.id}: ${e}`); } } + this.logger.log( + `Removed ${removed} of ${testVariations.length} TestVariations of ${project.name}, ${skipped} in use again, ${failed} failed` + ); + } + + private staleVariationsWhere(project: Project, removeAfter: Date): Prisma.TestVariationWhereInput { + return { + projectId: project.id, + updatedAt: { lte: removeAfter }, + testRuns: { none: { createdAt: { gt: removeAfter } } }, + branchName: { not: project.mainBranchName }, + }; } } diff --git a/src/shared/validation.audit.spec.ts b/src/shared/validation.audit.spec.ts new file mode 100644 index 00000000..cbfdb1cd --- /dev/null +++ b/src/shared/validation.audit.spec.ts @@ -0,0 +1,69 @@ +import { getMetadataStorage } from 'class-validator'; +import { ModifyBuildDto } from '../builds/dto/build-modify.dto'; +import { CreateBuildDto } from '../builds/dto/build-create.dto'; +import { CreateProjectDto } from '../projects/dto/create-project.dto'; +import { UpdateProjectDto } from '../projects/dto/update-project.dto'; +import { CreateTestRequestBase64Dto } from '../test-runs/dto/create-test-request-base64.dto'; +import { CreateTestRequestMultipartDto } from '../test-runs/dto/create-test-request-multipart.dto'; +import { CreateTestRequestDto } from '../test-runs/dto/create-test-request.dto'; +import { IgnoreAreaDto } from '../test-runs/dto/ignore-area.dto'; +import { UpdateIgnoreAreasDto } from '../test-runs/dto/update-ignore-area.dto'; +import { UpdateTestRunDto } from '../test-runs/dto/update-test.dto'; +import { MergeParams } from '../test-variations/types'; +import { AssignRoleDto } from '../users/dto/assign-role.dto'; +import { ChangePasswordDto } from '../users/dto/change-password.dto'; +import { CreateUserDto } from '../users/dto/user-create.dto'; +import { UserLoginRequestDto } from '../users/dto/user-login-request.dto'; +import { UpdateUserDto } from '../users/dto/user-update.dto'; +import { BaselineDataDto } from './dto/baseline-data.dto'; + +type DtoClass = abstract new (...args: any[]) => unknown; + +const declared = (dto: DtoClass): string[] => + ((Reflect.getMetadata('swagger/apiModelPropertiesArray', dto.prototype) ?? []) as string[]).map((name) => + name.replace(/^:/, '') + ); +const validated = (dto: DtoClass): Set => + new Set( + getMetadataStorage() + .getTargetValidationMetadatas(dto, '', true, false) + .map((m) => m.propertyName) + ); + +describe.each([ + IgnoreAreaDto, + UpdateIgnoreAreasDto, + UpdateTestRunDto, + BaselineDataDto, + CreateTestRequestDto, + CreateTestRequestBase64Dto, + CreateTestRequestMultipartDto, + CreateBuildDto, + ModifyBuildDto, + CreateProjectDto, + UpdateProjectDto, + CreateUserDto, + UserLoginRequestDto, + UpdateUserDto, + AssignRoleDto, + ChangePasswordDto, +])('%p', (dto) => { + it('validates every declared property', () => { + const properties = declared(dto); + expect(properties).not.toEqual([]); + expect(properties.filter((name) => !validated(dto).has(name))).toEqual([]); + }); +}); + +describe('MergeParams', () => { + it('validates every query parameter', () => { + expect([...validated(MergeParams)].sort()).toEqual(['fromBranch', 'projectId', 'toBranch']); + }); +}); + +describe('CreateTestRequestDto', () => { + it('validates sourceBranch, which the variation lookup reads', () => { + expect(validated(CreateTestRequestDto).has('sourceBranch')).toBe(true); + expect(validated(CreateTestRequestMultipartDto).has('sourceBranch')).toBe(true); + }); +}); diff --git a/src/shared/validation.spec.ts b/src/shared/validation.spec.ts new file mode 100644 index 00000000..8dab24f5 --- /dev/null +++ b/src/shared/validation.spec.ts @@ -0,0 +1,57 @@ +import { IsString } from 'class-validator'; +import { GlobalValidationPipe, ValidationOptions } from './validation'; + +class Plain { + @IsString() + a!: string; + + b?: string; +} + +@ValidationOptions({ forbidNonWhitelisted: true, transform: true }) +class Strict { + @IsString() + a!: string; +} + +class StrictChild extends Strict {} + +describe('GlobalValidationPipe', () => { + const pipe = new GlobalValidationPipe(); + + it('drops properties without a validator and keeps the declared ones', async () => { + expect(await pipe.transform({ a: 'x', b: 'y' }, { type: 'body', metatype: Plain })).toEqual({ a: 'x' }); + }); + + it('still rejects an invalid value', async () => { + await expect(pipe.transform({ a: 1 }, { type: 'body', metatype: Plain })).rejects.toThrow(); + }); + + describe('@ValidationOptions', () => { + it('rejects undeclared fields when the DTO says so', async () => { + await expect(pipe.transform({ a: 'x', b: 'y' }, { type: 'body', metatype: Strict })).rejects.toThrow(); + }); + + it('still whitelists beside the declared options and applies them', async () => { + const result = await pipe.transform({ a: 'x' }, { type: 'body', metatype: Strict }); + expect(result).toBeInstanceOf(Strict); + expect(result).toEqual({ a: 'x' }); + }); + + it('applies to a DTO that extends the decorated one', async () => { + await expect(pipe.transform({ a: 'x', b: 'y' }, { type: 'body', metatype: StrictChild })).rejects.toThrow(); + }); + }); + + it('passes a body without a metatype through the default whitelist pipe', async () => { + const body = { a: 1 }; + expect(await pipe.transform(body, { type: 'body' })).toBe(body); + }); + + it.each([ + ['an array', Array, [1, 'a']], + ['a primitive', String, 'text'], + ])('leaves %s alone', async (_name, metatype, value) => { + expect(await pipe.transform(value, { type: 'body', metatype })).toBe(value); + }); +}); diff --git a/src/shared/validation.ts b/src/shared/validation.ts new file mode 100644 index 00000000..7fff6495 --- /dev/null +++ b/src/shared/validation.ts @@ -0,0 +1,31 @@ +import { ArgumentMetadata, PipeTransform, ValidationPipe, ValidationPipeOptions } from '@nestjs/common'; + +const VALIDATION_OPTIONS = Symbol('validationOptions'); + +export const ValidationOptions = + (options: ValidationPipeOptions): ClassDecorator => + (target) => { + Reflect.defineMetadata(VALIDATION_OPTIONS, options, target); + }; + +export class GlobalValidationPipe implements PipeTransform { + private readonly defaultPipe = new ValidationPipe({ whitelist: true }); + private readonly pipes = new WeakMap(); + + transform(value: unknown, metadata: ArgumentMetadata): Promise { + return this.pipeFor(metadata.metatype).transform(value, metadata); + } + + private pipeFor(metatype: ArgumentMetadata['metatype']): ValidationPipe { + const options: ValidationPipeOptions | undefined = metatype && Reflect.getMetadata(VALIDATION_OPTIONS, metatype); + if (!metatype || !options) { + return this.defaultPipe; + } + let pipe = this.pipes.get(metatype); + if (!pipe) { + pipe = new ValidationPipe({ whitelist: true, ...options }); + this.pipes.set(metatype, pipe); + } + return pipe; + } +} diff --git a/src/shared/validation.whitelist.spec.ts b/src/shared/validation.whitelist.spec.ts new file mode 100644 index 00000000..4aecedb3 --- /dev/null +++ b/src/shared/validation.whitelist.spec.ts @@ -0,0 +1,158 @@ +import { BadRequestException, Type } from '@nestjs/common'; +import { ModifyBuildDto } from '../builds/dto/build-modify.dto'; +import { CreateBuildDto } from '../builds/dto/build-create.dto'; +import { UpdateProjectDto } from '../projects/dto/update-project.dto'; +import { CreateProjectDto } from '../projects/dto/create-project.dto'; +import { CreateTestRequestBase64Dto } from '../test-runs/dto/create-test-request-base64.dto'; +import { CreateTestRequestMultipartDto } from '../test-runs/dto/create-test-request-multipart.dto'; +import { UpdateIgnoreAreasDto } from '../test-runs/dto/update-ignore-area.dto'; +import { UpdateTestRunDto } from '../test-runs/dto/update-test.dto'; +import { MergeParams } from '../test-variations/types'; +import { AssignRoleDto } from '../users/dto/assign-role.dto'; +import { ChangePasswordDto } from '../users/dto/change-password.dto'; +import { CreateUserDto } from '../users/dto/user-create.dto'; +import { UserLoginRequestDto } from '../users/dto/user-login-request.dto'; +import { UpdateUserDto } from '../users/dto/user-update.dto'; +import { GlobalValidationPipe } from './validation'; + +const run = async ( + value: unknown, + metatype: Type, + type: 'body' | 'query' = 'body' +): Promise> => + (await new GlobalValidationPipe().transform(value, { type, metatype, data: undefined })) as Record; +const BUILD_ID = '146e7a8d-89f0-4565-aa2c-e61efabb0afd'; +const PROJECT_ID = '3bc4a5bc-006e-4d43-8e4e-eaa132627fcc'; +const area = { x: 1, y: 2, width: 3, height: 4 }; + +describe('global whitelist', () => { + it('UpdateTestRunDto keeps only the comment', async () => { + const body = await run({ comment: 'c', baselineName: 'x', ignoreAreas: 'bad' }, UpdateTestRunDto); + expect(body).toEqual({ comment: 'c' }); + }); + + it('UpdateIgnoreAreasDto keeps the id the web UI selects areas by', async () => { + const body = await run( + { ids: ['a'], ignoreAreas: [{ ...area, id: '1', junk: 'x' }], junk: 'x' }, + UpdateIgnoreAreasDto + ); + expect(body).toEqual({ ids: ['a'], ignoreAreas: [{ ...area, id: '1' }] }); + }); + + it('CreateTestRequestBase64Dto keeps sourceBranch and the area ids', async () => { + const body = await run( + { + buildId: BUILD_ID, + projectId: PROJECT_ID, + name: 'n', + branchName: 'b', + sourceBranch: 'release', + imageBase64: 'abc', + ignoreAreas: [{ ...area, id: '1', junk: 'x' }], + junk: 'x', + }, + CreateTestRequestBase64Dto + ); + expect(body).toMatchObject({ sourceBranch: 'release', imageBase64: 'abc', ignoreAreas: [{ ...area, id: '1' }] }); + expect(body).not.toHaveProperty('junk'); + expect(body.ignoreAreas[0]).not.toHaveProperty('junk'); + }); + + describe('CreateTestRequestMultipartDto', () => { + const form = { + buildId: BUILD_ID, + projectId: PROJECT_ID, + name: 'n', + branchName: 'b', + image: { fieldname: 'image', buffer: Buffer.from([1]) }, + }; + + it('is parsed once, with the image, the areas and sourceBranch kept', async () => { + const body = await run( + { + ...form, + merge: 'true', + diffTollerancePercent: '1.5', + ignoreAreas: JSON.stringify([{ ...area, id: '1' }]), + sourceBranch: 'release', + junk: 'x', + }, + CreateTestRequestMultipartDto + ); + expect(Buffer.isBuffer(body.image.buffer)).toBe(true); + expect(body.merge).toBe(true); + expect(body.diffTollerancePercent).toBe(1.5); + expect(body.ignoreAreas).toEqual([{ ...area, id: '1' }]); + expect(body.sourceBranch).toBe('release'); + expect(body).not.toHaveProperty('junk'); + }); + + it('turns empty form fields into the defaults', async () => { + const body = await run( + { ...form, merge: '', diffTollerancePercent: '', ignoreAreas: '' }, + CreateTestRequestMultipartDto + ); + expect(body).toMatchObject({ merge: false, diffTollerancePercent: 0, ignoreAreas: [] }); + }); + }); + + describe('ModifyBuildDto', () => { + it('still rejects fields it does not declare', async () => { + await expect(run({ branchName: 'x' }, ModifyBuildDto)).rejects.toBeInstanceOf(BadRequestException); + }); + + it('passes the declared fields', async () => { + expect(await run({ isRunning: false }, ModifyBuildDto)).toEqual({ isRunning: false }); + }); + + it('keeps an empty body empty', async () => { + expect(await run(undefined, ModifyBuildDto)).toBeUndefined(); + }); + }); + + it('MergeParams keeps its three query parameters', async () => { + const body = await run({ projectId: PROJECT_ID, fromBranch: 'a', toBranch: 'b', junk: 'x' }, MergeParams, 'query'); + expect(body).toEqual({ projectId: PROJECT_ID, fromBranch: 'a', toBranch: 'b' }); + }); + + describe('projects', () => { + const project = { + id: PROJECT_ID, + name: 'p', + mainBranchName: 'master', + autoApproveFeature: true, + imageComparison: 'pixelmatch', + maxBuildAllowed: 5, + maxBranchLifetime: 7, + imageComparisonConfig: '{}', + bulkApproveVariations: false, + bulkApproveGroupBy: 'name', + }; + + it('UpdateProjectDto drops the counters and dates, keeps the ten settings', async () => { + const body = await run({ ...project, buildsCounter: 3, createdAt: '2020-01-01', junk: 'x' }, UpdateProjectDto); + expect(body).toEqual(project); + }); + + it('CreateProjectDto has no id', async () => { + const { id: _id, ...fields } = project; + expect(await run({ ...project, junk: 'x' }, CreateProjectDto)).toEqual(fields); + }); + }); + + it.each([ + ['CreateBuildDto', CreateBuildDto, { project: 'p', branchName: 'b', ciBuildId: 'c' }], + ['CreateUserDto', CreateUserDto, { email: 'a@b.co', password: 'p', firstName: 'f', lastName: 'l' }], + ['UserLoginRequestDto', UserLoginRequestDto, { email: 'a@b.co', password: 'p' }], + ['UpdateUserDto', UpdateUserDto, { email: 'a@b.co', firstName: 'f', lastName: 'l' }], + ['AssignRoleDto', AssignRoleDto, { id: PROJECT_ID, role: 'editor' }], + ['ChangePasswordDto', ChangePasswordDto, { password: 'p' }], + ] as const)('%s keeps its declared fields and drops the rest', async (_name, dto, fields) => { + expect(await run({ ...fields, junk: 'x' }, dto as Type)).toEqual(fields); + }); + + it('leaves an array body of ids alone', async () => { + const ids = ['a', 'b']; + expect(await run(ids, Array as unknown as Type)).toBe(ids); + }); +}); diff --git a/src/shared/wrapper-type.ts b/src/shared/wrapper-type.ts index 50b5520a..8c799e3a 100644 --- a/src/shared/wrapper-type.ts +++ b/src/shared/wrapper-type.ts @@ -1,8 +1,2 @@ -/** - * The type of a dependency injected with forwardRef. A plain class type would - * be emitted into the constructor's design:paramtypes metadata, and with - * circular imports that class is not initialised yet when the decorator runs: - * tsc's output reads it as undefined, SWC's live bindings throw. Behind an - * alias the metadata says Object, and forwardRef hands Nest the real class. - */ +// Hides the type from design:paramtypes: a circular import is not initialised there yet and SWC throws export type WrapperType = T; diff --git a/src/static/aws/s3.service.spec.ts b/src/static/aws/s3.service.spec.ts index 73ed4d80..eb304a6b 100644 --- a/src/static/aws/s3.service.spec.ts +++ b/src/static/aws/s3.service.spec.ts @@ -6,7 +6,6 @@ import { S3Client, } from '@aws-sdk/client-s3'; import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; -import { PNG } from 'pngjs'; import { Readable } from 'stream'; import { AWSS3Service, URL_WINDOW_SECONDS } from './s3.service'; import { generateNewImageName } from '../utils'; @@ -31,12 +30,28 @@ jest.mock('../utils', () => ({ generateNewImageName: jest.fn(), })); +const notFound = (props: object) => Object.assign(new Error('not found'), props); +const notFoundCases = [ + ['a NoSuchKey error', { name: 'NoSuchKey' }], + ['an HTTP 404', { $metadata: { httpStatusCode: 404 } }], +] as const; +const defaultTimeouts = { + requestHandler: { connectionTimeout: 5000, requestTimeout: 60000, throwOnRequestTimeout: true }, +}; + describe('AWSS3Service', () => { const originalAwsBucket = process.env.AWS_S3_BUCKET_NAME; + const originalEnv = { ...process.env }; beforeEach(() => { jest.clearAllMocks(); process.env.AWS_S3_BUCKET_NAME = 'vrt-bucket'; + mockSend.mockResolvedValue({}); + (getSignedUrl as jest.Mock).mockResolvedValue('https://signed-url'); + }); + + afterEach(() => { + process.env = { ...originalEnv }; }); afterAll(() => { @@ -46,7 +61,6 @@ describe('AWSS3Service', () => { describe('saveImage', () => { it('uploads the image buffer and returns the generated image name', async () => { (generateNewImageName as jest.Mock).mockReturnValue('generated.screenshot.png'); - mockSend.mockResolvedValue({}); const service = new AWSS3Service(); const imageBuffer = Buffer.from('png-data'); @@ -82,49 +96,7 @@ describe('AWSS3Service', () => { }); }); - describe('getImage', () => { - it('returns null when the file name is missing', async () => { - const service = new AWSS3Service(); - - await expect(service.getImage('')).resolves.toBeNull(); - expect(mockSend).not.toHaveBeenCalled(); - }); - - it('reads the image from S3 and parses it as PNG', async () => { - const service = new AWSS3Service(); - const png = new PNG({ width: 1, height: 1 }); - const pngBuffer = PNG.sync.write(png); - const stream = { - toArray: jest.fn().mockResolvedValue([pngBuffer.subarray(0, 8), pngBuffer.subarray(8)]), - } as unknown as Readable; - mockSend.mockResolvedValue({ Body: stream }); - - const result = await service.getImage('baseline.png'); - - expect(GetObjectCommand).toHaveBeenCalledWith({ - Bucket: 'vrt-bucket', - Key: 'baseline.png', - }); - expect(result).toMatchObject({ width: 1, height: 1 }); - }); - - it('logs failures and returns undefined when the image cannot be read', async () => { - const service = new AWSS3Service(); - const loggerSpy = jest.spyOn((service as any).logger, 'error').mockImplementation(); - mockSend.mockRejectedValue(new Error('download failed')); - - await expect(service.getImage('baseline.png')).resolves.toBeUndefined(); - - expect(loggerSpy).toHaveBeenCalledWith( - 'Error from read : Cannot get image: baseline.png. Error: download failed' - ); - }); - }); - describe('getImageUrl', () => { - // A browser caches by URL. Signing from the current instant produced a - // different URL on every request, so the bytes it had just downloaded - // could never be reused and every view of an image cost a fresh download. const signingDateOfLastCall = (): Date => (getSignedUrl as jest.Mock).mock.calls.at(-1)[2].signingDate; afterEach(() => { @@ -133,7 +105,6 @@ describe('AWSS3Service', () => { it('returns a signed URL for the requested object', async () => { const service = new AWSS3Service(); - (getSignedUrl as jest.Mock).mockResolvedValue('https://signed-url'); const result = await service.getImageUrl('image.png'); @@ -143,47 +114,57 @@ describe('AWSS3Service', () => { ); }); - it('hands out the same URL for an image asked for again soon after', async () => { + it('signs for twice the window, from the start of the current window, with a matching cache header', async () => { + jest.spyOn(Date, 'now').mockReturnValue(Date.parse('2026-09-01T10:42:31.500Z')); const service = new AWSS3Service(); - (getSignedUrl as jest.Mock).mockResolvedValue('https://signed-url'); - const nowSpy = jest.spyOn(Date, 'now'); - nowSpy.mockReturnValue(Date.parse('2026-09-01T10:00:05Z')); await service.getImageUrl('image.png'); - const first = signingDateOfLastCall(); - nowSpy.mockReturnValue(Date.parse('2026-09-01T10:42:31Z')); + expect(URL_WINDOW_SECONDS).toBe(3600); + expect(GetObjectCommand).toHaveBeenCalledWith({ + Bucket: 'vrt-bucket', + Key: 'image.png', + ResponseCacheControl: 'private, max-age=3600', + }); + expect(getSignedUrl).toHaveBeenCalledWith(expect.objectContaining({ send: mockSend }), expect.anything(), { + expiresIn: 7200, + signingDate: new Date('2026-09-01T10:00:00.000Z'), + }); + }); + + it('signs against "undefined" when no bucket is configured (startup warns about it)', async () => { + delete process.env.AWS_S3_BUCKET_NAME; + const service = new AWSS3Service(); + await service.getImageUrl('image.png'); - // asserted to be a real instant first: two undefineds are also equal - expect(first).toBeInstanceOf(Date); - expect(signingDateOfLastCall()).toEqual(first); + expect(GetObjectCommand).toHaveBeenCalledWith(expect.objectContaining({ Bucket: 'undefined' })); }); - it('moves on to a new URL once the window has passed', async () => { + it.each([ + ['hands out the same URL for an image asked for again soon after', '10:00:05', '10:42:31', true], + ['moves on to a new URL once the window has passed', '10:42:31', '11:00:01', false], + ])('%s', async (_, firstTime, secondTime, same) => { const service = new AWSS3Service(); - (getSignedUrl as jest.Mock).mockResolvedValue('https://signed-url'); const nowSpy = jest.spyOn(Date, 'now'); - nowSpy.mockReturnValue(Date.parse('2026-09-01T10:42:31Z')); + nowSpy.mockReturnValue(Date.parse(`2026-09-01T${firstTime}Z`)); await service.getImageUrl('image.png'); const first = signingDateOfLastCall(); - nowSpy.mockReturnValue(Date.parse('2026-09-01T11:00:01Z')); + nowSpy.mockReturnValue(Date.parse(`2026-09-01T${secondTime}Z`)); await service.getImageUrl('image.png'); - expect(signingDateOfLastCall()).not.toEqual(first); + expect(first).toBeInstanceOf(Date); + if (same) { + expect(signingDateOfLastCall()).toEqual(first); + } else { + expect(signingDateOfLastCall()).not.toEqual(first); + } }); - // a URL handed out at the very end of a window must still be usable for a - // whole window afterwards, or the browser's cached copy outlives its URL it('keeps a URL alive well past the window it was signed in', async () => { const service = new AWSS3Service(); - (getSignedUrl as jest.Mock).mockResolvedValue('https://signed-url'); - // The worst case, when the least life is left: minted at the last second - // of a window. Pinned rather than read live, or a run that straddles the - // hour boundary asks the clock on the far side of it and sees a shorter - // life than the URL actually has. const lastSecondOfWindow = Date.parse('2026-09-01T10:59:59Z'); jest.spyOn(Date, 'now').mockReturnValue(lastSecondOfWindow); @@ -194,11 +175,8 @@ describe('AWSS3Service', () => { expect(livesUntil - lastSecondOfWindow).toBeGreaterThanOrEqual(3600 * 1000); }); - // S3 answers with no cache headers of its own, so a stable URL alone still - // leaves the browser guessing whether it may keep the bytes it('asks S3 to answer with a cache header', async () => { const service = new AWSS3Service(); - (getSignedUrl as jest.Mock).mockResolvedValue('https://signed-url'); await service.getImageUrl('image.png'); @@ -209,17 +187,16 @@ describe('AWSS3Service', () => { }); describe('constructor', () => { - it('creates one S3 client with the default credential chain', () => { + it('creates one S3 client with the default credential chain and request timeouts', () => { new AWSS3Service(); expect(S3Client).toHaveBeenCalledTimes(1); - expect(S3Client).toHaveBeenCalledWith(); + expect(S3Client).toHaveBeenCalledWith(defaultTimeouts); }); it('reads the bucket name when the service is constructed, not on each call', async () => { const service = new AWSS3Service(); process.env.AWS_S3_BUCKET_NAME = 'changed-later'; - mockSend.mockResolvedValue({}); await service.deleteImage('image.png'); @@ -228,8 +205,6 @@ describe('AWSS3Service', () => { }); describe('getImageBuffer', () => { - const notFound = (props: object) => Object.assign(new Error('not found'), props); - it.each(['', undefined, null])('returns null for an empty name (%p) without calling S3', async (fileName) => { const service = new AWSS3Service(); @@ -246,10 +221,7 @@ describe('AWSS3Service', () => { expect(mockSend).toHaveBeenCalledWith({ input: { Bucket: 'vrt-bucket', Key: 'image.png' }, type: 'get' }); }); - it.each([ - ['a NoSuchKey error', { name: 'NoSuchKey' }], - ['an HTTP 404', { $metadata: { httpStatusCode: 404 } }], - ])('returns null for %s', async (_label, props) => { + it.each(notFoundCases)('returns null for %s', async (_label, props) => { const service = new AWSS3Service(); jest.spyOn((service as any).logger, 'error').mockImplementation(); mockSend.mockRejectedValue(notFound(props)); @@ -281,29 +253,41 @@ describe('AWSS3Service', () => { }); }); - describe('getImage (missing object)', () => { - it('returns undefined when the object does not exist', async () => { + describe('getImageStream', () => { + it('returns the body of the object without reading it', async () => { const service = new AWSS3Service(); - jest.spyOn((service as any).logger, 'error').mockImplementation(); - mockSend.mockRejectedValue(Object.assign(new Error('nope'), { name: 'NoSuchKey' })); + const body = Readable.from([Buffer.from('ab')]); + mockSend.mockResolvedValue({ Body: body }); - await expect(service.getImage('missing.png')).resolves.toBeUndefined(); + await expect(service.getImageStream('image.png')).resolves.toBe(body); + expect(mockSend).toHaveBeenCalledWith({ input: { Bucket: 'vrt-bucket', Key: 'image.png' }, type: 'get' }); }); - it('logs and returns undefined when the object is not a valid PNG', async () => { + it('returns null when the response has no body', async () => { const service = new AWSS3Service(); - const loggerSpy = jest.spyOn((service as any).logger, 'error').mockImplementation(); - mockSend.mockResolvedValue({ Body: { toArray: jest.fn().mockResolvedValue([Buffer.from('garbage')]) } }); - await expect(service.getImage('corrupt.png')).resolves.toBeUndefined(); - expect(loggerSpy).toHaveBeenCalledWith(expect.stringContaining('Cannot get image: corrupt.png.')); + await expect(service.getImageStream('image.png')).resolves.toBeNull(); + }); + + it.each(notFoundCases)('returns null for %s', async (_label, props) => { + const service = new AWSS3Service(); + mockSend.mockRejectedValue(notFound(props)); + + await expect(service.getImageStream('missing.png')).resolves.toBeNull(); + }); + + it('rethrows any other failure', async () => { + const service = new AWSS3Service(); + const error = Object.assign(new Error('Access Denied'), { name: 'AccessDenied' }); + mockSend.mockRejectedValue(error); + + await expect(service.getImageStream('image.png')).rejects.toBe(error); }); }); describe('copyImage', () => { it('copies the source object inside the bucket under a newly generated name', async () => { (generateNewImageName as jest.Mock).mockReturnValue('generated.baseline.png'); - mockSend.mockResolvedValue({}); const service = new AWSS3Service(); await expect(service.copyImage('baseline', 'source.png')).resolves.toBe('generated.baseline.png'); @@ -328,44 +312,6 @@ describe('AWSS3Service', () => { }); }); - describe('getImageUrl (exact parameters)', () => { - afterEach(() => { - jest.restoreAllMocks(); - }); - - it('signs for twice the window, from the start of the current window, with a matching cache header', async () => { - jest.spyOn(Date, 'now').mockReturnValue(Date.parse('2026-09-01T10:42:31.500Z')); - (getSignedUrl as jest.Mock).mockResolvedValue('https://signed-url'); - const service = new AWSS3Service(); - - await service.getImageUrl('image.png'); - - expect(URL_WINDOW_SECONDS).toBe(3600); - expect(GetObjectCommand).toHaveBeenCalledWith({ - Bucket: 'vrt-bucket', - Key: 'image.png', - ResponseCacheControl: 'private, max-age=3600', - }); - expect(getSignedUrl).toHaveBeenCalledWith(expect.objectContaining({ send: mockSend }), expect.anything(), { - expiresIn: 7200, - signingDate: new Date('2026-09-01T10:00:00.000Z'), - }); - }); - - // NOTE: current behavior — the bucket is interpolated into a template - // string here, so a missing AWS_S3_BUCKET_NAME becomes the literal - // "undefined" (the other commands pass undefined through). - it('signs against a bucket literally named "undefined" when no bucket is configured', async () => { - delete process.env.AWS_S3_BUCKET_NAME; - (getSignedUrl as jest.Mock).mockResolvedValue('https://signed-url'); - const service = new AWSS3Service(); - - await service.getImageUrl('image.png'); - - expect(GetObjectCommand).toHaveBeenCalledWith(expect.objectContaining({ Bucket: 'undefined' })); - }); - }); - describe('deleteImage', () => { it('returns false when the image name is missing', async () => { const service = new AWSS3Service(); @@ -376,7 +322,6 @@ describe('AWSS3Service', () => { it('deletes the object and returns true', async () => { const service = new AWSS3Service(); - mockSend.mockResolvedValue({}); await expect(service.deleteImage('image.png')).resolves.toBe(true); diff --git a/src/static/aws/s3.service.ts b/src/static/aws/s3.service.ts index a99a4228..594d13ae 100644 --- a/src/static/aws/s3.service.ts +++ b/src/static/aws/s3.service.ts @@ -1,4 +1,3 @@ -import { PNG, PNGWithMetadata } from 'pngjs'; import { Logger } from '@nestjs/common'; import { Static } from '../static.interface'; import { @@ -12,8 +11,6 @@ import { Readable } from 'stream'; import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; import { generateNewImageName } from '../utils'; -// How long one pre-signed URL is reused before a new one is minted. Also the -// lifetime S3 is asked to advertise on the object itself. export const URL_WINDOW_SECONDS = 3600; export class AWSS3Service implements Static { @@ -23,7 +20,13 @@ export class AWSS3Service implements Static { private s3Client: S3Client; constructor() { - this.s3Client = new S3Client(); + this.s3Client = new S3Client({ + requestHandler: { + connectionTimeout: 5_000, + requestTimeout: 60_000, + throwOnRequestTimeout: true, + }, + }); this.logger.log('AWS S3 service is being used for file storage.'); } @@ -44,19 +47,6 @@ export class AWSS3Service implements Static { } } - async getImage(fileName: string | null): Promise { - if (!fileName) return null; - try { - // the comparison pipeline treats an unreadable image as a missing - // baseline, so storage failures stay contained here - const imageBuffer = await this.getImageBuffer(fileName); - if (!imageBuffer) return undefined; - return PNG.sync.read(imageBuffer); - } catch (ex) { - this.logger.error(`Error from read : Cannot get image: ${fileName}. ${ex}`); - } - } - async getImageBuffer(fileName: string | null): Promise { if (!fileName) return null; try { @@ -66,16 +56,30 @@ export class AWSS3Service implements Static { return Buffer.concat(await stream.toArray()); } catch (ex) { this.logger.error(`Error from read : Cannot get image: ${fileName}. ${ex}`); - // only a missing object means "no image"; credentials, throttling and - // network failures have to stay errors instead of reading as absence - const error = ex as { name?: string; $metadata?: { httpStatusCode?: number } } | undefined; - if (error?.name === 'NoSuchKey' || error?.$metadata?.httpStatusCode === 404) { + if (this.isNotFound(ex)) { return null; } throw ex; } } + async getImageStream(fileName: string): Promise { + try { + const { Body } = await this.s3Client.send( + new GetObjectCommand({ Bucket: this.AWS_S3_BUCKET_NAME, Key: fileName }) + ); + return (Body as Readable | undefined) ?? null; + } catch (ex) { + if (this.isNotFound(ex)) return null; + throw ex; + } + } + + private isNotFound(ex: unknown): boolean { + const error = ex as { name?: string; $metadata?: { httpStatusCode?: number } } | undefined; + return error?.name === 'NoSuchKey' || error?.$metadata?.httpStatusCode === 404; + } + async copyImage(type: 'screenshot' | 'diff' | 'baseline', sourceImageName: string): Promise { const imageName = generateNewImageName(type); try { @@ -92,23 +96,6 @@ export class AWSS3Service implements Static { } } - /** - * A pre-signed URL for the object, stable for the length of a window. - * - * A browser caches by URL. Signing from the current instant gave every - * request a different URL, so the bytes it had just downloaded could never be - * reused: reopening a screen, or stepping back one, paid for the whole image - * again. Signing from the start of a fixed window instead makes every request - * inside that window produce the very same URL, which a cache can hit. - * - * The signature is given twice the window to live, so a URL handed out at the - * last second of one window is still good for a whole window afterwards. - * - * `ResponseCacheControl` makes S3 answer with a cache header of its own — - * without it a stable URL still leaves the browser guessing, since the bucket - * sets none. An image name is unique per upload and its bytes never change, - * so the only thing bounding the lifetime is the signature. - */ async getImageUrl(imageName: string): Promise { const command = new GetObjectCommand({ Bucket: `${this.AWS_S3_BUCKET_NAME}`, diff --git a/src/static/hdd/hdd.service.spec.ts b/src/static/hdd/hdd.service.spec.ts index 8ad62fad..358d7091 100644 --- a/src/static/hdd/hdd.service.spec.ts +++ b/src/static/hdd/hdd.service.spec.ts @@ -5,8 +5,11 @@ import { PNG } from 'pngjs'; import { HDD_IMAGE_PATH } from './constants'; import { HddService } from './hdd.service'; -// Keeps the spec out of the real imageUploads/ folder: the directory is a -// fresh temp one, still named imageUploads so the resolved paths look alike. +jest.mock('fs', () => { + const actual = jest.requireActual('fs'); + return { ...actual, mkdirSync: jest.fn(actual.mkdirSync), existsSync: jest.fn(actual.existsSync) }; +}); + jest.mock('./constants', () => ({ HDD_IMAGE_PATH: `${require('path').join(require('os').tmpdir(), `vrt-hdd-spec-${process.pid}`, 'imageUploads')}/`, })); @@ -16,10 +19,11 @@ const pngBuffer = (width = 2, height = 1) => PNG.sync.write(new PNG({ width, hei const writeImage = (name: string, data: Buffer) => fs.writeFile(path.join(imageDir, name), data); describe('HddService', () => { - const service = new HddService(); + let service: HddService; let errorSpy: jest.SpyInstance; beforeEach(() => { + service = new HddService(); errorSpy = jest.spyOn(Logger.prototype, 'error').mockImplementation(); }); @@ -39,7 +43,6 @@ describe('HddService', () => { expect(service.getImagePath('ABC123.screenshot.png')).toMatch(/imageUploads\/ABC123\.screenshot\.png$/); }); - // a name may begin with dots without climbing out of the directory it.each(['..thumbnail.png', '...png', '.hidden.png'])( 'resolves a dotted name inside the directory: %s', (imageName) => { @@ -67,12 +70,51 @@ describe('HddService', () => { expect(existsSync(imageDir)).toBe(true); }); - it('returns null for a missing image', async () => { - await expect(service.getImageBuffer('definitely-missing.png')).resolves.toBeNull(); + it('creates the image directory once per instance', () => { + (mkdirSync as jest.Mock).mockClear(); + (existsSync as jest.Mock).mockClear(); + + service.getImagePath('a.screenshot.png'); + service.getImagePath('b.screenshot.png'); + + expect(mkdirSync).toHaveBeenCalledTimes(1); + expect(existsSync).not.toHaveBeenCalled(); + }); + + it.each([ + ['getImageBuffer', (name: string) => service.getImageBuffer(name)], + ['getImageStream', (name: string) => service.getImageStream(name)], + ])('%s returns null for a missing image', async (_, read) => { + await expect(read('definitely-missing.png')).resolves.toBeNull(); }); - it('propagates a traversal attempt instead of reading the file', async () => { - await expect(service.getImageBuffer('../../etc/passwd')).rejects.toThrow(/outside of the image directory/); + it.each([ + ['getImageBuffer', (name: string) => service.getImageBuffer(name)], + ['getImageStream', (name: string) => service.getImageStream(name)], + ['copyImage', (name: string) => service.copyImage('baseline', name)], + ])('%s rejects a path outside the image directory instead of reading the file', async (_, read) => { + await expect(read('../../etc/passwd')).rejects.toThrow(/outside of the image directory/); + }); + + describe('getImageStream', () => { + const readAll = async (stream: NodeJS.ReadableStream | null) => { + const chunks: Buffer[] = []; + for await (const chunk of stream!) chunks.push(chunk as Buffer); + return Buffer.concat(chunks); + }; + + it('streams the bytes of the stored file', async () => { + const data = pngBuffer(4, 4); + await writeImage('streamed.screenshot.png', data); + + await expect(readAll(await service.getImageStream('streamed.screenshot.png'))).resolves.toEqual(data); + }); + + it('rethrows a failure other than a missing file', async () => { + await writeImage('plain-file.png', pngBuffer()); + + await expect(service.getImageStream('plain-file.png/inside.png')).rejects.toMatchObject({ code: 'ENOTDIR' }); + }); }); describe('generateNewImage', () => { @@ -90,9 +132,7 @@ describe('HddService', () => { await expect(service.getImageUrl('image.png')).resolves.toBe('/image.png'); }); - // NOTE: current behavior — unlike the other methods, the name is not - // checked for traversal here (the URL is only handed to the browser). - it('does not validate the image name', async () => { + it('builds the URL without validating the name (the controller validates it)', async () => { await expect(service.getImageUrl('../secret')).resolves.toBe('/../secret'); }); }); @@ -116,40 +156,13 @@ describe('HddService', () => { await expect(fs.readFile(path.join(imageDir, imageName), 'utf8')).resolves.toBe('HELLO'); }); - // NOTE: current behavior — HddService itself does not check the format; - // only StaticService rejects non-PNG uploads. - it('accepts a buffer that is not a PNG', async () => { + it('stores any buffer (StaticService checks the format)', async () => { const imageName = await service.saveImage('diff', Buffer.from('not a png')); await expect(fs.readFile(path.join(imageDir, imageName), 'utf8')).resolves.toBe('not a png'); }); }); - describe('getImage', () => { - it('decodes a stored PNG', async () => { - await writeImage('decodable.png', pngBuffer(3, 2)); - - const image = await service.getImage('decodable.png'); - - expect(image).toMatchObject({ width: 3, height: 2 }); - }); - - it('returns undefined for a missing image', async () => { - await expect(service.getImage('missing.png')).resolves.toBeUndefined(); - }); - - it('returns undefined for an empty name', async () => { - await expect(service.getImage('')).resolves.toBeUndefined(); - }); - - it('logs and returns undefined for a file that is not a valid PNG', async () => { - await writeImage('corrupt.png', Buffer.from('garbage')); - - await expect(service.getImage('corrupt.png')).resolves.toBeUndefined(); - expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('Cannot decode image: corrupt.png.')); - }); - }); - describe('getImageBuffer', () => { it('returns the stored bytes', async () => { await writeImage('bytes.png', Buffer.from([1, 2, 3])); @@ -183,10 +196,6 @@ describe('HddService', () => { it('rejects when the source image is missing', async () => { await expect(service.copyImage('baseline', 'missing.png')).rejects.toMatchObject({ code: 'ENOENT' }); }); - - it('rejects a source name pointing outside the image directory', async () => { - await expect(service.copyImage('baseline', '../outside.png')).rejects.toThrow(/outside of the image directory/); - }); }); describe('deleteImage', () => { @@ -201,8 +210,6 @@ describe('HddService', () => { await expect(service.deleteImage('')).resolves.toBeUndefined(); }); - // NOTE: current behavior — failures are only logged and still reported as - // a successful delete. it('logs and still returns true when the file is missing', async () => { await expect(service.deleteImage('never-existed.png')).resolves.toBe(true); expect(errorSpy).toHaveBeenCalledWith(expect.objectContaining({ code: 'ENOENT' })); diff --git a/src/static/hdd/hdd.service.ts b/src/static/hdd/hdd.service.ts index 71e14f11..5c45e865 100644 --- a/src/static/hdd/hdd.service.ts +++ b/src/static/hdd/hdd.service.ts @@ -1,7 +1,7 @@ import { Injectable, Logger } from '@nestjs/common'; import path from 'path'; -import { promises as fs, mkdirSync, existsSync } from 'fs'; -import { PNG, PNGWithMetadata } from 'pngjs'; +import { promises as fs, mkdirSync } from 'fs'; +import { Readable } from 'stream'; import { Static } from '../static.interface'; import { HDD_IMAGE_PATH } from './constants'; import { generateNewImageName } from '../utils'; @@ -9,6 +9,7 @@ import { generateNewImageName } from '../utils'; @Injectable() export class HddService implements Static { private readonly logger: Logger = new Logger(HddService.name); + private directoryReady = false; generateNewImage(type: 'screenshot' | 'diff' | 'baseline'): { imageName: string; imagePath: string } { const imageName = generateNewImageName(type); @@ -19,13 +20,12 @@ export class HddService implements Static { } getImagePath(imageName: string): string { - this.ensureDirectoryExistence(HDD_IMAGE_PATH); + if (!this.directoryReady) { + mkdirSync(HDD_IMAGE_PATH, { recursive: true }); + this.directoryReady = true; + } const root = path.resolve(HDD_IMAGE_PATH); const imagePath = path.resolve(root, imageName); - // image names reach here straight from the request, so a traversal value - // would otherwise read any file the process can see - // only a leading parent-directory step means the name climbs out: a file - // whose name merely starts with dots stays inside const relativeToRoot = path.relative(root, imagePath); const climbsOut = relativeToRoot === '..' || relativeToRoot.startsWith(`..${path.sep}`) || path.isAbsolute(relativeToRoot); @@ -41,28 +41,16 @@ export class HddService implements Static { async saveImage(type: 'screenshot' | 'diff' | 'baseline', imageBuffer: Buffer): Promise { const { imageName, imagePath } = this.generateNewImage(type); - await fs.writeFile(imagePath, new Uint8Array(imageBuffer.buffer, imageBuffer.byteOffset, imageBuffer.byteLength)); + await fs.writeFile(imagePath, imageBuffer); return imageName; } - async getImage(imageName: string | null): Promise { - const imageBuffer = await this.getImageBuffer(imageName); - if (!imageBuffer) return; - try { - return PNG.sync.read(imageBuffer); - } catch (ex) { - this.logger.error(`Cannot decode image: ${imageName}. ${ex}`); - } - } - async getImageBuffer(imageName: string | null): Promise { if (!imageName) return null; try { return await fs.readFile(this.getImagePath(imageName)); } catch (ex) { this.logger.error(`Cannot get image: ${imageName}. ${ex}`); - // an absent file is the only case that means "no image"; a permission or - // I/O failure has to stay an error rather than read as a missing image if ((ex as NodeJS.ErrnoException | undefined)?.code === 'ENOENT') { return null; } @@ -70,6 +58,16 @@ export class HddService implements Static { } } + async getImageStream(imageName: string): Promise { + const imagePath = this.getImagePath(imageName); + try { + return (await fs.open(imagePath, 'r')).createReadStream(); + } catch (ex) { + if ((ex as NodeJS.ErrnoException | undefined)?.code === 'ENOENT') return null; + throw ex; + } + } + async copyImage(type: 'screenshot' | 'diff' | 'baseline', sourceImageName: string): Promise { const { imageName, imagePath } = this.generateNewImage(type); await fs.copyFile(this.getImagePath(sourceImageName), imagePath); @@ -85,14 +83,4 @@ export class HddService implements Static { } return true; } - - private ensureDirectoryExistence(dir: string) { - const filePath = path.resolve(dir); - if (existsSync(filePath)) { - return true; - } else { - mkdirSync(dir, { recursive: true }); - this.ensureDirectoryExistence(dir); - } - } } diff --git a/src/static/static.controller.spec.ts b/src/static/static.controller.spec.ts index 99ecec66..ac0add42 100644 --- a/src/static/static.controller.spec.ts +++ b/src/static/static.controller.spec.ts @@ -1,16 +1,45 @@ import { Test, TestingModule } from '@nestjs/testing'; import { BadRequestException, Logger, NotFoundException } from '@nestjs/common'; import { Response } from 'express'; +import { PassThrough, Readable } from 'stream'; import { StaticController } from './static.controller'; import { StaticService } from './static.service'; - -const initController = async (getImageBufferMock = jest.fn(), getImageUrlMock = jest.fn()) => { +import { generateNewImageName } from './utils'; + +const IMAGE = 'ABCDEFGHJKMNPQRSTVWXYZ012345.screenshot.png'; + +const ACCEPTED_NAMES = [ + generateNewImageName('screenshot'), + generateNewImageName('diff'), + generateNewImageName('baseline'), + '1612345678901.screenshot.png', + 'V1StGXR8_Z5jdHi6B-myT.diff.png', + '0123456789ABCDEFGHJKMNPQRSTV.baseline.png', +]; + +const REJECTED_NAMES = [ + '\\evil.com', + '/\\evil.com', + 'image.png', + 'a.b.screenshot.png', + 'x.screenshot.png.exe', + '..', + '.', + '', + '../../etc/passwd', + 'nested/image.png', + '/etc/passwd', + '/example.com', + '//example.com', +]; + +const initController = async (getImageStreamMock = jest.fn(), getImageUrlMock = jest.fn()) => { const module: TestingModule = await Test.createTestingModule({ controllers: [StaticController], providers: [ { provide: StaticService, - useValue: { getImageBuffer: getImageBufferMock, getImageUrl: getImageUrlMock }, + useValue: { getImageStream: getImageStreamMock, getImageUrl: getImageUrlMock }, }, ], }).compile(); @@ -24,76 +53,81 @@ const responseMock = () => { return res as unknown as Response & { set: jest.Mock; send: jest.Mock; redirect: jest.Mock; status: jest.Mock }; }; +const streamResponse = () => { + const res = Object.assign(new PassThrough(), { set: jest.fn() }); + const chunks: Buffer[] = []; + res.on('data', (chunk: Buffer) => chunks.push(chunk)); + return { res: res as unknown as Response & { set: jest.Mock }, body: () => Buffer.concat(chunks) }; +}; + describe('download', () => { - it('answers with the image bytes', async () => { - const imageBuffer = Buffer.from([1, 2, 3]); - const getImageBufferMock = jest.fn().mockResolvedValueOnce(imageBuffer); - const controller = await initController(getImageBufferMock); - const res = responseMock(); + it('streams the image bytes', async () => { + const getImageStreamMock = jest.fn().mockResolvedValueOnce(Readable.from([Buffer.from([1, 2]), Buffer.from([3])])); + const controller = await initController(getImageStreamMock); + const { res, body } = streamResponse(); - await controller.download('image.png', res); + await controller.download(IMAGE, res); - expect(getImageBufferMock).toHaveBeenCalledWith('image.png'); + expect(getImageStreamMock).toHaveBeenCalledWith(IMAGE); expect(res.set).toHaveBeenCalledWith({ 'Content-Type': 'image/png', - 'Content-Disposition': 'attachment; filename="image.png"', + 'Content-Disposition': `attachment; filename="${IMAGE}"`, + 'Cache-Control': 'private, max-age=31536000, immutable', }); - expect(res.send).toHaveBeenCalledWith(imageBuffer); + expect(body()).toEqual(Buffer.from([1, 2, 3])); }); it('answers 404 for an image that is not there', async () => { - const getImageBufferMock = jest.fn().mockResolvedValueOnce(null); - const controller = await initController(getImageBufferMock); + const getImageStreamMock = jest.fn().mockResolvedValueOnce(null); + const controller = await initController(getImageStreamMock); + const { res } = streamResponse(); - await expect(controller.download('missing.png', responseMock())).rejects.toThrow(NotFoundException); + await expect(controller.download(IMAGE, res)).rejects.toThrow(new NotFoundException(`Image not found: ${IMAGE}`)); + expect(res.set).not.toHaveBeenCalled(); }); - // a name carrying a path could otherwise read any file the process can see - it.each(['../../etc/passwd', '..', '.', '', 'nested/image.png', '/etc/passwd'])( - 'rejects %p without touching storage', - async (fileName) => { - const getImageBufferMock = jest.fn(); - const controller = await initController(getImageBufferMock); - - await expect(controller.download(fileName, responseMock())).rejects.toThrow(BadRequestException); - expect(getImageBufferMock).not.toHaveBeenCalled(); - } - ); -}); - -describe('getUrlAndRedirect', () => { - // Without this the browser asks the API for a location on every single view. - // Storage then hands back a freshly signed, and therefore different, URL, and - // the image it already holds is dead weight — every view a fresh download. - it('lets the browser reuse the redirect it was given', async () => { - const getImageUrlMock = jest.fn().mockResolvedValueOnce('https://storage/image.png'); - const controller = await initController(jest.fn(), getImageUrlMock); - const res = responseMock(); + it('warns instead of throwing when the stream fails midway', async () => { + const warnSpy = jest.spyOn(Logger.prototype, 'warn').mockImplementation(); + const failing = new Readable({ + read() { + this.push(Buffer.from([1])); + this.destroy(new Error('disk gone')); + }, + }); + const controller = await initController(jest.fn().mockResolvedValueOnce(failing)); + const { res } = streamResponse(); - await controller.getUrlAndRedirect('image.png', res); + await expect(controller.download(IMAGE, res)).resolves.toBeUndefined(); - expect(res.set).toHaveBeenCalledWith('Cache-Control', expect.stringContaining('max-age=')); - expect(res.redirect).toHaveBeenCalledWith('https://storage/image.png'); + expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining(`Download of ${IMAGE} stopped`)); + warnSpy.mockRestore(); }); - // the redirect may not outlive the URL it points at - it('stops reusing the redirect before the signed URL can expire', async () => { - const controller = await initController(jest.fn(), jest.fn().mockResolvedValueOnce('https://storage/image.png')); - const res = responseMock(); + it.each(ACCEPTED_NAMES)('accepts the generated name %s', async (fileName) => { + const controller = await initController(jest.fn().mockResolvedValueOnce(Readable.from([Buffer.from([1])]))); + const { res } = streamResponse(); - await controller.getUrlAndRedirect('image.png', res); + await expect(controller.download(fileName, res)).resolves.toBeUndefined(); + }); + + it.each(REJECTED_NAMES)('rejects %p without touching storage', async (fileName) => { + const getImageStreamMock = jest.fn(); + const controller = await initController(getImageStreamMock); - const [, value] = res.set.mock.calls.find(([header]) => header === 'Cache-Control'); - expect(Number(/max-age=(\d+)/.exec(value)![1])).toBeLessThan(3600); + await expect(controller.download(fileName, responseMock())).rejects.toThrow(BadRequestException); + expect(getImageStreamMock).not.toHaveBeenCalled(); }); +}); - it('sets a private cache header of exactly 3000 seconds before redirecting', async () => { +describe('getUrlAndRedirect', () => { + it('lets the browser reuse the redirect for exactly 3000 seconds, short of the signed URL expiry', async () => { const controller = await initController(jest.fn(), jest.fn().mockResolvedValueOnce('https://storage/image.png')); const res = responseMock(); - await controller.getUrlAndRedirect('image.png', res); + await controller.getUrlAndRedirect(IMAGE, res); expect(res.set).toHaveBeenCalledWith('Cache-Control', 'private, max-age=3000'); + expect(res.redirect).toHaveBeenCalledWith('https://storage/image.png'); expect(res.set.mock.invocationCallOrder[0]).toBeLessThan(res.redirect.mock.invocationCallOrder[0]); }); @@ -103,30 +137,34 @@ describe('getUrlAndRedirect', () => { const controller = await initController(jest.fn(), jest.fn().mockRejectedValueOnce(error)); const res = responseMock(); - await controller.getUrlAndRedirect('image.png', res); + await controller.getUrlAndRedirect(IMAGE, res); expect(res.status).toHaveBeenCalledWith(500); expect(res.send).toHaveBeenCalledWith('Error occurred while getting the file.'); expect(res.redirect).not.toHaveBeenCalled(); expect(res.set).not.toHaveBeenCalled(); - expect(errorSpy).toHaveBeenCalledWith('Error fetching file from S3:image.png', error); + expect(errorSpy).toHaveBeenCalledWith(`Error fetching file from S3:${IMAGE}`, error); errorSpy.mockRestore(); }); - // a leading slash would come back as a protocol-relative Location such as - // '//example.com', sending the browser to another host - it.each(['/example.com', '//example.com', 'nested/image.png', '..', '.', ''])( - 'refuses %p before asking storage for a URL', - async (fileName) => { - const getImageUrlMock = jest.fn(); - const controller = await initController(jest.fn(), getImageUrlMock); - const res = responseMock(); - - await expect(controller.getUrlAndRedirect(fileName, res)).rejects.toThrow( - new BadRequestException('Invalid image name') - ); - expect(getImageUrlMock).not.toHaveBeenCalled(); - expect(res.redirect).not.toHaveBeenCalled(); - } - ); + it.each(ACCEPTED_NAMES)('accepts the generated name %s', async (fileName) => { + const controller = await initController(jest.fn(), jest.fn().mockResolvedValueOnce('https://storage/x')); + const res = responseMock(); + + await controller.getUrlAndRedirect(fileName, res); + + expect(res.redirect).toHaveBeenCalledWith('https://storage/x'); + }); + + it.each(REJECTED_NAMES)('refuses %p before asking storage for a URL', async (fileName) => { + const getImageUrlMock = jest.fn(); + const controller = await initController(jest.fn(), getImageUrlMock); + const res = responseMock(); + + await expect(controller.getUrlAndRedirect(fileName, res)).rejects.toThrow( + new BadRequestException('Invalid image name') + ); + expect(getImageUrlMock).not.toHaveBeenCalled(); + expect(res.redirect).not.toHaveBeenCalled(); + }); }); diff --git a/src/static/static.controller.ts b/src/static/static.controller.ts index 5f86d83f..59ab51dc 100644 --- a/src/static/static.controller.ts +++ b/src/static/static.controller.ts @@ -1,20 +1,15 @@ import { BadRequestException, Controller, Get, Logger, NotFoundException, Param, Res } from '@nestjs/common'; import { Response } from 'express'; import { ApiOkResponse, ApiTags } from '@nestjs/swagger'; -import { basename } from 'path'; +import { pipeline } from 'stream/promises'; import { StaticService } from './static.service'; -// Shorter than the window a signed URL is minted for (URL_WINDOW_SECONDS), so a -// cached redirect always still points at a URL that works. const REDIRECT_MAX_AGE_SECONDS = 3000; -// A stored image is always a plain file name, so anything carrying a path is -// rejected before it reaches storage — on the redirect route a name like -// '/example.com' would otherwise send the browser to another host. basename -// keeps '.' and '..' as they are, so both are named here rather than reaching -// a backend that would answer 500 for them, or ask storage for a directory. +export const IMAGE_NAME_PATTERN = /^[0-9A-Za-z_-]{1,64}\.(screenshot|diff|baseline)\.png$/; + const assertImageName = (fileName: string): void => { - if (!fileName || fileName === '.' || fileName === '..' || fileName !== basename(fileName)) { + if (!IMAGE_NAME_PATTERN.test(fileName)) { throw new BadRequestException('Invalid image name'); } }; @@ -23,6 +18,7 @@ const assertImageName = (fileName: string): void => { @Controller('images') export class StaticController { private readonly logger: Logger = new Logger(StaticController.name); + constructor(private staticService: StaticService) {} @Get('/:fileName') @@ -31,12 +27,6 @@ export class StaticController { assertImageName(fileName); try { const url = await this.staticService.getImageUrl(fileName); - // The redirect has to be reusable too, or the browser comes back here for - // a location on every single view — and a storage that signs its URLs - // hands back a different one each time, making the copy the browser - // already holds worthless. Kept well inside the signature's life so the - // redirect can never outlive what it points at. Private, because a signed - // URL is an access grant and has no business in a shared cache. res.set('Cache-Control', `private, max-age=${REDIRECT_MAX_AGE_SECONDS}`); res.redirect(url); } catch (error) { @@ -45,25 +35,24 @@ export class StaticController { } } - /** - * Serves the image bytes from this origin instead of redirecting to storage. - * Redirected pre-signed S3 URLs carry no CORS headers, so a browser `fetch` - * (used to build the bulk download zip) is blocked; an `` tag is not, - * which is why display keeps using the redirect above. - */ @Get('/:fileName/download') @ApiOkResponse() async download(@Param('fileName') fileName: string, @Res() res: Response) { assertImageName(fileName); - const imageBuffer = await this.staticService.getImageBuffer(fileName); - if (!imageBuffer) { + const stream = await this.staticService.getImageStream(fileName); + if (!stream) { throw new NotFoundException(`Image not found: ${fileName}`); } res.set({ 'Content-Type': 'image/png', 'Content-Disposition': `attachment; filename="${fileName}"`, + 'Cache-Control': 'private, max-age=31536000, immutable', }); - res.send(imageBuffer); + try { + await pipeline(stream, res); + } catch (error) { + this.logger.warn(`Download of ${fileName} stopped: ${error}`); + } } } diff --git a/src/static/static.interface.ts b/src/static/static.interface.ts index c7a38d41..492fb31f 100644 --- a/src/static/static.interface.ts +++ b/src/static/static.interface.ts @@ -1,9 +1,9 @@ -import { PNGWithMetadata } from 'pngjs'; +import { Readable } from 'stream'; export interface Static { saveImage(type: 'screenshot' | 'diff' | 'baseline', imageBuffer: Buffer): Promise; - getImage(fileName: string | null): Promise; getImageBuffer(fileName: string | null): Promise; + getImageStream(fileName: string): Promise; copyImage(type: 'screenshot' | 'diff' | 'baseline', sourceImageName: string): Promise; deleteImage(imageName: string | null): Promise; getImageUrl(imageName: string): Promise; diff --git a/src/static/static.service.spec.ts b/src/static/static.service.spec.ts index 5db2437d..e9943a9d 100644 --- a/src/static/static.service.spec.ts +++ b/src/static/static.service.spec.ts @@ -1,10 +1,13 @@ +import { BadRequestException } from '@nestjs/common'; import { Test, TestingModule } from '@nestjs/testing'; +import { Readable } from 'stream'; import { PNG } from 'pngjs'; import { StaticService } from './static.service'; import { ConfigService } from '@nestjs/config'; import { StaticModule } from './static.module'; import { StaticFactoryService } from './static.factory'; import { Static } from './static.interface'; +import { ImageTooLargeException } from './utils'; describe('StaticService', () => { let service: StaticService; @@ -35,8 +38,8 @@ describe('StaticService', () => { beforeEach(() => { backend = { saveImage: jest.fn().mockResolvedValue('saved.png'), - getImage: jest.fn().mockResolvedValue({ width: 1 }), getImageBuffer: jest.fn().mockResolvedValue(Buffer.from('bytes')), + getImageStream: jest.fn().mockResolvedValue(Readable.from([])), copyImage: jest.fn().mockResolvedValue('copied.png'), deleteImage: jest.fn().mockResolvedValue(true), getImageUrl: jest.fn().mockResolvedValue('https://storage/image.png'), @@ -65,38 +68,92 @@ describe('StaticService', () => { await expect(service.saveImage('diff', signatureOnly)).resolves.toBe('saved.png'); }); + describe('image size limit', () => { + const originalPixels = process.env.MAX_IMAGE_PIXELS; + + afterEach(() => { + if (originalPixels === undefined) { + delete process.env.MAX_IMAGE_PIXELS; + } else { + process.env.MAX_IMAGE_PIXELS = originalPixels; + } + }); + + it('rejects a screenshot above MAX_IMAGE_PIXELS with 422 before reaching the backend', async () => { + process.env.MAX_IMAGE_PIXELS = '5'; + const png = PNG.sync.write(new PNG({ width: 3, height: 2 })); + + await expect(service.saveImage('screenshot', png)).rejects.toBeInstanceOf(ImageTooLargeException); + expect(backend.saveImage).not.toHaveBeenCalled(); + }); + + it('stores a baseline or a diff of any size', async () => { + process.env.MAX_IMAGE_PIXELS = '5'; + const png = PNG.sync.write(new PNG({ width: 3, height: 2 })); + + await expect(service.saveImage('baseline', png)).resolves.toBe('saved.png'); + await expect(service.saveImage('diff', png)).resolves.toBe('saved.png'); + }); + }); + it.each([ ['an empty buffer', Buffer.alloc(0)], ['a truncated signature', PNG_SIGNATURE.subarray(0, 7)], ['a JPEG', Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0, 0, 0, 0, 0])], ])('rejects %s without reaching the backend', async (_label, buffer) => { - await expect(service.saveImage('baseline', buffer)).rejects.toThrow('Cannot parse image as PNG file'); + const result = service.saveImage('baseline', buffer); + + await expect(result).rejects.toBeInstanceOf(BadRequestException); + await expect(result).rejects.toThrow('Cannot parse image as PNG file'); expect(backend.saveImage).not.toHaveBeenCalled(); }); - it('delegates getImage', async () => { - await expect(service.getImage('a.png')).resolves.toEqual({ width: 1 }); - expect(backend.getImage).toHaveBeenCalledWith('a.png'); + it('delegates getImageStream', async () => { + const stream = Readable.from([]); + backend.getImageStream.mockResolvedValue(stream); + + await expect(service.getImageStream('a.png')).resolves.toBe(stream); + expect(backend.getImageStream).toHaveBeenCalledWith('a.png'); }); - it('delegates getImageBuffer', async () => { - await expect(service.getImageBuffer('a.png')).resolves.toEqual(Buffer.from('bytes')); - expect(backend.getImageBuffer).toHaveBeenCalledWith('a.png'); + it.each([ + ['getImageBuffer', ['a.png'], Buffer.from('bytes')], + ['copyImage', ['baseline', 'a.png'], 'copied.png'], + ['deleteImage', ['a.png'], true], + ['getImageUrl', ['a.png'], 'https://storage/image.png'], + ] as const)('delegates %s', async (method, args, result) => { + await expect((service[method] as (...a: readonly string[]) => Promise)(...args)).resolves.toEqual( + result + ); + expect(backend[method]).toHaveBeenCalledWith(...args); }); - it('delegates copyImage', async () => { - await expect(service.copyImage('baseline', 'a.png')).resolves.toBe('copied.png'); - expect(backend.copyImage).toHaveBeenCalledWith('baseline', 'a.png'); + it('deleteImages skips empty names and deletes the rest', async () => { + await service.deleteImages(['a.png', null, '', undefined, 'b.png']); + + expect(backend.deleteImage.mock.calls).toEqual([['a.png'], ['b.png']]); }); - it('delegates deleteImage', async () => { - await expect(service.deleteImage('a.png')).resolves.toBe(true); - expect(backend.deleteImage).toHaveBeenCalledWith('a.png'); + it('deleteImages accepts any iterable and caps the concurrency', async () => { + let running = 0; + let peak = 0; + backend.deleteImage.mockImplementation(async () => { + peak = Math.max(peak, ++running); + await new Promise((resolve) => setImmediate(resolve)); + running--; + return true; + }); + + await service.deleteImages(new Set(['1', '2', '3', '4', '5']), 2); + + expect(backend.deleteImage).toHaveBeenCalledTimes(5); + expect(peak).toBe(2); }); - it('delegates getImageUrl', async () => { - await expect(service.getImageUrl('a.png')).resolves.toBe('https://storage/image.png'); - expect(backend.getImageUrl).toHaveBeenCalledWith('a.png'); + it('deleteImages rejects when a delete fails', async () => { + backend.deleteImage.mockRejectedValue(new Error('disk gone')); + + await expect(service.deleteImages(['a.png'])).rejects.toThrow('disk gone'); }); it('passes backend failures through', async () => { diff --git a/src/static/static.service.ts b/src/static/static.service.ts index ce5e40ad..5eae0a5d 100644 --- a/src/static/static.service.ts +++ b/src/static/static.service.ts @@ -1,12 +1,12 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { PNGWithMetadata } from 'pngjs'; +import { BadRequestException, Injectable } from '@nestjs/common'; +import { Readable } from 'stream'; import { StaticFactoryService } from './static.factory'; -import { isPngBuffer } from './utils'; +import { assertPngWithinPixelLimit, isPngBuffer } from './utils'; import { Static } from './static.interface'; +import { mapWithConcurrency } from '../utils/map-with-concurrency'; @Injectable() export class StaticService { - private readonly logger: Logger = new Logger(StaticService.name); private readonly staticService: Static; constructor(private staticFactoryService: StaticFactoryService) { @@ -14,21 +14,23 @@ export class StaticService { } async saveImage(type: 'screenshot' | 'diff' | 'baseline', imageBuffer: Buffer): Promise { - // validate here so every storage backend rejects non-PNG uploads if (!isPngBuffer(imageBuffer)) { - throw new Error('Cannot parse image as PNG file'); + throw new BadRequestException('Cannot parse image as PNG file'); + } + if (type === 'screenshot') { + assertPngWithinPixelLimit(imageBuffer); } return this.staticService.saveImage(type, imageBuffer); } - async getImage(imageName: string | null): Promise { - return this.staticService.getImage(imageName); - } - async getImageBuffer(imageName: string | null): Promise { return this.staticService.getImageBuffer(imageName); } + async getImageStream(imageName: string): Promise { + return this.staticService.getImageStream(imageName); + } + async copyImage(type: 'screenshot' | 'diff' | 'baseline', sourceImageName: string): Promise { return this.staticService.copyImage(type, sourceImageName); } @@ -37,6 +39,11 @@ export class StaticService { return this.staticService.deleteImage(imageName); } + async deleteImages(names: Iterable, concurrency = 16): Promise { + const present = [...names].filter((name): name is string => !!name); + await mapWithConcurrency(present, concurrency, (name) => this.deleteImage(name)); + } + async getImageUrl(imageName: string): Promise { return this.staticService.getImageUrl(imageName); } diff --git a/src/static/utils.spec.ts b/src/static/utils.spec.ts index cbdeae63..0e998674 100644 --- a/src/static/utils.spec.ts +++ b/src/static/utils.spec.ts @@ -1,4 +1,21 @@ -import { generateNewImageName, isHddStaticServiceConfigured, isPngBuffer, isS3ServiceConfigured } from './utils'; +import { readFileSync } from 'fs'; +import { join } from 'path'; +import { crc32 } from 'zlib'; +import { PNG } from 'pngjs'; +import { + asBuffer, + assertPngWithinPixelLimit, + assertWithinPixelLimit, + DEFAULT_MAX_IMAGE_PIXELS, + generateNewImageName, + ImageTooLargeException, + isByteEqualWellFormedPng, + isHddStaticServiceConfigured, + isPngBuffer, + isWellFormedPng, + maxImagePixels, + readPngDimensions, +} from './utils'; describe('static utils', () => { const originalStaticService = process.env.STATIC_SERVICE; @@ -12,17 +29,14 @@ describe('static utils', () => { }); describe('storage detection from STATIC_SERVICE', () => { - // NOTE: current behavior — with STATIC_SERVICE unset both helpers answer - // true, and for an unknown value both answer false, while - // StaticFactoryService falls back to HDD in both cases. - it.each<[string | undefined, boolean, boolean]>([ - [undefined, true, true], - ['', true, true], - ['hdd', true, false], - ['s3', false, true], - ['S3', false, false], - ['gcs', false, false], - ])('STATIC_SERVICE=%p: hdd %p, s3 %p', (value, hdd, s3) => { + it.each<[string | undefined, boolean]>([ + [undefined, true], + ['', true], + ['hdd', true], + ['s3', false], + ['S3', false], + ['gcs', false], + ])('STATIC_SERVICE=%p: hdd %p', (value, hdd) => { if (value === undefined) { delete process.env.STATIC_SERVICE; } else { @@ -30,7 +44,6 @@ describe('static utils', () => { } expect(isHddStaticServiceConfigured()).toBe(hdd); - expect(isS3ServiceConfigured()).toBe(s3); }); }); @@ -52,18 +65,12 @@ describe('static utils', () => { describe('isPngBuffer', () => { const signature = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]; - it('accepts a buffer that is exactly the PNG signature', () => { - expect(isPngBuffer(Buffer.from(signature))).toBe(true); - }); - - it('accepts a buffer that starts with the PNG signature', () => { - expect(isPngBuffer(Buffer.from([...signature, 1, 2, 3]))).toBe(true); - }); - - it('accepts a signature found at the start of a buffer view', () => { - const backing = Buffer.from([0, 0, ...signature, 9]); - - expect(isPngBuffer(backing.subarray(2))).toBe(true); + it.each([ + ['a buffer that is exactly the PNG signature', Buffer.from(signature)], + ['a buffer that starts with the PNG signature', Buffer.from([...signature, 1, 2, 3])], + ['a signature found at the start of a buffer view', Buffer.from([0, 0, ...signature, 9]).subarray(2)], + ])('accepts %s', (_label, buffer) => { + expect(isPngBuffer(buffer)).toBe(true); }); it.each([ @@ -75,4 +82,242 @@ describe('static utils', () => { expect(isPngBuffer(Buffer.from(bytes))).toBe(false); }); }); + + describe('PNG header helpers', () => { + const tiny = () => PNG.sync.write(new PNG({ width: 3, height: 2 })); + const viewOf = (png: Buffer) => { + const view = new Uint8Array(png.length + 4); + view.set(png, 4); + return view.subarray(4); + }; + const real = () => readFileSync(join(__dirname, '../../test/image.png')); + const originalPixels = process.env.MAX_IMAGE_PIXELS; + + afterEach(() => { + if (originalPixels === undefined) { + delete process.env.MAX_IMAGE_PIXELS; + } else { + process.env.MAX_IMAGE_PIXELS = originalPixels; + } + }); + + const refreshCrc = (b: Buffer, offset: number) => { + const length = b.readUInt32BE(offset); + b.writeUInt32BE(crc32(b.subarray(offset + 4, offset + 8 + length)), offset + 8 + length); + }; + + const chunk = (type: string, data: Buffer) => { + const out = Buffer.alloc(12 + data.length); + out.writeUInt32BE(data.length, 0); + out.write(type, 4, 'latin1'); + data.copy(out, 8); + out.writeUInt32BE(crc32(out.subarray(4, 8 + data.length)), 8 + data.length); + return out; + }; + + describe('readPngDimensions', () => { + it('reads the size from IHDR', () => { + expect(readPngDimensions(tiny())).toStrictEqual({ width: 3, height: 2 }); + }); + + it('accepts a Uint8Array view, as it arrives after postMessage', () => { + expect(readPngDimensions(viewOf(tiny()))).toStrictEqual({ width: 3, height: 2 }); + }); + + it.each([ + ['a buffer that is only a signature-less string', () => Buffer.from('png')], + ['a buffer shorter than a header', () => tiny().subarray(0, 32)], + ['a JPEG signature', () => Buffer.concat([Buffer.from([0xff, 0xd8, 0xff, 0xe0]), Buffer.alloc(40)])], + [ + 'a first chunk that is not a 13-byte IHDR', + () => { + const png = tiny(); + png.write('IHDX', 12, 'latin1'); + return png; + }, + ], + [ + 'a zero width', + () => { + const png = tiny(); + png.writeUInt32BE(0, 16); + return png; + }, + ], + ])('returns null for %s', (_label, make) => { + expect(readPngDimensions(make())).toBeNull(); + }); + }); + + describe('asBuffer', () => { + it('returns a Buffer as is', () => { + const buffer = Buffer.from([1, 2, 3]); + + expect(asBuffer(buffer)).toBe(buffer); + }); + + it('wraps a Uint8Array view without copying or widening it', () => { + const backing = new Uint8Array([9, 1, 2, 3, 9]); + const view = backing.subarray(1, 4); + + const buffer = asBuffer(view); + + expect(Buffer.isBuffer(buffer)).toBe(true); + expect([...buffer]).toEqual([1, 2, 3]); + expect(buffer.buffer).toBe(backing.buffer); + }); + }); + + describe('isByteEqualWellFormedPng', () => { + it('accepts two identical well-formed PNGs, Buffer or Uint8Array', () => { + expect(isByteEqualWellFormedPng(tiny(), tiny())).toBe(true); + expect(isByteEqualWellFormedPng(new Uint8Array(tiny()), tiny())).toBe(true); + }); + + it('rejects PNGs that differ', () => { + const other = tiny(); + other[other.length - 20] ^= 0xff; + + expect(isByteEqualWellFormedPng(tiny(), other)).toBe(false); + }); + + it('rejects an identical pair that is not a well-formed PNG', () => { + const truncated = tiny().subarray(0, tiny().length - 5); + + expect(isByteEqualWellFormedPng(truncated, Buffer.from(truncated))).toBe(false); + }); + }); + + describe('isWellFormedPng', () => { + it.each([ + ['a generated PNG', tiny], + ['a real screenshot', real], + ])('accepts %s', (_label, make) => { + expect(isWellFormedPng(make())).toBe(true); + }); + + it('accepts a Uint8Array view', () => { + expect(isWellFormedPng(viewOf(tiny()))).toBe(true); + }); + + const withChunk = (type: string, tail = true) => { + const png = tiny(); + return Buffer.concat([ + png.subarray(0, 33), + chunk(type, Buffer.from([1, 2])), + ...(tail ? [png.subarray(33)] : []), + ]); + }; + const withHeaderByte = (position: number, value: number) => () => { + const png = tiny(); + png[position] = value; + refreshCrc(png, 8); + return png; + }; + + it.each([ + ['a file cut short by 5 bytes', () => tiny().subarray(0, tiny().length - 5)], + ['a file that ends before IEND is complete, on a chunk boundary', () => tiny().subarray(0, tiny().length - 12)], + [ + 'a flipped bit in the middle (CRC mismatch)', + () => { + const png = tiny(); + png[Math.floor(png.length / 2)] ^= 0x01; + return png; + }, + ], + ['an extra byte after IEND', () => Buffer.concat([tiny(), Buffer.from([0])])], + ['a string', () => Buffer.from('png')], + ['a bit depth PNG does not define', withHeaderByte(24, 3)], + ['a color type PNG does not define', withHeaderByte(25, 5)], + ['an unknown compression method', withHeaderByte(26, 1)], + ['an unknown filter method', withHeaderByte(27, 1)], + ['an unknown interlace method', withHeaderByte(28, 2)], + ['an unknown critical chunk', () => withChunk('ABCD')], + ['a chunk type that is not four letters', () => withChunk('ab1D')], + [ + 'a chunk that claims more data than the buffer holds', + () => { + const png = tiny(); + png.writeUInt32BE(0x7fffffff, 33); + return png; + }, + ], + ['a file without IEND', () => withChunk('abCD', false)], + ])('rejects %s', (_label, make) => { + expect(isWellFormedPng(make())).toBe(false); + }); + + it('accepts an unknown ancillary chunk with a valid CRC', () => { + expect(isWellFormedPng(withChunk('abCD'))).toBe(true); + }); + }); + + describe('maxImagePixels', () => { + it('defaults to 100 million pixels', () => { + delete process.env.MAX_IMAGE_PIXELS; + + expect(maxImagePixels()).toBe(DEFAULT_MAX_IMAGE_PIXELS); + expect(DEFAULT_MAX_IMAGE_PIXELS).toBe(100_000_000); + }); + + it.each([ + ['0', 0], + ['2500', 2500], + ['abc', DEFAULT_MAX_IMAGE_PIXELS], + ['-5', DEFAULT_MAX_IMAGE_PIXELS], + [' ', DEFAULT_MAX_IMAGE_PIXELS], + ['', DEFAULT_MAX_IMAGE_PIXELS], + ])('MAX_IMAGE_PIXELS=%p gives %p', (value, expected) => { + process.env.MAX_IMAGE_PIXELS = value; + + expect(maxImagePixels()).toBe(expected); + }); + }); + + describe('assertPngWithinPixelLimit', () => { + it('rejects a PNG above the limit with 422 and names its size', () => { + process.env.MAX_IMAGE_PIXELS = '5'; + + let error: unknown; + try { + assertPngWithinPixelLimit(tiny()); + } catch (e) { + error = e; + } + + expect(error).toBeInstanceOf(ImageTooLargeException); + expect((error as ImageTooLargeException).getStatus()).toBe(422); + expect((error as ImageTooLargeException).message).toContain('3x2'); + }); + + it('lets a PNG exactly at the limit through', () => { + process.env.MAX_IMAGE_PIXELS = '6'; + + expect(() => assertPngWithinPixelLimit(tiny())).not.toThrow(); + }); + + it('lets anything through when the limit is 0', () => { + process.env.MAX_IMAGE_PIXELS = '0'; + + expect(() => assertPngWithinPixelLimit(tiny())).not.toThrow(); + expect(() => assertWithinPixelLimit(100_000, 100_000)).not.toThrow(); + }); + + it('leaves a malformed header to the decoder', () => { + process.env.MAX_IMAGE_PIXELS = '1'; + + expect(() => assertPngWithinPixelLimit(Buffer.from('png'))).not.toThrow(); + }); + + it('rejects a tiny file whose header claims a huge canvas', () => { + delete process.env.MAX_IMAGE_PIXELS; + const png = tiny(); + png.writeUInt32BE(30000, 16); + png.writeUInt32BE(30000, 20); + + expect(() => assertPngWithinPixelLimit(png)).toThrow(ImageTooLargeException); + }); + }); + }); }); diff --git a/src/static/utils.ts b/src/static/utils.ts index 259f7954..f0d66cd3 100644 --- a/src/static/utils.ts +++ b/src/static/utils.ts @@ -1,23 +1,85 @@ +import { UnprocessableEntityException } from '@nestjs/common'; +import { crc32 } from 'zlib'; import { generateRandomKey } from '../utils/random-key'; +import { nonNegativeNumber } from '../utils/env'; export function isHddStaticServiceConfigured() { return !process.env.STATIC_SERVICE || process.env.STATIC_SERVICE === 'hdd'; } -export function isS3ServiceConfigured() { - return !process.env.STATIC_SERVICE || process.env.STATIC_SERVICE === 's3'; -} - export function generateNewImageName(type: 'screenshot' | 'diff' | 'baseline'): string { return `${generateRandomKey()}.${type}.png`; } const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); -// Signature check instead of a full decode: parsing megapixel PNGs just to -// validate them blocks the event loop on every upload. +// Signature only: fully decoding megapixel PNGs blocks the event loop on every upload export function isPngBuffer(imageBuffer: Buffer): boolean { return ( imageBuffer.length >= PNG_SIGNATURE.length && imageBuffer.subarray(0, PNG_SIGNATURE.length).equals(PNG_SIGNATURE) ); } + +export const DEFAULT_MAX_IMAGE_PIXELS = 100_000_000; + +export function maxImagePixels(): number { + return nonNegativeNumber(process.env.MAX_IMAGE_PIXELS, DEFAULT_MAX_IMAGE_PIXELS); +} + +export function asBuffer(bytes: Buffer | Uint8Array): Buffer { + return Buffer.isBuffer(bytes) ? bytes : Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength); +} + +export function readPngDimensions(buffer: Buffer | Uint8Array): { width: number; height: number } | null { + const b = asBuffer(buffer); + if (!isPngBuffer(b) || b.length < 33) return null; + if (b.readUInt32BE(8) !== 13 || b.toString('latin1', 12, 16) !== 'IHDR') return null; + const width = b.readUInt32BE(16); + const height = b.readUInt32BE(20); + return width > 0 && height > 0 ? { width, height } : null; +} + +const CRITICAL_CHUNKS = new Set(['IHDR', 'PLTE', 'IDAT', 'IEND']); + +export function isWellFormedPng(buffer: Buffer | Uint8Array): boolean { + const b = asBuffer(buffer); + if (!readPngDimensions(b)) return false; + const [bitDepth, colorType, compression, filter, interlace] = [b[24], b[25], b[26], b[27], b[28]]; + if (![1, 2, 4, 8, 16].includes(bitDepth) || ![0, 2, 3, 4, 6].includes(colorType)) return false; + if (compression !== 0 || filter !== 0 || interlace > 1) return false; + let offset = 8; + while (offset + 12 <= b.length) { + const length = b.readUInt32BE(offset); + const end = offset + 12 + length; + if (end > b.length) return false; + const type = b.toString('latin1', offset + 4, offset + 8); + if (!/^[A-Za-z]{4}$/.test(type)) return false; + if (type[0] === type[0].toUpperCase() && !CRITICAL_CHUNKS.has(type)) return false; + if (crc32(b.subarray(offset + 4, offset + 8 + length)) !== b.readUInt32BE(offset + 8 + length)) return false; + if (type === 'IEND') return end === b.length; + offset = end; + } + return false; +} + +export function isByteEqualWellFormedPng(a: Buffer | Uint8Array, b: Buffer | Uint8Array): boolean { + return asBuffer(a).equals(asBuffer(b)) && isWellFormedPng(a); +} + +export class ImageTooLargeException extends UnprocessableEntityException { + constructor(width: number, height: number, limit: number) { + super(`Image is ${width}x${height} (${width * height} pixels), above MAX_IMAGE_PIXELS=${limit}`); + } +} + +export function assertWithinPixelLimit(width: number, height: number): void { + const limit = maxImagePixels(); + if (limit > 0 && width * height > limit) { + throw new ImageTooLargeException(width, height, limit); + } +} + +export function assertPngWithinPixelLimit(buffer: Buffer | Uint8Array): void { + const size = readPngDimensions(buffer); + if (size) assertWithinPixelLimit(size.width, size.height); +} diff --git a/src/test-runs/diffResult.ts b/src/test-runs/diffResult.ts index 168fc9bb..f6617916 100644 --- a/src/test-runs/diffResult.ts +++ b/src/test-runs/diffResult.ts @@ -1,44 +1,19 @@ import { TestStatus } from '@prisma/client'; -/** - * A signature and the comparison settings it was computed under. A signature - * only means anything under its own threshold, so comparing one taken at a - * different setting quietly makes grouping worse — the settings travel with it - * so a stored one can be discarded when the project's config moves on. - */ export interface StampedSignature { threshold: number; includeAA: boolean; signature: number[]; } -// Without a baseline nothing was compared, and only diffName is set (to null). export interface DiffResult { status: TestStatus | undefined; diffName: string | null; pixelMisMatchCount: number | undefined; diffPercent: number | undefined; isSameDimension: boolean | undefined; - /** - * Optional array of analysis insights (e.g., from VLM or other AI services) - * Each string represents a distinct observation or difference - * Can be displayed as bullet points in UI - */ vlmDescription?: string; - /** - * Position-independent colour signature of the change, produced by the same - * pass that produced the diff, carrying the settings it was produced under. - * Stored on the run so the variations dialog can group a screen's locales - * without decoding their screenshots all over again. Absent when nothing - * changed, when the dimensions differ, or when the comparison was not - * pixelmatch. - */ changeSignature?: StampedSignature; - /** - * Names the small copies of the checkpoint and the diff were saved under, for - * the card grids to draw instead of the full-size files. Absent when the - * comparison produced no diff, or was not pixelmatch. - */ imageThumbnailName?: string; diffThumbnailName?: string; } diff --git a/src/test-runs/dto/create-test-request-base64.dto.ts b/src/test-runs/dto/create-test-request-base64.dto.ts index a604dc99..c4b4f210 100644 --- a/src/test-runs/dto/create-test-request-base64.dto.ts +++ b/src/test-runs/dto/create-test-request-base64.dto.ts @@ -1,11 +1,9 @@ import { ApiProperty } from '@nestjs/swagger'; -import { Transform } from 'class-transformer'; -import { IsBase64 } from 'class-validator'; +import { IsString } from 'class-validator'; import { CreateTestRequestDto } from './create-test-request.dto'; export class CreateTestRequestBase64Dto extends CreateTestRequestDto { @ApiProperty() - @Transform(({ value }) => value.replace(/(\r\n|\n|\r)/gm, '')) - @IsBase64() + @IsString() imageBase64!: string; } diff --git a/src/test-runs/dto/create-test-request-multipart.dto.ts b/src/test-runs/dto/create-test-request-multipart.dto.ts index 19322f10..96b1d45a 100644 --- a/src/test-runs/dto/create-test-request-multipart.dto.ts +++ b/src/test-runs/dto/create-test-request-multipart.dto.ts @@ -1,33 +1,54 @@ import { ApiPropertyOptional, OmitType } from '@nestjs/swagger'; import { ApiFile } from '../../shared/api-file.decorator'; import { CreateTestRequestDto } from './create-test-request.dto'; -import { Transform, Type } from 'class-transformer'; +import { plainToInstance, Transform } from 'class-transformer'; import { IgnoreAreaDto } from './ignore-area.dto'; -import { IsBoolean, IsNumber, IsOptional } from 'class-validator'; +import { ValidationOptions } from '../../shared/validation'; +import { IsArray, IsBoolean, IsDefined, IsNumber, IsObject, IsOptional, ValidateNested } from 'class-validator'; +const parseIgnoreAreas = (value: unknown): unknown => { + if (!value) { + return []; + } + if (typeof value === 'string') { + try { + value = JSON.parse(value); + } catch { + return value; + } + } + return Array.isArray(value) + ? value.map((item) => (item !== null && typeof item === 'object' ? plainToInstance(IgnoreAreaDto, item) : item)) + : value; +}; + +@ValidationOptions({ transform: true, forbidUnknownValues: true }) export class CreateTestRequestMultipartDto extends OmitType(CreateTestRequestDto, [ 'ignoreAreas', 'merge', 'diffTollerancePercent', ]) { @ApiFile() + @IsDefined() + @IsObject() image!: Express.Multer.File; @ApiPropertyOptional() @IsOptional() @IsNumber() - @Transform(({ value }) => parseFloat(value) || 0) + @Transform(({ value }) => parseFloat(value) || 0, { toClassOnly: true }) diffTollerancePercent?: number; @ApiPropertyOptional() @IsOptional() @IsBoolean() - @Transform(({ value }) => value == 'true') + @Transform(({ value }) => value === true || value === 'true', { toClassOnly: true }) merge?: boolean; @ApiPropertyOptional({ type: String }) @IsOptional() - @Type(() => String) - @Transform(({ value }) => (value ? JSON.parse(value) : [])) + @IsArray() + @ValidateNested({ each: true }) + @Transform(({ value }) => parseIgnoreAreas(value), { toClassOnly: true }) ignoreAreas?: IgnoreAreaDto[]; } diff --git a/src/test-runs/dto/create-test-request.dto.spec.ts b/src/test-runs/dto/create-test-request.dto.spec.ts index e2276049..d880164d 100644 --- a/src/test-runs/dto/create-test-request.dto.spec.ts +++ b/src/test-runs/dto/create-test-request.dto.spec.ts @@ -1,4 +1,5 @@ import { plainToInstance } from 'class-transformer'; +import { GlobalValidationPipe } from '../../shared/validation'; import { validate } from 'class-validator'; import { CreateTestRequestDto } from './create-test-request.dto'; import { CreateTestRequestBase64Dto } from './create-test-request-base64.dto'; @@ -9,23 +10,23 @@ const BUILD_ID = '146e7a8d-89f0-4565-aa2c-e61efabb0afd'; const PROJECT_ID = '3bc4a5bc-006e-4d43-8e4e-eaa132627fcc'; const area = { x: 1, y: 2, width: 3, height: 4 }; +const base = { + buildId: BUILD_ID, + projectId: PROJECT_ID, + name: 'name', + branchName: 'develop', +}; + const failedProperties = async (dto: object): Promise => (await validate(dto)).map((error) => error.property).sort(); describe('CreateTestRequestDto', () => { - const valid = { - buildId: BUILD_ID, - projectId: PROJECT_ID, - name: 'name', - branchName: 'develop', - }; - it('accepts the minimal request', async () => { - expect(await failedProperties(plainToInstance(CreateTestRequestDto, valid))).toEqual([]); + expect(await failedProperties(plainToInstance(CreateTestRequestDto, base))).toEqual([]); }); it('turns ignore areas into IgnoreAreaDto instances', () => { - const dto = plainToInstance(CreateTestRequestDto, { ...valid, ignoreAreas: [area] }); + const dto = plainToInstance(CreateTestRequestDto, { ...base, ignoreAreas: [area] }); expect(dto.ignoreAreas![0]).toBeInstanceOf(IgnoreAreaDto); expect(dto.ignoreAreas![0]).toEqual(area); @@ -33,7 +34,7 @@ describe('CreateTestRequestDto', () => { it('rejects ids that are not uuids, and malformed optional fields', async () => { const dto = plainToInstance(CreateTestRequestDto, { - ...valid, + ...base, buildId: 'not a uuid', projectId: 'not a uuid', diffTollerancePercent: '1', @@ -44,48 +45,43 @@ describe('CreateTestRequestDto', () => { expect(await failedProperties(dto)).toEqual(['buildId', 'comment', 'diffTollerancePercent', 'merge', 'projectId']); }); - it('validates every ignore area', async () => { - const dto = plainToInstance(CreateTestRequestDto, { ...valid, ignoreAreas: [area, { ...area, x: 'one' }] }); + it('baseates every ignore area', async () => { + const dto = plainToInstance(CreateTestRequestDto, { ...base, ignoreAreas: [area, { ...area, x: 'one' }] }); expect(await failedProperties(dto)).toEqual(['ignoreAreas']); }); it('rejects ignore areas that are not an array', async () => { - const dto = plainToInstance(CreateTestRequestDto, { ...valid, ignoreAreas: area }); + const dto = plainToInstance(CreateTestRequestDto, { ...base, ignoreAreas: area }); expect(await failedProperties(dto)).toContain('ignoreAreas'); }); }); describe('CreateTestRequestBase64Dto', () => { - const valid = { - buildId: BUILD_ID, - projectId: PROJECT_ID, - name: 'name', - branchName: 'develop', - }; + it('keeps the image as sent: Buffer.from ignores line breaks', async () => { + const imageBase64 = 'aGVs\r\nbG8g\nd29y\rbGQ='; + const dto = plainToInstance(CreateTestRequestBase64Dto, { ...base, imageBase64 }); + + expect(dto.imageBase64).toBe(imageBase64); + expect(await failedProperties(dto)).toEqual([]); + }); - it('strips line breaks from the image before validating it', async () => { - const dto = plainToInstance(CreateTestRequestBase64Dto, { ...valid, imageBase64: 'aGVs\r\nbG8g\nd29y\rbGQ=' }); + it('accepts any string; the controller rejects what is not a PNG', async () => { + const dto = plainToInstance(CreateTestRequestBase64Dto, { ...base, imageBase64: 'not base64!' }); - expect(dto.imageBase64).toBe('aGVsbG8gd29ybGQ='); expect(await failedProperties(dto)).toEqual([]); }); - it('rejects an image that is not base64', async () => { - const dto = plainToInstance(CreateTestRequestBase64Dto, { ...valid, imageBase64: 'not base64!' }); + it.each([[null], [5]])('rejects a missing, null or numeric image (%p) without throwing', async (imageBase64) => { + const dto = plainToInstance(CreateTestRequestBase64Dto, { ...base, imageBase64 }); expect(await failedProperties(dto)).toEqual(['imageBase64']); }); }); describe('CreateTestRequestMultipartDto', () => { - const valid = { - buildId: BUILD_ID, - projectId: PROJECT_ID, - name: 'name', - branchName: 'develop', - }; + const valid = { ...base, image: { fieldname: 'image', buffer: Buffer.from('x') } }; it.each([ ['1.5', 1.5], @@ -103,26 +99,27 @@ describe('CreateTestRequestMultipartDto', () => { ['false', false], ['TRUE', false], ['1', false], + [true, true], ])('reads merge %p from the form as %p', (value, expected) => { const dto = plainToInstance(CreateTestRequestMultipartDto, { ...valid, merge: value }); expect(dto.merge).toBe(expected); }); - it('reads ignore areas from a JSON string', () => { - const dto = plainToInstance(CreateTestRequestMultipartDto, { ...valid, ignoreAreas: JSON.stringify([area]) }); + it.each([ + ['a JSON string', JSON.stringify([area]), [area]], + ['an empty string as an empty list', '', []], + ])('reads ignore areas from %s', (_case, ignoreAreas, expected) => { + const dto = plainToInstance(CreateTestRequestMultipartDto, { ...valid, ignoreAreas }); - expect(dto.ignoreAreas).toEqual([area]); + expect(dto.ignoreAreas).toEqual(expected); }); - it('reads empty ignore areas as an empty list', () => { - const dto = plainToInstance(CreateTestRequestMultipartDto, { ...valid, ignoreAreas: '' }); - - expect(dto.ignoreAreas).toEqual([]); - }); + it('keeps ignore areas that are not JSON as text, so validation rejects them', async () => { + const dto = plainToInstance(CreateTestRequestMultipartDto, { ...valid, ignoreAreas: '[{' }); - it('throws on ignore areas that are not JSON', () => { - expect(() => plainToInstance(CreateTestRequestMultipartDto, { ...valid, ignoreAreas: '[{' })).toThrow(SyntaxError); + expect(dto.ignoreAreas).toBe('[{'); + expect(await failedProperties(dto)).toContain('ignoreAreas'); }); it('leaves absent form fields undefined', () => { @@ -135,13 +132,6 @@ describe('CreateTestRequestMultipartDto', () => { }).toEqual({ diffTollerancePercent: undefined, merge: undefined, ignoreAreas: undefined }); }); - it('reads a JSON boolean merge as false', () => { - const dto = plainToInstance(CreateTestRequestMultipartDto, { ...valid, merge: true }); - - // NOTE: current behavior — only the string 'true' counts, `true == 'true'` is false - expect(dto.merge).toBe(false); - }); - it('accepts a request with the form fields converted', async () => { const dto = plainToInstance(CreateTestRequestMultipartDto, { ...valid, @@ -152,4 +142,69 @@ describe('CreateTestRequestMultipartDto', () => { expect(await failedProperties(dto)).toEqual([]); }); + + it('rejects a request without the image', async () => { + const { image: _image, ...withoutImage } = valid; + + expect(await failedProperties(plainToInstance(CreateTestRequestMultipartDto, withoutImage))).toEqual(['image']); + }); + + it('rejects a text field named image', async () => { + const dto = plainToInstance(CreateTestRequestMultipartDto, { ...valid, image: 'text' }); + + expect(await failedProperties(dto)).toEqual(['image']); + }); + + it.each([ + ['a string coordinate', '[{"x":"1","y":1,"width":1,"height":1}]'], + ['a coordinate beyond the bounds', '[{"x":-2e6,"y":1,"width":1,"height":1}]'], + ['an object instead of an array', '{"x":1,"y":1,"width":1,"height":1}'], + ])('validates every parsed ignore area: rejects %s', async (_, ignoreAreas) => { + const dto = plainToInstance(CreateTestRequestMultipartDto, { ...valid, ignoreAreas }); + + expect(await failedProperties(dto)).toEqual(['ignoreAreas']); + }); + + it('validates every parsed ignore area: accepts fractional ones as IgnoreAreaDto', async () => { + const dto = plainToInstance(CreateTestRequestMultipartDto, { + ...valid, + ignoreAreas: '[{"x":1.5,"y":1,"width":1,"height":1}]', + }); + + expect(await failedProperties(dto)).toEqual([]); + expect(dto.ignoreAreas![0]).toBeInstanceOf(IgnoreAreaDto); + }); + + it('takes already parsed ignore areas as they are', () => { + const dto = plainToInstance(CreateTestRequestMultipartDto, { ...valid, ignoreAreas: [area] }); + + expect(dto.ignoreAreas).toEqual([area]); + expect(dto.ignoreAreas![0]).toBeInstanceOf(IgnoreAreaDto); + }); + + it('keeps the id and every other key of an area', () => { + const withExtras = { id: '1', ...area, extra: 2 }; + const dto = plainToInstance(CreateTestRequestMultipartDto, { ...valid, ignoreAreas: JSON.stringify([withExtras]) }); + + expect(dto.ignoreAreas).toEqual([withExtras]); + }); + + it('is parsed once by the global pipe, with the form values converted', async () => { + const meta = { type: 'body' as const, metatype: CreateTestRequestMultipartDto, data: '' }; + const body = { + ...valid, + merge: 'true', + diffTollerancePercent: '1.5', + ignoreAreas: '[{"id":"7","x":1.5,"y":2,"width":3,"height":4}]', + image: { fieldname: 'image', buffer: Buffer.from('png') }, + }; + + const result = (await new GlobalValidationPipe().transform(body, meta)) as CreateTestRequestMultipartDto; + + expect(result.merge).toBe(true); + expect(result.diffTollerancePercent).toBe(1.5); + expect(result.ignoreAreas).toEqual([{ id: '7', x: 1.5, y: 2, width: 3, height: 4 }]); + expect(Buffer.isBuffer(result.image.buffer)).toBe(true); + expect(result.image.buffer.toString()).toBe('png'); + }); }); diff --git a/src/test-runs/dto/create-test-request.dto.ts b/src/test-runs/dto/create-test-request.dto.ts index 1b22ac9a..211285fb 100644 --- a/src/test-runs/dto/create-test-request.dto.ts +++ b/src/test-runs/dto/create-test-request.dto.ts @@ -34,4 +34,9 @@ export class CreateTestRequestDto extends BaselineDataDto { @IsOptional() @IsString() comment?: string | null; + + @ApiPropertyOptional() + @IsOptional() + @IsString() + sourceBranch?: string; } diff --git a/src/test-runs/dto/ignore-area.dto.spec.ts b/src/test-runs/dto/ignore-area.dto.spec.ts new file mode 100644 index 00000000..8899292c --- /dev/null +++ b/src/test-runs/dto/ignore-area.dto.spec.ts @@ -0,0 +1,38 @@ +import { plainToInstance } from 'class-transformer'; +import { validate } from 'class-validator'; +import { IgnoreAreaDto } from './ignore-area.dto'; + +const area = { x: 1, y: 2, width: 3, height: 4 }; + +const failedProperties = async (dto: object): Promise => + (await validate(dto)).map((error) => error.property).sort(); + +describe('IgnoreAreaDto', () => { + it('accepts fractional and negative coordinates within the bounds', async () => { + const dto = plainToInstance(IgnoreAreaDto, { x: -10.5, y: 0.25, width: 3.3, height: 4 }); + + expect(await failedProperties(dto)).toEqual([]); + }); + + it('rejects x or y beyond a million either way', async () => { + expect(await failedProperties(plainToInstance(IgnoreAreaDto, { ...area, x: -1e6 - 1 }))).toEqual(['x']); + expect(await failedProperties(plainToInstance(IgnoreAreaDto, { ...area, y: 1e6 + 1 }))).toEqual(['y']); + }); + + it('does not bound the size', async () => { + const dto = plainToInstance(IgnoreAreaDto, { ...area, width: 1e9, height: -1e9 }); + + expect(await failedProperties(dto)).toEqual([]); + }); + + it.each([['abc'], [5]])('keeps the id the frontend sends (%p), whatever its type', async (id) => { + const dto = plainToInstance(IgnoreAreaDto, { ...area, id }); + + expect(await validate(dto, { whitelist: true })).toEqual([]); + expect(dto.id).toBe(id); + }); + + it('does not require the id', async () => { + expect(await failedProperties(plainToInstance(IgnoreAreaDto, area))).toEqual([]); + }); +}); diff --git a/src/test-runs/dto/ignore-area.dto.ts b/src/test-runs/dto/ignore-area.dto.ts index 36cc6c51..24f8bbf4 100644 --- a/src/test-runs/dto/ignore-area.dto.ts +++ b/src/test-runs/dto/ignore-area.dto.ts @@ -1,13 +1,23 @@ -import { ApiProperty } from '@nestjs/swagger'; -import { IsNumber } from 'class-validator'; +import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; +import { IsNumber, IsOptional, Max, Min } from 'class-validator'; + +export const MAX_IGNORE_AREA_COORDINATE = 1_000_000; export class IgnoreAreaDto { + @ApiPropertyOptional() + @IsOptional() + readonly id?: string; + @ApiProperty() @IsNumber() + @Min(-MAX_IGNORE_AREA_COORDINATE) + @Max(MAX_IGNORE_AREA_COORDINATE) readonly x!: number; @ApiProperty() @IsNumber() + @Min(-MAX_IGNORE_AREA_COORDINATE) + @Max(MAX_IGNORE_AREA_COORDINATE) readonly y!: number; @ApiProperty() diff --git a/src/test-runs/dto/testRun-paginated.dto.spec.ts b/src/test-runs/dto/testRun-paginated.dto.spec.ts index 3177bf6b..3720b26a 100644 --- a/src/test-runs/dto/testRun-paginated.dto.spec.ts +++ b/src/test-runs/dto/testRun-paginated.dto.spec.ts @@ -2,7 +2,6 @@ import { Pagination } from '../../shared/dto/pagination.dto'; import { PaginatedTestRunDto } from './testRun-paginated.dto'; import { TestRunDto } from './testRun.dto'; -// metadata keys @nestjs/swagger stores @ApiProperty under (not exported from the package entry point) const API_MODEL_PROPERTIES = 'swagger/apiModelProperties'; const API_MODEL_PROPERTIES_ARRAY = 'swagger/apiModelPropertiesArray'; diff --git a/src/test-runs/dto/testRun.dto.ts b/src/test-runs/dto/testRun.dto.ts index d61694ef..2662cde2 100644 --- a/src/test-runs/dto/testRun.dto.ts +++ b/src/test-runs/dto/testRun.dto.ts @@ -8,7 +8,6 @@ export class TestRunDto { buildId: string; @ApiProperty() imageName: string; - // small copies for the card grids; null on runs ingested before these existed @ApiPropertyOptional() imageThumbnailName?: string | null; @ApiPropertyOptional() diff --git a/src/test-runs/dto/testRunResult.dto.spec.ts b/src/test-runs/dto/testRunResult.dto.spec.ts index 5fcb51f1..589aee19 100644 --- a/src/test-runs/dto/testRunResult.dto.spec.ts +++ b/src/test-runs/dto/testRunResult.dto.spec.ts @@ -40,14 +40,13 @@ describe('TestRunResultDto', () => { expect(result.baselineName).toBe('baseline of the variation'); }); - it('builds the link even when the frontend url is not configured', () => { + it('renders a missing APP_FRONTEND_URL as "undefined" (startup warns about it)', () => { delete process.env.APP_FRONTEND_URL; const testRun = generateTestRun(); const testVariation = generateTestVariation({ projectId: 'project-id' }); const result = new TestRunResultDto(testRun, testVariation); - // NOTE: current behavior — a missing APP_FRONTEND_URL is rendered as the literal string "undefined" expect(result.url).toBe(`undefined/project-id?buildId=${testRun.buildId}&testId=${testRun.id}`); }); }); diff --git a/src/test-runs/dto/testRunResult.dto.ts b/src/test-runs/dto/testRunResult.dto.ts index 30742640..85fcd0c7 100644 --- a/src/test-runs/dto/testRunResult.dto.ts +++ b/src/test-runs/dto/testRunResult.dto.ts @@ -6,11 +6,11 @@ export class TestRunResultDto extends TestRunDto { @ApiPropertyOptional() pixelMisMatchCount?: number | null; @ApiPropertyOptional() - vlmDescription?: string | null; + override vlmDescription?: string | null; @ApiProperty() url: string; @ApiProperty() - baselineName: string | null; + override baselineName: string | null; constructor(testRun: TestRun, testVariation: TestVariation) { super(testRun); diff --git a/src/test-runs/dto/update-ignore-area.dto.spec.ts b/src/test-runs/dto/update-ignore-area.dto.spec.ts index 09b14441..caf9bb64 100644 --- a/src/test-runs/dto/update-ignore-area.dto.spec.ts +++ b/src/test-runs/dto/update-ignore-area.dto.spec.ts @@ -21,4 +21,10 @@ describe('UpdateIgnoreAreasDto', () => { expect(await failedProperties(dto)).toEqual(['ids', 'ignoreAreas']); }); + + it('rejects ids and areas that are not arrays', async () => { + const dto = plainToInstance(UpdateIgnoreAreasDto, { ids: 'abc', ignoreAreas: area }); + + expect(await failedProperties(dto)).toEqual(['ids', 'ignoreAreas']); + }); }); diff --git a/src/test-runs/dto/update-ignore-area.dto.ts b/src/test-runs/dto/update-ignore-area.dto.ts index 68d11932..5a072859 100644 --- a/src/test-runs/dto/update-ignore-area.dto.ts +++ b/src/test-runs/dto/update-ignore-area.dto.ts @@ -1,14 +1,16 @@ import { ApiProperty } from '@nestjs/swagger'; import { IgnoreAreaDto } from './ignore-area.dto'; import { Type } from 'class-transformer'; -import { ValidateNested, IsString } from 'class-validator'; +import { IsArray, ValidateNested, IsString } from 'class-validator'; export class UpdateIgnoreAreasDto { @ApiProperty({ type: String, isArray: true }) + @IsArray() @IsString({ each: true }) readonly ids!: string[]; @ApiProperty({ type: IgnoreAreaDto, isArray: true }) + @IsArray() @Type(() => IgnoreAreaDto) @ValidateNested({ each: true }) readonly ignoreAreas!: IgnoreAreaDto[]; diff --git a/src/test-runs/test-runs.controller.spec.ts b/src/test-runs/test-runs.controller.spec.ts index f476b687..ff863d4e 100644 --- a/src/test-runs/test-runs.controller.spec.ts +++ b/src/test-runs/test-runs.controller.spec.ts @@ -1,8 +1,16 @@ import { Test, TestingModule } from '@nestjs/testing'; -import { GUARDS_METADATA, INTERCEPTORS_METADATA, PIPES_METADATA } from '@nestjs/common/constants'; -import { NotFoundException, ValidationPipe } from '@nestjs/common'; -import { Role, TestStatus, User } from '@prisma/client'; -import { TestRunsController } from './test-runs.controller'; +import { + GUARDS_METADATA, + HTTP_CODE_METADATA, + INTERCEPTORS_METADATA, + PATH_METADATA, + PIPES_METADATA, + ROUTE_ARGS_METADATA, +} from '@nestjs/common/constants'; +import { BadRequestException, NotFoundException, ParseArrayPipe } from '@nestjs/common'; +import { PNG } from 'pngjs'; +import { Role, User } from '@prisma/client'; +import { TestRunsController, multipartLimits, parseIds, parseUuidIds } from './test-runs.controller'; import { TestRunsService } from './test-runs.service'; import { TestVariationsService } from '../test-variations/test-variations.service'; import { PrismaService } from '../prisma/prisma.service'; @@ -15,6 +23,9 @@ import { CreateTestRequestBase64Dto } from './dto/create-test-request-base64.dto import { CreateTestRequestMultipartDto } from './dto/create-test-request-multipart.dto'; import { UpdateIgnoreAreasDto } from './dto/update-ignore-area.dto'; +const png = PNG.sync.write(new PNG({ width: 1, height: 1 })); +const request = { buildId: 'build', projectId: 'project', name: 'name', branchName: 'develop' }; + const initController = async (testRunsService: Partial> = {}) => { const module: TestingModule = await Test.createTestingModule({ controllers: [TestRunsController], @@ -28,45 +39,41 @@ const initController = async (testRunsService: Partial(TestRunsController); }; +const stub = async (method: keyof TestRunsService, result?: unknown) => { + const mock = jest.fn().mockResolvedValue(result); + return { controller: await initController({ [method]: mock }), mock }; +}; + describe('TestRuns Controller', () => { - let controller: TestRunsController; const user = { id: 'user id' } as User; const ignoreAreas = [{ x: 1, y: 2, width: 3, height: 4 }]; - beforeEach(async () => { - controller = await initController(); - }); - - it('should be defined', () => { - expect(controller).toBeDefined(); + it('should be defined', async () => { + expect(await initController()).toBeDefined(); }); it('lists the test runs of a build', async () => { const list = [{ id: 'a' }, { id: 'b' }]; - const findMany = jest.fn().mockResolvedValueOnce(list); - controller = await initController({ findMany }); + const { controller, mock } = await stub('findMany', list); const result = await controller.get('build id'); - expect(findMany).toHaveBeenCalledWith('build id'); + expect(mock).toHaveBeenCalledWith('build id'); expect(result).toBe(list); }); - it('returns the details of one test run as the service gives them', async () => { + it('returns the run as stored, with its variation (kept for existing clients)', async () => { const testRun = generateTestRun(); - const findOne = jest.fn().mockResolvedValueOnce(testRun); - controller = await initController({ findOne }); + const { controller, mock } = await stub('findOne', testRun); const result = await controller.getDetails(testRun.id); - expect(findOne).toHaveBeenCalledWith(testRun.id); - // NOTE: current behavior — returned raw (no TestRunDto mapping), so it also carries testVariation etc. + expect(mock).toHaveBeenCalledWith(testRun.id); expect(result).toBe(testRun); }); it('throws NotFoundException for an unknown test run', async () => { - const findOne = jest.fn().mockResolvedValueOnce(null); - controller = await initController({ findOne }); + const { controller } = await stub('findOne', null); await expect(controller.getDetails('missing')).rejects.toThrow( new NotFoundException('Test run not found: missing') @@ -76,97 +83,78 @@ describe('TestRuns Controller', () => { describe('approveTestRun', () => { it('approves every id one after another with merge flag, no auto-approve and the current user', async () => { const order: string[] = []; - const approve = jest.fn().mockImplementation(async (id: string) => { + const { controller, mock } = await stub('approve'); + mock.mockImplementation(async (id: string) => { order.push(`start ${id}`); await new Promise((resolve) => setTimeout(resolve, 0)); order.push(`end ${id}`); }); - controller = await initController({ approve }); const result = await controller.approveTestRun(user, ['1', '2'], true); - expect(approve).toHaveBeenNthCalledWith(1, '1', true, false, user.id); - expect(approve).toHaveBeenNthCalledWith(2, '2', true, false, user.id); + expect(mock).toHaveBeenNthCalledWith(1, '1', true, false, user.id); + expect(mock).toHaveBeenNthCalledWith(2, '2', true, false, user.id); expect(order).toEqual(['start 1', 'end 1', 'start 2', 'end 2']); expect(result).toBeUndefined(); }); it('stops at the first failing id', async () => { - const approve = jest.fn().mockRejectedValueOnce(new Error('boom')); - controller = await initController({ approve }); + const { controller, mock } = await stub('approve'); + mock.mockRejectedValueOnce(new Error('boom')); await expect(controller.approveTestRun(user, ['1', '2'], false)).rejects.toThrow('boom'); - expect(approve).toHaveBeenCalledTimes(1); - expect(approve).toHaveBeenCalledWith('1', false, false, user.id); + expect(mock).toHaveBeenCalledTimes(1); + expect(mock).toHaveBeenCalledWith('1', false, false, user.id); }); it('does nothing for an empty list', async () => { - const approve = jest.fn(); - controller = await initController({ approve }); + const { controller, mock } = await stub('approve'); await controller.approveTestRun(user, [], false); - expect(approve).not.toHaveBeenCalled(); + expect(mock).not.toHaveBeenCalled(); }); }); it('returns the matching variations of a test run', async () => { const group = { variations: [{ id: 'a' }], skipped: [] as unknown[] }; - const getMatchingVariations = jest.fn().mockResolvedValueOnce(group); - controller = await initController({ getMatchingVariations }); + const { controller, mock } = await stub('getMatchingVariations', group); const result = await controller.getMatchingSiblings('a'); - expect(getMatchingVariations).toHaveBeenCalledWith('a'); + expect(mock).toHaveBeenCalledWith('a'); expect(result).toBe(group); }); - it('rejects every id by setting its status to failed', async () => { - const setStatus = jest.fn().mockResolvedValue(undefined); - controller = await initController({ setStatus }); + it.each(['reject', 'delete'] as const)('%s goes through every id one by one', async (method) => { + const { controller, mock } = await stub(method); - const result = await controller.reject(['1', '2']); + const result = await controller[method](['1', '2']); - expect(setStatus.mock.calls).toEqual([ - ['1', TestStatus.failed], - ['2', TestStatus.failed], - ]); + expect(mock.mock.calls).toEqual([['1'], ['2']]); expect(result).toBeUndefined(); }); - it('deletes every id', async () => { - const deleteMock = jest.fn().mockResolvedValue(undefined); - controller = await initController({ delete: deleteMock }); + it('revert passes the ids to the service as one request and answers 204', async () => { + const { controller, mock } = await stub('revert'); - const result = await controller.delete(['1', '2']); + const result = await controller.revert(['1', '2']); - expect(deleteMock.mock.calls).toEqual([['1'], ['2']]); + expect(mock).toHaveBeenCalledTimes(1); + expect(mock).toHaveBeenCalledWith(['1', '2']); expect(result).toBeUndefined(); + expect(Reflect.getMetadata(HTTP_CODE_METADATA, TestRunsController.prototype.revert)).toBe(204); + expect(Reflect.getMetadata(PATH_METADATA, TestRunsController.prototype.revert)).toBe('revert'); }); - it('replaces the ignore areas of every id with the same areas', async () => { - const updateIgnoreAreas = jest.fn().mockResolvedValue(undefined); - controller = await initController({ updateIgnoreAreas }); + it.each(['updateIgnoreAreas', 'addIgnoreAreas'] as const)('%s hands the same areas to every id', async (method) => { + const { controller, mock } = await stub(method); const data: UpdateIgnoreAreasDto = { ids: ['1', '2'], ignoreAreas }; - const result = await controller.updateIgnoreAreas(data); + const result = await controller[method](data); - expect(updateIgnoreAreas.mock.calls).toEqual([ - ['1', ignoreAreas], - ['2', ignoreAreas], - ]); - expect(result).toBeUndefined(); - }); - - it('adds the same ignore areas to every id', async () => { - const addIgnoreAreas = jest.fn().mockResolvedValue(undefined); - controller = await initController({ addIgnoreAreas }); - const data: UpdateIgnoreAreasDto = { ids: ['1', '2'], ignoreAreas }; - - const result = await controller.addIgnoreAreas(data); - - expect(addIgnoreAreas.mock.calls).toEqual([ + expect(mock.mock.calls).toEqual([ ['1', ignoreAreas], ['2', ignoreAreas], ]); @@ -175,54 +163,126 @@ describe('TestRuns Controller', () => { it('updates the comment of a test run and returns what the service returns', async () => { const testRun = generateTestRun({ comment: 'new' }); - const update = jest.fn().mockResolvedValueOnce(testRun); - controller = await initController({ update }); + const { controller, mock } = await stub('update', testRun); const result = await controller.update(testRun.id, { comment: 'new' }); - expect(update).toHaveBeenCalledWith(testRun.id, { comment: 'new' }); + expect(mock).toHaveBeenCalledWith(testRun.id, { comment: 'new' }); expect(result).toBe(testRun); }); - it('decodes the base64 image and posts the test run', async () => { + it('passes the request and the decoded image to the service', async () => { const response = { id: 'result' }; - const postTestRun = jest.fn().mockResolvedValueOnce(response); - controller = await initController({ postTestRun }); - const createTestRequestDto = { - buildId: 'build', - projectId: 'project', - name: 'name', - branchName: 'develop', - imageBase64: Buffer.from('png bytes').toString('base64'), - } as CreateTestRequestBase64Dto; + const { controller, mock } = await stub('postTestRun', response); + const createTestRequestDto = { ...request, imageBase64: png.toString('base64') } as CreateTestRequestBase64Dto; const result = await controller.postTestRun(createTestRequestDto); - expect(postTestRun).toHaveBeenCalledWith({ createTestRequestDto, imageBuffer: Buffer.from('png bytes') }); - // NOTE: current behavior — the whole dto, imageBase64 included, is passed on to the service - expect(postTestRun.mock.calls[0][0].createTestRequestDto.imageBase64).toBe(createTestRequestDto.imageBase64); + expect(mock).toHaveBeenCalledWith({ createTestRequestDto, imageBuffer: png }); + expect(mock.mock.calls[0][0].createTestRequestDto.imageBase64).toBe(createTestRequestDto.imageBase64); expect(result).toBe(response); }); it('posts a multipart test run with the uploaded file buffer', async () => { const response = { id: 'result' }; - const postTestRun = jest.fn().mockResolvedValueOnce(response); - controller = await initController({ postTestRun }); - const buffer = Buffer.from('png bytes'); + const { controller, mock } = await stub('postTestRun', response); const createTestRequestDto = { - buildId: 'build', - projectId: 'project', - name: 'name', - branchName: 'develop', - image: { buffer } as Express.Multer.File, + ...request, + image: { buffer: png } as Express.Multer.File, } as CreateTestRequestMultipartDto; const result = await controller.postTestRunMultipart(createTestRequestDto); - expect(postTestRun).toHaveBeenCalledWith({ createTestRequestDto, imageBuffer: buffer }); + expect(mock).toHaveBeenCalledWith({ createTestRequestDto, imageBuffer: png }); expect(result).toBe(response); }); + it.each([ + [ + 'a base64 image that is not a PNG', + (controller: TestRunsController) => + controller.postTestRun({ imageBase64: Buffer.from('GIF89a').toString('base64') } as CreateTestRequestBase64Dto), + ], + [ + 'a multipart image without a buffer', + (controller: TestRunsController) => + controller.postTestRunMultipart({ image: {} as Express.Multer.File } as CreateTestRequestMultipartDto), + ], + ])('rejects %s and posts nothing', async (_case, post) => { + const { controller, mock } = await stub('postTestRun'); + + await expect(post(controller)).rejects.toThrow(BadRequestException); + expect(mock).not.toHaveBeenCalled(); + }); + + it.each(['approveTestRun', 'reject', 'delete', 'revert'] as const)( + 'validates the bulk ids of %s as an array', + (method) => { + const args = Reflect.getMetadata(ROUTE_ARGS_METADATA, TestRunsController, method) as Record< + string, + { data?: unknown; pipes: unknown[] } + >; + + const pipes = Object.values(args).flatMap((arg) => arg.pipes); + + expect(pipes.some((pipe) => pipe instanceof ParseArrayPipe)).toBe(true); + } + ); + + it('parseIds rejects a body that is not an array and passes an array through', async () => { + const pipe = parseIds(); + + await expect(pipe.transform({ ids: [] }, { type: 'body' })).rejects.toThrow(BadRequestException); + await expect(pipe.transform(['a', 'b'], { type: 'body' })).resolves.toEqual(['a', 'b']); + }); + + it('parseUuidIds rejects ids that are not uuids and passes uuids through', () => { + const pipe = parseUuidIds(); + const id = '10fb5e02-64e0-4cf5-9f17-c00ab3c96658'; + + expect(pipe.transform([id], { type: 'body' })).toEqual([id]); + expect(() => pipe.transform([id, 'abc', '1'], { type: 'body' })).toThrow( + new BadRequestException('Not a uuid: abc, 1') + ); + }); + + it('revert validates the ids as uuids', () => { + const args = Reflect.getMetadata(ROUTE_ARGS_METADATA, TestRunsController, 'revert') as Record< + string, + { pipes: unknown[] } + >; + + expect(Object.values(args).flatMap((arg) => arg.pipes)).toHaveLength(2); + }); + + describe('multipartLimits', () => { + afterEach(() => { + delete process.env.MULTIPART_FILE_SIZE_LIMIT; + }); + + it('defaults to 100 MiB', () => { + expect(multipartLimits()).toEqual({ fileSize: 104857600 }); + }); + + it('reads the limit from the environment on every call', () => { + process.env.MULTIPART_FILE_SIZE_LIMIT = '1000'; + + expect(multipartLimits()).toEqual({ fileSize: 1000 }); + }); + + it('falls back to the default for a value that is not a number', () => { + process.env.MULTIPART_FILE_SIZE_LIMIT = 'abc'; + + expect(multipartLimits()).toEqual({ fileSize: 104857600 }); + }); + + it('falls back to the default for a negative value', () => { + process.env.MULTIPART_FILE_SIZE_LIMIT = '-1'; + + expect(multipartLimits()).toEqual({ fileSize: 104857600 }); + }); + }); + describe('route protection', () => { const guardsOf = (method: keyof TestRunsController) => Reflect.getMetadata(GUARDS_METADATA, TestRunsController.prototype[method]); @@ -237,7 +297,7 @@ describe('TestRuns Controller', () => { } ); - it.each(['approveTestRun', 'reject', 'delete', 'updateIgnoreAreas', 'addIgnoreAreas', 'update'] as const)( + it.each(['approveTestRun', 'reject', 'revert', 'delete', 'updateIgnoreAreas', 'addIgnoreAreas', 'update'] as const)( '%s needs a logged in admin or editor', (method) => { expect(guardsOf(method)).toEqual([JwtAuthGuard, RoleGuard]); @@ -250,19 +310,13 @@ describe('TestRuns Controller', () => { expect(rolesOf(method)).toEqual([Role.admin, Role.editor]); }); - it('moves the uploaded image into the body and validates the multipart request', () => { + it('moves the uploaded image into the body; the DTO carries its own validation options', () => { const method = TestRunsController.prototype.postTestRunMultipart; const interceptors = Reflect.getMetadata(INTERCEPTORS_METADATA, method); - const pipes = Reflect.getMetadata(PIPES_METADATA, method); expect(interceptors).toHaveLength(2); expect(interceptors[1]).toBe(FileToBodyInterceptor); - expect(pipes).toHaveLength(1); - expect(pipes[0]).toBeInstanceOf(ValidationPipe); - expect(pipes[0]).toMatchObject({ - isTransformEnabled: true, - validatorOptions: expect.objectContaining({ forbidUnknownValues: true }), - }); + expect(Reflect.getMetadata(PIPES_METADATA, method)).toBeUndefined(); }); }); }); diff --git a/src/test-runs/test-runs.controller.ts b/src/test-runs/test-runs.controller.ts index c9aaa69e..2505eeba 100644 --- a/src/test-runs/test-runs.controller.ts +++ b/src/test-runs/test-runs.controller.ts @@ -9,11 +9,13 @@ import { Post, ParseBoolPipe, UseInterceptors, - UsePipes, - ValidationPipe, Logger, Patch, NotFoundException, + BadRequestException, + ParseArrayPipe, + HttpCode, + PipeTransform, } from '@nestjs/common'; import { ApiTags, @@ -25,9 +27,10 @@ import { ApiConsumes, ApiBody, ApiOperation, + ApiNoContentResponse, } from '@nestjs/swagger'; import { JwtAuthGuard } from '../auth/guards/auth.guard'; -import { Role, TestRun, TestStatus, User } from '@prisma/client'; +import { Role, TestRun, User } from '@prisma/client'; import { TestRunsService } from './test-runs.service'; import { TestRunResultDto } from './dto/testRunResult.dto'; import { ApiGuard } from '../auth/guards/api.guard'; @@ -41,6 +44,35 @@ import { UpdateTestRunDto } from './dto/update-test.dto'; import { CurrentUser } from '../shared/current-user.decorator'; import { RoleGuard } from '../auth/guards/role.guard'; import { Roles } from '../shared/roles.decorator'; +import { isPngBuffer } from '../static/utils'; +import { positiveNumber } from '../utils/env'; +import { isUUID } from 'class-validator'; + +const DEFAULT_MULTIPART_FILE_SIZE_LIMIT = 100 * 1024 * 1024; + +// read per request: decorators run before ConfigModule has loaded .env +export const multipartLimits = () => ({ + fileSize: positiveNumber(process.env.MULTIPART_FILE_SIZE_LIMIT, DEFAULT_MULTIPART_FILE_SIZE_LIMIT), +}); + +export const parseIds = () => new ParseArrayPipe({ items: String }); + +export const parseUuidIds = (): PipeTransform => ({ + transform: (ids) => { + const invalid = ids.filter((id) => !isUUID(id)); + if (invalid.length) { + throw new BadRequestException(`Not a uuid: ${invalid.join(', ')}`); + } + return ids; + }, +}); + +function pngOrBadRequest(imageBuffer: unknown): Buffer { + if (!Buffer.isBuffer(imageBuffer) || !isPngBuffer(imageBuffer)) { + throw new BadRequestException('Image is not a PNG file'); + } + return imageBuffer; +} @ApiTags('test-runs') @Controller('test-runs') @@ -77,7 +109,7 @@ export class TestRunsController { @Roles(Role.admin, Role.editor) async approveTestRun( @CurrentUser() user: User, - @Body() ids: string[], + @Body(parseIds()) ids: string[], @Query('merge', new ParseBoolPipe()) merge: boolean ): Promise { this.logger.debug(`Going to approve TestRuns: ${ids}`); @@ -110,19 +142,30 @@ export class TestRunsController { @ApiBearerAuth() @UseGuards(JwtAuthGuard, RoleGuard) @Roles(Role.admin, Role.editor) - async reject(@Body() ids: string[]): Promise { + async reject(@Body(parseIds()) ids: string[]): Promise { this.logger.debug(`Going to reject TestRuns: ${ids}`); for (const id of ids) { - await this.testRunsService.setStatus(id, TestStatus.failed); + await this.testRunsService.reject(id); } } + @Post('revert') + @HttpCode(204) + @ApiNoContentResponse() + @ApiBearerAuth() + @UseGuards(JwtAuthGuard, RoleGuard) + @Roles(Role.admin, Role.editor) + revert(@Body(parseIds(), parseUuidIds()) ids: string[]): Promise { + this.logger.debug(`Going to revert TestRuns: ${ids}`); + return this.testRunsService.revert(ids); + } + @Post('delete') @ApiOkResponse({ type: undefined }) @ApiBearerAuth() @UseGuards(JwtAuthGuard, RoleGuard) @Roles(Role.admin, Role.editor) - async delete(@Body() ids: string[]): Promise { + async delete(@Body(parseIds()) ids: string[]): Promise { this.logger.debug(`Going to delete TestRuns: ${ids}`); for (const id of ids) { await this.testRunsService.delete(id); @@ -169,8 +212,8 @@ export class TestRunsController { @ApiOkResponse({ type: TestRunResultDto }) @UseGuards(ApiGuard, RoleGuard) @Roles(Role.admin, Role.editor) - postTestRun(@Body() createTestRequestDto: CreateTestRequestBase64Dto): Promise { - const imageBuffer = Buffer.from(createTestRequestDto.imageBase64, 'base64'); + async postTestRun(@Body() createTestRequestDto: CreateTestRequestBase64Dto): Promise { + const imageBuffer = pngOrBadRequest(Buffer.from(createTestRequestDto.imageBase64, 'base64')); return this.testRunsService.postTestRun({ createTestRequestDto, imageBuffer, @@ -184,10 +227,9 @@ export class TestRunsController { @ApiConsumes('multipart/form-data') @UseGuards(ApiGuard, RoleGuard) @Roles(Role.admin, Role.editor) - @UseInterceptors(FileInterceptor('image'), FileToBodyInterceptor) - @UsePipes(new ValidationPipe({ transform: true, forbidUnknownValues: true })) - postTestRunMultipart(@Body() createTestRequestDto: CreateTestRequestMultipartDto): Promise { - const imageBuffer = createTestRequestDto.image.buffer; + @UseInterceptors(FileInterceptor('image', { limits: multipartLimits }), FileToBodyInterceptor) + async postTestRunMultipart(@Body() createTestRequestDto: CreateTestRequestMultipartDto): Promise { + const imageBuffer = pngOrBadRequest(createTestRequestDto.image.buffer); return this.testRunsService.postTestRun({ createTestRequestDto, imageBuffer }); } } diff --git a/src/test-runs/test-runs.service.spec.ts b/src/test-runs/test-runs.service.spec.ts index 0a869fd7..7c30bf57 100644 --- a/src/test-runs/test-runs.service.spec.ts +++ b/src/test-runs/test-runs.service.spec.ts @@ -1,1637 +1,1933 @@ -import { NotFoundException } from '@nestjs/common'; +import { ConflictException, Logger, NotFoundException } from '@nestjs/common'; import { mocked } from 'jest-mock'; -import { Test, TestingModule } from '@nestjs/testing'; +import { Test } from '@nestjs/testing'; import { SIGNATURE_CONCURRENCY, TestRunsService } from './test-runs.service'; import { SIGNATURE_LENGTH } from '../compare/libs/pixelmatch/signature.core'; import { PrismaService } from '../prisma/prisma.service'; import { StaticService } from '../static/static.service'; -import { Prisma, Project, TestStatus, TestRun, TestVariation } from '@prisma/client'; +import { Baseline, Prisma, Project, TestStatus, TestRun, TestVariation } from '@prisma/client'; import { CreateTestRequestDto } from './dto/create-test-request.dto'; import { TestRunResultDto } from './dto/testRunResult.dto'; import { DiffResult } from './diffResult'; import { EventsGateway } from '../shared/events/events.gateway'; import { TestVariationsService } from '../test-variations/test-variations.service'; import { TestRunDto } from '../test-runs/dto/testRun.dto'; -import { BuildsService } from '../builds/builds.service'; import { generateBaseline, generateTestRun, generateTestVariation, TEST_PROJECT } from '../_data_'; import { getTestVariationUniqueData } from '../utils'; -import { BaselineDataDto } from '../shared/dto/baseline-data.dto'; import { CompareService } from '../compare/compare.service'; -import { UpdateTestRunDto } from './dto/update-test.dto'; jest.mock('pixelmatch'); jest.mock('./dto/testRunResult.dto'); -const initService = async ({ - testRunDeleteMock = jest.fn(), - testRunUpdateMock = jest.fn(), - testRunFindUniqueMock = jest.fn(), - testRunFindManyMock = jest.fn(), - testRunCreateMock = jest.fn(), - testRunCountMock = jest.fn(), - getImageMock = jest.fn(), - saveImageMock = jest.fn(), - deleteImageMock = jest.fn(), - eventTestRunUpdatedMock = jest.fn(), - eventTestRunCreatedMock = jest.fn(), - eventTestRunDeletedMock = jest.fn(), - eventBuildUpdatedMock = jest.fn(), - eventBuildCreatedMock = jest.fn(), - testVariationCreateMock = jest.fn(), - testVariationFindManyMock = jest.fn(), - baselineCreateMock = jest.fn(), - testVariationFindMock = jest.fn(), - testVariationUpdateMock = jest.fn(), - testVariationGetDetailsMock = jest.fn(), - testVariationFindUniqueMock = jest.fn(), - projectFindUniqueMock = jest.fn(), - compareGetDiffMock = jest.fn(), - compareGetChangeSignatureMock = jest.fn(), - compareGetComparatorMock = jest.fn(), - getImageBufferMock = jest.fn(), - copyImageMock = jest.fn(), - testVariationServiceCreateMock = jest.fn(), - testVariationServiceFindUniqueMock = jest.fn(), - testVariationAddBaselineMock = jest.fn(), -}) => { - const module: TestingModule = await Test.createTestingModule({ +const createMocks = () => { + const m = { + testRun: { + delete: jest.fn(), + findMany: jest.fn(), + findUnique: jest.fn(), + create: jest.fn(), + update: jest.fn(), + count: jest.fn(), + updateMany: jest.fn().mockResolvedValue({ count: 1 }), + }, + testVariation: { count: jest.fn() }, + baseline: { count: jest.fn().mockResolvedValue(0) }, + project: { findUnique: jest.fn() }, + build: { findUnique: jest.fn() }, + static: { getImageBuffer: jest.fn(), copyImage: jest.fn(), saveImage: jest.fn(), deleteImage: jest.fn() }, + events: { testRunUpdated: jest.fn(), testRunCreated: jest.fn(), testRunDeleted: jest.fn() }, + tv: { + find: jest.fn(), + findBaselineHistory: jest.fn(), + findLatestBaselines: jest.fn(), + update: jest.fn(), + create: jest.fn(), + createOrTakeWinner: jest.fn(), + findUnique: jest.fn(), + addBaseline: jest.fn(), + }, + compare: { getDiff: jest.fn(), getChangeSignature: jest.fn(), getComparator: jest.fn() }, + tx: {} as object, + transaction: jest.fn(), + }; + m.tx = { testRun: { updateMany: m.testRun.updateMany } }; + m.transaction.mockImplementation((callback: (client: object) => unknown) => callback(m.tx)); + return m; +}; +type Mocks = ReturnType; + +const build = async () => { + const m = createMocks(); + const module = await Test.createTestingModule({ providers: [ TestRunsService, { provide: PrismaService, useValue: { - testRun: { - delete: testRunDeleteMock, - findMany: testRunFindManyMock, - findUnique: testRunFindUniqueMock, - create: testRunCreateMock, - update: testRunUpdateMock, - count: testRunCountMock, - }, - testVariation: { - create: testVariationCreateMock, - findMany: testVariationFindManyMock, - findUnique: testVariationFindUniqueMock, - }, - baseline: { - create: baselineCreateMock, - }, - project: { - findUnique: projectFindUniqueMock, - }, + $transaction: m.transaction, + testRun: m.testRun, + testVariation: m.testVariation, + baseline: m.baseline, + project: m.project, + build: m.build, }, }, { provide: StaticService, useValue: { - getImage: getImageMock, - getImageBuffer: getImageBufferMock, - copyImage: copyImageMock, - saveImage: saveImageMock, - deleteImage: deleteImageMock, - }, - }, - { - provide: EventsGateway, - useValue: { - testRunUpdated: eventTestRunUpdatedMock, - testRunCreated: eventTestRunCreatedMock, - testRunDeleted: eventTestRunDeletedMock, - buildUpdated: eventBuildUpdatedMock, - buildCreated: eventBuildCreatedMock, - }, - }, - { - provide: TestVariationsService, - useValue: { - find: testVariationFindMock, - getDetails: testVariationGetDetailsMock, - update: testVariationUpdateMock, - create: testVariationServiceCreateMock, - findUnique: testVariationServiceFindUniqueMock, - addBaseline: testVariationAddBaselineMock, - }, - }, - { - provide: BuildsService, - useValue: {}, - }, - { - provide: CompareService, - useValue: { - getDiff: compareGetDiffMock, - getChangeSignature: compareGetChangeSignatureMock, - getComparator: compareGetComparatorMock, + ...m.static, + deleteImages: async (names: Iterable) => { + for (const name of names) if (name) await m.static.deleteImage(name); + }, }, }, + { provide: EventsGateway, useValue: m.events }, + { provide: TestVariationsService, useValue: m.tv }, + { provide: CompareService, useValue: m.compare }, ], }).compile(); + return { service: module.get(TestRunsService), m }; +}; + +const prismaError = (code: string) => new Prisma.PrismaClientKnownRequestError(code, { code, clientVersion: 'test' }); + +const requestDto = (overrides: Partial = {}): CreateTestRequestDto => ({ + buildId: 'buildId', + projectId: 'projectId', + name: 'Test name', + branchName: 'develop', + ...overrides, +}); - return module.get(TestRunsService); +const okDiff: DiffResult = { + status: TestStatus.ok, + diffName: null, + pixelMisMatchCount: 0, + diffPercent: 0, + isSameDimension: true, +}; +const unresolvedDiff: DiffResult = { + status: TestStatus.unresolved, + diffName: 'diff image name', + pixelMisMatchCount: 11, + diffPercent: 22, + isSameDimension: true, }; + +const NO_VARIATION = 'No test variation found. Re-create test run'; + +const deletedNames = (m: Mocks) => m.static.deleteImage.mock.calls.map(([name]) => name).sort(); +const updatedData = (m: Mocks) => m.testRun.update.mock.calls[0][0].data; +const diffInputs = (m: Mocks) => m.compare.getDiff.mock.calls.map(([input]) => input); + describe('TestRunsService', () => { - let service: TestRunsService; const imageBuffer = Buffer.from('Image'); const ignoreAreas = [{ x: 1, y: 2, width: 10, height: 20 }]; + const saveDiff = async (diff: DiffResult | null, id = 'some id') => { + const { service, m } = await build(); + m.testRun.update.mockResolvedValueOnce(generateTestRun()); + await service.saveDiffResult(id, diff); + return m; + }; + it('findOne', async () => { - const id = 'some id'; - const testRunFindUniqueMock = jest.fn(); - service = await initService({ - testRunFindUniqueMock, - }); + const { service, m } = await build(); - service.findOne(id); + await service.findOne('some id'); - expect(testRunFindUniqueMock).toHaveBeenCalledWith({ - where: { id }, - include: { - testVariation: true, - }, - }); + expect(m.testRun.findUnique).toHaveBeenCalledWith({ where: { id: 'some id' }, include: { testVariation: true } }); }); - it('setStatus', async () => { - const testRun = { - id: 'id', - imageName: 'imageName', - diffTollerancePercent: 12, - status: TestStatus.new, - buildId: 'buildId', - testVariationId: 'testVariationId', - updatedAt: new Date(), - createdAt: new Date(), - name: 'test run name', - ignoreAreas: '[]', - }; - const testRunUpdateMock = jest.fn().mockResolvedValueOnce(testRun); - const eventTestRunUpdatedMock = jest.fn(); - service = await initService({ - testRunUpdateMock, - eventTestRunUpdatedMock, - }); - - await service.setStatus(testRun.id, TestStatus.failed); + describe('findOneOrThrow', () => { + it('returns the run', async () => { + const run = generateTestRun(); + const { service, m } = await build(); + m.testRun.findUnique.mockResolvedValueOnce(run); - expect(testRunUpdateMock).toHaveBeenCalledWith({ - where: { id: testRun.id }, - data: { - status: TestStatus.failed, - }, + await expect(service.findOneOrThrow(run.id)).resolves.toBe(run); }); - expect(eventTestRunUpdatedMock).toHaveBeenCalledWith(testRun); - }); - it('create', async () => { - const initCreateTestRequestDto: CreateTestRequestDto = { - buildId: 'buildId', - projectId: 'projectId', - name: 'Test name', - os: 'OS', - browser: 'browser', - viewport: 'viewport', - device: 'device', - customTags: '', - diffTollerancePercent: undefined, - branchName: 'develop', - merge: true, - ignoreAreas: [ - { - x: 1, - y: 2, - width: 100, - height: 200, - }, - ], - }; - const testRunWithResult = { - id: 'id', - imageName: 'imageName', - baselineName: 'baselineName', - diffTollerancePercent: 1, - ignoreAreas: '[]', - diffName: 'diffName', - status: TestStatus.unresolved, - }; + it('throws NotFoundException for an unknown run', async () => { + const { service, m } = await build(); + m.testRun.findUnique.mockResolvedValueOnce(null); - const testRun = { - id: 'id', - imageName: 'imageName', - baselineName: 'baselineName', - diffTollerancePercent: 1, - ignoreAreas: '[]', - diffName: null as string | null, - }; - const testVariation = { - id: '123', - projectId: 'project Id', - name: 'Test name', - baselineName: 'baselineName', - os: 'OS', - browser: 'browser', - viewport: 'viewport', - device: 'device', - customTags: '', - ignoreAreas: '[{"x":3,"y":4,"width":500,"height":600}]', - comment: 'some comment', - createdAt: new Date(), - updatedAt: new Date(), - branchName: 'master', - }; - const diffResult: DiffResult = { - status: TestStatus.unresolved, - diffName: 'diff image name', - pixelMisMatchCount: 11, - diffPercent: 22, - isSameDimension: true, - }; - const createTestRequestDto = initCreateTestRequestDto; - const testRunCreateMock = jest.fn().mockResolvedValueOnce(testRun); - const imageName = 'image name'; - const saveImageMock = jest.fn().mockReturnValueOnce(imageName); - const image = 'image'; - const baseline = 'baseline'; - const getImageMock = jest.fn().mockReturnValueOnce(baseline).mockReturnValueOnce(image); - const eventTestRunCreatedMock = jest.fn(); - const compareGetDiffMock = jest.fn().mockReturnValueOnce(diffResult); - service = await initService({ - testRunCreateMock, - saveImageMock, - getImageMock, - eventTestRunCreatedMock, - compareGetDiffMock, - }); - - const saveDiffResultMock = jest.fn(); - service.saveDiffResult = saveDiffResultMock.mockResolvedValueOnce(testRunWithResult); - const tryAutoApproveByPastBaselines = jest.fn(); - service['tryAutoApproveByPastBaselines'] = tryAutoApproveByPastBaselines.mockResolvedValueOnce(testRunWithResult); - const tryAutoApproveByNewBaselines = jest.fn(); - service['tryAutoApproveByNewBaselines'] = tryAutoApproveByNewBaselines.mockResolvedValueOnce(testRunWithResult); - - const result = await service.create({ testVariation, createTestRequestDto, imageBuffer }); - - expect(saveImageMock).toHaveBeenCalledWith('screenshot', imageBuffer); - expect(testRunCreateMock).toHaveBeenCalledWith({ - data: { - imageName, - testVariation: { - connect: { - id: testVariation.id, - }, - }, - build: { - connect: { - id: createTestRequestDto.buildId, - }, - }, - project: { - connect: { - id: createTestRequestDto.projectId, - }, - }, - name: testVariation.name, - browser: testVariation.browser, - device: testVariation.device, - os: testVariation.os, - viewport: testVariation.viewport, - customTags: testVariation.customTags, - baselineName: testVariation.baselineName, - ignoreAreas: testVariation.ignoreAreas, - tempIgnoreAreas: JSON.stringify(createTestRequestDto.ignoreAreas), - comment: testVariation.comment, - baselineBranchName: testVariation.branchName, - branchName: createTestRequestDto.branchName, - diffTollerancePercent: createTestRequestDto.diffTollerancePercent, - merge: createTestRequestDto.merge, - status: TestStatus.new, - }, + await expect(service.findOneOrThrow('missing')).rejects.toThrow( + new NotFoundException('Test run not found: missing') + ); }); - expect(result).toBe(testRun); }); - it('calculateDiff', async () => { - const testRun: TestRun = generateTestRun(); - const testRunUpdateMock = jest.fn(); - const deleteImageMock = jest.fn(); - const diffResult = { - id: 'test', + describe('create', () => { + const request = requestDto({ buildId: 'b', projectId: 'p' }); + + const initCreate = async () => { + const ctx = await build(); + ctx.m.static.saveImage.mockResolvedValueOnce('image.png'); + return ctx; }; - const compareGetDiffMock = jest.fn().mockReturnValueOnce(diffResult); - service = await initService({ - testRunUpdateMock, - deleteImageMock, - compareGetDiffMock, - }); - service.saveDiffResult = jest.fn(); - - await service.calculateDiff('projectId', testRun); - - expect(deleteImageMock).toHaveBeenCalledWith(testRun.diffName); - expect(compareGetDiffMock).toHaveBeenCalledWith({ - projectId: 'projectId', - data: { - image: testRun.imageName, - baseline: testRun.baselineName, - ignoreAreas: service['getAllIgnoteAreas'](testRun), - diffTollerancePercent: testRun.diffTollerancePercent, - saveDiffAsFile: true, - }, - }); - expect(service.saveDiffResult).toHaveBeenCalledWith(testRun.id, diffResult); - }); - describe('saveDiffResult', () => { - const testRun: TestRun = generateTestRun(); - it('no results', async () => { - const id = 'some id'; - const testRunUpdateMock = jest.fn().mockResolvedValueOnce(testRun); - const eventTestRunUpdatedMock = jest.fn(); - service = await initService({ - testRunUpdateMock, - eventTestRunUpdatedMock, + it('create', async () => { + const createTestRequestDto = requestDto({ + os: 'OS', + browser: 'browser', + viewport: 'viewport', + device: 'device', + customTags: '', + diffTollerancePercent: undefined, + merge: true, + ignoreAreas: [{ x: 1, y: 2, width: 100, height: 200 }], }); + const testVariation = generateTestVariation({ ignoreAreas: '[{"x":3,"y":4,"width":500,"height":600}]' }); + const testRun = generateTestRun(); + const { service, m } = await build(); + m.testRun.create.mockResolvedValueOnce(testRun); + m.static.saveImage.mockResolvedValueOnce('image name'); - await service.saveDiffResult(id, null); + const result = await service.create({ testVariation, createTestRequestDto, imageBuffer }); - expect(testRunUpdateMock).toHaveBeenCalledWith({ - where: { id }, + expect(m.static.saveImage).toHaveBeenCalledWith('screenshot', imageBuffer); + expect(m.testRun.create).toHaveBeenCalledWith({ data: { + imageName: 'image name', + testVariationId: testVariation.id, + buildId: createTestRequestDto.buildId, + projectId: createTestRequestDto.projectId, + name: testVariation.name, + browser: testVariation.browser, + device: testVariation.device, + os: testVariation.os, + viewport: testVariation.viewport, + customTags: testVariation.customTags, + baselineName: testVariation.baselineName, + ignoreAreas: testVariation.ignoreAreas, + tempIgnoreAreas: JSON.stringify(createTestRequestDto.ignoreAreas), + comment: testVariation.comment, + baselineBranchName: testVariation.branchName, + branchName: createTestRequestDto.branchName, + diffTollerancePercent: createTestRequestDto.diffTollerancePercent, + merge: createTestRequestDto.merge, status: TestStatus.new, - diffName: null, - pixelMisMatchCount: null, - diffPercent: null, - vlmDescription: null, - changeSignature: null, - imageThumbnailName: null, - diffThumbnailName: null, }, }); - expect(eventTestRunUpdatedMock).toHaveBeenCalledWith(testRun); + expect(result).toBe(testRun); }); - // Written here, next to the diff that decoded the screenshots anyway, so - // the variations dialog never has to decode them again. - it('stores the change signature the comparison produced', async () => { - const testRunUpdateMock = jest.fn().mockResolvedValueOnce(testRun); - service = await initService({ testRunUpdateMock }); + it('prefers the comment of the request over the one of the variation and leaves tempIgnoreAreas to the database default when the request has none', async () => { + const { service, m } = await initCreate(); + m.testRun.create.mockResolvedValueOnce(generateTestRun()); - await service.saveDiffResult('some id', { - status: TestStatus.unresolved, - diffName: 'diff image name', - pixelMisMatchCount: 11, - diffPercent: 22, - isSameDimension: true, - changeSignature: { threshold: 0.1, includeAA: true, signature: [0.25, 0.75] }, + const result = await service.create({ + testVariation: generateTestVariation({ comment: 'variation comment' }), + createTestRequestDto: requestDto({ comment: 'request comment' }), + imageBuffer, }); - expect(testRunUpdateMock.mock.calls[0][0].data).toMatchObject({ - changeSignature: JSON.stringify({ threshold: 0.1, includeAA: true, signature: [0.25, 0.75] }), - }); + const { data } = m.testRun.create.mock.calls[0][0]; + expect(data.comment).toBe('request comment'); + expect(data.imageName).toBe('image.png'); + expect(data.tempIgnoreAreas).toBeUndefined(); + expect(m.events.testRunCreated).toHaveBeenCalledWith(result); }); - it('stores no signature when the comparison produced none', async () => { - const testRunUpdateMock = jest.fn().mockResolvedValueOnce(testRun); - service = await initService({ testRunUpdateMock }); + it('create inserts with scalar foreign keys, no nested connect', async () => { + const { service, m } = await initCreate(); + m.testRun.create.mockResolvedValueOnce(generateTestRun()); - await service.saveDiffResult('some id', { - status: TestStatus.ok, - diffName: null, - pixelMisMatchCount: 0, - diffPercent: 0, - isSameDimension: true, + await service.create({ + testVariation: generateTestVariation({ id: 'v' }), + createTestRequestDto: request, + imageBuffer, }); - expect(testRunUpdateMock.mock.calls[0][0].data).toMatchObject({ changeSignature: null }); + const { data } = m.testRun.create.mock.calls[0][0]; + expect(data).toMatchObject({ buildId: 'b', projectId: 'p', testVariationId: 'v' }); + expect(data.build).toBeUndefined(); + expect(data.project).toBeUndefined(); + expect(data.testVariation).toBeUndefined(); }); - it('with results', async () => { - const diff: DiffResult = { - status: TestStatus.unresolved, - diffName: 'diff image name', - pixelMisMatchCount: 11, - diffPercent: 22, - isSameDimension: true, - vlmDescription: 'VLM detected significant color differences in the header section', - }; - const id = 'some id'; - const testRunUpdateMock = jest.fn().mockResolvedValueOnce(testRun); - const eventTestRunUpdatedMock = jest.fn(); - service = await initService({ - testRunUpdateMock, - eventTestRunUpdatedMock, - }); - - await service.saveDiffResult(id, diff); - - expect(testRunUpdateMock).toHaveBeenCalledWith({ - where: { id }, - data: { - status: diff.status, - diffName: diff.diffName, - pixelMisMatchCount: diff.pixelMisMatchCount, - diffPercent: diff.diffPercent, - vlmDescription: diff.vlmDescription, - changeSignature: null, - imageThumbnailName: null, - diffThumbnailName: null, - }, - }); - expect(eventTestRunUpdatedMock).toHaveBeenCalledWith(testRun); - }); - }); + it('a run whose build disappeared is a 404', async () => { + const { service, m } = await initCreate(); + m.testRun.create.mockRejectedValueOnce(prismaError('P2003')); - it('findMany', async () => { - const buildId = 'some id'; - const testRun: TestRun = generateTestRun({ - vlmDescription: 'VLM analysis completed', - }); - const testRunFindManyMock = jest.fn().mockResolvedValueOnce([testRun]); - service = await initService({ - testRunFindManyMock, + await expect( + service.create({ testVariation: generateTestVariation(), createTestRequestDto: request, imageBuffer }) + ).rejects.toBeInstanceOf(NotFoundException); + expect(m.events.testRunCreated).not.toHaveBeenCalled(); }); - const result = await service.findMany(buildId); + it('create rethrows other errors', async () => { + const failure = new Error('connection lost'); + const { service, m } = await initCreate(); + m.testRun.create.mockRejectedValueOnce(failure); - expect(testRunFindManyMock).toHaveBeenCalledWith({ - where: { buildId }, + await expect( + service.create({ testVariation: generateTestVariation(), createTestRequestDto: request, imageBuffer }) + ).rejects.toBe(failure); }); - expect(result).toEqual([new TestRunDto(testRun)]); }); - it('delete', async () => { - const id = 'some id'; - const testRun = { - diffName: 'diffName', - imageName: 'imageName', - }; - const findOneMock = jest.fn().mockResolvedValueOnce(testRun); - const deleteImageMock = jest.fn(); - const testRunDeleteMock = jest.fn(); - const eventTestRunDeletedMock = jest.fn(); - service = await initService({ - deleteImageMock, - testRunDeleteMock, - eventTestRunDeletedMock, - }); - service.findOne = findOneMock; - - await service.delete(id); - - expect(findOneMock).toHaveBeenCalledWith(id); - expect(deleteImageMock).toHaveBeenNthCalledWith(1, testRun.diffName); - expect(deleteImageMock).toHaveBeenNthCalledWith(2, testRun.imageName); - expect(testRunDeleteMock).toHaveBeenCalledWith({ - where: { id }, - }); - expect(eventTestRunDeletedMock).toHaveBeenCalledWith(testRun); - }); + describe('calculateDiff', () => { + it('calculateDiff', async () => { + const testRun = generateTestRun(); + const diffResult = { id: 'test' }; + const { service, m } = await build(); + m.compare.getDiff.mockReturnValueOnce(diffResult); + service.saveDiffResult = jest.fn(); + + await service.calculateDiff('projectId', testRun); - it('delete not found', async () => { - const id = 'some id'; - const findOneMock = jest.fn().mockResolvedValueOnce(undefined); - const deleteImageMock = jest.fn(); - const testRunDeleteMock = jest.fn(); - const eventTestRunDeletedMock = jest.fn(); - service = await initService({ - deleteImageMock, - testRunDeleteMock, - eventTestRunDeletedMock, + expect(m.static.deleteImage).toHaveBeenCalledWith(testRun.diffName); + expect(m.compare.getDiff).toHaveBeenCalledWith({ + projectId: 'projectId', + data: { + image: testRun.imageName, + baseline: testRun.baselineName, + ignoreAreas: service['getAllIgnoreAreas'](testRun), + diffTollerancePercent: testRun.diffTollerancePercent, + saveDiffAsFile: true, + }, + }); + expect(service.saveDiffResult).toHaveBeenCalledWith(testRun.id, diffResult); }); - service.findOne = findOneMock; - const result = await service.delete(id); + it('compares with the permanent and temporary ignore areas together, returns the saved run and asks storage to drop the previous diff and thumbnails, missing ones included', async () => { + const permanent = { x: 1, y: 1, width: 1, height: 1 }; + const temporary = { x: 2, y: 2, width: 2, height: 2 }; + const testRun = generateTestRun({ + ignoreAreas: JSON.stringify([permanent]), + tempIgnoreAreas: JSON.stringify([temporary]), + }); + const saved = generateTestRun({ id: 'saved' }); + const { service, m } = await build(); + m.compare.getDiff.mockResolvedValueOnce({ status: TestStatus.ok }); + m.testRun.update.mockResolvedValueOnce(saved); - expect(deleteImageMock).not.toHaveBeenCalled(); - expect(testRunDeleteMock).not.toHaveBeenCalled(); - expect(result).toBeUndefined(); - }); + const result = await service.calculateDiff('projectId', testRun); - it('updateIgnoreAreas', async () => { - const testRun = { - id: 'testRunId', - testVariationId: 'testVariationId', - diffName: 'diffName', - imageName: 'imageName', - ignoreAreas: JSON.stringify(ignoreAreas), - }; - const testVariation = { - id: 'testVariationId', - projectId: 'someProjectId', - }; - const testRunUpdateMock = jest.fn().mockResolvedValueOnce(testRun); - const testVariationUpdateMock = jest.fn().mockResolvedValueOnce(testVariation); - service = await initService({ - testRunUpdateMock, - testVariationUpdateMock, + expect(diffInputs(m)[0].data.ignoreAreas).toEqual([permanent, temporary]); + expect(result).toBe(saved); + expect(m.static.deleteImage.mock.calls).toEqual([[testRun.diffName], [null], [null]]); }); - service.calculateDiff = jest.fn(); - await service.updateIgnoreAreas(testRun.id, ignoreAreas); + it('calculateDiff passes the bytes it is given', async () => { + const { service, m } = await build(); + m.compare.getDiff.mockResolvedValueOnce(null); + m.testRun.update.mockResolvedValueOnce({}); - expect(testRunUpdateMock).toHaveBeenCalledWith({ - where: { id: testRun.id }, - data: { - ignoreAreas: JSON.stringify(ignoreAreas), - }, - }); - expect(testVariationUpdateMock).toHaveBeenCalledWith(testRun.testVariationId, { - ignoreAreas: testRun.ignoreAreas, - }); - expect(service.calculateDiff).toHaveBeenCalledWith(testVariation.projectId, testRun); - }); + await service.calculateDiff('projectId', generateTestRun(), imageBuffer); - it('update', async () => { - const testRun = { - id: 'testRunId', - testVariationId: 'testVariationId', - diffName: 'diffName', - imageName: 'imageName', - }; - const data: UpdateTestRunDto = { - comment: 'random comment', - }; - const testRunUpdateMock = jest.fn().mockResolvedValueOnce(testRun); - const testVariationUpdateMock = jest.fn(); - const eventTestRunUpdatedMock = jest.fn(); - service = await initService({ - testRunUpdateMock, - eventTestRunUpdatedMock, - testVariationUpdateMock, + expect(diffInputs(m)[0].data.imageBuffer).toBe(imageBuffer); }); - await service.update(testRun.id, data); + it('calculateDiff hands the project it is given to the comparison', async () => { + const { service, m } = await build(); + m.compare.getDiff.mockResolvedValueOnce(null); + m.testRun.update.mockResolvedValueOnce({}); - expect(testRunUpdateMock).toHaveBeenCalledWith({ - where: { id: testRun.id }, - data: { - comment: data.comment, - }, + await service.calculateDiff('projectId', generateTestRun(), imageBuffer, TEST_PROJECT); + + expect(diffInputs(m)[0].project).toBe(TEST_PROJECT); }); - expect(testVariationUpdateMock).toHaveBeenCalledWith(testRun.testVariationId, data); - expect(eventTestRunUpdatedMock).toHaveBeenCalledWith(testRun); - }); - it('postTestRun', async () => { - const createTestRequestDto: CreateTestRequestDto = { - buildId: 'buildId', - projectId: 'projectId', - name: 'Test name', - os: 'OS', - browser: 'browser', - viewport: 'viewport', - device: 'device', - branchName: 'develop', - customTags: 'customTags', - comment: 'new comment', - }; - const testVariation: TestVariation = { - id: '123', - projectId: 'project Id', - name: 'Test name', - baselineName: 'baselineName', - os: 'OS', - browser: 'browser', - viewport: 'viewport', - device: 'device', - customTags: '', - ignoreAreas: '[]', - comment: 'some comment', - createdAt: new Date(), - updatedAt: new Date(), - branchName: 'master', - }; - const testRun: TestRun = generateTestRun(); - const projectFindUniqueMock = jest.fn().mockResolvedValueOnce(TEST_PROJECT); - const testVariationFindMock = jest.fn().mockResolvedValueOnce(testVariation); - const testRunFindManyMock = jest.fn().mockResolvedValueOnce([testRun]); - const deleteMock = jest.fn(); - const createMock = jest.fn().mockResolvedValueOnce(testRun); - const service = await initService({ - projectFindUniqueMock, - testVariationFindMock, - testRunFindManyMock, - }); - service.delete = deleteMock; - service.create = createMock; - service.calculateDiff = jest.fn().mockResolvedValueOnce(testRun); - service['tryAutoApproveByPastBaselines'] = jest.fn().mockResolvedValueOnce(testRun); - service['tryAutoApproveByNewBaselines'] = jest.fn().mockResolvedValueOnce(testRun); - const baselineData: BaselineDataDto = { - ...getTestVariationUniqueData(createTestRequestDto), - branchName: createTestRequestDto.branchName, - }; + it.each([ + ['unreadable', '{}', 'not json'], + ['null', 'null', 'null'], + ])('treats %s stored ignore areas as none', async (_case, stored, temporary) => { + const testRun = generateTestRun({ ignoreAreas: stored, tempIgnoreAreas: temporary }); + const { service, m } = await build(); + m.compare.getDiff.mockResolvedValueOnce(null); + m.testRun.update.mockResolvedValueOnce(testRun); - await service.postTestRun({ createTestRequestDto, imageBuffer }); + await service.calculateDiff('projectId', testRun); - expect(testVariationFindMock).toHaveBeenCalledWith(createTestRequestDto); - expect(testRunFindManyMock).toHaveBeenCalledWith({ - where: { - buildId: createTestRequestDto.buildId, - ...baselineData, - NOT: { OR: [{ status: TestStatus.approved }, { status: TestStatus.autoApproved }] }, - }, + expect(diffInputs(m)[0].data.ignoreAreas).toEqual([]); + expect(updatedData(m).status).toBe(TestStatus.new); }); - expect(deleteMock).toHaveBeenCalledWith(testRun.id); - expect(createMock).toHaveBeenCalledWith({ testVariation, createTestRequestDto, imageBuffer }); - expect(service.calculateDiff).toHaveBeenCalledWith(createTestRequestDto.projectId, testRun); - expect(service['tryAutoApproveByPastBaselines']).toHaveBeenCalledWith({ testVariation, testRun }); - expect(service['tryAutoApproveByNewBaselines']).toHaveBeenCalledWith({ testVariation, testRun }); - expect(mocked(TestRunResultDto)).toHaveBeenCalledWith(testRun, testVariation); }); - describe('tryAutoApproveByNewBaselines', () => { - it('skip if ok', async () => { - const testRun = generateTestRun({ status: TestStatus.ok }); - service = await initService({}); + describe('saveDiffResult', () => { + it('no results', async () => { + const m = await saveDiff(null); - const result = await service['tryAutoApproveByNewBaselines']({ - testVariation: generateTestVariation(), - testRun, + expect(m.testRun.update).toHaveBeenCalledWith({ + where: { id: 'some id' }, + data: { + status: TestStatus.new, + diffName: null, + pixelMisMatchCount: null, + diffPercent: null, + vlmDescription: null, + changeSignature: null, + imageThumbnailName: null, + diffThumbnailName: null, + }, }); - - expect(result).toBe(testRun); + expect(m.events.testRunUpdated).toHaveBeenCalled(); }); - it('should not auto approve', async () => { - const testRun = generateTestRun({ status: TestStatus.unresolved }); - const baseline = generateBaseline(); - const testRunFindManyMock = jest.fn().mockResolvedValueOnce([generateTestRun({ status: TestStatus.approved })]); - const testVariationGetDetailsMock = jest.fn().mockResolvedValueOnce(generateTestVariation({}, [baseline])); - service = await initService({ testRunFindManyMock, testVariationGetDetailsMock }); - service['shouldAutoApprove'] = jest.fn().mockResolvedValueOnce(false); + it('stores the change signature the comparison produced', async () => { + const changeSignature = { threshold: 0.1, includeAA: true, signature: [0.25, 0.75] }; - const result = await service['tryAutoApproveByNewBaselines']({ - testVariation: generateTestVariation(), - testRun, - }); + const m = await saveDiff({ ...unresolvedDiff, changeSignature }); - expect(service['shouldAutoApprove']).toHaveBeenCalled(); - expect(result).toBe(testRun); + expect(updatedData(m)).toMatchObject({ changeSignature: JSON.stringify(changeSignature) }); }); - it('should auto approve', async () => { - const testRun = generateTestRun({ status: TestStatus.unresolved }); - const baseline = generateBaseline(); - const testRunFindManyMock = jest.fn().mockResolvedValueOnce([generateTestRun({ status: TestStatus.approved })]); - const testVariationGetDetailsMock = jest.fn().mockResolvedValueOnce(generateTestVariation({}, [baseline])); - service = await initService({ testRunFindManyMock, testVariationGetDetailsMock }); - service['shouldAutoApprove'] = jest.fn().mockResolvedValueOnce(true); - service.approve = jest.fn().mockResolvedValueOnce({ - ...testRun, - status: TestStatus.autoApproved, - }); + it('with results', async () => { + const diff: DiffResult = { + ...unresolvedDiff, + vlmDescription: 'VLM detected significant color differences in the header section', + }; - const result = await service['tryAutoApproveByNewBaselines']({ - testVariation: generateTestVariation(), - testRun, + const m = await saveDiff(diff); + + expect(m.testRun.update).toHaveBeenCalledWith({ + where: { id: 'some id' }, + data: { + status: diff.status, + diffName: diff.diffName, + pixelMisMatchCount: diff.pixelMisMatchCount, + diffPercent: diff.diffPercent, + vlmDescription: diff.vlmDescription, + changeSignature: null, + imageThumbnailName: null, + diffThumbnailName: null, + }, }); + expect(m.events.testRunUpdated).toHaveBeenCalled(); + }); + + it('clears the analysis of a previous comparison when this one has none', async () => { + const m = await saveDiff(okDiff, 'id'); - expect(result).toStrictEqual({ - ...testRun, - status: TestStatus.autoApproved, + expect(updatedData(m)).toEqual({ + diffName: null, + pixelMisMatchCount: 0, + diffPercent: 0, + status: TestStatus.ok, + vlmDescription: null, + changeSignature: null, + imageThumbnailName: null, + diffThumbnailName: null, }); }); }); - describe('tryAutoApproveByPastBaselines', () => { - it('skip if ok', async () => { - const testRun = generateTestRun({ status: TestStatus.ok }); - service = await initService({}); + it('findMany', async () => { + const testRun = generateTestRun({ vlmDescription: 'VLM analysis completed' }); + const { service, m } = await build(); + m.testRun.findMany.mockResolvedValueOnce([testRun]); - const result = await service['tryAutoApproveByPastBaselines']({ - testVariation: generateTestVariation(), - testRun, - }); + const result = await service.findMany('some id'); - expect(result).toBe(testRun); - }); + expect(m.testRun.findMany).toHaveBeenCalledWith({ where: { buildId: 'some id' } }); + expect(result).toEqual([new TestRunDto(testRun)]); + }); - it('skip if the branch the same as baseline', async () => { - const testRun = generateTestRun({ status: TestStatus.unresolved, branchName: 'a', baselineBranchName: 'a' }); - service = await initService({}); + describe('delete', () => { + it('delete', async () => { + const testRun = generateTestRun({ id: 'some id', diffName: 'diffName', imageName: 'imageName' }); + const { service, m } = await build(); + m.testRun.delete.mockResolvedValueOnce(testRun); - const result = await service['tryAutoApproveByPastBaselines']({ - testVariation: generateTestVariation(), - testRun, - }); + const result = await service.delete('some id'); + expect(m.static.deleteImage).toHaveBeenCalledWith(testRun.diffName); + expect(m.static.deleteImage).toHaveBeenCalledWith(testRun.imageName); + expect(m.testRun.delete).toHaveBeenCalledWith({ where: { id: 'some id' } }); + expect(m.events.testRunDeleted).toHaveBeenCalledWith(testRun); expect(result).toBe(testRun); }); - it('should not auto approve', async () => { - const testRun = generateTestRun({ status: TestStatus.unresolved }); - const baseline = generateBaseline(); - const testVariationGetDetailsMock = jest - .fn() - .mockResolvedValueOnce(generateTestVariation({}, [baseline, baseline])); - service = await initService({ testVariationGetDetailsMock }); - service['shouldAutoApprove'] = jest.fn().mockResolvedValueOnce(false); + it('delete not found', async () => { + const { service, m } = await build(); + m.testRun.delete.mockRejectedValueOnce(prismaError('P2025')); - const result = await service['tryAutoApproveByPastBaselines']({ - testVariation: generateTestVariation(), - testRun, - }); + const result = await service.delete('some id'); - expect(service['shouldAutoApprove']).toHaveBeenCalled(); - expect(result).toBe(testRun); + expect(m.static.deleteImage).not.toHaveBeenCalled(); + expect(m.events.testRunDeleted).not.toHaveBeenCalled(); + expect(result).toBeUndefined(); }); - it('should auto approve', async () => { - const testRun = generateTestRun({ status: TestStatus.unresolved }); - const baseline = generateBaseline(); - const testVariationGetDetailsMock = jest - .fn() - .mockResolvedValueOnce(generateTestVariation({}, [baseline, baseline])); - service = await initService({ testVariationGetDetailsMock }); - service['shouldAutoApprove'] = jest.fn().mockResolvedValueOnce(true); - service.approve = jest.fn().mockResolvedValueOnce({ - ...testRun, - status: TestStatus.autoApproved, + it('delete removes the row before the images', async () => { + const order: string[] = []; + const testRun = generateTestRun(); + const { service, m } = await build(); + m.testRun.delete.mockImplementationOnce(async () => { + order.push('row'); + return testRun; }); - - const result = await service['tryAutoApproveByPastBaselines']({ - testVariation: generateTestVariation(), - testRun, + m.static.deleteImage.mockImplementation(() => { + order.push('image'); }); - expect(result).toStrictEqual({ - ...testRun, - status: TestStatus.autoApproved, - }); + await service.delete(testRun.id); + + expect(order).toEqual(['row', 'image', 'image']); + }); + + it.each([ + ['another prisma error', prismaError('P2003')], + ['any other error', new Error('connection lost')], + ['a non-prisma error carrying the P2025 code', Object.assign(new Error('gone'), { code: 'P2025' })], + ])('rethrows %s and announces nothing', async (_case, error) => { + const { service, m } = await build(); + m.testRun.delete.mockRejectedValueOnce(error); + + await expect(service.delete('id')).rejects.toBe(error); + + expect(m.events.testRunDeleted).not.toHaveBeenCalled(); + expect(m.static.deleteImage).not.toHaveBeenCalled(); }); }); describe('thumbnails', () => { - it('stores the names the comparison saved them under', async () => { - const testRunUpdateMock = jest.fn().mockResolvedValueOnce(generateTestRun()); - service = await initService({ testRunUpdateMock }); + const thumbnailed = (overrides: Partial = {}) => + generateTestRun({ + diffName: 'old-diff.png', + imageThumbnailName: 'old-image.thumb.png', + diffThumbnailName: 'old-diff.thumb.png', + ...overrides, + }); - await service.saveDiffResult('some id', { - status: TestStatus.unresolved, + it('stores the names the comparison saved them under', async () => { + const m = await saveDiff({ + ...unresolvedDiff, diffName: 'diff.png', - pixelMisMatchCount: 11, - diffPercent: 22, - isSameDimension: true, imageThumbnailName: 'image.thumb.png', diffThumbnailName: 'diff.thumb.png', }); - expect(testRunUpdateMock.mock.calls[0][0].data).toMatchObject({ + expect(updatedData(m)).toMatchObject({ imageThumbnailName: 'image.thumb.png', diffThumbnailName: 'diff.thumb.png', }); }); - // Recalculating replaces the names on the row, so whatever they pointed at - // before has to go with the old diff — otherwise every ignore-area edit - // leaves two more objects nothing references. it('removes the previous ones when the diff is recalculated', async () => { - const testRun = generateTestRun({ - diffName: 'old-diff.png', - imageThumbnailName: 'old-image.thumb.png', - diffThumbnailName: 'old-diff.thumb.png', - }); - const deleteImageMock = jest.fn(); - service = await initService({ - deleteImageMock, - compareGetDiffMock: jest.fn().mockResolvedValueOnce({ status: TestStatus.ok }), - testRunUpdateMock: jest.fn().mockResolvedValueOnce(testRun), - }); + const testRun = thumbnailed(); + const { service, m } = await build(); + m.compare.getDiff.mockResolvedValueOnce({ status: TestStatus.ok }); + m.testRun.update.mockResolvedValueOnce(testRun); await service.calculateDiff('projectId', testRun); - expect(deleteImageMock.mock.calls.map(([name]) => name).sort()).toEqual( - ['old-diff.png', 'old-diff.thumb.png', 'old-image.thumb.png'].sort() - ); + expect(deletedNames(m)).toEqual(['old-diff.png', 'old-diff.thumb.png', 'old-image.thumb.png']); }); - // Deleting before the replacement exists means a failed comparison leaves - // the row pointing at files that are gone — the reviewer opens the run and - // sees broken pictures, with nothing to recover from. it('keeps the previous ones when the comparison fails', async () => { - const testRun = generateTestRun({ - diffName: 'old-diff.png', - imageThumbnailName: 'old-image.thumb.png', - diffThumbnailName: 'old-diff.thumb.png', - }); - const deleteImageMock = jest.fn(); - service = await initService({ - deleteImageMock, - compareGetDiffMock: jest.fn().mockRejectedValueOnce(new Error('storage is down')), - }); + const { service, m } = await build(); + m.compare.getDiff.mockRejectedValueOnce(new Error('storage is down')); - await expect(service.calculateDiff('projectId', testRun)).rejects.toThrow('storage is down'); + await expect(service.calculateDiff('projectId', thumbnailed())).rejects.toThrow('storage is down'); - expect(deleteImageMock).not.toHaveBeenCalled(); + expect(m.static.deleteImage).not.toHaveBeenCalled(); }); - // two extra objects per run: left behind on delete they would outlive every - // build that ever referenced them, and nothing would ever collect them it('removes them with the run they belong to', async () => { - const testRun = generateTestRun({ + const testRun = thumbnailed({ imageName: 'image.png', diffName: 'diff.png', imageThumbnailName: 'image.thumb.png', diffThumbnailName: 'diff.thumb.png', }); - const deleteImageMock = jest.fn(); - service = await initService({ - testRunFindUniqueMock: jest.fn().mockResolvedValueOnce(testRun), - testRunDeleteMock: jest.fn().mockResolvedValueOnce(testRun), - deleteImageMock, - }); + const { service, m } = await build(); + m.testRun.findUnique.mockResolvedValueOnce(testRun); + m.testRun.delete.mockResolvedValueOnce(testRun); await service.delete(testRun.id); - expect(deleteImageMock.mock.calls.map(([name]) => name).sort()).toEqual( - ['diff.png', 'diff.thumb.png', 'image.png', 'image.thumb.png'].sort() - ); + expect(deletedNames(m)).toEqual(['diff.png', 'diff.thumb.png', 'image.png', 'image.thumb.png']); }); }); - describe('getMatchingVariations', () => { - // Full-length histograms, because a stored one is only reused when its shape - // matches what this build produces. Orthogonal, so they never match. - const histogram = (hot: number): number[] => - Array.from({ length: SIGNATURE_LENGTH }, (_unused, index) => (index === hot ? 1 : 0)); - const SAME_PALETTE = histogram(0); - const OTHER_PALETTE = histogram(SIGNATURE_LENGTH - 1); + describe('ignore areas and comments', () => { + const findsVariation = (m: Mocks, testVariationId: string | null = 'testVariationId') => + m.testRun.findUnique.mockResolvedValue({ testVariationId }); - // A sibling of the reviewed screen. imageName doubles as the run's identity - // in the signature mocks below, which see buffers rather than runs. - const sibling = (id: string, overrides: Partial = {}): TestRun => - generateTestRun({ - id, - status: TestStatus.unresolved, - name: 'Screen A', - baselineName: `${id}.baseline.png`, - imageName: `${id}.screenshot.png`, - customTags: id, - diffPercent: 12, - ...overrides, - }); - - // The pool takes PNG bytes, so the fake static service hands back the image - // name as its own content and the signature mock reads the name back out. - const initMatchingService = async ({ - testRun, - siblings, - signatureOf, - }: { - testRun: TestRun; - siblings: TestRun[]; - signatureOf: (imageName: string) => Promise | number[] | null; - }) => { - const compareGetChangeSignatureMock = jest.fn().mockImplementation(async (input) => ({ - signature: await signatureOf(input.image.toString()), - })); - const built = await initService({ - testRunFindUniqueMock: jest.fn().mockResolvedValue({ ...testRun, testVariation: generateTestVariation() }), - testRunFindManyMock: jest.fn().mockResolvedValue(siblings), - projectFindUniqueMock: jest - .fn() - .mockResolvedValue({ bulkApproveVariations: true, bulkApproveGroupBy: 'customTags' }), - getImageBufferMock: jest.fn().mockImplementation(async (name: string) => Buffer.from(name)), - compareGetChangeSignatureMock, - }); - return { service: built, compareGetChangeSignatureMock }; - }; + it('updateIgnoreAreas', async () => { + const testRun = { id: 'testRunId', testVariationId: 'testVariationId', ignoreAreas: JSON.stringify(ignoreAreas) }; + const { service, m } = await build(); + m.testRun.update.mockResolvedValueOnce(testRun); + m.tv.update.mockResolvedValueOnce({ id: 'testVariationId', projectId: 'someProjectId' }); + findsVariation(m); + service.calculateDiff = jest.fn(); - it('matches same-palette, same-size siblings and skips different pattern / far larger change', async () => { - const testRun = sibling('target', { customTags: '' }); - const matching = sibling('locale-a'); - const bigger = sibling('locale-b', { diffPercent: 30 }); - const different = sibling('locale-c'); + await service.updateIgnoreAreas(testRun.id, ignoreAreas); - const built = await initMatchingService({ - testRun, - siblings: [matching, bigger, different], - signatureOf: (imageName) => (imageName.startsWith('locale-c') ? OTHER_PALETTE : SAME_PALETTE), + expect(m.testRun.update).toHaveBeenCalledWith({ + where: { id: testRun.id }, + data: { ignoreAreas: JSON.stringify(ignoreAreas) }, }); + expect(m.tv.update).toHaveBeenCalledWith(testRun.testVariationId, { ignoreAreas: testRun.ignoreAreas }); + expect(service.calculateDiff).toHaveBeenCalledWith('someProjectId', testRun); + }); - const result = await built.service.getMatchingVariations(testRun.id); + it('returns the recalculated run', async () => { + const recalculated = generateTestRun({ id: 'recalculated' }); + const { service, m } = await build(); + m.testRun.update.mockResolvedValueOnce(generateTestRun()); + m.tv.update.mockResolvedValueOnce(generateTestVariation({ projectId: 'p' })); + findsVariation(m, 'v'); + service.calculateDiff = jest.fn().mockResolvedValueOnce(recalculated); - expect(result.variations.map((variation) => variation.id)).toEqual([testRun.id, matching.id]); - expect(result.skipped.map((item) => ({ id: item.id, reason: item.reason }))).toEqual([ - { id: bigger.id, reason: 'different change size' }, - { id: different.id, reason: 'different change pattern' }, - ]); + expect(await service.updateIgnoreAreas('id', ignoreAreas)).toBe(recalculated); }); - // The whole point of storing it: a build's screenshots are fetched and - // decoded once at ingest, never again to group a screen at review time. - it('groups from the signatures stored at ingest, without decoding anything', async () => { - const signed = (id: string, signature: number[], overrides = {}) => - sibling(id, { - changeSignature: JSON.stringify({ threshold: 0.1, includeAA: true, signature }), - ...overrides, - }); - const testRun = signed('target', SAME_PALETTE, { customTags: '' }); - const matching = signed('locale-a', SAME_PALETTE); - const different = signed('locale-c', OTHER_PALETTE); + describe('addIgnoreAreas', () => { + const newArea = { x: 5, y: 6, width: 7, height: 8 }; - const built = await initMatchingService({ - testRun, - siblings: [matching, different], - signatureOf: () => { - throw new Error('should not be computing a signature'); - }, - }); + it.each([ + ['appends to the existing areas', JSON.stringify(ignoreAreas), [...ignoreAreas, newArea]], + ['starts from none when they are null', 'null', [newArea]], + ])('%s', async (_case, stored, expected) => { + const updated = generateTestRun({ id: 'updated' }); + const { service, m } = await build(); + m.testRun.findUnique.mockResolvedValueOnce(generateTestRun({ ignoreAreas: stored })); + service.updateIgnoreAreas = jest.fn().mockResolvedValueOnce(updated); - const result = await built.service.getMatchingVariations(testRun.id); + const result = await service.addIgnoreAreas('id', [newArea]); - expect(built.compareGetChangeSignatureMock).not.toHaveBeenCalled(); - expect(result.variations.map((variation) => variation.id)).toEqual([testRun.id, matching.id]); - expect(result.skipped.map((item) => item.reason)).toEqual(['different change pattern']); + expect(m.testRun.findUnique).toHaveBeenCalledWith({ where: { id: 'id' }, include: { testVariation: true } }); + expect(service.updateIgnoreAreas).toHaveBeenCalledWith('id', expected); + expect(result).toBe(updated); + }); }); - // A signature only means anything under the threshold it was computed with. - // The in-memory memo has always keyed on that; the stored one has to too, - // or changing a project's comparison config silently mixes signatures from - // two different settings and grouping quietly gets worse. - it('recomputes when the project config has moved on since ingest', async () => { - const stale = JSON.stringify({ threshold: 0.9, includeAA: false, signature: SAME_PALETTE }); - const testRun = sibling('target', { customTags: '', changeSignature: stale }); - const matching = sibling('locale-a', { changeSignature: stale }); + it('update', async () => { + const testRun = { id: 'testRunId', testVariationId: 'testVariationId' }; + const { service, m } = await build(); + m.testRun.update.mockResolvedValueOnce(testRun); + findsVariation(m); - const built = await initMatchingService({ - testRun, - siblings: [matching], - signatureOf: () => SAME_PALETTE, + await service.update(testRun.id, { comment: 'random comment' }); + + expect(m.testRun.update).toHaveBeenCalledWith({ + where: { id: testRun.id }, + data: { comment: 'random comment' }, }); + expect(m.tv.update).toHaveBeenCalledWith(testRun.testVariationId, { comment: 'random comment' }); + expect(m.events.testRunUpdated).toHaveBeenCalledWith(testRun); + }); + + it('returns the updated run', async () => { + const testRun = generateTestRun(); + const { service, m } = await build(); + m.testRun.update.mockResolvedValueOnce(testRun); + findsVariation(m, 'v'); - const result = await built.service.getMatchingVariations(testRun.id); + expect(await service.update(testRun.id, { comment: 'c' })).toBe(testRun); + }); + + it('update hands only the comment to the variation', async () => { + const { service, m } = await build(); + m.testRun.update.mockResolvedValueOnce(generateTestRun()); + findsVariation(m, 'v'); - const asked = built.compareGetChangeSignatureMock.mock.calls.map(([input]) => input.image.toString()); - expect(asked.sort()).toEqual([testRun.imageName, matching.imageName].sort()); - expect(result.variations.map((variation) => variation.id)).toEqual([testRun.id, matching.id]); + await service.update('id', { comment: 'c', baselineName: 'evil.png', ignoreAreas: 'not json' } as never); + + expect(m.tv.update).toHaveBeenCalledWith('v', { comment: 'c' }); }); - // A signature is a fixed-length histogram. One of the wrong length compared - // against one of the right length gives a similarity score computed over - // undefined entries — a number, silently meaningless. The same self- - // invalidating guard as the config stamp: if the shape moves on, recompute. - it.each([ - ['the wrong length', [1, 0]], - ['values that are not numbers', new Array(SIGNATURE_LENGTH).fill('nope')], - ])('recomputes a stored signature with %s', async (_case, signature) => { - const stored = JSON.stringify({ threshold: 0.1, includeAA: true, signature }); - const testRun = sibling('target', { customTags: '', changeSignature: stored }); + describe.each([ + ['update', (svc: TestRunsService) => svc.update('id', { comment: 'c' })], + ['updateIgnoreAreas', (svc: TestRunsService) => svc.updateIgnoreAreas('id', ignoreAreas)], + ])('%s of a run that cannot be edited', (_name, call) => { + it('fails with a conflict before writing anything when the run has no variation', async () => { + const { service, m } = await build(); + findsVariation(m, null); - const built = await initMatchingService({ - testRun, - siblings: [], - signatureOf: () => SAME_PALETTE, + const result = call(service); + + await expect(result).rejects.toBeInstanceOf(ConflictException); + await expect(result).rejects.toThrow(NO_VARIATION); + expect(m.testRun.update).not.toHaveBeenCalled(); }); - await built.service.getMatchingVariations(testRun.id); + it('fails with NotFoundException before writing anything for an unknown id', async () => { + const { service, m } = await build(); + m.testRun.findUnique.mockResolvedValue(null); - expect(built.compareGetChangeSignatureMock).toHaveBeenCalled(); + await expect(call(service)).rejects.toBeInstanceOf(NotFoundException); + expect(m.testRun.update).not.toHaveBeenCalled(); + }); }); + }); - // builds ingested before the column existed, and runs compared by something - // other than pixelmatch, still have to group - it('falls back to computing for a run with nothing stored', async () => { - const testRun = sibling('target', { - customTags: '', - changeSignature: JSON.stringify({ threshold: 0.1, includeAA: true, signature: SAME_PALETTE }), - }); - const unsigned = sibling('locale-a'); + describe('postTestRun', () => { + const createTestRequestDto = requestDto(); - const built = await initMatchingService({ - testRun, - siblings: [unsigned], - signatureOf: () => SAME_PALETTE, + const initPostService = async ({ + project = { ...TEST_PROJECT, autoApproveFeature: false }, + existingVariation = null, + previousRuns = [], + buildRow = { projectId: createTestRequestDto.projectId }, + }: { + project?: Project | null; + existingVariation?: TestVariation | null; + previousRuns?: TestRun[]; + buildRow?: { projectId: string } | null; + } = {}) => { + const testRun = generateTestRun(); + const diffed = generateTestRun({ status: TestStatus.unresolved }); + const createdVariation = generateTestVariation({ id: 'created variation' }); + const { service, m } = await build(); + m.project.findUnique.mockResolvedValue(project); + m.build.findUnique.mockResolvedValue(buildRow); + m.tv.find.mockResolvedValue(existingVariation); + m.tv.createOrTakeWinner.mockResolvedValue(createdVariation); + m.testRun.findMany.mockResolvedValue(previousRuns); + m.static.saveImage.mockResolvedValue('img.png'); + service.delete = jest.fn(); + const insert = jest.fn().mockResolvedValue(testRun); + service['insertTestRun'] = insert; + service.calculateDiff = jest.fn().mockResolvedValue(diffed); + service['tryAutoApproveByPastBaselines'] = jest.fn(); + service['tryAutoApproveByNewBaselines'] = jest.fn(); + return { service, m, testRun, diffed, createdVariation, insert }; + }; + type PostContext = Awaited>; + const post = (service: TestRunsService) => service.postTestRun({ createTestRequestDto, imageBuffer }); + + beforeEach(() => mocked(TestRunResultDto).mockClear()); + + it('postTestRun', async () => { + const request = requestDto({ + os: 'OS', + browser: 'browser', + viewport: 'viewport', + device: 'device', + customTags: 'customTags', + comment: 'new comment', + }); + const testVariation = generateTestVariation(); + const previousRun = generateTestRun({ id: 'previous' }); + const { service, m, insert, testRun, diffed } = await initPostService({ + project: TEST_PROJECT, + existingVariation: testVariation, + previousRuns: [previousRun], }); + const afterPast = generateTestRun({ id: 'after past' }); + const afterNew = generateTestRun({ id: 'after new' }); + service['tryAutoApproveByPastBaselines'] = jest.fn().mockResolvedValueOnce(afterPast); + service['tryAutoApproveByNewBaselines'] = jest.fn().mockResolvedValueOnce(afterNew); - const result = await built.service.getMatchingVariations(testRun.id); + await service.postTestRun({ createTestRequestDto: request, imageBuffer }); - const asked = built.compareGetChangeSignatureMock.mock.calls.map(([input]) => input.image.toString()); - expect(asked).toEqual([unsigned.imageName]); - expect(result.variations.map((variation) => variation.id)).toEqual([testRun.id, unsigned.id]); + expect(m.tv.find).toHaveBeenCalledWith(request, TEST_PROJECT.mainBranchName); + expect(m.testRun.findMany).toHaveBeenCalledWith({ + where: { + buildId: request.buildId, + ...getTestVariationUniqueData(request), + branchName: request.branchName, + NOT: { OR: [{ status: TestStatus.approved }, { status: TestStatus.autoApproved }] }, + }, + }); + expect(service.delete).toHaveBeenCalledWith(previousRun.id); + expect(insert).toHaveBeenCalledWith({ testVariation, createTestRequestDto: request, imageName: 'img.png' }); + expect(service.calculateDiff).toHaveBeenCalledWith(request.projectId, testRun, imageBuffer, TEST_PROJECT); + expect(service['tryAutoApproveByPastBaselines']).toHaveBeenCalledWith({ + testVariation, + testRun: diffed, + project: TEST_PROJECT, + imageBuffer, + }); + expect(service['tryAutoApproveByNewBaselines']).toHaveBeenCalledWith({ + testVariation, + testRun: afterPast, + project: TEST_PROJECT, + imageBuffer, + }); + expect(mocked(TestRunResultDto)).toHaveBeenCalledWith(afterNew, testVariation); }); - it('skips a far larger change without paying for its signature', async () => { - const testRun = sibling('target', { customTags: '' }); - const bigger = sibling('locale-b', { diffPercent: 30 }); + it('creates the test variation when there is none yet', async () => { + const { service, m, testRun, diffed, createdVariation, insert } = await initPostService(); + + await post(service); - const built = await initMatchingService({ + expect(m.tv.createOrTakeWinner).toHaveBeenCalledWith(createTestRequestDto); + expect(m.static.saveImage).toHaveBeenCalledWith('screenshot', imageBuffer); + expect(insert).toHaveBeenCalledWith({ + testVariation: createdVariation, + createTestRequestDto, + imageName: 'img.png', + }); + expect(service.calculateDiff).toHaveBeenCalledWith( + createTestRequestDto.projectId, testRun, - siblings: [bigger], - signatureOf: () => SAME_PALETTE, + imageBuffer, + expect.objectContaining({ id: TEST_PROJECT.id }) + ); + expect(mocked(TestRunResultDto)).toHaveBeenCalledWith(diffed, createdVariation); + }); + + it('deletes nothing when there is no previous unapproved run', async () => { + const { service } = await initPostService({ existingVariation: generateTestVariation() }); + + await post(service); + + expect(service.delete).not.toHaveBeenCalled(); + }); + + it('only deletes the first previous unapproved run', async () => { + const { service } = await initPostService({ + existingVariation: generateTestVariation(), + previousRuns: [generateTestRun({ id: 'first' }), generateTestRun({ id: 'second' })], }); - await built.service.getMatchingVariations(testRun.id); + await post(service); - const asked = built.compareGetChangeSignatureMock.mock.calls.map(([input]) => input.image.toString()); - expect(asked).toEqual([testRun.imageName]); + expect(service.delete).toHaveBeenCalledTimes(1); + expect(service.delete).toHaveBeenCalledWith('first'); }); - it('says a sibling has no diff rather than blaming its change size', async () => { - const testRun = sibling('target', { customTags: '' }); - const unchanged = sibling('locale-b', { diffPercent: 0 }); + it('skips auto approval when the project has it turned off', async () => { + const variation = generateTestVariation(); + const { service, diffed } = await initPostService({ existingVariation: variation }); - const built = await initMatchingService({ - testRun, - siblings: [unchanged], - signatureOf: () => SAME_PALETTE, - }); + await post(service); + + expect(service['tryAutoApproveByPastBaselines']).not.toHaveBeenCalled(); + expect(service['tryAutoApproveByNewBaselines']).not.toHaveBeenCalled(); + expect(mocked(TestRunResultDto)).toHaveBeenCalledWith(diffed, variation); + }); + + it('chains the two auto approvals, each one seeing the result of the previous step', async () => { + const variation = generateTestVariation(); + const project = { ...TEST_PROJECT, autoApproveFeature: true }; + const { service, diffed } = await initPostService({ project, existingVariation: variation }); + const afterPast = generateTestRun({ id: 'after past' }); + const afterNew = generateTestRun({ id: 'after new' }); + service['tryAutoApproveByPastBaselines'] = jest.fn().mockResolvedValueOnce(afterPast); + service['tryAutoApproveByNewBaselines'] = jest.fn().mockResolvedValueOnce(afterNew); - const result = await built.service.getMatchingVariations(testRun.id); + await post(service); - expect(result.skipped.map((item) => item.reason)).toEqual(['no diff to match']); + expect(service['tryAutoApproveByPastBaselines']).toHaveBeenCalledWith({ + testVariation: variation, + testRun: diffed, + project, + imageBuffer, + }); + expect(service['tryAutoApproveByNewBaselines']).toHaveBeenCalledWith({ + testVariation: variation, + testRun: afterPast, + project, + imageBuffer, + }); + expect(mocked(TestRunResultDto)).toHaveBeenCalledWith(afterNew, variation); }); - it('asks for the siblings signatures at once rather than one after another', async () => { - const testRun = sibling('target', { customTags: '' }); - const siblings = ['locale-a', 'locale-b', 'locale-c'].map((id) => sibling(id)); + it('throws NotFoundException before storing anything when the project does not exist', async () => { + const { service, m } = await initPostService({ project: null, existingVariation: generateTestVariation() }); - let inFlight = 0; - let peakInFlight = 0; - const built = await initMatchingService({ - testRun, - siblings, - signatureOf: async () => { - inFlight += 1; - peakInFlight = Math.max(peakInFlight, inFlight); - await new Promise((resolve) => setTimeout(resolve, 0)); - inFlight -= 1; - return SAME_PALETTE; - }, + await expect(post(service)).rejects.toThrow( + new NotFoundException(`Project not found: ${createTestRequestDto.projectId}`) + ); + expect(m.static.saveImage).not.toHaveBeenCalled(); + }); + + describe('storing the screenshot alongside the lookups', () => { + it('stores the screenshot while the variation is being looked up', async () => { + let resolve!: (value: TestVariation) => void; + const { service, m } = await initPostService(); + m.tv.find.mockReturnValueOnce(new Promise((r) => (resolve = r))); + + const posting = post(service); + await new Promise(setImmediate); + + expect(m.static.saveImage).toHaveBeenCalledWith('screenshot', imageBuffer); + resolve(generateTestVariation()); + await posting; + }); + + it.each<[string, Parameters[0], (ctx: PostContext, failure: Error) => void]>([ + ['the variation lookup fails', {}, ({ m }, failure) => m.tv.find.mockRejectedValueOnce(failure)], + [ + 'the previous run lookup fails', + { existingVariation: generateTestVariation() }, + ({ m }, failure) => m.testRun.findMany.mockRejectedValueOnce(failure), + ], + [ + 'the previous run cannot be deleted', + { existingVariation: generateTestVariation(), previousRuns: [generateTestRun({ id: 'previous' })] }, + ({ service }, failure) => { + service.delete = jest.fn().mockRejectedValueOnce(failure); + }, + ], + ])('removes the stored screenshot when %s', async (_case, options, arrange) => { + const failure = new Error('lookup failed'); + const ctx = await initPostService(options); + arrange(ctx, failure); + + await expect(post(ctx.service)).rejects.toBe(failure); + + expect(ctx.m.static.deleteImage).toHaveBeenCalledWith('img.png'); + expect(ctx.insert).not.toHaveBeenCalled(); }); - await built.service.getMatchingVariations(testRun.id); + it('passes the save error on without removing anything', async () => { + const failure = new Error('not a png'); + const { service, m, insert } = await initPostService({ existingVariation: generateTestVariation() }); + m.static.saveImage.mockRejectedValueOnce(failure); - expect(peakInFlight).toBe(siblings.length + 1); - }); + await expect(post(service)).rejects.toBe(failure); - it('keeps the number of images being decoded at once bounded', async () => { - const testRun = sibling('target', { customTags: '' }); - const siblings = Array.from({ length: SIGNATURE_CONCURRENCY * 3 }, (_, index) => sibling(`locale-${index}`)); + expect(m.static.deleteImage).not.toHaveBeenCalled(); + expect(insert).not.toHaveBeenCalled(); + }); - let inFlight = 0; - let peakInFlight = 0; - const built = await initMatchingService({ - testRun, - siblings, - signatureOf: async () => { - inFlight += 1; - peakInFlight = Math.max(peakInFlight, inFlight); - await new Promise((resolve) => setTimeout(resolve, 0)); - inFlight -= 1; - return SAME_PALETTE; - }, + it('removes the stored screenshot when the insert fails', async () => { + const failure = new Error('insert failed'); + const { service, m, insert } = await initPostService({ existingVariation: generateTestVariation() }); + insert.mockRejectedValueOnce(failure); + + await expect(post(service)).rejects.toBe(failure); + + expect(m.static.deleteImage).toHaveBeenCalledWith('img.png'); }); - await built.service.getMatchingVariations(testRun.id); + it('deletes the previous unapproved run before inserting', async () => { + const order: string[] = []; + const { service, insert } = await initPostService({ + existingVariation: generateTestVariation(), + previousRuns: [generateTestRun({ id: 'previous' })], + }); + service.delete = jest.fn(async (id: string) => { + order.push(`delete ${id}`); + return undefined; + }); + insert.mockImplementationOnce(async () => { + order.push('insert'); + return generateTestRun(); + }); - expect(peakInFlight).toBe(SIGNATURE_CONCURRENCY); + await post(service); + + expect(order).toEqual(['delete previous', 'insert']); + }); }); - it('reuses the signatures it already computed when the dialog is reopened', async () => { - const testRun = sibling('target', { customTags: '' }); - const matching = sibling('locale-a'); + it('postTestRun hands the uploaded bytes to the diff', async () => { + const { service, testRun } = await initPostService({ existingVariation: generateTestVariation() }); + + await post(service); - const built = await initMatchingService({ + expect(service.calculateDiff).toHaveBeenCalledWith( + createTestRequestDto.projectId, testRun, - siblings: [matching], - signatureOf: () => SAME_PALETTE, - }); + imageBuffer, + expect.objectContaining({ id: TEST_PROJECT.id }) + ); + expect(service.calculateDiff).toHaveBeenCalledTimes(1); + }); - await built.service.getMatchingVariations(testRun.id); - const afterFirst = built.compareGetChangeSignatureMock.mock.calls.length; - const second = await built.service.getMatchingVariations(testRun.id); + it('rejects an unknown build before storing anything', async () => { + const { service, m } = await initPostService({ buildRow: null, existingVariation: generateTestVariation() }); - expect(afterFirst).toBe(2); - expect(built.compareGetChangeSignatureMock).toHaveBeenCalledTimes(2); - expect(second.variations.map((variation) => variation.id)).toEqual([testRun.id, matching.id]); + await expect(post(service)).rejects.toThrow( + new NotFoundException(`Build not found: ${createTestRequestDto.buildId}`) + ); + expect(m.static.saveImage).not.toHaveBeenCalled(); }); - it('recomputes a signature once the run gained an ignore area', async () => { - const testRun = sibling('target', { customTags: '' }); - const matching = sibling('locale-a'); + describe('a build of another project', () => { + const message = `Build ${createTestRequestDto.buildId} does not belong to project ${createTestRequestDto.projectId}`; - const built = await initMatchingService({ - testRun, - siblings: [matching], - signatureOf: () => SAME_PALETTE, + it('is accepted with a warning', async () => { + const warn = jest.spyOn(Logger.prototype, 'warn').mockImplementation(); + const { service, diffed, createdVariation } = await initPostService({ buildRow: { projectId: 'other' } }); + + await post(service); + + expect(warn).toHaveBeenCalledWith(message); + expect(mocked(TestRunResultDto)).toHaveBeenCalledWith(diffed, createdVariation); + warn.mockRestore(); }); + }); - await built.service.getMatchingVariations(testRun.id); - matching.ignoreAreas = JSON.stringify([{ x: 0, y: 0, width: 10, height: 10 }]); - await built.service.getMatchingVariations(testRun.id); + it("fails before storing anything when the project's comparator cannot run here", async () => { + const { service, m, insert } = await initPostService({ existingVariation: generateTestVariation() }); + m.compare.getComparator.mockImplementation(() => { + throw new Error('Odiff can only be used with HDD static service.'); + }); - expect(built.compareGetChangeSignatureMock).toHaveBeenCalledTimes(3); + await expect(post(service)).rejects.toThrow('Odiff can only be used with HDD static service.'); + expect(m.static.saveImage).not.toHaveBeenCalled(); + expect(insert).not.toHaveBeenCalled(); + expect(service.delete).not.toHaveBeenCalled(); }); }); - describe('postTestRun branches', () => { - const createTestRequestDto: CreateTestRequestDto = { - buildId: 'buildId', - projectId: 'projectId', - name: 'Test name', - branchName: 'develop', + describe('reject', () => { + const initRejectService = async (before: TestRun | null, after?: TestRun, count = 1) => { + const ctx = await build(); + ctx.m.testRun.findUnique.mockResolvedValueOnce(before).mockResolvedValueOnce(after); + ctx.m.testRun.updateMany.mockResolvedValue({ count }); + return ctx; }; - const initPostService = async ({ - project = { ...TEST_PROJECT, autoApproveFeature: false } as Project | null, - existingVariation = null as TestVariation | null, - previousRuns = [] as TestRun[], - }) => { - const testRun = generateTestRun(); - const diffed = generateTestRun({ status: TestStatus.unresolved }); - const createdVariation = generateTestVariation({ id: 'created variation' }); - const testVariationServiceCreateMock = jest.fn().mockResolvedValueOnce(createdVariation); - const testRunFindManyMock = jest.fn().mockResolvedValueOnce(previousRuns); - const built = await initService({ - projectFindUniqueMock: jest.fn().mockResolvedValueOnce(project), - testVariationFindMock: jest.fn().mockResolvedValueOnce(existingVariation), - testVariationServiceCreateMock, - testRunFindManyMock, - }); - built.delete = jest.fn(); - built.create = jest.fn().mockResolvedValueOnce(testRun); - built.calculateDiff = jest.fn().mockResolvedValueOnce(diffed); - built['tryAutoApproveByPastBaselines'] = jest.fn(); - built['tryAutoApproveByNewBaselines'] = jest.fn(); - return { built, testRun, diffed, createdVariation, testVariationServiceCreateMock, testRunFindManyMock }; - }; + it('reject records the previous status', async () => { + const run = generateTestRun({ status: TestStatus.unresolved }); + const rejected = { ...run, status: TestStatus.failed, statusBeforeDecision: TestStatus.unresolved }; + const { service, m } = await initRejectService(run, rejected); - beforeEach(() => mocked(TestRunResultDto).mockClear()); + const result = await service.reject(run.id); - it('creates the test variation when there is none yet', async () => { - const { built, testRun, diffed, createdVariation, testVariationServiceCreateMock } = await initPostService({}); + expect(m.testRun.updateMany).toHaveBeenCalledWith({ + where: { id: run.id, status: TestStatus.unresolved }, + data: { status: TestStatus.failed, statusBeforeDecision: TestStatus.unresolved }, + }); + expect(m.events.testRunUpdated).toHaveBeenCalledWith(rejected); + expect(result).toBe(rejected); + }); - await built.postTestRun({ createTestRequestDto, imageBuffer }); + it('rejecting a failed run changes nothing', async () => { + const run = generateTestRun({ status: TestStatus.failed }); + const { service, m } = await initRejectService(run); - expect(testVariationServiceCreateMock).toHaveBeenCalledWith({ createTestRequestDto }); - expect(built.create).toHaveBeenCalledWith({ testVariation: createdVariation, createTestRequestDto, imageBuffer }); - expect(built.calculateDiff).toHaveBeenCalledWith(createTestRequestDto.projectId, testRun); - expect(mocked(TestRunResultDto)).toHaveBeenCalledWith(diffed, createdVariation); + const result = await service.reject(run.id); + + expect(m.testRun.updateMany).not.toHaveBeenCalled(); + expect(m.events.testRunUpdated).not.toHaveBeenCalled(); + expect(result).toBe(run); }); - it('deletes nothing when there is no previous unapproved run', async () => { - const { built } = await initPostService({ existingVariation: generateTestVariation() }); + it('reject of an unknown run is a 404', async () => { + const { service, m } = await initRejectService(null); - await built.postTestRun({ createTestRequestDto, imageBuffer }); + await expect(service.reject('missing')).rejects.toThrow(new NotFoundException('Test run not found: missing')); - expect(built.delete).not.toHaveBeenCalled(); + expect(m.testRun.updateMany).not.toHaveBeenCalled(); }); - it('only deletes the first previous unapproved run', async () => { - const first = generateTestRun({ id: 'first' }); - const second = generateTestRun({ id: 'second' }); - const { built } = await initPostService({ - existingVariation: generateTestVariation(), - previousRuns: [first, second], + it('a lost race emits nothing', async () => { + const run = generateTestRun({ status: TestStatus.unresolved }); + const current = { ...run, status: TestStatus.approved }; + const { service, m } = await initRejectService(run, current, 0); + + const result = await service.reject(run.id); + + expect(m.events.testRunUpdated).not.toHaveBeenCalled(); + expect(result).toBe(current); + }); + }); + + describe('revert', () => { + const approvedRun = (overrides: Partial = {}) => + generateTestRun({ + id: 'run-1', + status: TestStatus.approved, + statusBeforeDecision: TestStatus.unresolved, + branchName: 'master', + baselineBranchName: 'master', + projectId: 'project', + ...overrides, }); + const variationOf = (overrides: Partial = {}) => + generateTestVariation({ id: 'variation', projectId: 'project', baselineName: 'new.png', ...overrides }, []); + const baselineOf = (overrides: Partial = {}) => + generateBaseline({ + id: 'baseline-1', + testRunId: 'run-1', + testVariationId: 'variation', + baselineName: 'new.png', + ...overrides, + }); + const makeTx = () => ({ + testRun: { updateMany: jest.fn().mockResolvedValue({ count: 1 }), count: jest.fn().mockResolvedValue(0) }, + baseline: { + findMany: jest.fn().mockResolvedValue([]), + findUnique: jest.fn().mockResolvedValue(null), + findFirst: jest.fn().mockResolvedValue(null), + delete: jest.fn(), + count: jest.fn().mockResolvedValue(0), + }, + testVariation: { findUnique: jest.fn().mockResolvedValue(null), update: jest.fn(), delete: jest.fn() }, + }); + const claim = (run: TestRun, data: object) => ({ where: { id: run.id, status: run.status }, data }); + const restored = (status: TestStatus, extra: object = {}) => ({ status, statusBeforeDecision: null, ...extra }); + + const initRevertService = async (runs: TestRun[], counts: [number, number, number] = [0, 0, 0]) => { + const ctx = await build(); + const { m } = ctx; + const tx = makeTx(); + m.tx = tx; + m.testRun.findMany.mockResolvedValue(runs); + m.testRun.findUnique.mockImplementation((args: { where: { id: string } }) => + Promise.resolve({ ...runs.find((run) => run.id === args.where.id), testVariation: null }) + ); + m.testRun.count.mockResolvedValue(counts[0]); + m.testVariation.count.mockResolvedValue(counts[1]); + m.baseline.count.mockResolvedValue(counts[2]); + return { ...ctx, tx }; + }; + type RevertContext = Awaited>; + + const givenApproval = ({ tx }: RevertContext, previousName: string | null = 'old.png', branchName = 'master') => { + tx.baseline.findUnique.mockResolvedValue(baselineOf()); + tx.baseline.findFirst + .mockResolvedValueOnce(baselineOf()) + .mockResolvedValueOnce(previousName && baselineOf({ id: 'baseline-0', baselineName: previousName })); + tx.testVariation.findUnique.mockResolvedValue(variationOf({ branchName })); + }; - await built.postTestRun({ createTestRequestDto, imageBuffer }); + it('reverts a rejected run to the status it was decided from', async () => { + const run = generateTestRun({ status: TestStatus.failed, statusBeforeDecision: TestStatus.ok }); + const { service, tx } = await initRevertService([run]); - expect(built.delete).toHaveBeenCalledTimes(1); - expect(built.delete).toHaveBeenCalledWith('first'); + await service.revert([run.id]); + + expect(tx.testRun.updateMany).toHaveBeenCalledWith(claim(run, restored(TestStatus.ok))); + expect(tx.baseline.findUnique).not.toHaveBeenCalled(); }); - it('skips auto approval when the project has it turned off', async () => { - const variation = generateTestVariation(); - const { built, diffed } = await initPostService({ existingVariation: variation }); + it.each([ + ['has a baseline', 'baseline.png', TestStatus.unresolved], + ['has none', null, TestStatus.new], + ])( + 'a rejected run without a recorded status falls back by its baseline: run that %s', + async (_title, baselineName, expected) => { + const run = generateTestRun({ status: TestStatus.failed, statusBeforeDecision: null, baselineName }); + const { service, tx } = await initRevertService([run]); - await built.postTestRun({ createTestRequestDto, imageBuffer }); + await service.revert([run.id]); - expect(built['tryAutoApproveByPastBaselines']).not.toHaveBeenCalled(); - expect(built['tryAutoApproveByNewBaselines']).not.toHaveBeenCalled(); - expect(mocked(TestRunResultDto)).toHaveBeenCalledWith(diffed, variation); + expect(tx.testRun.updateMany).toHaveBeenCalledWith(claim(run, restored(expected))); + } + ); + + it('reverts a same-branch approval: deletes the baseline row, restores the previous baselineName and the status', async () => { + const run = approvedRun(); + const ctx = await initRevertService([run]); + const { service, m, tx } = ctx; + givenApproval(ctx); + + await service.revert([run.id]); + + expect(tx.baseline.findUnique).toHaveBeenCalledWith({ where: { testRunId: run.id } }); + expect(tx.baseline.findFirst).toHaveBeenNthCalledWith(1, { + where: { testVariationId: 'variation' }, + orderBy: { createdAt: 'desc' }, + }); + expect(tx.baseline.delete).toHaveBeenCalledWith({ where: { id: 'baseline-1' } }); + expect(tx.testVariation.update).toHaveBeenCalledWith({ + where: { id: 'variation' }, + data: { baselineName: 'old.png' }, + }); + expect(tx.testRun.updateMany).toHaveBeenCalledWith(claim(run, restored(TestStatus.unresolved))); + expect(m.tv.findUnique).not.toHaveBeenCalled(); }); - it('chains the two auto approvals, each one seeing the result of the previous step', async () => { - const variation = generateTestVariation(); - const { built, diffed } = await initPostService({ - project: { ...TEST_PROJECT, autoApproveFeature: true }, - existingVariation: variation, + it('reverting the first approval of a screen leaves the variation without a baseline', async () => { + const run = approvedRun(); + const ctx = await initRevertService([run]); + givenApproval(ctx, null); + + await ctx.service.revert([run.id]); + + expect(ctx.tx.testVariation.update).toHaveBeenCalledWith({ + where: { id: 'variation' }, + data: { baselineName: null }, }); - const afterPast = generateTestRun({ id: 'after past' }); - const afterNew = generateTestRun({ id: 'after new' }); - built['tryAutoApproveByPastBaselines'] = jest.fn().mockResolvedValueOnce(afterPast); - built['tryAutoApproveByNewBaselines'] = jest.fn().mockResolvedValueOnce(afterNew); + }); - await built.postTestRun({ createTestRequestDto, imageBuffer }); + it('reverts a merge approval into the main variation without moving the run', async () => { + const run = approvedRun({ branchName: 'feature', baselineBranchName: 'master', merge: true }); + const ctx = await initRevertService([run]); + const { tx } = ctx; + givenApproval(ctx, null); - expect(built['tryAutoApproveByPastBaselines']).toHaveBeenCalledWith({ - testVariation: variation, - testRun: diffed, + await ctx.service.revert([run.id]); + + expect(tx.testVariation.update).toHaveBeenCalledWith({ + where: { id: 'variation' }, + data: { baselineName: null }, }); - expect(built['tryAutoApproveByNewBaselines']).toHaveBeenCalledWith({ - testVariation: variation, - testRun: afterPast, + expect(tx.testRun.updateMany).toHaveBeenCalledWith(claim(run, restored(TestStatus.unresolved))); + expect(tx.testVariation.delete).not.toHaveBeenCalled(); + }); + + describe('of a feature-branch approval', () => { + const featureRun = () => approvedRun({ branchName: 'feature', baselineBranchName: 'master' }); + const mainVariation = generateTestVariation({ id: 'main variation', branchName: 'master' }, []); + + const givenFeatureApproval = (ctx: RevertContext, previousName: string | null = null) => { + givenApproval(ctx, previousName, 'feature'); + ctx.m.tv.findUnique.mockResolvedValue(mainVariation); + }; + + it('moves the run back to the main variation and deletes the feature variation it created', async () => { + const run = featureRun(); + const ctx = await initRevertService([run]); + const { service, m, tx } = ctx; + givenFeatureApproval(ctx); + + await service.revert([run.id]); + + expect(m.tv.findUnique).toHaveBeenCalledWith( + { projectId: 'project', branchName: 'master', ...getTestVariationUniqueData(run) }, + tx + ); + expect(tx.testRun.updateMany).toHaveBeenCalledWith( + claim(run, restored(TestStatus.unresolved, { testVariationId: 'main variation' })) + ); + expect(tx.testRun.count).toHaveBeenCalledWith({ where: { testVariationId: 'variation' } }); + expect(tx.baseline.count).toHaveBeenCalledWith({ where: { testVariationId: 'variation' } }); + expect(tx.testVariation.delete).toHaveBeenCalledWith({ where: { id: 'variation' } }); + expect(tx.testVariation.update).not.toHaveBeenCalled(); + }); + + it.each([ + ['other runs', 1, 0], + ['other baselines', 0, 1], + ])( + 'keeps the feature variation when it has %s, restoring its previous baselineName', + async (_title, runCount, baselineCount) => { + const run = featureRun(); + const ctx = await initRevertService([run]); + const { tx } = ctx; + givenFeatureApproval(ctx, 'old.png'); + tx.testRun.count.mockResolvedValue(runCount); + tx.baseline.count.mockResolvedValue(baselineCount); + + await ctx.service.revert([run.id]); + + expect(tx.testVariation.delete).not.toHaveBeenCalled(); + expect(tx.testVariation.update).toHaveBeenCalledWith({ + where: { id: 'variation' }, + data: { baselineName: 'old.png' }, + }); + } + ); + + it('409 when the main variation was deleted', async () => { + const run = featureRun(); + const ctx = await initRevertService([run]); + const { service, m, tx } = ctx; + givenFeatureApproval(ctx); + m.tv.findUnique.mockResolvedValue(null); + + await expect(service.revert([run.id])).rejects.toThrow(new ConflictException(NO_VARIATION)); + + expect(tx.testRun.updateMany).not.toHaveBeenCalled(); + expect(m.static.deleteImage).not.toHaveBeenCalled(); + expect(m.events.testRunUpdated).not.toHaveBeenCalled(); }); - expect(mocked(TestRunResultDto)).toHaveBeenCalledWith(afterNew, variation); }); - it('throws NotFoundException before storing anything when the project does not exist', async () => { - const { built } = await initPostService({ project: null, existingVariation: generateTestVariation() }); + it.each([ + ['a later baseline exists', () => baselineOf({ id: 'baseline-2' }), 'new.png'], + ['the variation points at another file', () => baselineOf(), 'other.png'], + ])('409 when a later approval moved the baseline on: %s', async (_title, latest, variationBaselineName) => { + const run = approvedRun(); + const { service, m, tx } = await initRevertService([run]); + tx.baseline.findUnique.mockResolvedValue(baselineOf()); + tx.baseline.findFirst.mockResolvedValue(latest()); + tx.testVariation.findUnique.mockResolvedValue(variationOf({ baselineName: variationBaselineName })); + + await expect(service.revert([run.id])).rejects.toThrow( + new ConflictException(`Test run ${run.id}: a later approval changed this baseline`) + ); - await expect(built.postTestRun({ createTestRequestDto, imageBuffer })).rejects.toThrow( - new NotFoundException(`Project not found: ${createTestRequestDto.projectId}`) + expect(tx.baseline.delete).not.toHaveBeenCalled(); + expect(tx.testVariation.update).not.toHaveBeenCalled(); + expect(tx.testRun.updateMany).not.toHaveBeenCalled(); + expect(m.static.deleteImage).not.toHaveBeenCalled(); + expect(m.events.testRunUpdated).not.toHaveBeenCalled(); + }); + + it.each([TestStatus.autoApproved, TestStatus.new, TestStatus.unresolved, TestStatus.ok])( + '409 for a %s run', + async (status) => { + const run = generateTestRun({ status }); + const { service, m, tx } = await initRevertService([run]); + + await expect(service.revert([run.id])).rejects.toThrow( + new ConflictException(`Test run ${run.id} is ${status}: only approved or rejected runs can be reverted`) + ); + + expect(tx.testRun.updateMany).not.toHaveBeenCalled(); + expect(m.events.testRunUpdated).not.toHaveBeenCalled(); + } + ); + + it('409 when the approval has no baseline row', async () => { + const run = approvedRun(); + const { service, tx } = await initRevertService([run]); + + await expect(service.revert([run.id])).rejects.toThrow( + new ConflictException(`Test run ${run.id} has no baseline to revert`) ); - expect(built.create).not.toHaveBeenCalled(); + + expect(tx.testRun.updateMany).not.toHaveBeenCalled(); }); - it("fails before storing anything when the project's comparator cannot run here", async () => { - const { built } = await initPostService({ existingVariation: generateTestVariation() }); - built['compareService'].getComparator = jest.fn(() => { - throw new Error('Odiff can only be used with HDD static service.'); + it('409 for an empty list', async () => { + const { service, m } = await initRevertService([]); + + await expect(service.revert([])).rejects.toThrow(new ConflictException('Nothing to revert')); + + expect(m.testRun.findMany).not.toHaveBeenCalled(); + expect(m.transaction).not.toHaveBeenCalled(); + }); + + it('404 for an unknown id, and then no transaction runs', async () => { + const run = generateTestRun({ status: TestStatus.failed }); + const { service, m } = await initRevertService([run]); + + await expect(service.revert([run.id, 'missing 1', 'missing 2'])).rejects.toThrow( + new NotFoundException('Test run not found: missing 1, missing 2') + ); + + expect(m.transaction).not.toHaveBeenCalled(); + }); + + it('409 when the run changed status meanwhile', async () => { + const run = generateTestRun({ status: TestStatus.failed }); + const { service, m, tx } = await initRevertService([run]); + tx.testRun.updateMany.mockResolvedValue({ count: 0 }); + + await expect(service.revert([run.id])).rejects.toThrow( + new ConflictException(`Test run ${run.id} changed while reverting, try again`) + ); + + expect(m.events.testRunUpdated).not.toHaveBeenCalled(); + }); + + describe('the freed baseline file', () => { + it.each([ + ['a run compared against it', [1, 0, 0]], + ['a variation pointing at it', [0, 1, 0]], + ['another baseline row', [0, 0, 1]], + ] as [string, [number, number, number]][])('is kept while %s', async (_title, counts) => { + const run = approvedRun(); + const ctx = await initRevertService([run], counts); + givenApproval(ctx); + + await ctx.service.revert([run.id]); + + expect(ctx.m.static.deleteImage).not.toHaveBeenCalled(); }); - await expect(built.postTestRun({ createTestRequestDto, imageBuffer })).rejects.toThrow( - 'Odiff can only be used with HDD static service.' + it('is deleted, after the commit, when nothing references it', async () => { + const run = approvedRun({ name: 'screen' }); + const ctx = await initRevertService([run]); + const { service, m } = ctx; + givenApproval(ctx); + + await service.revert([run.id]); + + expect(m.testRun.count).toHaveBeenCalledWith({ where: { name: 'screen', baselineName: 'new.png' } }); + expect(m.testVariation.count).toHaveBeenCalledWith({ where: { baselineName: 'new.png' } }); + expect(m.baseline.count).toHaveBeenCalledWith({ where: { baselineName: 'new.png' } }); + expect(m.static.deleteImage).toHaveBeenCalledTimes(1); + expect(m.static.deleteImage).toHaveBeenCalledWith('new.png'); + expect(m.static.deleteImage.mock.invocationCallOrder[0]).toBeGreaterThan( + m.transaction.mock.invocationCallOrder[0] + ); + }); + + it('is not touched when a rejection is reverted', async () => { + const run = generateTestRun({ status: TestStatus.failed }); + const { service, m } = await initRevertService([run]); + + await service.revert([run.id]); + + expect(m.static.deleteImage).not.toHaveBeenCalled(); + }); + }); + + it('emits testRun_updated for every run after the commit, nothing on a 409', async () => { + const rejected = generateTestRun({ id: 'run-2', status: TestStatus.failed }); + const approved = approvedRun(); + const ctx = await initRevertService([rejected, approved]); + const { service, m } = ctx; + givenApproval(ctx); + m.testRun.findMany + .mockResolvedValueOnce([rejected, approved]) + .mockResolvedValueOnce([{ ...approved, status: TestStatus.unresolved }, rejected]); + + await service.revert(['run-2', approved.id, 'run-2']); + + expect(m.testRun.findMany).toHaveBeenLastCalledWith({ where: { id: { in: ['run-2', approved.id] } } }); + expect(m.testRun.findUnique).not.toHaveBeenCalled(); + expect(m.events.testRunUpdated).toHaveBeenCalledWith( + expect.objectContaining({ id: approved.id, status: TestStatus.unresolved }) + ); + expect(m.events.testRunUpdated).toHaveBeenCalledTimes(2); + expect(m.events.testRunUpdated).toHaveBeenNthCalledWith(1, expect.objectContaining({ id: 'run-2' })); + expect(m.events.testRunUpdated).toHaveBeenNthCalledWith(2, expect.objectContaining({ id: approved.id })); + expect(m.events.testRunUpdated.mock.invocationCallOrder[0]).toBeGreaterThan( + m.transaction.mock.invocationCallOrder[0] ); - expect(built.create).not.toHaveBeenCalled(); - expect(built.delete).not.toHaveBeenCalled(); + + const failing = await initRevertService([generateTestRun({ status: TestStatus.new })]); + await expect(failing.service.revert(['10fb5e02-64e0-4cf5-9f17-c00ab3c96658'])).rejects.toThrow(ConflictException); + expect(failing.m.events.testRunUpdated).not.toHaveBeenCalled(); + }); + + it('reverts two approvals of one screen in one request, latest first', async () => { + const older = approvedRun({ id: 'run-1' }); + const newer = approvedRun({ id: 'run-2' }); + const baselineOlder = baselineOf({ baselineName: 'first.png', createdAt: new Date(1000) }); + const baselineNewer = baselineOf({ + id: 'baseline-2', + testRunId: 'run-2', + baselineName: 'second.png', + createdAt: new Date(2000), + }); + const { service, m, tx } = await initRevertService([older, newer]); + tx.baseline.findMany.mockResolvedValue([baselineOlder, baselineNewer]); + tx.baseline.findUnique.mockImplementation(({ where }: { where: { testRunId: string } }) => + Promise.resolve(where.testRunId === 'run-1' ? baselineOlder : baselineNewer) + ); + tx.baseline.findFirst + .mockResolvedValueOnce(baselineNewer) + .mockResolvedValueOnce(baselineOlder) + .mockResolvedValueOnce(baselineOlder) + .mockResolvedValueOnce(null); + tx.testVariation.findUnique + .mockResolvedValueOnce(variationOf({ baselineName: 'second.png' })) + .mockResolvedValueOnce(variationOf({ baselineName: 'first.png' })); + + await service.revert(['run-1', 'run-2']); + + expect(tx.baseline.findMany).toHaveBeenCalledWith({ + where: { testRunId: { in: ['run-1', 'run-2'] } }, + select: { testRunId: true, createdAt: true }, + }); + expect(tx.baseline.delete.mock.calls).toEqual([ + [{ where: { id: 'baseline-2' } }], + [{ where: { id: 'baseline-1' } }], + ]); + expect(tx.testVariation.update.mock.calls).toEqual([ + [{ where: { id: 'variation' }, data: { baselineName: 'first.png' } }], + [{ where: { id: 'variation' }, data: { baselineName: null } }], + ]); + expect(m.static.deleteImage.mock.calls).toEqual([['second.png'], ['first.png']]); }); }); describe('approve', () => { const copiedBaselineName = 'copied.baseline.png'; - const initApproveService = async (testRun: object | null) => { - const testRunFindUniqueMock = jest.fn().mockResolvedValue(testRun); - const testRunUpdateMock = jest.fn().mockResolvedValue(testRun); - const copyImageMock = jest.fn().mockResolvedValue(copiedBaselineName); - const testVariationServiceFindUniqueMock = jest.fn(); - const testVariationServiceCreateMock = jest.fn(); - const testVariationUpdateMock = jest.fn(); - const testVariationAddBaselineMock = jest.fn(); - const eventTestRunUpdatedMock = jest.fn(); - const built = await initService({ - testRunFindUniqueMock, - testRunUpdateMock, - copyImageMock, - testVariationServiceFindUniqueMock, - testVariationServiceCreateMock, - testVariationUpdateMock, - testVariationAddBaselineMock, - eventTestRunUpdatedMock, - }); - return { - built, - testRunFindUniqueMock, - testRunUpdateMock, - copyImageMock, - testVariationServiceFindUniqueMock, - testVariationServiceCreateMock, - testVariationUpdateMock, - testVariationAddBaselineMock, - eventTestRunUpdatedMock, - }; + const initApproveService = async ( + testRun: object | null, + { updateManyCount = 1, baselineCount = 0 }: { updateManyCount?: number; baselineCount?: number } = {} + ) => { + const ctx = await build(); + ctx.m.testRun.findUnique.mockResolvedValue(testRun); + ctx.m.testRun.updateMany.mockResolvedValue({ count: updateManyCount }); + ctx.m.baseline.count.mockResolvedValue(baselineCount); + ctx.m.static.copyImage.mockResolvedValue(copiedBaselineName); + return ctx; }; const mainVariation = generateTestVariation({ id: 'main variation', branchName: 'master' }); + const sameBranchRun = (overrides: Partial = {}) => ({ + ...generateTestRun({ branchName: 'master', baselineBranchName: 'master', ...overrides }), + testVariation: mainVariation, + }); + const featureRun = (overrides: Partial = {}) => ({ + ...generateTestRun({ branchName: 'feature', baselineBranchName: 'master', ...overrides }), + testVariation: mainVariation, + }); + const expectClaim = (m: Mocks, run: TestRun, status: TestStatus) => + expect(m.testRun.updateMany).toHaveBeenCalledWith({ + where: { id: run.id, status: run.status }, + data: { status, statusBeforeDecision: run.status }, + }); + const expectNoWrites = (m: Mocks) => { + expect(m.static.copyImage).not.toHaveBeenCalled(); + expect(m.tv.addBaseline).not.toHaveBeenCalled(); + expect(m.testRun.updateMany).not.toHaveBeenCalled(); + }; + + it('approve of a run without variation is a 409', async () => { + const { service, m } = await initApproveService({ ...generateTestRun(), testVariation: null }); - it('throws when the run has no test variation', async () => { - const { built, copyImageMock } = await initApproveService({ ...generateTestRun(), testVariation: null }); + const result = service.approve('id'); - await expect(built.approve('id')).rejects.toThrow('No test variation found. Re-create test run'); - expect(copyImageMock).not.toHaveBeenCalled(); + await expect(result).rejects.toBeInstanceOf(ConflictException); + await expect(result).rejects.toThrow(NO_VARIATION); + expect(m.static.copyImage).not.toHaveBeenCalled(); }); it('throws NotFoundException when the run does not exist', async () => { - const { built } = await initApproveService(null); + const { service } = await initApproveService(null); - await expect(built.approve('id')).rejects.toThrow(new NotFoundException('Test run not found: id')); + await expect(service.approve('id')).rejects.toThrow(new NotFoundException('Test run not found: id')); }); it('copies the image as the new baseline of the same-branch variation and marks the run approved', async () => { - const testRun = { - ...generateTestRun({ branchName: 'master', baselineBranchName: 'master', imageName: 'image.png' }), - testVariation: mainVariation, - }; - const mocks = await initApproveService(testRun); + const testRun = sameBranchRun({ imageName: 'image.png' }); + const { service, m } = await initApproveService(testRun); - const result = await mocks.built.approve(testRun.id, false, false, 'user id'); + const result = await service.approve(testRun.id, false, false, 'user id'); - expect(mocks.copyImageMock).toHaveBeenCalledWith('baseline', 'image.png'); - expect(mocks.testVariationServiceFindUniqueMock).not.toHaveBeenCalled(); - expect(mocks.testVariationAddBaselineMock).toHaveBeenCalledWith({ - id: mainVariation.id, - userId: 'user id', - testRunId: testRun.id, - baselineName: copiedBaselineName, - }); - expect(mocks.testRunUpdateMock).toHaveBeenCalledWith({ - where: { id: testRun.id }, - data: { status: TestStatus.approved }, - }); - expect(mocks.eventTestRunUpdatedMock).toHaveBeenCalledWith(testRun); + expect(m.static.copyImage).toHaveBeenCalledWith('baseline', 'image.png'); + expect(m.tv.findUnique).not.toHaveBeenCalled(); + expect(m.tv.addBaseline).toHaveBeenCalledWith( + { id: mainVariation.id, userId: 'user id', testRunId: testRun.id, baselineName: copiedBaselineName }, + expect.anything() + ); + expectClaim(m, testRun, TestStatus.approved); + expect(m.events.testRunUpdated).toHaveBeenCalledWith(testRun); expect(result).toBe(testRun); }); + it('claims the run and adds the baseline in one transaction', async () => { + const testRun = sameBranchRun({ status: TestStatus.failed }); + const { service, m } = await initApproveService(testRun); + + await service.approve(testRun.id); + + expectClaim(m, testRun, TestStatus.approved); + const [, tx] = m.tv.addBaseline.mock.calls[0]; + expect(tx).toHaveProperty('testRun.updateMany', m.testRun.updateMany); + expect(m.events.testRunUpdated).toHaveBeenCalledWith(testRun); + }); + + it('approve records the status it decided from', async () => { + const testRun = sameBranchRun({ status: TestStatus.unresolved }); + const { service, m } = await initApproveService(testRun); + + await service.approve(testRun.id); + + expectClaim(m, testRun, TestStatus.approved); + }); + it('creates a feature-branch variation when approving a feature-branch run against the main baseline', async () => { - const testRun = { - ...generateTestRun({ - branchName: 'feature', - baselineBranchName: 'master', - baselineName: 'old.baseline.png', - ignoreAreas: '[{"x":1}]', - comment: 'run comment', - }), - testVariation: mainVariation, - }; - const mocks = await initApproveService(testRun); + const testRun = featureRun({ + baselineName: 'old.baseline.png', + ignoreAreas: '[{"x":1}]', + comment: 'run comment', + }); + const { service, m } = await initApproveService(testRun); const created = generateTestVariation({ id: 'created feature variation', branchName: 'feature' }); const updated = generateTestVariation({ id: 'updated feature variation', branchName: 'feature' }); - mocks.testVariationServiceFindUniqueMock.mockResolvedValueOnce(null); - mocks.testVariationServiceCreateMock.mockResolvedValueOnce(created); - mocks.testVariationUpdateMock.mockResolvedValueOnce(updated); - - await mocks.built.approve(testRun.id, false, false, 'user id'); - - // NOTE: current behavior — the whole run (including the nested testVariation) is spread into findUnique - expect(mocks.testVariationServiceFindUniqueMock).toHaveBeenCalledWith({ ...testRun }); - expect(mocks.testVariationServiceCreateMock).toHaveBeenCalledWith({ - testRunId: testRun.id, - createTestRequestDto: { - projectId: testRun.projectId, - branchName: 'feature', - ...getTestVariationUniqueData(testRun), - }, - }); - // NOTE: current behavior — carries over the run's *previous* baselineName, the new copy only - // reaches the variation through addBaseline below - expect(mocks.testVariationUpdateMock).toHaveBeenCalledWith( + m.tv.findUnique.mockResolvedValueOnce(null); + m.tv.create.mockResolvedValueOnce(created); + m.tv.update.mockResolvedValueOnce(updated); + const key = { projectId: mainVariation.projectId, branchName: 'feature', ...getTestVariationUniqueData(testRun) }; + + await service.approve(testRun.id, false, false, 'user id'); + + expect(m.tv.findUnique).toHaveBeenCalledWith(key, expect.anything()); + expect(m.tv.create).toHaveBeenCalledWith({ createTestRequestDto: key }, expect.anything()); + expect(m.tv.update).toHaveBeenCalledWith( created.id, - { baselineName: 'old.baseline.png', ignoreAreas: '[{"x":1}]', comment: 'run comment' }, - testRun.id + { ignoreAreas: '[{"x":1}]', comment: 'run comment' }, + testRun.id, + expect.anything() ); - expect(mocks.testVariationAddBaselineMock).toHaveBeenCalledWith({ - id: updated.id, - userId: 'user id', - testRunId: testRun.id, - baselineName: copiedBaselineName, - }); - expect(mocks.testRunUpdateMock).toHaveBeenCalledWith({ - where: { id: testRun.id }, - data: { status: TestStatus.approved }, - }); + expect(m.tv.addBaseline).toHaveBeenCalledWith( + { id: updated.id, userId: 'user id', testRunId: testRun.id, baselineName: copiedBaselineName }, + expect.anything() + ); + expectClaim(m, testRun, TestStatus.approved); }); it('reuses the existing feature-branch variation', async () => { - const testRun = { - ...generateTestRun({ branchName: 'feature', baselineBranchName: 'master' }), - testVariation: mainVariation, - }; - const mocks = await initApproveService(testRun); + const testRun = featureRun(); + const { service, m } = await initApproveService(testRun); const existing = generateTestVariation({ id: 'existing feature variation', branchName: 'feature' }); - mocks.testVariationServiceFindUniqueMock.mockResolvedValueOnce(existing); - mocks.testVariationUpdateMock.mockResolvedValueOnce(existing); + m.tv.findUnique.mockResolvedValueOnce(existing); + m.tv.update.mockResolvedValueOnce(existing); - await mocks.built.approve(testRun.id); + await service.approve(testRun.id); - expect(mocks.testVariationServiceCreateMock).not.toHaveBeenCalled(); - expect(mocks.testVariationUpdateMock).toHaveBeenCalledWith(existing.id, expect.any(Object), testRun.id); - expect(mocks.testVariationAddBaselineMock).toHaveBeenCalledWith( - expect.objectContaining({ id: existing.id, userId: undefined }) + expect(m.tv.create).not.toHaveBeenCalled(); + expect(m.tv.update).toHaveBeenCalledWith(existing.id, expect.any(Object), testRun.id, expect.anything()); + expect(m.tv.addBaseline).toHaveBeenCalledWith( + expect.objectContaining({ id: existing.id, userId: undefined }), + expect.anything() ); }); it('adds the baseline straight to the main variation when merging', async () => { - const testRun = { - ...generateTestRun({ branchName: 'feature', baselineBranchName: 'master' }), - testVariation: mainVariation, - }; - const mocks = await initApproveService(testRun); + const testRun = featureRun(); + const { service, m } = await initApproveService(testRun); - await mocks.built.approve(testRun.id, true); + await service.approve(testRun.id, true); - expect(mocks.testVariationServiceFindUniqueMock).not.toHaveBeenCalled(); - expect(mocks.testVariationUpdateMock).not.toHaveBeenCalled(); - expect(mocks.testVariationAddBaselineMock).toHaveBeenCalledWith( - expect.objectContaining({ id: mainVariation.id, baselineName: copiedBaselineName }) + expect(m.tv.findUnique).not.toHaveBeenCalled(); + expect(m.tv.update).not.toHaveBeenCalled(); + expect(m.tv.addBaseline).toHaveBeenCalledWith( + expect.objectContaining({ id: mainVariation.id, baselineName: copiedBaselineName }), + expect.anything() ); - expect(mocks.testRunUpdateMock).toHaveBeenCalledWith({ - where: { id: testRun.id }, - data: { status: TestStatus.approved }, - }); + expectClaim(m, testRun, TestStatus.approved); }); - it('auto approves a feature-branch run without touching any variation', async () => { - const testRun = { - ...generateTestRun({ branchName: 'feature', baselineBranchName: 'master' }), - testVariation: mainVariation, - }; - const mocks = await initApproveService(testRun); + it('auto approving a feature-branch run copies no baseline image', async () => { + const testRun = featureRun(); + const { service, m } = await initApproveService(testRun); - await mocks.built.approve(testRun.id, false, true); + await service.approve(testRun.id, false, true); - // NOTE: current behavior — the baseline image is still copied even though nothing references it - expect(mocks.copyImageMock).toHaveBeenCalledWith('baseline', testRun.imageName); - expect(mocks.testVariationServiceFindUniqueMock).not.toHaveBeenCalled(); - expect(mocks.testVariationAddBaselineMock).not.toHaveBeenCalled(); - expect(mocks.testRunUpdateMock).toHaveBeenCalledWith({ - where: { id: testRun.id }, - data: { status: TestStatus.autoApproved }, - }); + expect(m.static.copyImage).not.toHaveBeenCalled(); + expect(m.tv.findUnique).not.toHaveBeenCalled(); + expect(m.tv.addBaseline).not.toHaveBeenCalled(); + expectClaim(m, testRun, TestStatus.autoApproved); }); it('auto approves a same-branch run by adding the baseline', async () => { - const testRun = { - ...generateTestRun({ branchName: 'master', baselineBranchName: 'master' }), - testVariation: mainVariation, - }; - const mocks = await initApproveService(testRun); + const testRun = sameBranchRun(); + const { service, m } = await initApproveService(testRun); - await mocks.built.approve(testRun.id, false, true); + await service.approve(testRun.id, false, true); - expect(mocks.testVariationAddBaselineMock).toHaveBeenCalledWith({ - id: mainVariation.id, - userId: undefined, - testRunId: testRun.id, - baselineName: copiedBaselineName, - }); - expect(mocks.testRunUpdateMock).toHaveBeenCalledWith({ - where: { id: testRun.id }, - data: { status: TestStatus.autoApproved }, - }); + expect(m.tv.addBaseline).toHaveBeenCalledWith( + { id: mainVariation.id, userId: undefined, testRunId: testRun.id, baselineName: copiedBaselineName }, + expect.anything() + ); + expectClaim(m, testRun, TestStatus.autoApproved); }); - }); - describe('create details', () => { - it('prefers the comment of the request over the one of the variation', async () => { - const testRunCreateMock = jest.fn().mockResolvedValueOnce(generateTestRun()); - const eventTestRunCreatedMock = jest.fn(); - service = await initService({ - testRunCreateMock, - eventTestRunCreatedMock, - saveImageMock: jest.fn().mockResolvedValueOnce('image.png'), - }); + it('re-approving an already approved feature-branch run changes nothing', async () => { + const testRun = featureRun({ status: TestStatus.approved }); + const { service, m } = await initApproveService(testRun); - const result = await service.create({ - testVariation: generateTestVariation({ comment: 'variation comment' }), - createTestRequestDto: { - buildId: 'buildId', - projectId: 'projectId', - name: 'Test name', - branchName: 'develop', - comment: 'request comment', - }, - imageBuffer, - }); + const result = await service.approve(testRun.id); - const { data } = testRunCreateMock.mock.calls[0][0]; - expect(data.comment).toBe('request comment'); - expect(data.imageName).toBe('image.png'); - // NOTE: current behavior — absent ignore areas are stored as undefined (JSON.stringify(undefined)) - expect(data.tempIgnoreAreas).toBeUndefined(); - expect(eventTestRunCreatedMock).toHaveBeenCalledWith(result); + expect(result).toBe(testRun); + expectNoWrites(m); + expect(m.tv.update).not.toHaveBeenCalled(); + expect(m.events.testRunUpdated).not.toHaveBeenCalled(); }); - }); - describe('calculateDiff details', () => { - it('compares with the permanent and temporary ignore areas together and returns the saved run', async () => { - const permanent = { x: 1, y: 1, width: 1, height: 1 }; - const temporary = { x: 2, y: 2, width: 2, height: 2 }; - const testRun = generateTestRun({ - ignoreAreas: JSON.stringify([permanent]), - tempIgnoreAreas: JSON.stringify([temporary]), - }); - const saved = generateTestRun({ id: 'saved' }); - const compareGetDiffMock = jest.fn().mockResolvedValueOnce({ status: TestStatus.ok }); - const deleteImageMock = jest.fn(); - service = await initService({ - compareGetDiffMock, - deleteImageMock, - testRunUpdateMock: jest.fn().mockResolvedValueOnce(saved), - }); + it('auto approving an already auto approved run changes nothing', async () => { + const testRun = featureRun({ status: TestStatus.autoApproved }); + const { service, m } = await initApproveService(testRun); + + const result = await service.approve(testRun.id, false, true); + + expect(result).toBe(testRun); + expect(m.baseline.count).not.toHaveBeenCalled(); + expect(m.static.copyImage).not.toHaveBeenCalled(); + expect(m.testRun.updateMany).not.toHaveBeenCalled(); + }); + + it('manually approving an autoApproved run that already has a baseline is a no-op', async () => { + const testRun = sameBranchRun({ status: TestStatus.autoApproved }); + const { service, m } = await initApproveService(testRun, { baselineCount: 1 }); + + const result = await service.approve(testRun.id); + + expect(m.baseline.count).toHaveBeenCalledWith({ where: { testRunId: testRun.id } }); + expect(result).toBe(testRun); + expectNoWrites(m); + }); + + it('manually approving an autoApproved feature-branch run without a baseline approves it', async () => { + const testRun = featureRun({ status: TestStatus.autoApproved }); + const { service, m } = await initApproveService(testRun, { baselineCount: 0 }); + m.tv.findUnique.mockResolvedValueOnce(null); + m.tv.create.mockResolvedValueOnce(generateTestVariation({ id: 'feature variation' })); + m.tv.update.mockResolvedValueOnce(generateTestVariation({ id: 'feature variation' })); + + await service.approve(testRun.id); + + expectClaim(m, testRun, TestStatus.approved); + expect(m.tv.addBaseline).toHaveBeenCalledWith( + expect.objectContaining({ id: 'feature variation', baselineName: copiedBaselineName }), + expect.anything() + ); + }); + + it('a lost claim deletes the copy and changes nothing', async () => { + const testRun = sameBranchRun(); + const { service, m } = await initApproveService(testRun, { updateManyCount: 0 }); + + const result = await service.approve(testRun.id); + + expect(m.tv.addBaseline).not.toHaveBeenCalled(); + expect(m.static.deleteImage).toHaveBeenCalledWith(copiedBaselineName); + expect(m.events.testRunUpdated).not.toHaveBeenCalled(); + expect(result).toBe(testRun); + }); + + it('a lost claim returns the run it read when the run is gone meanwhile', async () => { + const testRun = sameBranchRun(); + const { service, m } = await initApproveService(testRun, { updateManyCount: 0 }); + m.testRun.findUnique.mockResolvedValueOnce(testRun).mockResolvedValueOnce(null); + + expect(await service.approve(testRun.id)).toBe(testRun); + }); - const result = await service.calculateDiff('projectId', testRun); + it('deletes the copy when the transaction fails', async () => { + const testRun = sameBranchRun(); + const { service, m } = await initApproveService(testRun); + m.tv.addBaseline.mockRejectedValueOnce(new Error('db')); - expect(compareGetDiffMock.mock.calls[0][0].data.ignoreAreas).toEqual([permanent, temporary]); - expect(result).toBe(saved); - // NOTE: current behavior — deleteImage is also asked to delete the missing (null) thumbnails - expect(deleteImageMock.mock.calls).toEqual([[testRun.diffName], [null], [null]]); + await expect(service.approve(testRun.id)).rejects.toThrow('db'); + + expect(m.static.deleteImage).toHaveBeenCalledWith(copiedBaselineName); + expect(m.events.testRunUpdated).not.toHaveBeenCalled(); }); - it('treats null ignore areas as none', async () => { - const testRun = generateTestRun({ ignoreAreas: 'null', tempIgnoreAreas: 'null' }); - const compareGetDiffMock = jest.fn().mockResolvedValueOnce(null); - const testRunUpdateMock = jest.fn().mockResolvedValueOnce(testRun); - service = await initService({ compareGetDiffMock, testRunUpdateMock }); + it('retries once when a concurrent approval created the feature variation', async () => { + const testRun = featureRun(); + const { service, m } = await initApproveService(testRun); + m.static.copyImage.mockResolvedValueOnce('copy1.png').mockResolvedValueOnce('copy2.png'); + const winner = generateTestVariation({ id: 'winner', branchName: 'feature' }); + m.tv.findUnique.mockResolvedValueOnce(null).mockResolvedValueOnce(winner); + m.tv.create.mockRejectedValueOnce(prismaError('P2002')); + m.tv.update.mockResolvedValueOnce(winner); - await service.calculateDiff('projectId', testRun); + const result = await service.approve(testRun.id); - expect(compareGetDiffMock.mock.calls[0][0].data.ignoreAreas).toEqual([]); - expect(testRunUpdateMock.mock.calls[0][0].data.status).toBe(TestStatus.new); + expect(result).toBe(testRun); + expect(m.static.deleteImage).toHaveBeenCalledWith('copy1.png'); + expect(m.tv.addBaseline).toHaveBeenCalledTimes(1); + expect(m.tv.addBaseline).toHaveBeenCalledWith( + expect.objectContaining({ id: winner.id, baselineName: 'copy2.png' }), + expect.anything() + ); }); - }); - describe('saveDiffResult details', () => { - it('clears the analysis of a previous comparison when this one has none', async () => { - const testRunUpdateMock = jest.fn().mockResolvedValueOnce(generateTestRun()); - service = await initService({ testRunUpdateMock }); + it('rethrows a second P2002', async () => { + const testRun = featureRun(); + const { service, m } = await initApproveService(testRun); + m.tv.findUnique.mockResolvedValue(null); + m.tv.create.mockRejectedValue(prismaError('P2002')); - await service.saveDiffResult('id', { - status: TestStatus.ok, - diffName: null, - pixelMisMatchCount: 0, - diffPercent: 0, - isSameDimension: true, - }); + await expect(service.approve(testRun.id)).rejects.toMatchObject({ code: 'P2002' }); - expect(testRunUpdateMock.mock.calls[0][0].data).toEqual({ - diffName: null, - pixelMisMatchCount: 0, - diffPercent: 0, - status: TestStatus.ok, - vlmDescription: null, - changeSignature: null, - imageThumbnailName: null, - diffThumbnailName: null, - }); + expect(m.tv.create).toHaveBeenCalledTimes(2); + expect(m.tv.addBaseline).not.toHaveBeenCalled(); }); }); - describe('delete failures', () => { - const initDeleteService = async (deleteError: unknown) => { - const testRun = generateTestRun(); - const eventTestRunDeletedMock = jest.fn(); - const built = await initService({ - testRunFindUniqueMock: jest.fn().mockResolvedValueOnce(testRun), - testRunDeleteMock: jest.fn().mockRejectedValueOnce(deleteError), - eventTestRunDeletedMock, + describe('auto approval', () => { + const tryNew = (service: TestRunsService, testRun: TestRun, testVariation = generateTestVariation()) => + service['tryAutoApproveByNewBaselines']({ testVariation, testRun, project: TEST_PROJECT }); + const tryPast = (service: TestRunsService, testRun: TestRun, testVariation = generateTestVariation()) => + service['tryAutoApproveByPastBaselines']({ testVariation, testRun, project: TEST_PROJECT }); + const imageBytes = Buffer.from('x'); + + describe.each([ + [ + 'tryAutoApproveByNewBaselines', + tryNew, + (m: Mocks) => { + m.testRun.findMany.mockResolvedValueOnce([{ testVariationId: 'other variation' }]); + m.tv.findLatestBaselines.mockResolvedValueOnce([generateBaseline()]); + }, + ], + [ + 'tryAutoApproveByPastBaselines', + tryPast, + (m: Mocks) => m.tv.findBaselineHistory.mockResolvedValueOnce([generateBaseline(), generateBaseline()]), + ], + ])('%s', (_name, run, arrange) => { + it('skip if ok', async () => { + const testRun = generateTestRun({ status: TestStatus.ok }); + const { service } = await build(); + + expect(await run(service, testRun)).toBe(testRun); }); - return { built, testRun, eventTestRunDeletedMock }; - }; - it('treats a run that is already gone as deleted, without announcing it', async () => { - const { built, testRun, eventTestRunDeletedMock } = await initDeleteService( - new Prisma.PrismaClientKnownRequestError('gone', { code: 'P2025', clientVersion: 'test' }) - ); + it('should not auto approve', async () => { + const testRun = generateTestRun({ status: TestStatus.unresolved }); + const { service, m } = await build(); + arrange(m); + service['shouldAutoApprove'] = jest.fn().mockResolvedValueOnce(false); - const result = await built.delete(testRun.id); + const result = await run(service, testRun); - expect(result).toBeUndefined(); - expect(eventTestRunDeletedMock).not.toHaveBeenCalled(); + expect(service['shouldAutoApprove']).toHaveBeenCalled(); + expect(result).toBe(testRun); + }); + + it('should auto approve', async () => { + const testRun = generateTestRun({ status: TestStatus.unresolved }); + const { service, m } = await build(); + arrange(m); + service['shouldAutoApprove'] = jest.fn().mockResolvedValueOnce(true); + service.approve = jest.fn().mockResolvedValueOnce({ ...testRun, status: TestStatus.autoApproved }); + + const result = await run(service, testRun); + + expect(result).toStrictEqual({ ...testRun, status: TestStatus.autoApproved }); + }); }); - it.each([ - [ - 'another prisma error', - new Prisma.PrismaClientKnownRequestError('fk', { code: 'P2003', clientVersion: 'test' }), - ], - ['any other error', new Error('connection lost')], - ['a non-prisma error carrying the P2025 code', Object.assign(new Error('gone'), { code: 'P2025' })], - ])('swallows %s and reports the run as deleted', async (_case, error) => { - const { built, testRun, eventTestRunDeletedMock } = await initDeleteService(error); + it('does not look again once the past baselines approved the run', async () => { + const testRun = generateTestRun({ status: TestStatus.autoApproved }); + const { service, m } = await build(); - const result = await built.delete(testRun.id); + const result = await tryNew(service, testRun); - // NOTE: current behavior — every delete failure other than P2025 is silently ignored and the run is - // announced as deleted although it may still exist + expect(m.testRun.findMany).not.toHaveBeenCalled(); expect(result).toBe(testRun); - expect(eventTestRunDeletedMock).toHaveBeenCalledWith(testRun); }); - }); - describe('addIgnoreAreas', () => { - const newArea = { x: 5, y: 6, width: 7, height: 8 }; - - it.each([ - ['appends to the existing areas', JSON.stringify(ignoreAreas), [...ignoreAreas, newArea]], - ['starts from none when they are null', 'null', [newArea]], - ])('%s', async (_case, stored, expected) => { - const testRunFindUniqueMock = jest.fn().mockResolvedValueOnce(generateTestRun({ ignoreAreas: stored })); - service = await initService({ testRunFindUniqueMock }); - const updated = generateTestRun({ id: 'updated' }); - service.updateIgnoreAreas = jest.fn().mockResolvedValueOnce(updated); + it('skip if the branch the same as baseline', async () => { + const testRun = generateTestRun({ status: TestStatus.unresolved, branchName: 'a', baselineBranchName: 'a' }); + const { service } = await build(); + + expect(await tryPast(service, testRun)).toBe(testRun); + }); + + describe('details', () => { + it('by past baselines skips the latest baseline and approves on the first older one that matches', async () => { + const testRun = generateTestRun({ status: TestStatus.unresolved, branchName: 'feature' }); + const testVariation = generateTestVariation({ id: 'variation', projectId: 'project' }); + const { service, m } = await build(); + m.tv.findBaselineHistory.mockResolvedValueOnce( + ['latest', 'older', 'oldest'].map((name) => generateBaseline({ id: name, baselineName: `${name}.png` })) + ); + m.compare.getDiff.mockResolvedValueOnce({ status: TestStatus.ok }); + m.static.getImageBuffer.mockResolvedValue(imageBytes); + const approved = generateTestRun({ status: TestStatus.autoApproved }); + service.approve = jest.fn().mockResolvedValueOnce(approved); + + const result = await tryPast(service, testRun, testVariation); + + expect(m.tv.findBaselineHistory).toHaveBeenCalledWith('variation'); + expect(m.compare.getDiff).toHaveBeenCalledTimes(1); + expect(diffInputs(m)[0].data.baseline).toBe('older.png'); + expect(service.approve).toHaveBeenCalledWith(testRun.id, false, true); + expect(result).toBe(approved); + }); + + it('by past baselines has nothing to try with a single baseline', async () => { + const testRun = generateTestRun({ status: TestStatus.unresolved, branchName: 'feature' }); + const { service, m } = await build(); + m.tv.findBaselineHistory.mockResolvedValueOnce([generateBaseline()]); + service['shouldAutoApprove'] = jest.fn(); + + const result = await tryPast(service, testRun); + + expect(service['shouldAutoApprove']).not.toHaveBeenCalled(); + expect(result).toBe(testRun); + }); + + it('by new baselines looks for approved runs against the same baseline and compares with their latest baseline', async () => { + const testRun = generateTestRun({ status: TestStatus.unresolved }); + const testVariation = generateTestVariation({ id: 'variation', projectId: 'project', baselineName: 'b.png' }); + const { service, m } = await build(); + m.testRun.findMany.mockResolvedValueOnce([{ testVariationId: 'other variation' }]); + m.tv.findLatestBaselines.mockResolvedValueOnce([ + generateBaseline({ id: 'latest', baselineName: 'latest.png' }), + ]); + m.compare.getDiff.mockResolvedValueOnce({ status: TestStatus.ok }); + m.static.getImageBuffer.mockResolvedValue(imageBytes); + service.approve = jest.fn().mockResolvedValueOnce(testRun); + + await tryNew(service, testRun, testVariation); + + expect(m.testRun.findMany).toHaveBeenCalledWith({ + where: { + ...getTestVariationUniqueData(testVariation), + baselineName: 'b.png', + status: TestStatus.approved, + testVariation: { projectId: 'project' }, + }, + select: { testVariationId: true }, + }); + expect(m.tv.findLatestBaselines).toHaveBeenCalledWith(['other variation']); + expect(diffInputs(m)[0].data.baseline).toBe('latest.png'); + expect(service.approve).toHaveBeenCalledWith(testRun.id, false, true); + }); - const result = await service.addIgnoreAreas('id', [newArea]); + it('by new baselines asks once per variation, however many runs it approved', async () => { + const { service, m } = await build(); + m.testRun.findMany.mockResolvedValueOnce( + ['v1', 'v2', 'v1', null].map((testVariationId) => ({ testVariationId })) + ); + m.tv.findLatestBaselines.mockResolvedValueOnce([]); - expect(testRunFindUniqueMock).toHaveBeenCalledWith({ where: { id: 'id' }, include: { testVariation: true } }); - expect(service.updateIgnoreAreas).toHaveBeenCalledWith('id', expected); - expect(result).toBe(updated); - }); - }); + await tryNew(service, generateTestRun({ status: TestStatus.unresolved })); - describe('updateIgnoreAreas details', () => { - it('returns the recalculated run', async () => { - const testRun = generateTestRun(); - const recalculated = generateTestRun({ id: 'recalculated' }); - service = await initService({ - testRunUpdateMock: jest.fn().mockResolvedValueOnce(testRun), - testVariationUpdateMock: jest.fn().mockResolvedValueOnce(generateTestVariation({ projectId: 'p' })), + expect(m.tv.findLatestBaselines).toHaveBeenCalledWith(['v1', 'v2']); }); - service.calculateDiff = jest.fn().mockResolvedValueOnce(recalculated); - expect(await service.updateIgnoreAreas(testRun.id, ignoreAreas)).toBe(recalculated); + it.each([ + ['skips a variation without baselines instead of failing', [{ testVariationId: 'v1' }], ['v1']], + ['keeps the run when nothing was approved before', [], []], + ])('by new baselines %s', async (_case, approvedRuns, variationIds) => { + const testRun = generateTestRun({ status: TestStatus.unresolved }); + const { service, m } = await build(); + m.testRun.findMany.mockResolvedValueOnce(approvedRuns); + m.tv.findLatestBaselines.mockResolvedValueOnce([]); + + const result = await tryNew(service, testRun); + + expect(m.tv.findLatestBaselines).toHaveBeenCalledWith(variationIds); + expect(m.compare.getDiff).not.toHaveBeenCalled(); + expect(result).toBe(testRun); + }); }); - }); - describe('update details', () => { - it('returns the updated run', async () => { - const testRun = generateTestRun(); - service = await initService({ testRunUpdateMock: jest.fn().mockResolvedValueOnce(testRun) }); + describe('decides as before', () => { + const legacyPast = (history: string[], ok: Set) => history.slice(1).some((name) => ok.has(name)); + const legacyNew = (runVariations: string[], newest: Record, ok: Set) => + runVariations.some((id) => ok.has(newest[id])); - expect(await service.update(testRun.id, { comment: 'c' })).toBe(testRun); + const testRun = generateTestRun({ status: TestStatus.unresolved, branchName: 'feature' }); + const testVariation = generateTestVariation({ id: 'variation', projectId: 'project' }); + + const initWith = async (ok: Set, arrange: (m: Mocks) => void) => { + const { service, m } = await build(); + arrange(m); + m.compare.getDiff.mockImplementation(async ({ data }) => ({ + status: ok.has(data.baseline) ? TestStatus.ok : TestStatus.unresolved, + })); + m.static.getImageBuffer.mockResolvedValue(Buffer.from('image')); + service.approve = jest.fn().mockResolvedValue(generateTestRun({ status: TestStatus.autoApproved })); + return { service, compared: () => diffInputs(m).map(({ data }) => data.baseline as string) }; + }; + + it.each<[string, string[], string[]]>([ + ['no history', [], []], + ['a single baseline that would match', ['a'], ['a']], + ['an older baseline that matches', ['a', 'b'], ['b']], + ['a repeated name before a match', ['a', 'b', 'b', 'c'], ['c']], + ['a repeated name without a match', ['a', 'b', 'b'], []], + ['the newest name repeated in the past', ['a', 'a'], ['a']], + ['a match on the newest baseline only', ['a', 'b', 'c'], ['a']], + ])('by past baselines: %s', async (_name, history, matching) => { + const ok = new Set(matching); + const { service, compared } = await initWith(ok, (m) => + m.tv.findBaselineHistory.mockResolvedValue( + history.map((baselineName, i) => generateBaseline({ id: `b${i}`, baselineName })) + ) + ); + + await tryPast(service, testRun, testVariation); + + expect(jest.mocked(service.approve).mock.calls.length > 0).toBe(legacyPast(history, ok)); + expect(new Set(compared()).size).toBe(compared().length); + expect(compared().length).toBeLessThanOrEqual(new Set(history.slice(1)).size); + }); + + it.each<[string, string[], Record, string[]]>([ + ['two runs of one variation, a match on the other', ['v1', 'v1', 'v2'], { v1: 'x', v2: 'y' }, ['y']], + ['two variations with the same newest baseline, no match', ['v1', 'v2'], { v1: 'x', v2: 'x' }, []], + ['no approved runs', [], {}, []], + ])('by new baselines: %s', async (_name, runVariations, newest, matching) => { + const ok = new Set(matching); + const { service, compared } = await initWith(ok, (m) => { + m.testRun.findMany.mockResolvedValue(runVariations.map((testVariationId) => ({ testVariationId }))); + m.tv.findLatestBaselines.mockImplementation(async (ids: string[]) => + ids.flatMap((id) => (newest[id] ? [generateBaseline({ id: `b-${id}`, baselineName: newest[id] })] : [])) + ); + }); + + await tryNew(service, testRun, testVariation); + + expect(jest.mocked(service.approve).mock.calls.length > 0).toBe(legacyNew(runVariations, newest, ok)); + expect(new Set(compared()).size).toBe(compared().length); + expect(compared().length).toBeLessThanOrEqual(new Set(runVariations.map((id) => newest[id])).size); + }); }); }); @@ -1642,17 +1938,27 @@ describe('TestRunsService', () => { diffTollerancePercent: 3, }); const baseline = generateBaseline({ baselineName: 'past.baseline.png' }); + const project = { ...TEST_PROJECT, id: 'projectId' }; + const bytes = Buffer.from('image bytes'); it('compares the run against the given baseline without saving a diff, and approves an ok result', async () => { - const compareGetDiffMock = jest.fn().mockResolvedValueOnce({ status: TestStatus.ok }); - service = await initService({ compareGetDiffMock }); + const { service, m } = await build(); + m.compare.getDiff.mockResolvedValueOnce({ status: TestStatus.ok }); - const result = await service['shouldAutoApprove']({ projectId: 'projectId', baseline, testRun }); + const result = await service['shouldAutoApprove']({ + project, + baseline, + testRun, + imageBuffer: bytes, + ignoreAreas, + }); - expect(compareGetDiffMock).toHaveBeenCalledWith({ + expect(m.compare.getDiff).toHaveBeenCalledWith({ projectId: 'projectId', + project, data: { image: 'image.png', + imageBuffer: bytes, baseline: 'past.baseline.png', ignoreAreas, diffTollerancePercent: 3, @@ -1663,104 +1969,117 @@ describe('TestRunsService', () => { }); it('does not approve any other result', async () => { - service = await initService({ - compareGetDiffMock: jest.fn().mockResolvedValueOnce({ status: TestStatus.unresolved }), - }); - - const result = await service['shouldAutoApprove']({ projectId: 'projectId', baseline, testRun }); + const { service, m } = await build(); + m.compare.getDiff.mockResolvedValueOnce({ status: TestStatus.unresolved }); - // NOTE: current behavior — resolves to undefined, not false - expect(result).toBeUndefined(); + expect(await service['shouldAutoApprove']({ project, baseline, testRun, ignoreAreas })).toBe(false); }); }); - describe('auto approval details', () => { - it('by past baselines skips the latest baseline and approves on the first older one that matches', async () => { - const testRun = generateTestRun({ status: TestStatus.unresolved, branchName: 'feature' }); - const latest = generateBaseline({ id: 'latest' }); - const older = generateBaseline({ id: 'older' }); - const oldest = generateBaseline({ id: 'oldest' }); - const testVariation = generateTestVariation({ id: 'variation', projectId: 'project' }); - const testVariationGetDetailsMock = jest - .fn() - .mockResolvedValueOnce(generateTestVariation({}, [latest, older, oldest])); - service = await initService({ testVariationGetDetailsMock }); - service['shouldAutoApprove'] = jest.fn().mockResolvedValueOnce(true); - const approved = generateTestRun({ status: TestStatus.autoApproved }); - service.approve = jest.fn().mockResolvedValueOnce(approved); + describe('autoApproveOnFirstMatch', () => { + const testRun = generateTestRun({ + status: TestStatus.unresolved, + imageName: 'image.png', + ignoreAreas: JSON.stringify(ignoreAreas), + diffTollerancePercent: 3, + }); + const ref = (baselineName: string) => ({ id: `id of ${baselineName}`, baselineName }); + const refs = (...names: string[]) => names.map(ref); + const firstMatch = (service: TestRunsService, baselines: ReturnType, uploaded?: Buffer) => + service['autoApproveOnFirstMatch'](TEST_PROJECT, testRun, baselines, uploaded); + const initFirstMatch = async () => { + const ctx = await build(); + ctx.m.compare.getDiff.mockResolvedValue({ status: TestStatus.unresolved }); + return ctx; + }; + const expectBytes = (m: Mocks, times: number, expected: Buffer | undefined) => { + expect(m.compare.getDiff).toHaveBeenCalledTimes(times); + diffInputs(m).forEach(({ data }) => expect(data.imageBuffer).toBe(expected)); + }; - const result = await service['tryAutoApproveByPastBaselines']({ testVariation, testRun }); + it('compares each baseline name once', async () => { + const { service, m } = await initFirstMatch(); + m.static.getImageBuffer.mockResolvedValue(Buffer.from('x')); - expect(testVariationGetDetailsMock).toHaveBeenCalledWith('variation'); - expect(service['shouldAutoApprove']).toHaveBeenCalledTimes(1); - expect(service['shouldAutoApprove']).toHaveBeenCalledWith({ projectId: 'project', baseline: older, testRun }); - expect(service.approve).toHaveBeenCalledWith(testRun.id, false, true); - expect(result).toBe(approved); + const result = await firstMatch(service, refs('a', 'b', 'a', 'c')); + + expect(diffInputs(m).map(({ data }) => data.baseline)).toStrictEqual(['a', 'b', 'c']); + expect(result).toBe(testRun); }); - it('by past baselines has nothing to try with a single baseline', async () => { - const testRun = generateTestRun({ status: TestStatus.unresolved, branchName: 'feature' }); - service = await initService({ - testVariationGetDetailsMock: jest.fn().mockResolvedValueOnce(generateTestVariation({}, [generateBaseline()])), - }); - service['shouldAutoApprove'] = jest.fn(); + it('reads the run image once for all candidates', async () => { + const bytes = Buffer.from('image bytes'); + const { service, m } = await initFirstMatch(); + m.static.getImageBuffer.mockResolvedValue(bytes); - const result = await service['tryAutoApproveByPastBaselines']({ - testVariation: generateTestVariation(), - testRun, - }); + await firstMatch(service, refs('a', 'b', 'c')); + + expect(m.static.getImageBuffer).toHaveBeenCalledTimes(1); + expect(m.static.getImageBuffer).toHaveBeenCalledWith('image.png'); + expectBytes(m, 3, bytes); + }); + + it('uses the uploaded bytes without reading the storage', async () => { + const uploaded = Buffer.from('uploaded bytes'); + const { service, m } = await initFirstMatch(); + + await firstMatch(service, refs('a', 'b'), uploaded); + + expect(m.static.getImageBuffer).not.toHaveBeenCalled(); + expectBytes(m, 2, uploaded); + }); + + it('lets each comparison read the image when reading it up front fails', async () => { + const { service, m } = await initFirstMatch(); + m.static.getImageBuffer.mockRejectedValueOnce(new Error('s3')); - expect(service['shouldAutoApprove']).not.toHaveBeenCalled(); + const result = await firstMatch(service, refs('a', 'b')); + + expectBytes(m, 2, undefined); expect(result).toBe(testRun); }); - it('by new baselines looks for approved runs against the same baseline and compares with their latest baseline', async () => { - const testRun = generateTestRun({ status: TestStatus.unresolved }); - const testVariation = generateTestVariation({ id: 'variation', projectId: 'project', baselineName: 'b.png' }); - const approvedRun = generateTestRun({ status: TestStatus.approved, testVariationId: 'other variation' }); - const latest = generateBaseline({ id: 'latest' }); - const testRunFindManyMock = jest.fn().mockResolvedValueOnce([approvedRun]); - const testVariationGetDetailsMock = jest - .fn() - .mockResolvedValueOnce(generateTestVariation({}, [latest, generateBaseline({ id: 'older' })])); - service = await initService({ testRunFindManyMock, testVariationGetDetailsMock }); - service['shouldAutoApprove'] = jest.fn().mockResolvedValueOnce(true); - service.approve = jest.fn().mockResolvedValueOnce(testRun); + it('lets each comparison read the image when the storage has none', async () => { + const { service, m } = await initFirstMatch(); + m.static.getImageBuffer.mockResolvedValue(null); - await service['tryAutoApproveByNewBaselines']({ testVariation, testRun }); + await firstMatch(service, refs('a')); - expect(testRunFindManyMock).toHaveBeenCalledWith({ - where: { - ...getTestVariationUniqueData(testVariation), - baselineName: 'b.png', - status: TestStatus.approved, - testVariation: { projectId: 'project' }, - }, - }); - expect(testVariationGetDetailsMock).toHaveBeenCalledWith('other variation'); - expect(service['shouldAutoApprove']).toHaveBeenCalledWith({ projectId: 'project', baseline: latest, testRun }); - expect(service.approve).toHaveBeenCalledWith(testRun.id, false, true); + expect(diffInputs(m)[0].data.imageBuffer).toBeUndefined(); }); - it('by new baselines keeps the run when nothing was approved before', async () => { - const testRun = generateTestRun({ status: TestStatus.unresolved }); - service = await initService({ testRunFindManyMock: jest.fn().mockResolvedValueOnce([]) }); - service['shouldAutoApprove'] = jest.fn(); + it('reads nothing without candidates', async () => { + const { service, m } = await build(); - const result = await service['tryAutoApproveByNewBaselines']({ - testVariation: generateTestVariation(), - testRun, - }); + const result = await firstMatch(service, []); - expect(service['shouldAutoApprove']).not.toHaveBeenCalled(); + expect(m.static.getImageBuffer).not.toHaveBeenCalled(); + expect(m.compare.getDiff).not.toHaveBeenCalled(); expect(result).toBe(testRun); }); + + it('stops at the first match and approves', async () => { + const { service, m } = await build(); + m.compare.getDiff.mockImplementation(async ({ data }) => ({ + status: data.baseline === 'b' ? TestStatus.ok : TestStatus.unresolved, + })); + m.static.getImageBuffer.mockResolvedValue(Buffer.from('x')); + const approved = generateTestRun({ status: TestStatus.autoApproved }); + service.approve = jest.fn().mockResolvedValueOnce(approved); + + const result = await firstMatch(service, refs('a', 'b', 'c')); + + expect(m.compare.getDiff).toHaveBeenCalledTimes(2); + expect(service.approve).toHaveBeenCalledWith(testRun.id, false, true); + expect(result).toBe(approved); + }); }); - describe('getMatchingVariations edge cases', () => { + describe('getMatchingVariations', () => { const histogram = (hot: number): number[] => Array.from({ length: SIGNATURE_LENGTH }, (_unused, index) => (index === hot ? 1 : 0)); const SAME_PALETTE = histogram(0); + const OTHER_PALETTE = histogram(SIGNATURE_LENGTH - 1); const sibling = (id: string, overrides: Partial = {}): TestRun => generateTestRun({ @@ -1773,76 +2092,218 @@ describe('TestRunsService', () => { diffPercent: 12, ...overrides, }); + const target = (overrides: Partial = {}) => sibling('target', { customTags: '', ...overrides }); + const stamped = (signature: unknown[], stamp = { threshold: 0.1, includeAA: true }) => + JSON.stringify({ ...stamp, signature }); - const initEdgeService = async ({ + type Result = Awaited>; + const variationIds = (result: Result) => result.variations.map((variation) => variation.id); + const skippedReasons = (result: Result) => result.skipped.map((item) => item.reason); + const skippedPairs = (result: Result) => result.skipped.map((item) => [item.id, item.reason]); + + const initMatching = async ({ testRun, siblings = [], - project = { bulkApproveVariations: true, bulkApproveGroupBy: 'customTags', imageComparisonConfig: null } as - object | null, - signatureOf = (() => SAME_PALETTE) as (imageName: string) => Promise | number[] | null, - getImageBufferMock = jest.fn().mockImplementation(async (name: string) => Buffer.from(name)), + project = { bulkApproveVariations: true, bulkApproveGroupBy: 'customTags', imageComparisonConfig: null }, + signatureOf = () => SAME_PALETTE, + imageBufferOf = async (name: string) => Buffer.from(name), }: { testRun: TestRun | null; siblings?: TestRun[]; project?: object | null; signatureOf?: (imageName: string) => Promise | number[] | null; - getImageBufferMock?: jest.Mock; + imageBufferOf?: (name: string) => Promise; }) => { - const compareGetChangeSignatureMock = jest.fn().mockImplementation(async (input) => ({ + const { service, m } = await build(); + m.compare.getChangeSignature.mockImplementation(async (input) => ({ signature: await signatureOf(input.image.toString()), })); - const projectFindUniqueMock = jest.fn().mockResolvedValue(project); - const testRunFindManyMock = jest.fn().mockResolvedValue(siblings); - const built = await initService({ - testRunFindUniqueMock: jest.fn().mockResolvedValue(testRun), - testRunFindManyMock, - projectFindUniqueMock, - getImageBufferMock, - compareGetChangeSignatureMock, - }); - return { built, compareGetChangeSignatureMock, projectFindUniqueMock, testRunFindManyMock }; + m.testRun.findUnique.mockResolvedValue(testRun); + m.testRun.findMany.mockResolvedValue(siblings); + m.project.findUnique.mockResolvedValue(project); + m.static.getImageBuffer.mockImplementation(imageBufferOf); + return { + service, + m, + matches: () => service.getMatchingVariations(testRun?.id ?? 'missing'), + signed: () => m.compare.getChangeSignature.mock.calls.map(([input]) => input.image.toString()), + }; + }; + + const trackConcurrency = () => { + let inFlight = 0; + let peak = 0; + return { + peak: () => peak, + signatureOf: async () => { + inFlight += 1; + peak = Math.max(peak, inFlight); + await new Promise((resolve) => setTimeout(resolve, 0)); + inFlight -= 1; + return SAME_PALETTE; + }, + }; }; + it('matches same-palette, same-size siblings and skips different pattern / far larger change', async () => { + const testRun = target(); + const matching = sibling('locale-a'); + const bigger = sibling('locale-b', { diffPercent: 30 }); + const different = sibling('locale-c'); + const { matches } = await initMatching({ + testRun, + siblings: [matching, bigger, different], + signatureOf: (imageName) => (imageName.startsWith('locale-c') ? OTHER_PALETTE : SAME_PALETTE), + }); + + const result = await matches(); + + expect(variationIds(result)).toEqual([testRun.id, matching.id]); + expect(result.skipped.map((item) => ({ id: item.id, reason: item.reason }))).toEqual([ + { id: bigger.id, reason: 'different change size' }, + { id: different.id, reason: 'different change pattern' }, + ]); + }); + + it('groups from the signatures stored at ingest, without decoding anything', async () => { + const signed = (id: string, signature: number[], overrides = {}) => + sibling(id, { changeSignature: stamped(signature), ...overrides }); + const testRun = signed('target', SAME_PALETTE, { customTags: '' }); + const matching = signed('locale-a', SAME_PALETTE); + const different = signed('locale-c', OTHER_PALETTE); + const ctx = await initMatching({ + testRun, + siblings: [matching, different], + signatureOf: () => { + throw new Error('should not be computing a signature'); + }, + }); + + const result = await ctx.matches(); + + expect(ctx.m.compare.getChangeSignature).not.toHaveBeenCalled(); + expect(variationIds(result)).toEqual([testRun.id, matching.id]); + expect(skippedReasons(result)).toEqual(['different change pattern']); + }); + + it('recomputes when the project config has moved on since ingest', async () => { + const stale = stamped(SAME_PALETTE, { threshold: 0.9, includeAA: false }); + const testRun = target({ changeSignature: stale }); + const matching = sibling('locale-a', { changeSignature: stale }); + const ctx = await initMatching({ testRun, siblings: [matching] }); + + const result = await ctx.matches(); + + expect(ctx.signed().sort()).toEqual([testRun.imageName, matching.imageName].sort()); + expect(variationIds(result)).toEqual([testRun.id, matching.id]); + }); + + it('falls back to computing for a run with nothing stored', async () => { + const testRun = target({ changeSignature: stamped(SAME_PALETTE) }); + const unsigned = sibling('locale-a'); + const ctx = await initMatching({ testRun, siblings: [unsigned] }); + + const result = await ctx.matches(); + + expect(ctx.signed()).toEqual([unsigned.imageName]); + expect(variationIds(result)).toEqual([testRun.id, unsigned.id]); + }); + + it('skips a far larger change without paying for its signature', async () => { + const testRun = target(); + const ctx = await initMatching({ testRun, siblings: [sibling('locale-b', { diffPercent: 30 })] }); + + await ctx.matches(); + + expect(ctx.signed()).toEqual([testRun.imageName]); + }); + + it('says a sibling has no diff rather than blaming its change size', async () => { + const ctx = await initMatching({ testRun: target(), siblings: [sibling('locale-b', { diffPercent: 0 })] }); + + expect(skippedReasons(await ctx.matches())).toEqual(['no diff to match']); + }); + + it('asks for the siblings signatures at once rather than one after another', async () => { + const siblings = ['locale-a', 'locale-b', 'locale-c'].map((id) => sibling(id)); + const tracked = trackConcurrency(); + const ctx = await initMatching({ testRun: target(), siblings, signatureOf: tracked.signatureOf }); + + await ctx.matches(); + + expect(tracked.peak()).toBe(siblings.length + 1); + }); + + it('keeps the number of images being decoded at once bounded', async () => { + const siblings = Array.from({ length: SIGNATURE_CONCURRENCY * 3 }, (_, index) => sibling(`locale-${index}`)); + const tracked = trackConcurrency(); + const ctx = await initMatching({ testRun: target(), siblings, signatureOf: tracked.signatureOf }); + + await ctx.matches(); + + expect(tracked.peak()).toBe(SIGNATURE_CONCURRENCY); + }); + + it('reuses the signatures it already computed when the dialog is reopened', async () => { + const testRun = target(); + const matching = sibling('locale-a'); + const ctx = await initMatching({ testRun, siblings: [matching] }); + + await ctx.matches(); + const afterFirst = ctx.m.compare.getChangeSignature.mock.calls.length; + const second = await ctx.matches(); + + expect(afterFirst).toBe(2); + expect(ctx.m.compare.getChangeSignature).toHaveBeenCalledTimes(2); + expect(variationIds(second)).toEqual([testRun.id, matching.id]); + }); + + it('recomputes a signature once the run gained an ignore area', async () => { + const matching = sibling('locale-a'); + const ctx = await initMatching({ testRun: target(), siblings: [matching] }); + + await ctx.matches(); + matching.ignoreAreas = JSON.stringify([{ x: 0, y: 0, width: 10, height: 10 }]); + await ctx.matches(); + + expect(ctx.m.compare.getChangeSignature).toHaveBeenCalledTimes(3); + }); + it('throws for an unknown test run', async () => { - const { built } = await initEdgeService({ testRun: null }); + const ctx = await initMatching({ testRun: null }); - await expect(built.getMatchingVariations('missing')).rejects.toThrow( - new NotFoundException('Test run not found: missing') - ); + await expect(ctx.matches()).rejects.toThrow(new NotFoundException('Test run not found: missing')); }); it('returns only the run itself when the run has no project', async () => { const testRun = sibling('target', { projectId: null }); - const { built, projectFindUniqueMock, testRunFindManyMock } = await initEdgeService({ testRun }); + const ctx = await initMatching({ testRun }); - const result = await built.getMatchingVariations(testRun.id); + const result = await ctx.matches(); - expect(projectFindUniqueMock).not.toHaveBeenCalled(); - expect(testRunFindManyMock).not.toHaveBeenCalled(); + expect(ctx.m.project.findUnique).not.toHaveBeenCalled(); + expect(ctx.m.testRun.findMany).not.toHaveBeenCalled(); expect(result).toEqual({ variations: [new TestRunDto(testRun)], skipped: [] }); }); it('returns only the run itself when the project has bulk approval turned off', async () => { const testRun = sibling('target'); - const { built, projectFindUniqueMock, testRunFindManyMock } = await initEdgeService({ - testRun, - project: { bulkApproveVariations: false }, - }); + const ctx = await initMatching({ testRun, project: { bulkApproveVariations: false } }); - const result = await built.getMatchingVariations(testRun.id); + const result = await ctx.matches(); - expect(projectFindUniqueMock).toHaveBeenCalledWith({ where: { id: testRun.projectId } }); - expect(testRunFindManyMock).not.toHaveBeenCalled(); + expect(ctx.m.project.findUnique).toHaveBeenCalledWith({ where: { id: testRun.projectId } }); + expect(ctx.m.testRun.findMany).not.toHaveBeenCalled(); expect(result).toEqual({ variations: [new TestRunDto(testRun)], skipped: [] }); }); it('looks for unresolved and new siblings that differ only in customTags by default', async () => { const testRun = sibling('target', { os: 'linux', device: null, browser: 'chrome', viewport: '800x600' }); - const { built, testRunFindManyMock } = await initEdgeService({ testRun }); + const ctx = await initMatching({ testRun }); - await built.getMatchingVariations(testRun.id); + await ctx.matches(); - expect(testRunFindManyMock).toHaveBeenCalledWith({ + expect(ctx.m.testRun.findMany).toHaveBeenCalledWith({ where: { id: { not: testRun.id }, buildId: testRun.buildId, @@ -1862,25 +2323,24 @@ describe('TestRunsService', () => { ['viewport', { customTags: 'target', os: null, device: null, browser: 'chromium' }], ['an unknown axis', { os: null, device: null, browser: 'chromium', viewport: '1800x1600' }], ])('groups across %s when the project says so', async (groupBy, fixedAxes) => { - const testRun = sibling('target'); - const { built, testRunFindManyMock } = await initEdgeService({ - testRun, + const ctx = await initMatching({ + testRun: sibling('target'), project: { bulkApproveVariations: true, bulkApproveGroupBy: groupBy === 'an unknown axis' ? 'locale' : groupBy, }, }); - await built.getMatchingVariations(testRun.id); + await ctx.matches(); - const { where } = testRunFindManyMock.mock.calls[0][0]; + const { where } = ctx.m.testRun.findMany.mock.calls[0][0]; const axes = ['customTags', 'os', 'device', 'browser', 'viewport'].filter((axis) => axis in where); expect(Object.fromEntries(axes.map((axis) => [axis, where[axis]]))).toEqual(fixedAxes); }); it('signs with the threshold and anti-aliasing setting of the project config', async () => { const testRun = sibling('target', { ignoreAreas: JSON.stringify(ignoreAreas) }); - const { built, compareGetChangeSignatureMock } = await initEdgeService({ + const ctx = await initMatching({ testRun, project: { bulkApproveVariations: true, @@ -1888,9 +2348,9 @@ describe('TestRunsService', () => { }, }); - await built.getMatchingVariations(testRun.id); + await ctx.matches(); - expect(compareGetChangeSignatureMock).toHaveBeenCalledWith({ + expect(ctx.m.compare.getChangeSignature).toHaveBeenCalledWith({ baseline: Buffer.from(testRun.baselineName!), image: Buffer.from(testRun.imageName), ignoreAreas, @@ -1903,36 +2363,29 @@ describe('TestRunsService', () => { const testRun = sibling('target', { baselineName: null }); const unchanged = sibling('locale-a', { diffPercent: 0 }); const candidate = sibling('locale-b'); - const { built, compareGetChangeSignatureMock } = await initEdgeService({ - testRun, - siblings: [unchanged, candidate], - }); + const ctx = await initMatching({ testRun, siblings: [unchanged, candidate] }); - const result = await built.getMatchingVariations(testRun.id); + const result = await ctx.matches(); - expect(result.variations.map((variation) => variation.id)).toEqual([testRun.id]); - expect(result.skipped.map((item) => [item.id, item.reason])).toEqual([ + expect(variationIds(result)).toEqual([testRun.id]); + expect(skippedPairs(result)).toEqual([ [unchanged.id, 'no reference diff'], [candidate.id, 'no reference diff'], ]); - // NOTE: current behavior — the candidate is still signed although there is nothing to match it against - expect(compareGetChangeSignatureMock.mock.calls.map(([input]) => input.image.toString())).toEqual([ - candidate.imageName, - ]); + expect(ctx.signed()).toEqual([candidate.imageName]); }); it('treats a missing screenshot as no reference diff', async () => { - const testRun = sibling('target'); - const { built, compareGetChangeSignatureMock } = await initEdgeService({ - testRun, + const ctx = await initMatching({ + testRun: sibling('target'), siblings: [sibling('locale-a')], - getImageBufferMock: jest.fn().mockResolvedValue(null), + imageBufferOf: async () => null, }); - const result = await built.getMatchingVariations(testRun.id); + const result = await ctx.matches(); - expect(compareGetChangeSignatureMock).not.toHaveBeenCalled(); - expect(result.skipped.map((item) => item.reason)).toEqual(['no reference diff']); + expect(ctx.m.compare.getChangeSignature).not.toHaveBeenCalled(); + expect(skippedReasons(result)).toEqual(['no reference diff']); }); it('skips a sibling that has nothing to compare with no diff to match', async () => { @@ -1940,35 +2393,34 @@ describe('TestRunsService', () => { const noBaseline = sibling('locale-a', { baselineName: null }); const noSignature = sibling('locale-b'); const matching = sibling('locale-c'); - const { built } = await initEdgeService({ + const ctx = await initMatching({ testRun, siblings: [noBaseline, noSignature, matching], signatureOf: (imageName) => (imageName.startsWith('locale-b') ? null : SAME_PALETTE), }); - const result = await built.getMatchingVariations(testRun.id); + const result = await ctx.matches(); - expect(result.variations.map((variation) => variation.id)).toEqual([testRun.id, matching.id]); - expect(result.skipped.map((item) => [item.id, item.reason])).toEqual([ + expect(variationIds(result)).toEqual([testRun.id, matching.id]); + expect(skippedPairs(result)).toEqual([ [noBaseline.id, 'no diff to match'], [noSignature.id, 'no diff to match'], ]); }); it('lists skipped siblings in the order they were found, whichever check skipped them', async () => { - const testRun = sibling('target'); const otherPattern = sibling('locale-a'); const unchanged = sibling('locale-b', { diffPercent: 0 }); const bigger = sibling('locale-c', { diffPercent: 100 }); - const { built } = await initEdgeService({ - testRun, + const ctx = await initMatching({ + testRun: sibling('target'), siblings: [otherPattern, unchanged, bigger], - signatureOf: (imageName) => (imageName.startsWith('locale-a') ? histogram(SIGNATURE_LENGTH - 1) : SAME_PALETTE), + signatureOf: (imageName) => (imageName.startsWith('locale-a') ? OTHER_PALETTE : SAME_PALETTE), }); - const result = await built.getMatchingVariations(testRun.id); + const result = await ctx.matches(); - expect(result.skipped.map((item) => [item.id, item.reason])).toEqual([ + expect(skippedPairs(result)).toEqual([ [otherPattern.id, 'different change pattern'], [unchanged.id, 'no diff to match'], [bigger.id, 'different change size'], @@ -1986,72 +2438,48 @@ describe('TestRunsService', () => { ])('with the reviewed change at %p%% a sibling at %p%% %s', async (reference, size, outcome) => { const testRun = sibling('target', { diffPercent: reference }); const other = sibling('locale-a', { diffPercent: size }); - const { built } = await initEdgeService({ testRun, siblings: [other] }); + const ctx = await initMatching({ testRun, siblings: [other] }); - const result = await built.getMatchingVariations(testRun.id); + const result = await ctx.matches(); if (outcome === 'matches') { - expect(result.variations.map((variation) => variation.id)).toEqual([testRun.id, other.id]); + expect(variationIds(result)).toEqual([testRun.id, other.id]); } else { - expect(result.skipped.map((item) => item.reason)).toEqual([outcome]); + expect(skippedReasons(result)).toEqual([outcome]); } }); it('recomputes and warns when the stored signature cannot be read', async () => { - const testRun = sibling('target', { changeSignature: '{not json' }); - const { built, compareGetChangeSignatureMock } = await initEdgeService({ testRun }); - const warn = jest.spyOn(built['logger'], 'warn').mockImplementation(() => undefined); + const ctx = await initMatching({ testRun: sibling('target', { changeSignature: '{not json' }) }); + const warn = jest.spyOn(ctx.service['logger'], 'warn').mockImplementation(() => undefined); - await built.getMatchingVariations(testRun.id); + await ctx.matches(); expect(warn).toHaveBeenCalledWith(expect.stringContaining('Ignoring unreadable stored change signature')); - expect(compareGetChangeSignatureMock).toHaveBeenCalledTimes(1); - }); - - it('recomputes a stored signature that has no histogram at all', async () => { - const testRun = sibling('target', { changeSignature: 'null' }); - const { built, compareGetChangeSignatureMock } = await initEdgeService({ testRun }); - - await built.getMatchingVariations(testRun.id); - - expect(compareGetChangeSignatureMock).toHaveBeenCalledTimes(1); + expect(ctx.m.compare.getChangeSignature).toHaveBeenCalledTimes(1); }); - // the default project config is threshold 0.1 with antialiasing ignored - const stamped = (signature: unknown[], stamp = { threshold: 0.1, includeAA: true }) => - JSON.stringify({ ...stamp, signature }); - it('reuses a stored signature stamped with the current config', async () => { - const testRun = sibling('target', { changeSignature: stamped(SAME_PALETTE) }); - const { built, compareGetChangeSignatureMock } = await initEdgeService({ testRun }); - - await built.getMatchingVariations(testRun.id); - - expect(compareGetChangeSignatureMock).not.toHaveBeenCalled(); - }); - - it.each([ - ['only the threshold changed', { threshold: 0.2, includeAA: true }], - ['only the antialiasing setting changed', { threshold: 0.1, includeAA: false }], - ])('recomputes a stored signature when %s', async (_case, stamp) => { - const testRun = sibling('target', { changeSignature: stamped(SAME_PALETTE, stamp) }); - const { built, compareGetChangeSignatureMock } = await initEdgeService({ testRun }); + const ctx = await initMatching({ testRun: sibling('target', { changeSignature: stamped(SAME_PALETTE) }) }); - await built.getMatchingVariations(testRun.id); + await ctx.matches(); - expect(compareGetChangeSignatureMock).toHaveBeenCalledTimes(1); + expect(ctx.m.compare.getChangeSignature).not.toHaveBeenCalled(); }); it.each([ + ['only the threshold changed', stamped(SAME_PALETTE, { threshold: 0.2, includeAA: true })], + ['only the antialiasing setting changed', stamped(SAME_PALETTE, { threshold: 0.1, includeAA: false })], + ['the wrong length', stamped([1, 0])], + ['values that are not numbers', stamped(new Array(SIGNATURE_LENGTH).fill('nope'))], ['a single value that is not a number', stamped([1, ...new Array(SIGNATURE_LENGTH - 1).fill('nope')])], ['a value that is not finite', stamped(SAME_PALETTE).replace('[1,', '[1e999,')], - ])('recomputes a stored signature with %s', async (_case, changeSignature) => { - const testRun = sibling('target', { changeSignature }); - const { built, compareGetChangeSignatureMock } = await initEdgeService({ testRun }); + ])('recomputes a stored signature when %s', async (_case, changeSignature) => { + const ctx = await initMatching({ testRun: sibling('target', { changeSignature }) }); - await built.getMatchingVariations(testRun.id); + await ctx.matches(); - expect(compareGetChangeSignatureMock).toHaveBeenCalledTimes(1); + expect(ctx.m.compare.getChangeSignature).toHaveBeenCalledTimes(1); }); it.each([ @@ -2059,20 +2487,19 @@ describe('TestRunsService', () => { ['an empty stored signature', ''], ['a stored json null', 'null'], ])('computes quietly for %s', async (_case, changeSignature) => { - const testRun = sibling('target', { changeSignature }); - const { built, compareGetChangeSignatureMock } = await initEdgeService({ testRun }); - const warn = jest.spyOn(built['logger'], 'warn').mockImplementation(() => undefined); + const ctx = await initMatching({ testRun: sibling('target', { changeSignature }) }); + const warn = jest.spyOn(ctx.service['logger'], 'warn').mockImplementation(() => undefined); - await built.getMatchingVariations(testRun.id); + await ctx.matches(); expect(warn).not.toHaveBeenCalled(); - expect(compareGetChangeSignatureMock).toHaveBeenCalledTimes(1); + expect(ctx.m.compare.getChangeSignature).toHaveBeenCalledTimes(1); }); it('does not remember a failed signature', async () => { const testRun = sibling('target'); let calls = 0; - const { built, compareGetChangeSignatureMock } = await initEdgeService({ + const ctx = await initMatching({ testRun, signatureOf: () => { calls += 1; @@ -2083,61 +2510,53 @@ describe('TestRunsService', () => { }, }); - await expect(built.getMatchingVariations(testRun.id)).rejects.toThrow('worker crashed'); - const result = await built.getMatchingVariations(testRun.id); + await expect(ctx.matches()).rejects.toThrow('worker crashed'); + const result = await ctx.matches(); - expect(compareGetChangeSignatureMock).toHaveBeenCalledTimes(2); - expect(result.variations.map((variation) => variation.id)).toEqual([testRun.id]); + expect(ctx.m.compare.getChangeSignature).toHaveBeenCalledTimes(2); + expect(variationIds(result)).toEqual([testRun.id]); }); }); describe('signature cache', () => { - beforeEach(async () => { - service = await initService({}); - }); - - it('forgets the oldest entry once it holds more than 2000', () => { + const filledCache = async (size = 2000) => { + const { service } = await build(); const cache = service['signatureCache']; - for (let index = 0; index <= 2000; index++) { + for (let index = 0; index < size; index++) { cache.set(`key ${index}`, Promise.resolve([index])); } + return cache; + }; + + it('forgets the oldest entry once it holds more than 2000', async () => { + const cache = await filledCache(2001); expect(cache.get('key 0')).toBeUndefined(); expect(cache.get('key 1')).toBeDefined(); expect(cache.get('key 2000')).toBeDefined(); }); - it('does not take up room for a key that was looked up but never set', () => { - const cache = service['signatureCache']; - for (let index = 0; index < 2000; index++) { - cache.set(`key ${index}`, Promise.resolve([index])); - } + it('does not take up room for a key that was looked up but never set', async () => { + const cache = await filledCache(); expect(cache.get('absent')).toBeUndefined(); - // replacing an existing key keeps the size, so nothing is evicted cache.set('key 0', Promise.resolve([0])); expect(cache.get('key 1')).toBeDefined(); }); - it('keeps an entry that was read recently', () => { - const cache = service['signatureCache']; - for (let index = 0; index < 2000; index++) { - cache.set(`key ${index}`, Promise.resolve([index])); - } + it('keeps an entry that was read recently', async () => { + const cache = await filledCache(); - cache.get('key 0'); + void cache.get('key 0'); cache.set('one more', Promise.resolve([])); expect(cache.get('key 0')).toBeDefined(); expect(cache.get('key 1')).toBeUndefined(); }); - it('moves a re-set entry to the back', () => { - const cache = service['signatureCache']; - for (let index = 0; index < 2000; index++) { - cache.set(`key ${index}`, Promise.resolve([index])); - } + it('moves a re-set entry to the back', async () => { + const cache = await filledCache(); const replacement = Promise.resolve([42]); cache.set('key 0', replacement); diff --git a/src/test-runs/test-runs.service.ts b/src/test-runs/test-runs.service.ts index 03e042f8..41d27fe8 100644 --- a/src/test-runs/test-runs.service.ts +++ b/src/test-runs/test-runs.service.ts @@ -1,15 +1,18 @@ -import { forwardRef, Inject, Injectable, Logger, NotFoundException } from '@nestjs/common'; +import { ConflictException, forwardRef, Inject, Injectable, Logger, NotFoundException } from '@nestjs/common'; import { CreateTestRequestDto } from './dto/create-test-request.dto'; import { IgnoreAreaDto } from './dto/ignore-area.dto'; import { StaticService } from '../static/static.service'; import { PrismaService } from '../prisma/prisma.service'; -import { Baseline, Prisma, TestRun, TestStatus, TestVariation } from '@prisma/client'; +import { isPrismaError } from '../prisma/prisma-errors'; +import { Prisma, Project, TestRun, TestStatus, TestVariation } from '@prisma/client'; import { DiffResult, StampedSignature } from './diffResult'; import { EventsGateway } from '../shared/events/events.gateway'; import { TestRunResultDto } from '../test-runs/dto/testRunResult.dto'; -import { TestVariationsService } from '../test-variations/test-variations.service'; +import { BaselineRef, TestVariationsService } from '../test-variations/test-variations.service'; import { TestRunDto } from './dto/testRun.dto'; -import { getTestVariationUniqueData } from '../utils'; +import { getTestVariationUniqueData, parseIgnoreAreas, testRunImageNames, variationKey } from '../utils'; +import { BoundedCache } from '../utils/bounded-cache'; +import { mapWithConcurrency } from '../utils/map-with-concurrency'; import { CompareService } from '../compare/compare.service'; import { UpdateTestRunDto } from './dto/update-test.dto'; import { parseConfig } from '../compare/utils'; @@ -39,12 +42,7 @@ export class TestRunsService { return list.map((item) => new TestRunDto(item)); } - async findOne(id: string): Promise< - | (TestRun & { - testVariation: TestVariation | null; - }) - | null - > { + async findOne(id: string): Promise { return this.prismaService.testRun.findUnique({ where: { id }, include: { @@ -53,6 +51,14 @@ export class TestRunsService { }); } + async findOneOrThrow(id: string): Promise { + const testRun = await this.findOne(id); + if (!testRun) { + throw testRunNotFound(id); + } + return testRun; + } + async postTestRun({ createTestRequestDto, imageBuffer, @@ -60,138 +66,317 @@ export class TestRunsService { createTestRequestDto: CreateTestRequestDto; imageBuffer: Buffer; }): Promise { - const project = await this.prismaService.project.findUnique({ where: { id: createTestRequestDto.projectId } }); + const [project, build] = await Promise.all([ + this.prismaService.project.findUnique({ where: { id: createTestRequestDto.projectId } }), + this.prismaService.build.findUnique({ where: { id: createTestRequestDto.buildId }, select: { projectId: true } }), + ]); if (!project) { throw new NotFoundException(`Project not found: ${createTestRequestDto.projectId}`); } - // A comparator that cannot run on this server (odiff without HDD storage) - // fails the request here, before the run is stored: afterwards it would - // stay in the build as "new", never to get a verdict. + if (!build) { + throw new NotFoundException(`Build not found: ${createTestRequestDto.buildId}`); + } + if (build.projectId !== createTestRequestDto.projectId) { + this.warnForeignBuild(createTestRequestDto); + } this.compareService.getComparator(project.imageComparison); - let testVariation = await this.testVariationService.find(createTestRequestDto); - // creates variatioin if does not exist - if (!testVariation) { - testVariation = await this.testVariationService.create({ - createTestRequestDto, - }); + // allSettled, not all: a rejection must not leave the other promises unhandled or the screenshot orphaned + const [savedImage, foundVariation, previousRuns] = await Promise.allSettled([ + this.staticService.saveImage('screenshot', imageBuffer), + this.findOrCreateVariation(createTestRequestDto, project.mainBranchName), + this.prismaService.testRun.findMany({ + where: { + buildId: createTestRequestDto.buildId, + branchName: createTestRequestDto.branchName, + ...getTestVariationUniqueData(createTestRequestDto), + NOT: { OR: [{ status: TestStatus.approved }, { status: TestStatus.autoApproved }] }, + }, + }), + ]); + if ( + savedImage.status === 'rejected' || + foundVariation.status === 'rejected' || + previousRuns.status === 'rejected' + ) { + if (savedImage.status === 'fulfilled') { + await this.staticService.deleteImage(savedImage.value); + } + throw [savedImage, foundVariation, previousRuns].find((result) => result.status === 'rejected')!.reason; } + const imageName = savedImage.value; + const testVariation = foundVariation.value; + const [previousTestRun] = previousRuns.value; - // delete previous test run if exists - const [previousTestRun] = await this.prismaService.testRun.findMany({ - where: { - buildId: createTestRequestDto.buildId, - branchName: createTestRequestDto.branchName, - ...getTestVariationUniqueData(createTestRequestDto), - NOT: { OR: [{ status: TestStatus.approved }, { status: TestStatus.autoApproved }] }, - }, - }); - if (!!previousTestRun) { - await this.delete(previousTestRun.id); + let testRun: TestRun; + try { + if (previousTestRun) { + await this.delete(previousTestRun.id); + } + testRun = await this.insertTestRun({ testVariation, createTestRequestDto, imageName }); + } catch (e) { + await this.staticService.deleteImage(imageName); + throw e; } + let testRunWithResult = await this.calculateDiff(createTestRequestDto.projectId, testRun, imageBuffer, project); - // create test run result - const testRun = await this.create({ testVariation, createTestRequestDto, imageBuffer }); - - // calculate diff - let testRunWithResult = await this.calculateDiff(createTestRequestDto.projectId, testRun); - - // try auto approve if (project.autoApproveFeature) { - testRunWithResult = await this.tryAutoApproveByPastBaselines({ testVariation, testRun: testRunWithResult }); - testRunWithResult = await this.tryAutoApproveByNewBaselines({ testVariation, testRun: testRunWithResult }); + testRunWithResult = await this.tryAutoApproveByPastBaselines({ + testVariation, + testRun: testRunWithResult, + project, + imageBuffer, + }); + testRunWithResult = await this.tryAutoApproveByNewBaselines({ + testVariation, + testRun: testRunWithResult, + project, + imageBuffer, + }); } return new TestRunResultDto(testRunWithResult, testVariation); } - /** - * Confirm difference for testRun - */ + private async findOrCreateVariation( + createTestRequestDto: CreateTestRequestDto, + mainBranchName: string + ): Promise { + return ( + (await this.testVariationService.find(createTestRequestDto, mainBranchName)) ?? + (await this.testVariationService.createOrTakeWinner(createTestRequestDto)) + ); + } + + private warnForeignBuild({ buildId, projectId }: CreateTestRequestDto): void { + this.logger.warn(`Build ${buildId} does not belong to project ${projectId}`); + } + async approve(id: string, merge = false, autoApprove = false, userId?: string): Promise { - this.logger.log(`Approving testRun: ${id} merge: ${merge} autoApprove: ${autoApprove}`); - const testRun = await this.findOne(id); - if (!testRun) { - throw new NotFoundException(`Test run not found: ${id}`); + try { + return await this.approveOnce(id, merge, autoApprove, userId); + } catch (e) { + if (isPrismaError(e, 'P2002')) { + return this.approveOnce(id, merge, autoApprove, userId); + } + throw e; } - let { testVariation } = testRun; + } + + private async approveOnce(id: string, merge: boolean, autoApprove: boolean, userId?: string): Promise { + this.logger.log(`Approving testRun: ${id} merge: ${merge} autoApprove: ${autoApprove}`); + const testRun = await this.findOneOrThrow(id); + const { testVariation } = testRun; if (!testVariation) { - throw new Error('No test variation found. Re-create test run'); - } - - // save new baseline as a byte-for-byte copy — decoding and re-encoding the - // PNG here blocked the event loop for every approved run - const baselineName = await this.staticService.copyImage('baseline', testRun.imageName); - - if (testRun.baselineBranchName !== testRun.branchName && !merge && !autoApprove) { - // replace main branch with feature branch test variation - // looked up by the run's own columns, which it copied from its variation - const featureBranchTestVariation = await this.testVariationService.findUnique( - testRun as Parameters[0] - ); - - if (!featureBranchTestVariation) { - testVariation = await this.testVariationService.create({ - testRunId: id, - createTestRequestDto: { - projectId: testRun.projectId!, - branchName: testRun.branchName, - ...getTestVariationUniqueData(testRun), + throw new ConflictException(NO_TEST_VARIATION); + } + if (await this.isAlreadyApproved(testRun, autoApprove)) { + return testRun; + } + + const toFeatureBranch = testRun.baselineBranchName !== testRun.branchName && !merge && !autoApprove; + const addsBaseline = !autoApprove || testRun.baselineBranchName === testRun.branchName; + const baselineName = addsBaseline ? await this.staticService.copyImage('baseline', testRun.imageName) : null; + + let claimed: boolean; + try { + claimed = await this.prismaService.$transaction(async (tx) => { + const { count } = await tx.testRun.updateMany({ + where: { id, status: testRun.status }, + data: { + status: autoApprove ? TestStatus.autoApproved : TestStatus.approved, + statusBeforeDecision: testRun.status, }, }); - } else { - testVariation = featureBranchTestVariation; + if (count === 0) { + return false; + } + let target: TestVariation = testVariation; + if (toFeatureBranch) { + const key = variationKey(testVariation.projectId, testRun.branchName, testRun); + const featureBranchTestVariation = + (await this.testVariationService.findUnique(key, tx)) ?? + (await this.testVariationService.create({ createTestRequestDto: key }, tx)); + target = await this.testVariationService.update( + featureBranchTestVariation.id, + { ignoreAreas: testRun.ignoreAreas, comment: testRun.comment }, + id, + tx + ); + } + if (baselineName) { + await this.testVariationService.addBaseline({ id: target.id, userId, testRunId: id, baselineName }, tx); + } + return true; + }); + } catch (e) { + await this.staticService.deleteImage(baselineName); + throw e; + } + if (!claimed) { + await this.staticService.deleteImage(baselineName); + return (await this.findOne(id)) ?? testRun; + } + + const approved = (await this.findOne(id))!; + this.eventsGateway.testRunUpdated(approved); + return approved; + } + + private async isAlreadyApproved(testRun: TestRun, autoApprove: boolean): Promise { + if (testRun.status === TestStatus.approved) { + return true; + } + if (testRun.status !== TestStatus.autoApproved) { + return false; + } + return autoApprove || (await this.prismaService.baseline.count({ where: { testRunId: testRun.id } })) > 0; + } + + async reject(id: string): Promise { + const testRun = await this.prismaService.testRun.findUnique({ where: { id } }); + if (!testRun) { + throw testRunNotFound(id); + } + if (testRun.status === TestStatus.failed) { + return testRun; + } + const { count } = await this.prismaService.testRun.updateMany({ + where: { id, status: testRun.status }, + data: { status: TestStatus.failed, statusBeforeDecision: testRun.status }, + }); + const current = (await this.prismaService.testRun.findUnique({ where: { id } }))!; + if (count > 0) { + this.eventsGateway.testRunUpdated(current); + } + return current; + } + + async revert(ids: string[]): Promise { + const uniqueIds = [...new Set(ids)]; + if (uniqueIds.length === 0) { + throw new ConflictException('Nothing to revert'); + } + const found = await this.prismaService.testRun.findMany({ where: { id: { in: uniqueIds } } }); + const byId = new Map(found.map((run) => [run.id, run])); + const missing = uniqueIds.filter((id) => !byId.has(id)); + if (missing.length) { + throw testRunNotFound(missing.join(', ')); + } + const runs = uniqueIds.map((id) => byId.get(id)!); + + const freedBaselines = await this.prismaService.$transaction(async (tx) => { + const freed: { name: string; baselineName: string }[] = []; + for (const run of await this.orderForRevert(runs, tx)) { + if (run.status === TestStatus.failed) { + await this.restoreStatus(run, {}, tx); + } else if (run.status === TestStatus.approved) { + freed.push({ name: run.name, baselineName: await this.revertApproval(run, tx) }); + } else { + throw new ConflictException( + `Test run ${run.id} is ${run.status}: only approved or rejected runs can be reverted` + ); + } } + return freed; + }); - // carry over data from testRun - testVariation = await this.testVariationService.update( - testVariation.id, - { - baselineName: testRun.baselineName, - ignoreAreas: testRun.ignoreAreas, - comment: testRun.comment, - }, - testRun.id - ); + for (const { name, baselineName } of freedBaselines) { + if (!(await this.isBaselineReferenced(name, baselineName))) { + await this.staticService.deleteImage(baselineName); + } } + const reverted = await this.prismaService.testRun.findMany({ where: { id: { in: uniqueIds } } }); + const revertedById = new Map(reverted.map((run) => [run.id, run])); + for (const id of uniqueIds) { + this.eventsGateway.testRunUpdated(revertedById.get(id)!); + } + } - if (!autoApprove || (autoApprove && testRun.baselineBranchName === testRun.branchName)) { - // add baseline - await this.testVariationService.addBaseline({ - id: testVariation.id, - userId, - testRunId: testRun.id, - baselineName, - }); + private async orderForRevert(runs: TestRun[], tx: Prisma.TransactionClient): Promise { + const approved = runs.filter((run) => run.status === TestStatus.approved); + const baselines = await tx.baseline.findMany({ + where: { testRunId: { in: approved.map((run) => run.id) } }, + select: { testRunId: true, createdAt: true }, + }); + const approvedAt = new Map(baselines.map((baseline) => [baseline.testRunId, baseline.createdAt.getTime()])); + approved.sort((a, b) => (approvedAt.get(b.id) ?? 0) - (approvedAt.get(a.id) ?? 0)); + return [...approved, ...runs.filter((run) => run.status !== TestStatus.approved)]; + } + + private async restoreStatus( + run: TestRun, + extra: { testVariationId?: string }, + tx: Prisma.TransactionClient + ): Promise { + const { count } = await tx.testRun.updateMany({ + where: { id: run.id, status: run.status }, + data: { + status: run.statusBeforeDecision ?? (run.baselineName ? TestStatus.unresolved : TestStatus.new), + statusBeforeDecision: null, + ...extra, + }, + }); + if (count === 0) { + throw new ConflictException(`Test run ${run.id} changed while reverting, try again`); } + } + + private async revertApproval(run: TestRun, tx: Prisma.TransactionClient): Promise { + const baseline = await tx.baseline.findUnique({ where: { testRunId: run.id } }); + if (!baseline) { + throw new ConflictException(`Test run ${run.id} has no baseline to revert`); + } + const latest = await latestBaseline(tx, baseline.testVariationId); + const variation = await tx.testVariation.findUnique({ where: { id: baseline.testVariationId } }); + if (latest?.id !== baseline.id || variation?.baselineName !== baseline.baselineName) { + throw new ConflictException(`Test run ${run.id}: a later approval changed this baseline`); + } + await tx.baseline.delete({ where: { id: baseline.id } }); + const previous = await latestBaseline(tx, variation.id); + const restoreBaselineName = () => + tx.testVariation.update({ where: { id: variation.id }, data: { baselineName: previous?.baselineName ?? null } }); + + const wasFeatureBranchApproval = + variation.branchName === run.branchName && run.baselineBranchName !== run.branchName; + if (!wasFeatureBranchApproval) { + await restoreBaselineName(); + await this.restoreStatus(run, {}, tx); + return baseline.baselineName; + } + + const mainVariation = await this.testVariationService.findUnique( + variationKey(variation.projectId, run.baselineBranchName!, run), + tx + ); + if (!mainVariation) { + throw new ConflictException(NO_TEST_VARIATION); + } + await this.restoreStatus(run, { testVariationId: mainVariation.id }, tx); + const [baselines, testRuns] = await Promise.all([ + tx.baseline.count({ where: { testVariationId: variation.id } }), + tx.testRun.count({ where: { testVariationId: variation.id } }), + ]); + if (baselines === 0 && testRuns === 0) { + await tx.testVariation.delete({ where: { id: variation.id } }); + } else { + await restoreBaselineName(); + } + return baseline.baselineName; + } + + private async isBaselineReferenced(name: string, baselineName: string): Promise { + const [runs, variations, baselines] = await Promise.all([ + this.prismaService.testRun.count({ where: { name, baselineName } }), + this.prismaService.testVariation.count({ where: { baselineName } }), + this.prismaService.baseline.count({ where: { baselineName } }), + ]); + return runs + variations + baselines > 0; + } - // update status - const status = autoApprove ? TestStatus.autoApproved : TestStatus.approved; - return this.setStatus(id, status); - } - - /** - * Finds sibling variations of a test run in the same build — same screen - * (name) and same test-variation axes except the one configured on the project - * as `bulkApproveGroupBy` (customTags by default, i.e. per-locale screenshots). - * The feature is opt-in per project via `bulkApproveVariations`. - * - * Matching uses a position-independent color signature of the changed pixels - * (which colors appeared/changed), so it survives per-locale layout shifts — - * e.g. a title wrapping to a different number of lines pushes the whole screen - * down, but the same change (a moved selection, a recolored button) still - * produces the same signature. A loose guard on change size (diffPercent) also - * skips variations whose change area is far larger/smaller — a same-palette but - * additional/different change — while tolerating per-locale text reflow. - * Siblings that don't match, or have no diff to compare, are reported as - * skipped. Used by {@link getMatchingVariations}. - */ private async findMatchingSiblings( id: string ): Promise<{ testRun: TestRun; matching: TestRun[]; skipped: SkippedSibling[] }> { - const testRun = await this.findOne(id); - if (!testRun) { - throw new NotFoundException(`Test run not found: ${id}`); - } + const testRun = await this.findOneOrThrow(id); const project = testRun.projectId ? await this.prismaService.project.findUnique({ where: { id: testRun.projectId } }) @@ -200,18 +385,13 @@ export class TestRunsService { return { testRun, matching: [], skipped: [] }; } - // Mirror the project's diff config so the change signature classifies the - // same way the normal pixelmatch diff does, rather than a hardcoded threshold. - const config: PixelmatchConfig = { - ...DEFAULT_CONFIG, - ...parseConfig(project.imageComparisonConfig, DEFAULT_CONFIG, this.logger), - }; + const config: PixelmatchConfig = parseConfig(project.imageComparisonConfig, DEFAULT_CONFIG, this.logger); const groupBy = resolveGroupByAxis(project.bulkApproveGroupBy); const fixedAxes: Record = {}; for (const axis of GROUP_BY_AXES) { if (axis !== groupBy) { - fixedAxes[axis] = (testRun as unknown as Record)[axis] ?? null; + fixedAxes[axis] = testRun[axis] ?? null; } } @@ -226,73 +406,46 @@ export class TestRunsService { }, }); - const matching: TestRun[] = []; - const skipped: SkippedSibling[] = []; - - // Comparing change sizes is free, computing a signature costs two decoded - // screenshots — so the cheap test goes first and takes those siblings out - // of the expensive pass entirely. - const candidates: TestRun[] = []; - for (const sibling of siblings) { - if (!sibling.diffPercent) { - // Nothing changed on this one, so there is no change to recognise — - // the same verdict the signature pass would have reached, for free. - skipped.push({ run: sibling, reason: 'no diff to match' }); - } else if (!magnitudesSimilar(testRun.diffPercent, sibling.diffPercent)) { - skipped.push({ run: sibling, reason: 'different change size' }); - } else { - candidates.push(sibling); - } - } - - // The reviewed run and every remaining sibling are signed in one bounded - // fan-out across the worker pool. Signing them one after another is what - // made this dialog take tens of seconds on a build with many locales. + const candidates = siblings.filter( + (sibling) => !!sibling.diffPercent && magnitudesSimilar(testRun.diffPercent, sibling.diffPercent) + ); const [referenceSignature, ...candidateSignatures] = await mapWithConcurrency( [testRun, ...candidates], SIGNATURE_CONCURRENCY, (run) => this.getChangeSignature(run, config) ); - // Nothing to match against: every sibling goes to manual review, whatever - // the cheap pass made of it. if (!referenceSignature) { return { testRun, - matching, + matching: [], skipped: siblings.map((run) => ({ run, reason: 'no reference diff' })), }; } - candidates.forEach((sibling, index) => { - const signature = candidateSignatures[index]; - if (!signature) { - skipped.push({ run: sibling, reason: 'no diff to match' }); + const signatures = new Map(candidates.map((sibling, index) => [sibling.id, candidateSignatures[index]] as const)); + const matching: TestRun[] = []; + const skipped: SkippedSibling[] = []; + for (const sibling of siblings) { + const signature = signatures.get(sibling.id); + let reason: string | null = null; + if (!sibling.diffPercent || signature === null) { + reason = 'no diff to match'; + } else if (signature === undefined) { + reason = 'different change size'; } else if (!signaturesMatch(referenceSignature, signature)) { - skipped.push({ run: sibling, reason: 'different change pattern' }); + reason = 'different change pattern'; + } + if (reason) { + skipped.push({ run: sibling, reason }); } else { matching.push(sibling); } - }); - - // The cheap pass ran first, so restore the order the siblings came in — - // the dialog lists them as one group and its order should not depend on - // which test rejected a sibling. - const orderOf = new Map(siblings.map((sibling, index) => [sibling.id, index] as const)); - skipped.sort((a, b) => orderOf.get(a.run.id)! - orderOf.get(b.run.id)!); + } return { testRun, matching, skipped }; } - /** - * Returns the group of variations for the reviewed run — the run itself plus - * the sibling variations whose change matches it — for the reviewer to confirm - * before a bulk approve/reject. Nothing is mutated here: the actual action is - * driven by the reviewer's explicit selection (regular approve/reject of the - * chosen ids), so a regression that merely looks similar can never be approved - * without a human seeing it. `skipped` lists siblings left out of the group - * (different/additional change, or no diff to compare). - */ async getMatchingVariations( id: string ): Promise<{ variations: TestRunDto[]; skipped: Array }> { @@ -302,19 +455,6 @@ export class TestRunsService { return { variations, skipped: skippedDto }; } - /** - * Position-independent color signature of the change between a test run's - * baseline and image. Null when there is no baseline, dimensions differ, or - * nothing changed. - * - * Normally this was written at ingest, beside the diff that had already - * decoded both screenshots, and grouping a screen costs nothing but the row - * it is read from. The rest of this is the fallback for runs that carry no - * stored signature — ingested before the column existed, or compared by - * something other than pixelmatch: their bytes go to the worker pool - * undecoded, and the result is memoized so reopening the dialog on an old - * build does not pay for the same decodes twice. - */ private async getChangeSignature(testRun: TestRun, config: PixelmatchConfig): Promise { const stored = parseStoredSignature(testRun.changeSignature, config, this.logger); if (stored) { @@ -323,9 +463,7 @@ export class TestRunsService { if (!testRun.baselineName) { return null; } - const ignoreAreas = this.getAllIgnoteAreas(testRun); - // Image names are unique per upload, so only the ignore areas and the - // project's diff config can change a signature under a stable pair. + const ignoreAreas = this.getAllIgnoreAreas(testRun); const key = [ testRun.baselineName, testRun.imageName, @@ -340,7 +478,6 @@ export class TestRunsService { const pending = this.computeChangeSignature(testRun, ignoreAreas, config); this.signatureCache.set(key, pending); - // A failed read must not be remembered as "no signature" forever. pending.catch(() => this.signatureCache.delete(key)); return pending; } @@ -367,75 +504,43 @@ export class TestRunsService { return signature; } - async setStatus(id: string, status: TestStatus): Promise { + async saveDiffResult(id: string, diffResult: DiffResult | null): Promise { const testRun = await this.prismaService.testRun.update({ where: { id }, data: { - status, + diffName: diffResult && diffResult.diffName, + pixelMisMatchCount: diffResult && diffResult.pixelMisMatchCount, + diffPercent: diffResult && diffResult.diffPercent, + status: diffResult ? diffResult.status : TestStatus.new, + // null, not undefined: Prisma skips undefined and would keep the previous analysis + vlmDescription: diffResult?.vlmDescription ?? null, + changeSignature: diffResult?.changeSignature ? JSON.stringify(diffResult.changeSignature) : null, + imageThumbnailName: diffResult?.imageThumbnailName ?? null, + diffThumbnailName: diffResult?.diffThumbnailName ?? null, }, }); - this.eventsGateway.testRunUpdated(testRun); - return (await this.findOne(id))!; - } - - // Nullable because it genuinely is: a comparison that produced nothing at all - // clears the row back to "new". The body has always handled that; the - // signature just did not say so. - async saveDiffResult(id: string, diffResult: DiffResult | null): Promise { - return this.prismaService.testRun - .update({ - where: { id }, - data: { - diffName: diffResult && diffResult.diffName, - pixelMisMatchCount: diffResult && diffResult.pixelMisMatchCount, - diffPercent: diffResult && diffResult.diffPercent, - status: diffResult ? diffResult.status : TestStatus.new, - // null, not undefined: Prisma skips undefined, which would leave the - // previous comparison's analysis on a run recomputed without one — - // e.g. one the rendering-noise filter now passes before VLM is asked - vlmDescription: diffResult?.vlmDescription ?? null, - // Always written, never merged: a recomputed diff — after the - // reviewer edits the ignore areas, say — must not leave the previous - // signature behind describing a change that no longer exists. - changeSignature: diffResult?.changeSignature ? JSON.stringify(diffResult.changeSignature) : null, - // Overwritten together with the diff they were made from, so a - // recomputed comparison never leaves a picture of the old change. - imageThumbnailName: diffResult?.imageThumbnailName ?? null, - diffThumbnailName: diffResult?.diffThumbnailName ?? null, - }, - }) - .then((testRun) => { - this.eventsGateway.testRunUpdated(testRun); - return testRun; - }); + return testRun; } - async calculateDiff(projectId: string, testRun: TestRun): Promise { - // The recomputed result replaces all three names on the row, so the old - // pictures go with it — leaving the thumbnails behind would strand two more - // objects on every ignore-area edit. - // - // They go *after* the new result is persisted, not before. Deleting first - // meant a comparison that failed left the row pointing at files that were - // already gone: the reviewer opens the run and finds broken pictures, with - // nothing to fall back on. Deleting late can only ever leak bytes, which is - // the cheaper of the two failures by a distance. + async calculateDiff(projectId: string, testRun: TestRun, imageBuffer?: Buffer, project?: Project): Promise { const previous = [testRun.diffName, testRun.imageThumbnailName, testRun.diffThumbnailName]; const diffResult = await this.compareService.getDiff({ projectId, + project, data: { image: testRun.imageName, + imageBuffer, baseline: testRun.baselineName, - ignoreAreas: this.getAllIgnoteAreas(testRun), + ignoreAreas: this.getAllIgnoreAreas(testRun), diffTollerancePercent: testRun.diffTollerancePercent, saveDiffAsFile: true, }, }); const saved = await this.saveDiffResult(testRun.id, diffResult); - previous.forEach((name) => this.staticService.deleteImage(name)); + for (const name of previous) void this.staticService.deleteImage(name); return saved; } @@ -448,163 +553,147 @@ export class TestRunsService { createTestRequestDto: CreateTestRequestDto; imageBuffer: Buffer; }): Promise { - // save image const imageName = await this.staticService.saveImage('screenshot', imageBuffer); + return this.insertTestRun({ testVariation, createTestRequestDto, imageName }); + } - const testRun = await this.prismaService.testRun.create({ - data: { - imageName, - testVariation: { - connect: { - id: testVariation.id, - }, - }, - build: { - connect: { - id: createTestRequestDto.buildId, - }, - }, - project: { - connect: { - id: createTestRequestDto.projectId, - }, + private async insertTestRun({ + testVariation, + createTestRequestDto, + imageName, + }: { + testVariation: TestVariation; + createTestRequestDto: CreateTestRequestDto; + imageName: string; + }): Promise { + let testRun: TestRun; + try { + testRun = await this.prismaService.testRun.create({ + data: { + imageName, + testVariationId: testVariation.id, + buildId: createTestRequestDto.buildId, + projectId: createTestRequestDto.projectId, + ...getTestVariationUniqueData(testVariation), + baselineName: testVariation.baselineName, + baselineBranchName: testVariation.branchName, + ignoreAreas: testVariation.ignoreAreas, + tempIgnoreAreas: JSON.stringify(createTestRequestDto.ignoreAreas), + comment: createTestRequestDto.comment || testVariation.comment, + diffTollerancePercent: createTestRequestDto.diffTollerancePercent, + branchName: createTestRequestDto.branchName, + merge: createTestRequestDto.merge, + status: TestStatus.new, }, - ...getTestVariationUniqueData(testVariation), - baselineName: testVariation.baselineName, - baselineBranchName: testVariation.branchName, - ignoreAreas: testVariation.ignoreAreas, - tempIgnoreAreas: JSON.stringify(createTestRequestDto.ignoreAreas), - comment: createTestRequestDto.comment || testVariation.comment, - diffTollerancePercent: createTestRequestDto.diffTollerancePercent, - branchName: createTestRequestDto.branchName, - merge: createTestRequestDto.merge, - status: TestStatus.new, - }, - }); - + }); + } catch (e) { + if (isPrismaError(e, 'P2003')) { + throw new NotFoundException( + `Build, project or test variation not found for test run ${createTestRequestDto.name}` + ); + } + throw e; + } this.eventsGateway.testRunCreated(testRun); return testRun; } async delete(id: string): Promise { this.logger.debug(`Going to remove TestRun ${id}`); - const testRun = await this.findOne(id); - - if (!testRun) { - this.logger.warn(`TestRun not found ${id}`); - return; - } - - await Promise.all([ - this.staticService.deleteImage(testRun.diffName), - this.staticService.deleteImage(testRun.imageName), - // left behind, these would outlive every build that referenced them and - // nothing would ever collect them - this.staticService.deleteImage(testRun.imageThumbnailName), - this.staticService.deleteImage(testRun.diffThumbnailName), - ]); - + let testRun: TestRun; try { - await this.prismaService.testRun.delete({ where: { id } }); + testRun = await this.prismaService.testRun.delete({ where: { id } }); } catch (e) { - if (e instanceof Prisma.PrismaClientKnownRequestError) { - // workaround https://github.com/Visual-Regression-Tracker/Visual-Regression-Tracker/issues/435 - if (e.code === 'P2025') { - this.logger.warn(`TestRun already deleted ${id}`); - return; - } + // workaround https://github.com/Visual-Regression-Tracker/Visual-Regression-Tracker/issues/435 + if (isPrismaError(e, 'P2025')) { + this.logger.warn(`TestRun not found ${id}`); + return; } + throw e; } - + await this.staticService.deleteImages(testRunImageNames(testRun)); this.logger.log(`TestRun deleted ${id}`); this.eventsGateway.testRunDeleted(testRun); return testRun; } async updateIgnoreAreas(id: string, ignoreAreas: IgnoreAreaDto[]): Promise { - return this.prismaService.testRun - .update({ - where: { id }, - data: { - ignoreAreas: JSON.stringify(ignoreAreas), - }, - }) - .then(async (testRun: TestRun) => { - const testVariation = await this.testVariationService.update(testRun.testVariationId!, { - ignoreAreas: testRun.ignoreAreas, - }); - return this.calculateDiff(testVariation.projectId, testRun); - }); + const testVariationId = await this.getTestVariationId(id); + const testRun = await this.prismaService.testRun.update({ + where: { id }, + data: { + ignoreAreas: JSON.stringify(ignoreAreas), + }, + }); + const testVariation = await this.testVariationService.update(testVariationId, { + ignoreAreas: testRun.ignoreAreas, + }); + return this.calculateDiff(testVariation.projectId, testRun); } async addIgnoreAreas(id: string, ignoreAreas: IgnoreAreaDto[]): Promise { - const testRun = await this.findOne(id); - if (!testRun) { - throw new NotFoundException(`Test run not found: ${id}`); - } - const oldIgnoreAreas: IgnoreAreaDto[] = JSON.parse(testRun.ignoreAreas) ?? []; - return this.updateIgnoreAreas(id, oldIgnoreAreas.concat(ignoreAreas)); + const testRun = await this.findOneOrThrow(id); + return this.updateIgnoreAreas(id, parseIgnoreAreas(testRun.ignoreAreas).concat(ignoreAreas)); } async update(id: string, data: UpdateTestRunDto): Promise { - return this.prismaService.testRun - .update({ - where: { id }, - data: { - comment: data.comment, - }, - }) - .then(async (testRun) => { - await this.testVariationService.update(testRun.testVariationId!, data); - this.eventsGateway.testRunUpdated(testRun); - return testRun; - }); + const testVariationId = await this.getTestVariationId(id); + const testRun = await this.prismaService.testRun.update({ + where: { id }, + data: { + comment: data.comment, + }, + }); + await this.testVariationService.update(testVariationId, { comment: data.comment }); + this.eventsGateway.testRunUpdated(testRun); + return testRun; + } + + private async getTestVariationId(id: string): Promise { + const testRun = await this.prismaService.testRun.findUnique({ + where: { id }, + select: { testVariationId: true }, + }); + if (!testRun) { + throw testRunNotFound(id); + } + if (!testRun.testVariationId) { + throw new ConflictException(NO_TEST_VARIATION); + } + return testRun.testVariationId; } - private getAllIgnoteAreas(testRun: TestRun): IgnoreAreaDto[] { - const ignoreAreas: IgnoreAreaDto[] = JSON.parse(testRun.ignoreAreas) ?? []; - const tempIgnoreAreas: IgnoreAreaDto[] = JSON.parse(testRun.tempIgnoreAreas) ?? []; - return ignoreAreas.concat(tempIgnoreAreas); + private getAllIgnoreAreas(testRun: TestRun): IgnoreAreaDto[] { + return parseIgnoreAreas(testRun.ignoreAreas).concat(parseIgnoreAreas(testRun.tempIgnoreAreas)); } - /** - * Reason: not rebased code from feature branch is compared agains new main branch baseline thus diff is expected - * Tries to find past baseline in main branch and autoApprove in case matched - * @param testVariation - * @param testRun - */ - private async tryAutoApproveByPastBaselines({ testRun, testVariation }: AutoApproveProps): Promise { + private async tryAutoApproveByPastBaselines({ + testRun, + testVariation, + project, + imageBuffer, + }: AutoApproveProps): Promise { if (testRun.status === TestStatus.ok || testRun.branchName === testRun.baselineBranchName) { return testRun; } this.logger.log(`Try AutoApproveByPastBaselines testRun: ${testRun.id}`); - const testVariationHistory = (await this.testVariationService.getDetails(testVariation.id))!; - // skip first baseline as it was used by default in general flow - for (const baseline of testVariationHistory.baselines.slice(1)) { - if (await this.shouldAutoApprove({ projectId: testVariation.projectId, baseline, testRun })) { - return this.approve(testRun.id, false, true); - } - } - - return testRun; + const [, ...pastBaselines] = await this.testVariationService.findBaselineHistory(testVariation.id); + return this.autoApproveOnFirstMatch(project, testRun, pastBaselines, imageBuffer); } - /** - * Reason: branch got another one merged thus diff is expected - * Tries to find latest baseline in test variation - * that has already approved test agains the same baseline image - * and autoApprove in case matched - * @param testVariation - * @param testRun - */ - private async tryAutoApproveByNewBaselines({ testVariation, testRun }: AutoApproveProps): Promise { - if (testRun.status === TestStatus.ok) { + private async tryAutoApproveByNewBaselines({ + testVariation, + testRun, + project, + imageBuffer, + }: AutoApproveProps): Promise { + if (testRun.status === TestStatus.ok || testRun.status === TestStatus.autoApproved) { return testRun; } this.logger.log(`Try AutoApproveByNewBaselines testRun: ${testRun.id}`); - const alreadyApprovedTestRuns: TestRun[] = await this.prismaService.testRun.findMany({ + const alreadyApprovedTestRuns = await this.prismaService.testRun.findMany({ where: { ...getTestVariationUniqueData(testVariation), baselineName: testVariation.baselineName, @@ -613,14 +702,37 @@ export class TestRunsService { projectId: testVariation.projectId, }, }, + select: { testVariationId: true }, }); + const testVariationIds = [ + ...new Set(alreadyApprovedTestRuns.flatMap(({ testVariationId }) => (testVariationId ? [testVariationId] : []))), + ]; + const latestBaselines = await this.testVariationService.findLatestBaselines(testVariationIds); + return this.autoApproveOnFirstMatch(project, testRun, latestBaselines, imageBuffer); + } - for (const approvedTestRun of alreadyApprovedTestRuns) { - // the query only matched runs whose variation exists - const approvedTestVariation = (await this.testVariationService.getDetails(approvedTestRun.testVariationId!))!; - const baseline = approvedTestVariation.baselines.shift()!; - - if (await this.shouldAutoApprove({ projectId: testVariation.projectId, baseline, testRun })) { + private async autoApproveOnFirstMatch( + project: Project, + testRun: TestRun, + baselines: BaselineRef[], + uploaded?: Buffer + ): Promise { + const seen = new Set(); + const candidates = baselines.filter(({ baselineName }) => !seen.has(baselineName) && !!seen.add(baselineName)); + if (candidates.length === 0) { + return testRun; + } + let imageBuffer = uploaded; + if (!imageBuffer) { + try { + imageBuffer = (await this.staticService.getImageBuffer(testRun.imageName)) ?? undefined; + } catch { + imageBuffer = undefined; + } + } + const ignoreAreas = this.getAllIgnoreAreas(testRun); + for (const baseline of candidates) { + if (await this.shouldAutoApprove({ project, baseline, testRun, imageBuffer, ignoreAreas })) { return this.approve(testRun.id, false, true); } } @@ -629,35 +741,44 @@ export class TestRunsService { } private async shouldAutoApprove({ - projectId, + project, baseline, testRun, + imageBuffer, + ignoreAreas, }: { - projectId: string; - baseline: Baseline; + project: Project; + baseline: BaselineRef; testRun: TestRun; - }): Promise { + imageBuffer?: Buffer; + ignoreAreas: IgnoreAreaDto[]; + }): Promise { const diffResult = await this.compareService.getDiff({ - projectId, + projectId: project.id, + project, data: { image: testRun.imageName, + imageBuffer, baseline: baseline.baselineName, - ignoreAreas: this.getAllIgnoteAreas(testRun), + ignoreAreas, diffTollerancePercent: testRun.diffTollerancePercent, saveDiffAsFile: false, }, }); - if (diffResult.status === TestStatus.ok) { - this.logger.log(`TestRun ${testRun.id} could be auto approved based on Baseline ${baseline.id}`); - return true; + if (diffResult.status !== TestStatus.ok) { + return false; } + this.logger.log(`TestRun ${testRun.id} could be auto approved based on Baseline ${baseline.id}`); + return true; } } interface AutoApproveProps { testVariation: TestVariation; testRun: TestRun; + project: Project; + imageBuffer?: Buffer; } interface SkippedSibling { @@ -665,84 +786,25 @@ interface SkippedSibling { reason: string; } -// Test-variation axes a project may bulk-approve across (the one that varies -// within a group). customTags (e.g. locales) is the default. +type TestRunWithVariation = TestRun & { testVariation: TestVariation | null }; + +const testRunNotFound = (id: string) => new NotFoundException(`Test run not found: ${id}`); + +const latestBaseline = (tx: Prisma.TransactionClient, testVariationId: string) => + tx.baseline.findFirst({ where: { testVariationId }, orderBy: { createdAt: 'desc' } }); + +const NO_TEST_VARIATION = 'No test variation found. Re-create test run'; + const GROUP_BY_AXES = ['customTags', 'os', 'device', 'browser', 'viewport'] as const; function resolveGroupByAxis(value: string | null | undefined): string { return value && (GROUP_BY_AXES as readonly string[]).includes(value) ? value : 'customTags'; } -// How many sibling screenshots may be in the worker pool at once for one -// variations dialog. The pool queue is shared with build ingestion, and every -// queued job holds two image buffers, so the fan-out stays bounded rather than -// handing the pool a job per locale. export const SIGNATURE_CONCURRENCY = 8; -// Signatures kept across requests, keyed by the image pair and diff settings. -// Roughly one build's worth of screens, at 64 floats each. const SIGNATURE_CACHE_SIZE = 2000; -/** - * Insertion-ordered cache with a hard size limit: reviewing build after build - * would otherwise grow the signature memo without bound. Reinserting a key - * refreshes its position, so what the reviewer keeps coming back to survives. - */ -class BoundedCache { - private readonly entries = new Map(); - - constructor(private readonly limit: number) {} - - get(key: string): T | undefined { - const value = this.entries.get(key); - if (value !== undefined) { - this.entries.delete(key); - this.entries.set(key, value); - } - return value; - } - - set(key: string, value: T): void { - this.entries.delete(key); - this.entries.set(key, value); - while (this.entries.size > this.limit) { - // the map is not empty while it is over the limit - this.entries.delete(this.entries.keys().next().value!); - } - } - - delete(key: string): void { - this.entries.delete(key); - } -} - -/** - * Runs `task` over every item with at most `limit` in flight, keeping the - * results in the items' order. - */ -async function mapWithConcurrency(items: T[], limit: number, task: (item: T) => Promise): Promise { - const results = new Array(items.length); - let next = 0; - const worker = async (): Promise => { - while (next < items.length) { - const index = next++; - results[index] = await task(items[index]); - } - }; - await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker)); - return results; -} - -/** - * The signature stored on a run, if it is still usable. Null — meaning - * "recompute" — when there is none, when it cannot be read, or when the - * project's comparison settings have moved on since it was written. - * - * That last case is the point. Comparing a signature taken at one threshold - * against a sibling's taken at another quietly makes grouping worse, and the - * reviewer has no way to tell why. The in-memory memo has always keyed on the - * same settings; this is the stored equivalent. - */ function parseStoredSignature( stored: string | null | undefined, config: PixelmatchConfig, @@ -753,9 +815,6 @@ function parseStoredSignature( } try { const parsed: StampedSignature = JSON.parse(stored); - // Shape first: a histogram of the wrong length, or one carrying anything - // that is not a number, compares against a current signature to produce a - // score that looks fine and means nothing. Recomputing is always safe. const wellFormed = Array.isArray(parsed?.signature) && parsed.signature.length === SIGNATURE_LENGTH && @@ -771,9 +830,6 @@ function parseStoredSignature( } } -// Same palette but a much larger/smaller change area signals a different or -// additional change (an extra element changed too), not just per-locale text -// reflow — which stays well under this ratio. Such variations go to manual review. const MATCH_MAGNITUDE_RATIO = 2; function magnitudesSimilar(a: number | null, b: number | null): boolean { diff --git a/src/test-variations/test-variations.controller.ts b/src/test-variations/test-variations.controller.ts index b3e42fd6..2a22e512 100644 --- a/src/test-variations/test-variations.controller.ts +++ b/src/test-variations/test-variations.controller.ts @@ -35,7 +35,7 @@ export class TestVariationsController { @ApiOkResponse({ type: TestVariationDto }) @ApiBearerAuth() @UseGuards(JwtAuthGuard) - getDetails(@Param('id', new ParseUUIDPipe()) id: string): Promise { + getDetails(@Param('id', new ParseUUIDPipe()) id: string): Promise { return this.testVariations.getDetails(id); } diff --git a/src/test-variations/test-variations.service.spec.ts b/src/test-variations/test-variations.service.spec.ts index 25324f5f..2e54edbd 100644 --- a/src/test-variations/test-variations.service.spec.ts +++ b/src/test-variations/test-variations.service.spec.ts @@ -1,101 +1,89 @@ import { NotFoundException } from '@nestjs/common'; -import { Test, TestingModule } from '@nestjs/testing'; +import { Test } from '@nestjs/testing'; import { TestVariationsService } from './test-variations.service'; import { PrismaService } from '../prisma/prisma.service'; -import { CreateTestRequestDto } from '../test-runs/dto/create-test-request.dto'; import { StaticService } from '../static/static.service'; -import { TestVariation, Baseline, Project, Build } from '@prisma/client'; +import { Prisma } from '@prisma/client'; import { PNG } from 'pngjs'; import { BuildsService } from '../builds/builds.service'; import { TestRunsService } from '../test-runs/test-runs.service'; import { generateTestVariation, TEST_PROJECT } from '../_data_'; -import { TestVariationUpdateDto } from './dto/test-variation-update.dto'; - -const initModule = async ({ - imageDeleteMock = jest.fn(), - getImageBufferMock = jest.fn(), - variationfindUniqueMock = jest.fn, - variationFindManyMock = jest.fn().mockReturnValue([]), - variationCreateMock = jest.fn(), - variationUpdateMock = jest.fn(), - variationDeleteMock = jest.fn(), - baselineDeleteMock = jest.fn(), - projectFindUniqueMock = jest.fn(), - buildFindOrCreateMock = jest.fn(), - buildUpdateMock = jest.fn(), - testRunCreateMock = jest.fn(), - testRunFindMany = jest.fn(), - testRunDeleteMock = jest.fn(), - testRuncalCulateDiffMock = jest.fn(), - $executeRawMock = jest.fn(), -}) => { - const module: TestingModule = await Test.createTestingModule({ + +const build = async () => { + const m = { + static: { getImageBuffer: jest.fn(), deleteImage: jest.fn() }, + builds: { findOrCreate: jest.fn(), update: jest.fn() }, + testRuns: { create: jest.fn(), calculateDiff: jest.fn() }, + prisma: { + $executeRaw: jest.fn(), + $transaction: jest.fn((operations: Promise[]) => Promise.all(operations)), + testVariation: { + findUnique: jest.fn(), + findMany: jest.fn().mockReturnValue([]), + create: jest.fn(), + update: jest.fn(), + delete: jest.fn(), + }, + baseline: { findMany: jest.fn().mockResolvedValue([]), deleteMany: jest.fn() }, + project: { findUnique: jest.fn() }, + }, + }; + const module = await Test.createTestingModule({ providers: [ TestVariationsService, { provide: StaticService, useValue: { - getImageBuffer: getImageBufferMock, - deleteImage: imageDeleteMock, - }, - }, - { - provide: BuildsService, - useValue: { - findOrCreate: buildFindOrCreateMock, - update: buildUpdateMock, - }, - }, - { - provide: TestRunsService, - useValue: { - delete: testRunDeleteMock, - create: testRunCreateMock, - calculateDiff: testRuncalCulateDiffMock, - }, - }, - { - provide: PrismaService, - useValue: { - $executeRaw: $executeRawMock, - testVariation: { - findUnique: variationfindUniqueMock, - findMany: variationFindManyMock, - create: variationCreateMock, - update: variationUpdateMock, - delete: variationDeleteMock, - }, - baseline: { - delete: baselineDeleteMock, - }, - project: { - findUnique: projectFindUniqueMock, - }, - testRun: { - findMany: testRunFindMany, + ...m.static, + deleteImages: async (names: Iterable) => { + for (const name of names) if (name) await m.static.deleteImage(name); }, }, }, + { provide: BuildsService, useValue: m.builds }, + { provide: TestRunsService, useValue: m.testRuns }, + { provide: PrismaService, useValue: m.prisma }, ], }).compile(); + return { service: module.get(TestVariationsService), m }; +}; - return module.get(TestVariationsService); +const baselineData = { + name: 'Test name', + os: 'OS', + browser: 'browser', + viewport: 'viewport', + device: 'device', + customTags: 'tag', }; +const findRequest = (overrides: object = {}) => ({ + ...baselineData, + projectId: TEST_PROJECT.id, + branchName: 'develop', + ...overrides, +}); +const keyOf = (variation: typeof baselineData & { projectId: string }, branchName: string) => ({ + name: variation.name, + os: variation.os, + device: variation.device, + browser: variation.browser, + viewport: variation.viewport, + customTags: variation.customTags, + branchName, + projectId: variation.projectId, +}); +const prismaError = (code: string) => new Prisma.PrismaClientKnownRequestError(code, { code, clientVersion: 'test' }); describe('TestVariationsService', () => { - let service: TestVariationsService; - const projectMock = TEST_PROJECT; - describe('getDetails', () => { it('can find one', async () => { - const id = 'test id'; - const variationfindUniqueMock = jest.fn().mockResolvedValueOnce(generateTestVariation({ id }, [])); - service = await initModule({ variationfindUniqueMock }); + const { service, m } = await build(); + m.prisma.testVariation.findUnique.mockResolvedValueOnce(generateTestVariation({ id: 'test id' }, [])); - await service.getDetails(id); + await service.getDetails('test id'); - expect(variationfindUniqueMock).toHaveBeenCalledWith({ - where: { id }, + expect(m.prisma.testVariation.findUnique).toHaveBeenCalledWith({ + where: { id: 'test id' }, include: { baselines: { include: { @@ -111,8 +99,8 @@ describe('TestVariationsService', () => { }); it('throws NotFoundException for an unknown id', async () => { - const variationfindUniqueMock = jest.fn().mockResolvedValueOnce(null); - service = await initModule({ variationfindUniqueMock }); + const { service, m } = await build(); + m.prisma.testVariation.findUnique.mockResolvedValueOnce(null); await expect(service.getDetails('missing')).rejects.toThrow( new NotFoundException('Test variation not found: missing') @@ -120,535 +108,161 @@ describe('TestVariationsService', () => { }); }); - describe('find', () => { - it('can find by main branch', async () => { - const createRequest: CreateTestRequestDto = { - buildId: 'buildId', - projectId: projectMock.id, - name: 'Test name', - os: 'OS', - browser: 'browser', - viewport: 'viewport', - device: 'device', - customTags: '', - branchName: 'develop', - }; - - const variationMock: TestVariation = { - id: '123', - projectId: projectMock.id, - name: 'Test name', - baselineName: 'baselineName', - os: 'OS', - browser: 'browser', - viewport: 'viewport', - device: 'device', - customTags: '', - ignoreAreas: '[]', - comment: 'some comment', - branchName: 'develop', - createdAt: new Date(), - updatedAt: new Date(), - }; - const projectFindUniqueMock = jest.fn().mockReturnValueOnce(projectMock); - service = await initModule({ projectFindUniqueMock }); - service.findUnique = jest.fn().mockResolvedValueOnce(variationMock).mockResolvedValueOnce(undefined); - - const result = await service.find(createRequest); - - expect(projectFindUniqueMock).toHaveBeenCalledWith({ where: { id: createRequest.projectId } }); - expect(service.findUnique).toHaveBeenNthCalledWith(1, { - name: createRequest.name, - projectId: createRequest.projectId, - os: createRequest.os, - browser: createRequest.browser, - viewport: createRequest.viewport, - device: createRequest.device, - customTags: createRequest.customTags, - branchName: projectMock.mainBranchName, + describe('findBaselineHistory', () => { + it('lists id and name of every baseline, newest first', async () => { + const history = [ + { id: 'b3', baselineName: 'c.png' }, + { id: 'b2', baselineName: 'b.png' }, + ]; + const { service, m } = await build(); + m.prisma.baseline.findMany.mockResolvedValueOnce(history); + + const result = await service.findBaselineHistory('variationId'); + + expect(m.prisma.baseline.findMany).toHaveBeenCalledWith({ + where: { testVariationId: 'variationId' }, + orderBy: { createdAt: 'desc' }, + select: { id: true, baselineName: true }, }); - expect(service.findUnique).toHaveBeenNthCalledWith(2, { - name: createRequest.name, - projectId: createRequest.projectId, - os: createRequest.os, - browser: createRequest.browser, - viewport: createRequest.viewport, - device: createRequest.device, - customTags: createRequest.customTags, - branchName: createRequest.branchName, - }); - expect(result).toBe(variationMock); - }); - - it('can find by current branch', async () => { - const createRequest: CreateTestRequestDto = { - buildId: 'buildId', - projectId: projectMock.id, - name: 'Test name', - os: 'OS', - browser: 'browser', - viewport: 'viewport', - device: 'device', - customTags: '', - branchName: 'develop', - }; - - const variationMock: TestVariation = { - id: '123', - projectId: projectMock.id, - name: 'Test name', - baselineName: 'baselineName', - os: 'OS', - browser: 'browser', - viewport: 'viewport', - device: 'device', - customTags: '', - ignoreAreas: '[]', - comment: 'some comment', - branchName: 'develop', - createdAt: new Date(), - updatedAt: new Date(), - }; - const projectFindUniqueMock = jest.fn().mockReturnValueOnce(projectMock); - service = await initModule({ projectFindUniqueMock }); - service.findUnique = jest.fn().mockResolvedValueOnce(undefined).mockResolvedValueOnce(variationMock); - - const result = await service.find(createRequest); - - expect(projectFindUniqueMock).toHaveBeenCalledWith({ where: { id: createRequest.projectId } }); - expect(service.findUnique).toHaveBeenNthCalledWith(1, { - name: createRequest.name, - projectId: createRequest.projectId, - os: createRequest.os, - browser: createRequest.browser, - viewport: createRequest.viewport, - device: createRequest.device, - customTags: createRequest.customTags, - branchName: projectMock.mainBranchName, - }); - expect(service.findUnique).toHaveBeenNthCalledWith(2, { - name: createRequest.name, - projectId: createRequest.projectId, - os: createRequest.os, - browser: createRequest.browser, - viewport: createRequest.viewport, - device: createRequest.device, - customTags: createRequest.customTags, - branchName: createRequest.branchName, - }); - expect(result).toBe(variationMock); - }); - - it('can find by current branch but main branch is more relevant', async () => { - const createRequest: CreateTestRequestDto = { - buildId: 'buildId', - projectId: projectMock.id, - name: 'Test name', - os: 'OS', - browser: 'browser', - viewport: 'viewport', - device: 'device', - customTags: '', - branchName: 'develop', - }; - - const variationMainMock: TestVariation = { - id: '123', - projectId: projectMock.id, - name: 'Test name', - baselineName: 'baselineName', - os: 'OS', - browser: 'browser', - viewport: 'viewport', - device: 'device', - customTags: '', - ignoreAreas: '[]', - comment: 'some comment', - branchName: 'master', - createdAt: new Date(), - updatedAt: new Date(), - }; - const variationFeatureMock: TestVariation = { - id: '123', - projectId: projectMock.id, - name: 'Test name', - baselineName: 'baselineName', - os: 'OS', - browser: 'browser', - viewport: 'viewport', - device: 'device', - customTags: '', - ignoreAreas: '[]', - comment: 'some comment', - branchName: 'develop', - createdAt: new Date(), - updatedAt: new Date(variationMainMock.updatedAt.getDate() - 1), - }; - const projectFindUniqueMock = jest.fn().mockReturnValueOnce(projectMock); - service = await initModule({ projectFindUniqueMock }); - service.findUnique = jest - .fn() - .mockResolvedValueOnce(variationMainMock) - .mockResolvedValueOnce(variationFeatureMock); - - const result = await service.find(createRequest); - - expect(projectFindUniqueMock).toHaveBeenCalledWith({ where: { id: createRequest.projectId } }); - expect(service.findUnique).toHaveBeenNthCalledWith(1, { - name: createRequest.name, - projectId: createRequest.projectId, - os: createRequest.os, - browser: createRequest.browser, - viewport: createRequest.viewport, - device: createRequest.device, - customTags: createRequest.customTags, - branchName: projectMock.mainBranchName, - }); - expect(service.findUnique).toHaveBeenNthCalledWith(2, { - name: createRequest.name, - projectId: createRequest.projectId, - os: createRequest.os, - browser: createRequest.browser, - viewport: createRequest.viewport, - device: createRequest.device, - customTags: createRequest.customTags, - branchName: createRequest.branchName, - }); - expect(result).toBe(variationMainMock); + expect(result).toBe(history); }); }); - it('create', async () => { - const createRequest: CreateTestRequestDto = { - buildId: 'buildId', - projectId: projectMock.id, - name: 'Test name', - os: 'OS', - browser: 'browser', - viewport: 'viewport', - device: 'device', - customTags: '', - branchName: 'develop', - }; + describe('findLatestBaselines', () => { + const row = (id: string) => ({ id, baselineName: `${id}.png`, testVariationId: id.split('-')[0] }); - const projectFindUniqueMock = jest.fn().mockReturnValueOnce(projectMock); - const variationCreateMock = jest.fn(); - service = await initModule({ projectFindUniqueMock, variationCreateMock }); - service.findUnique = jest.fn().mockResolvedValueOnce(undefined).mockResolvedValueOnce(undefined); - - await service.create({ - testRunId: 'testRunId', - createTestRequestDto: createRequest, - }); - - expect(variationCreateMock).toHaveBeenCalledWith({ - data: { - name: createRequest.name, - os: createRequest.os, - browser: createRequest.browser, - viewport: createRequest.viewport, - device: createRequest.device, - customTags: createRequest.customTags, - branchName: createRequest.branchName, - testRuns: { connect: { id: 'testRunId' } }, - project: { - connect: { - id: createRequest.projectId, - }, - }, - }, + it('takes the newest baseline of each variation in the given order', async () => { + const { service, m } = await build(); + m.prisma.baseline.findMany.mockResolvedValueOnce(['v2-new', 'v1-new', 'v2-old', 'v1-old'].map(row)); + + const result = await service.findLatestBaselines(['v1', 'v2']); + + expect(m.prisma.baseline.findMany).toHaveBeenCalledWith({ + where: { testVariationId: { in: ['v1', 'v2'] } }, + orderBy: { createdAt: 'desc' }, + select: { id: true, baselineName: true, testVariationId: true }, + }); + expect(result).toStrictEqual([ + { id: 'v1-new', baselineName: 'v1-new.png' }, + { id: 'v2-new', baselineName: 'v2-new.png' }, + ]); }); - }); - it('update', async () => { - const id = 'some id'; - const data: TestVariationUpdateDto = { - baselineName: 'baselineName', - comment: 'random comment', - ignoreAreas: '[]', - }; - const variationUpdateMock = jest.fn(); - service = await initModule({ - variationUpdateMock, + it('leaves out variations without baselines', async () => { + const { service, m } = await build(); + m.prisma.baseline.findMany.mockResolvedValueOnce([row('v2-new')]); + + expect(await service.findLatestBaselines(['v1', 'v2'])).toStrictEqual([ + { id: 'v2-new', baselineName: 'v2-new.png' }, + ]); }); - await service.update(id, data); + it('does not query for no ids', async () => { + const { service, m } = await build(); - expect(variationUpdateMock).toHaveBeenCalledWith({ - where: { id }, - data, + expect(await service.findLatestBaselines([])).toStrictEqual([]); + expect(m.prisma.baseline.findMany).not.toHaveBeenCalled(); }); }); - it('merge', async () => { - const fromBranch = 'develop'; - const targetBranch = 'test'; - const project: Project = TEST_PROJECT; - const build: Build = { - id: 'a9385fc1-884d-4f9f-915e-40da0e7773d5', - ciBuildId: 'ciBuildId', - number: null, - branchName: targetBranch, - status: null, - projectId: project.id, - updatedAt: new Date(), - createdAt: new Date(), - userId: null, - isRunning: true, - }; - const testVariation: TestVariation = { - id: '123', - projectId: project.id, - name: 'Test name', - baselineName: 'baselineName', - os: 'OS', - browser: 'browser', - viewport: 'viewport', - device: 'device', - customTags: '', - ignoreAreas: '[]', - comment: 'some comment', - createdAt: new Date(), - updatedAt: new Date(), - branchName: fromBranch, - }; - const testVariationSecond: TestVariation = { - id: '123', - projectId: project.id, - name: 'Test name second', - baselineName: 'baselineName', - os: 'OS', - browser: 'browser', - viewport: 'viewport', - device: 'device', - customTags: '', - ignoreAreas: '[]', - comment: 'some comment', - createdAt: new Date(), - updatedAt: new Date(), - branchName: fromBranch, - }; - const testVariationNoBaseline: TestVariation = { - id: '123', - projectId: project.id, - name: 'Test name', - baselineName: null, - os: 'OS', - browser: 'browser', - viewport: 'viewport', - device: 'device', - customTags: '', - ignoreAreas: '[]', - comment: 'some comment', - createdAt: new Date(), - updatedAt: new Date(), - branchName: fromBranch, - }; - const testVariationSourceBranch: TestVariation = { - id: '123', - projectId: project.id, - name: 'Test name', - baselineName: 'baselineName', - os: 'OS', - browser: 'browser', - viewport: 'viewport', - device: 'device', - customTags: '', - ignoreAreas: '[]', - comment: 'some comment', - createdAt: new Date(), - updatedAt: new Date(), - branchName: fromBranch, - }; - const testVariationTargetBranch: TestVariation = { - ...testVariationSourceBranch, - branchName: targetBranch, + describe('find', () => { + const initFind = async (...found: unknown[]) => { + const ctx = await build(); + ctx.service.findUnique = jest.fn(); + found.forEach((variation) => jest.mocked(ctx.service.findUnique).mockResolvedValueOnce(variation as never)); + return ctx; }; - const buildFindOrCreateMock = jest.fn().mockResolvedValueOnce(build); - const variationFindManyMock = jest - .fn() - .mockResolvedValueOnce([testVariation, testVariationSecond, testVariationNoBaseline]); - const image = new PNG({ - width: 10, - height: 10, - }); - const imageBuffer = PNG.sync.write(image); - const getImageBufferMock = jest - .fn() - .mockReturnValueOnce(imageBuffer) - .mockReturnValueOnce(imageBuffer) - .mockReturnValueOnce(null); - const testRunCreateMock = jest.fn(); - const buildUpdateMock = jest.fn(); - const testRuncalCulateDiffMock = jest.fn(); - const service = await initModule({ - buildFindOrCreateMock, - buildUpdateMock, - testRunCreateMock, - testRuncalCulateDiffMock, - variationFindManyMock, - getImageBufferMock, - }); - // neither screen has a variation on the target branch yet - service.findUnique = jest.fn().mockResolvedValueOnce(null).mockResolvedValueOnce(null); - service.create = jest - .fn() - .mockResolvedValueOnce(testVariationTargetBranch) - .mockResolvedValueOnce(testVariationTargetBranch); - - await service.merge(project.id, fromBranch, targetBranch); - - expect(buildFindOrCreateMock).toHaveBeenCalledWith({ - branchName: targetBranch, - projectId: project.id, - }); - expect(service.findUnique).toHaveBeenNthCalledWith(1, { - name: testVariation.name, - os: testVariation.os, - device: testVariation.device, - browser: testVariation.browser, - viewport: testVariation.viewport, - customTags: testVariation.customTags, - branchName: targetBranch, - projectId: project.id, - }); - expect(service.findUnique).toHaveBeenNthCalledWith(2, { - name: testVariationSecond.name, - os: testVariationSecond.os, - device: testVariationSecond.device, - browser: testVariationSecond.browser, - viewport: testVariationSecond.viewport, - customTags: testVariationSecond.customTags, - branchName: targetBranch, - projectId: project.id, - }); - expect(testRunCreateMock).toHaveBeenNthCalledWith(1, { - testVariation: testVariationTargetBranch, - createTestRequestDto: { - ...testVariation, - branchName: targetBranch, - buildId: build.id, - diffTollerancePercent: 0, - merge: true, - ignoreAreas: JSON.parse(testVariation.ignoreAreas), - }, - imageBuffer, - }); - expect(testRunCreateMock).toHaveBeenNthCalledWith(2, { - testVariation: testVariationTargetBranch, - createTestRequestDto: { - ...testVariationSecond, - branchName: targetBranch, - buildId: build.id, - diffTollerancePercent: 0, - merge: true, - ignoreAreas: JSON.parse(testVariationSecond.ignoreAreas), - }, - imageBuffer, + const main = generateTestVariation({ branchName: 'master', updatedAt: new Date(2000) }); + const current = generateTestVariation({ branchName: 'develop', updatedAt: new Date(1000) }); + + it.each([ + ['the main branch', [main, undefined], main], + ['the current branch', [undefined, current], current], + ['the current branch but main branch is more relevant', [main, current], main], + ])('can find by %s', async (_case, [mainResult, currentResult], expected) => { + const request = findRequest({ projectId: TEST_PROJECT.id }); + const { service, m } = await initFind(mainResult, currentResult); + m.prisma.project.findUnique.mockReturnValueOnce(TEST_PROJECT); + + const result = await service.find(request); + + expect(m.prisma.project.findUnique).toHaveBeenCalledWith({ where: { id: request.projectId } }); + expect(service.findUnique).toHaveBeenNthCalledWith(1, keyOf(request, TEST_PROJECT.mainBranchName)); + expect(service.findUnique).toHaveBeenNthCalledWith(2, keyOf(request, request.branchName)); + expect(result).toBe(expected); }); - expect(testRunCreateMock).toHaveBeenCalledTimes(2); - expect(buildUpdateMock).toHaveBeenCalledWith(build.id, { isRunning: false }); - }); - it('delete', async () => { - const testRunId = 'test run id'; - const testVariationId = 'test variation id'; - const variation: TestVariation & { - baselines: Baseline[]; - } = { - id: testVariationId, - projectId: 'project Id', - name: 'Test name', - baselineName: 'baselineName', - os: 'OS', - browser: 'browser', - viewport: 'viewport', - device: 'device', - customTags: '', - ignoreAreas: '[]', - comment: 'some comment', - branchName: 'develop', - createdAt: new Date(), - updatedAt: new Date(), - baselines: [ - { - id: 'baseline id 1', - baselineName: 'image name 1', - testVariationId: testVariationId, - testRunId: testRunId, - userId: 'userId', - createdAt: new Date(), - updatedAt: new Date(), - }, - ], - }; + it('returns undefined when neither branch has the variation', async () => { + const { service, m } = await initFind(null, null); + m.prisma.project.findUnique.mockResolvedValueOnce(TEST_PROJECT); - const $executeRawMock = jest.fn(); - const variationDeleteMock = jest.fn(); - const getDetailsMock = jest.fn().mockResolvedValueOnce(variation); - const deleteBaselineMock = jest.fn().mockResolvedValueOnce(variation.baselines[0]); - const service = await initModule({ - variationDeleteMock, - $executeRawMock, + await expect(service.find(findRequest())).resolves.toBeUndefined(); }); - service.getDetails = getDetailsMock; - service.deleteBaseline = deleteBaselineMock; - await service.delete(testVariationId); + it('only looks on the main branch when the build runs on it', async () => { + const variation = generateTestVariation({ branchName: TEST_PROJECT.mainBranchName }); + const { service, m } = await initFind(variation); + m.prisma.project.findUnique.mockResolvedValueOnce(TEST_PROJECT); + const request = findRequest({ branchName: TEST_PROJECT.mainBranchName }); - expect(service.getDetails).toHaveBeenCalledWith(testVariationId); - expect(service.deleteBaseline).toHaveBeenCalledWith(variation.baselines[0]); - expect($executeRawMock).toHaveBeenCalled(); - expect(variationDeleteMock).toHaveBeenCalledWith({ - where: { id: testVariationId }, + const result = await service.find(request); + + expect(result).toBe(variation); + expect(service.findUnique).toHaveBeenCalledTimes(1); + expect(service.findUnique).toHaveBeenCalledWith(request); }); - }); - it('deleteBaseline', async () => { - const baseline: Baseline = { - id: 'baseline id 1', - baselineName: 'image name 1', - testVariationId: 'test variation id', - testRunId: 'test run id 1', - userId: 'userId', - createdAt: new Date(), - updatedAt: new Date(), - }; + it('uses sourceBranch instead of the project main branch when given', async () => { + const { service, m } = await initFind(); + + await service.find(findRequest({ branchName: 'feature', sourceBranch: 'release' })); - const baselineDeleteMock = jest.fn(); - const imageDeleteMock = jest.fn(); - const service = await initModule({ - baselineDeleteMock, - imageDeleteMock, + expect(m.prisma.project.findUnique).not.toHaveBeenCalled(); + expect(service.findUnique).toHaveBeenNthCalledWith(1, expect.objectContaining({ branchName: 'release' })); + expect(service.findUnique).toHaveBeenNthCalledWith(2, expect.objectContaining({ branchName: 'feature' })); }); - await service.deleteBaseline(baseline); + it('takes the main branch from the caller instead of reading the project', async () => { + const { service, m } = await initFind(); - expect(imageDeleteMock).toHaveBeenCalledWith(baseline.baselineName); - expect(baselineDeleteMock).toHaveBeenCalledWith({ - where: { id: baseline.id }, + await service.find(findRequest({ branchName: 'feature' }), 'trunk'); + + expect(m.prisma.project.findUnique).not.toHaveBeenCalled(); + expect(service.findUnique).toHaveBeenNthCalledWith(1, expect.objectContaining({ branchName: 'trunk' })); + expect(service.findUnique).toHaveBeenNthCalledWith(2, expect.objectContaining({ branchName: 'feature' })); }); - }); - describe('characterization', () => { - const baselineData = { - name: 'Test name', - os: 'OS', - browser: 'browser', - viewport: 'viewport', - device: 'device', - customTags: 'tag', - }; + it('reads the project main branch when neither sourceBranch nor the caller gives one', async () => { + const { service, m } = await initFind(); + m.prisma.project.findUnique.mockResolvedValueOnce({ ...TEST_PROJECT, mainBranchName: 'trunk' }); + + await service.find(findRequest({ branchName: 'feature' })); + + expect(m.prisma.project.findUnique).toHaveBeenCalledWith({ where: { id: TEST_PROJECT.id } }); + expect(service.findUnique).toHaveBeenNthCalledWith(1, expect.objectContaining({ branchName: 'trunk' })); + }); + + it('keeps the current branch variation when it is as new as the main branch one', async () => { + const updatedAt = new Date('2024-01-01'); + const sameAgeMain = generateTestVariation({ id: 'main', updatedAt }); + const sameAgeCurrent = generateTestVariation({ id: 'current', updatedAt }); + const { service, m } = await initFind(sameAgeMain, sameAgeCurrent); + m.prisma.project.findUnique.mockResolvedValueOnce(TEST_PROJECT); + + await expect(service.find(findRequest())).resolves.toBe(sameAgeCurrent); + }); + }); - it('findUnique looks the variation up by its full composite key', async () => { + describe('findUnique', () => { + it('looks the variation up by its full composite key', async () => { const variation = generateTestVariation(); - const variationfindUniqueMock = jest.fn().mockResolvedValueOnce(variation); - service = await initModule({ variationfindUniqueMock }); + const { service, m } = await build(); + m.prisma.testVariation.findUnique.mockResolvedValueOnce(variation); const result = await service.findUnique({ ...baselineData, projectId: 'projectId', branchName: 'develop' }); expect(result).toBe(variation); - expect(variationfindUniqueMock).toHaveBeenCalledWith({ + expect(m.prisma.testVariation.findUnique).toHaveBeenCalledWith({ where: { projectId_name_browser_device_os_viewport_customTags_branchName: { projectId: 'projectId', @@ -658,74 +272,54 @@ describe('TestVariationsService', () => { }, }); }); + }); - describe('find', () => { - it('returns undefined when neither branch has the variation', async () => { - const projectFindUniqueMock = jest.fn().mockResolvedValueOnce(TEST_PROJECT); - service = await initModule({ projectFindUniqueMock }); - service.findUnique = jest.fn().mockResolvedValue(null); + describe('create', () => { + it('create stores the project, branch and screen', async () => { + const { service, m } = await build(); - // NOTE: current behavior — declared as `TestVariation | null` but resolves undefined - await expect( - service.find({ ...baselineData, projectId: TEST_PROJECT.id, branchName: 'develop' }) - ).resolves.toBeUndefined(); + await service.create({ + createTestRequestDto: { ...baselineData, projectId: 'projectId', branchName: 'develop' }, }); - it('only looks on the main branch when the build runs on it', async () => { - const variation = generateTestVariation({ branchName: TEST_PROJECT.mainBranchName }); - const projectFindUniqueMock = jest.fn().mockResolvedValueOnce(TEST_PROJECT); - service = await initModule({ projectFindUniqueMock }); - service.findUnique = jest.fn().mockResolvedValueOnce(variation); - - const result = await service.find({ - ...baselineData, - projectId: TEST_PROJECT.id, - branchName: TEST_PROJECT.mainBranchName, - }); - - expect(result).toBe(variation); - expect(service.findUnique).toHaveBeenCalledTimes(1); - expect(service.findUnique).toHaveBeenCalledWith({ - ...baselineData, - projectId: TEST_PROJECT.id, - branchName: TEST_PROJECT.mainBranchName, - }); + expect(m.prisma.testVariation.create.mock.calls[0][0].data).toStrictEqual({ + project: { connect: { id: 'projectId' } }, + branchName: 'develop', + ...baselineData, }); + }); - it('uses sourceBranch instead of the project main branch when given', async () => { - const projectFindUniqueMock = jest.fn().mockResolvedValueOnce(TEST_PROJECT); - service = await initModule({ projectFindUniqueMock }); - service.findUnique = jest.fn().mockResolvedValue(null); - - await service.find({ - ...baselineData, - projectId: TEST_PROJECT.id, - branchName: 'feature', - sourceBranch: 'release', - }); - - expect(projectFindUniqueMock).toHaveBeenCalledWith({ where: { id: TEST_PROJECT.id } }); - expect(service.findUnique).toHaveBeenNthCalledWith(1, expect.objectContaining({ branchName: 'release' })); - expect(service.findUnique).toHaveBeenNthCalledWith(2, expect.objectContaining({ branchName: 'feature' })); + it('create stores missing optional fields as empty strings', async () => { + const { service, m } = await build(); + + await service.create({ + createTestRequestDto: { name: 'only name', projectId: 'projectId', branchName: 'develop' }, }); - it('keeps the current branch variation when it is as new as the main branch one', async () => { - const updatedAt = new Date('2024-01-01'); - const main = generateTestVariation({ id: 'main', updatedAt }); - const current = generateTestVariation({ id: 'current', updatedAt }); - const projectFindUniqueMock = jest.fn().mockResolvedValueOnce(TEST_PROJECT); - service = await initModule({ projectFindUniqueMock }); - service.findUnique = jest.fn().mockResolvedValueOnce(main).mockResolvedValueOnce(current); - - await expect( - service.find({ ...baselineData, projectId: TEST_PROJECT.id, branchName: 'develop' }) - ).resolves.toBe(current); + expect(m.prisma.testVariation.create.mock.calls[0][0].data).toMatchObject({ + name: 'only name', + os: '', + device: '', + browser: '', + viewport: '', + customTags: '', }); }); + }); + + describe('update', () => { + it('update', async () => { + const data = { baselineName: 'baselineName', comment: 'random comment', ignoreAreas: '[]' }; + const { service, m } = await build(); + + await service.update('some id', data); + + expect(m.prisma.testVariation.update).toHaveBeenCalledWith({ where: { id: 'some id' }, data }); + }); it('update connects the test run when one is given', async () => { - const variationUpdateMock = jest.fn().mockResolvedValueOnce('updated'); - service = await initModule({ variationUpdateMock }); + const { service, m } = await build(); + m.prisma.testVariation.update.mockResolvedValueOnce('updated'); const result = await service.update( 'variationId', @@ -734,7 +328,7 @@ describe('TestVariationsService', () => { ); expect(result).toBe('updated'); - expect(variationUpdateMock).toHaveBeenCalledWith({ + expect(m.prisma.testVariation.update).toHaveBeenCalledWith({ where: { id: 'variationId' }, data: { baselineName: 'baseline.png', @@ -744,55 +338,15 @@ describe('TestVariationsService', () => { }, }); }); + }); - it('create leaves testRuns undefined without a test run id', async () => { - const variationCreateMock = jest.fn(); - service = await initModule({ variationCreateMock }); - - await service.create({ - createTestRequestDto: { - ...baselineData, - projectId: 'projectId', - branchName: 'develop', - } as CreateTestRequestDto, - }); - - expect(variationCreateMock.mock.calls[0][0].data).toStrictEqual({ - project: { connect: { id: 'projectId' } }, - testRuns: undefined, - branchName: 'develop', - ...baselineData, - }); - }); - - it('create stores missing optional fields as empty strings', async () => { - const variationCreateMock = jest.fn(); - service = await initModule({ variationCreateMock }); - - await service.create({ - createTestRequestDto: { - name: 'only name', - projectId: 'projectId', - branchName: 'develop', - } as CreateTestRequestDto, - }); - - expect(variationCreateMock.mock.calls[0][0].data).toMatchObject({ - name: 'only name', - os: '', - device: '', - browser: '', - viewport: '', - customTags: '', - }); - }); - + describe('addBaseline', () => { it.each([ ['a user', 'userId', { connect: { id: 'userId' } }], ['no user', undefined, undefined], - ])('addBaseline records a baseline for the test run with %s', async (_, userId, user) => { - const variationUpdateMock = jest.fn().mockResolvedValueOnce('updated'); - service = await initModule({ variationUpdateMock }); + ])('records a baseline for the test run with %s', async (_, userId, user) => { + const { service, m } = await build(); + m.prisma.testVariation.update.mockResolvedValueOnce('updated'); const result = await service.addBaseline({ id: 'variationId', @@ -802,7 +356,7 @@ describe('TestVariationsService', () => { }); expect(result).toBe('updated'); - expect(variationUpdateMock.mock.calls[0][0]).toStrictEqual({ + expect(m.prisma.testVariation.update.mock.calls[0][0]).toStrictEqual({ where: { id: 'variationId' }, data: { baselineName: 'baseline.png', @@ -816,39 +370,93 @@ describe('TestVariationsService', () => { }, }); }); + }); + + describe('merge', () => { + const project = TEST_PROJECT; + const fromBranch = 'develop'; + const targetBranch = 'test'; + const source = (overrides: object = {}) => + generateTestVariation({ id: '123', projectId: project.id, branchName: fromBranch, ...overrides }, []); + + it('merge', async () => { + const mergeBuild = { id: 'a9385fc1-884d-4f9f-915e-40da0e7773d5' }; + const testVariation = source(); + const testVariationSecond = source({ name: 'Test name second' }); + const testVariationNoBaseline = source({ baselineName: null }); + const testVariationTargetBranch = { ...source(), branchName: targetBranch }; + const imageBuffer = PNG.sync.write(new PNG({ width: 10, height: 10 })); + const { service, m } = await build(); + m.builds.findOrCreate.mockResolvedValueOnce(mergeBuild); + m.prisma.testVariation.findMany.mockResolvedValueOnce([ + testVariation, + testVariationSecond, + testVariationNoBaseline, + ]); + m.static.getImageBuffer + .mockReturnValueOnce(imageBuffer) + .mockReturnValueOnce(imageBuffer) + .mockReturnValueOnce(null); + m.prisma.project.findUnique.mockResolvedValue(project); + service.findUnique = jest.fn().mockResolvedValueOnce(null).mockResolvedValueOnce(null); + service.create = jest + .fn() + .mockResolvedValueOnce(testVariationTargetBranch) + .mockResolvedValueOnce(testVariationTargetBranch); + + await service.merge(project.id, fromBranch, targetBranch); + + expect(m.builds.findOrCreate).toHaveBeenCalledWith({ branchName: targetBranch, projectId: project.id }); + expect(service.findUnique).toHaveBeenNthCalledWith(1, keyOf(testVariation, targetBranch)); + expect(service.findUnique).toHaveBeenNthCalledWith(2, keyOf(testVariationSecond, targetBranch)); + [testVariation, testVariationSecond].forEach((variation, index) => + expect(m.testRuns.create).toHaveBeenNthCalledWith(index + 1, { + testVariation: testVariationTargetBranch, + createTestRequestDto: { + ...variation, + branchName: targetBranch, + buildId: mergeBuild.id, + diffTollerancePercent: 0, + merge: true, + ignoreAreas: JSON.parse(variation.ignoreAreas), + }, + imageBuffer, + }) + ); + expect(m.testRuns.create).toHaveBeenCalledTimes(2); + expect(m.prisma.project.findUnique).toHaveBeenCalledTimes(1); + expect(m.testRuns.calculateDiff).toHaveBeenCalledTimes(2); + expect(m.testRuns.calculateDiff).toHaveBeenCalledWith(project.id, undefined, imageBuffer, project); + expect(m.builds.update).toHaveBeenCalledWith(mergeBuild.id, { isRunning: false }); + }); it('merge reuses a variation that already lives on the target branch', async () => { - const source = generateTestVariation({ + const mergeSource = generateTestVariation({ branchName: 'develop', ignoreAreas: '[{"x":1,"y":2,"width":3,"height":4}]', }); const target = generateTestVariation({ id: 'target', branchName: 'master' }); - const build = { id: 'buildId' }; const imageBuffer = Buffer.from('png'); const testRun = { id: 'testRunId' }; - const buildFindOrCreateMock = jest.fn().mockResolvedValueOnce(build); - const variationFindManyMock = jest.fn().mockResolvedValueOnce([source]); - const getImageBufferMock = jest.fn().mockResolvedValueOnce(imageBuffer); - const testRunCreateMock = jest.fn().mockResolvedValueOnce(testRun); - const testRuncalCulateDiffMock = jest.fn(); - const buildUpdateMock = jest.fn().mockResolvedValueOnce('stopped build'); - service = await initModule({ - buildFindOrCreateMock, - variationFindManyMock, - getImageBufferMock, - testRunCreateMock, - testRuncalCulateDiffMock, - buildUpdateMock, - }); + const mergeProject = { ...TEST_PROJECT, id: 'projectId' }; + const { service, m } = await build(); + m.builds.findOrCreate.mockResolvedValueOnce({ id: 'buildId' }); + m.prisma.testVariation.findMany.mockResolvedValueOnce([mergeSource]); + m.static.getImageBuffer.mockResolvedValueOnce(imageBuffer); + m.testRuns.create.mockResolvedValueOnce(testRun); + m.builds.update.mockResolvedValueOnce('stopped build'); + m.prisma.project.findUnique.mockResolvedValue(mergeProject); service.findUnique = jest.fn().mockResolvedValueOnce(target); service.create = jest.fn(); const result = await service.merge('projectId', 'develop', 'master'); - expect(variationFindManyMock).toHaveBeenCalledWith({ where: { projectId: 'projectId', branchName: 'develop' } }); - expect(getImageBufferMock).toHaveBeenCalledWith(source.baselineName); + expect(m.prisma.testVariation.findMany).toHaveBeenCalledWith({ + where: { projectId: 'projectId', branchName: 'develop' }, + }); + expect(m.static.getImageBuffer).toHaveBeenCalledWith(mergeSource.baselineName); expect(service.create).not.toHaveBeenCalled(); - expect(testRunCreateMock).toHaveBeenCalledWith({ + expect(m.testRuns.create).toHaveBeenCalledWith({ testVariation: target, createTestRequestDto: expect.objectContaining({ branchName: 'master', @@ -859,46 +467,202 @@ describe('TestVariationsService', () => { }), imageBuffer, }); - expect(testRuncalCulateDiffMock).toHaveBeenCalledWith('projectId', testRun); - expect(buildUpdateMock).toHaveBeenCalledWith('buildId', { isRunning: false }); + expect(m.testRuns.calculateDiff).toHaveBeenCalledWith('projectId', testRun, imageBuffer, mergeProject); + expect(m.builds.update).toHaveBeenCalledWith('buildId', { isRunning: false }); expect(result).toBe('stopped build'); }); + it('merge into a deleted project answers 404 and stops its build', async () => { + const { service, m } = await build(); + m.builds.findOrCreate.mockResolvedValueOnce({ id: 'buildId' }); + m.builds.update.mockResolvedValueOnce('stopped build'); + m.prisma.project.findUnique.mockResolvedValue(null); + + await expect(service.merge('gone', 'develop', 'master')).rejects.toThrow( + new NotFoundException('Project not found: gone') + ); + + expect(m.builds.update).toHaveBeenCalledWith('buildId', { isRunning: false }); + expect(m.prisma.testVariation.findMany).not.toHaveBeenCalled(); + }); + it('merge stops its build when a step fails and passes the failure on', async () => { - const source = generateTestVariation({ branchName: 'develop' }); - const buildFindOrCreateMock = jest.fn().mockResolvedValueOnce({ id: 'buildId' }); - const variationFindManyMock = jest.fn().mockResolvedValueOnce([source]); - const getImageBufferMock = jest.fn().mockResolvedValueOnce(Buffer.from('png')); - const testRunCreateMock = jest.fn().mockRejectedValueOnce(new Error('storage down')); - const buildUpdateMock = jest.fn().mockResolvedValueOnce('stopped build'); - service = await initModule({ - buildFindOrCreateMock, - variationFindManyMock, - getImageBufferMock, - testRunCreateMock, - buildUpdateMock, - }); + const { service, m } = await build(); + m.builds.findOrCreate.mockResolvedValueOnce({ id: 'buildId' }); + m.prisma.testVariation.findMany.mockResolvedValueOnce([generateTestVariation({ branchName: 'develop' })]); + m.static.getImageBuffer.mockResolvedValueOnce(Buffer.from('png')); + m.testRuns.create.mockRejectedValueOnce(new Error('storage down')); + m.builds.update.mockResolvedValueOnce('stopped build'); + m.prisma.project.findUnique.mockResolvedValue({ ...TEST_PROJECT, id: 'projectId' }); service.findUnique = jest.fn().mockResolvedValueOnce(generateTestVariation({ branchName: 'master' })); await expect(service.merge('projectId', 'develop', 'master')).rejects.toThrow('storage down'); - expect(buildUpdateMock).toHaveBeenCalledWith('buildId', { isRunning: false }); + expect(m.builds.update).toHaveBeenCalledWith('buildId', { isRunning: false }); + }); + }); + + describe('delete', () => { + const twoBaselines = [{ baselineName: 'image 1' }, { baselineName: 'image 2' }]; + + it('removes the rows in one transaction before the files', async () => { + const order: string[] = []; + const { service, m } = await build(); + m.prisma.$transaction.mockImplementation(async (operations: Promise[]) => { + order.push('rows'); + return Promise.all(operations); + }); + m.static.deleteImage.mockImplementation(async () => { + order.push('file'); + }); + m.prisma.baseline.findMany.mockResolvedValueOnce(twoBaselines); + + await service.delete('variationId'); + + expect(m.prisma.baseline.findMany).toHaveBeenCalledWith({ + where: { testVariationId: 'variationId' }, + select: { baselineName: true }, + }); + expect(m.prisma.baseline.deleteMany).toHaveBeenCalledWith({ where: { testVariationId: 'variationId' } }); + expect(m.prisma.testVariation.delete).toHaveBeenCalledWith({ where: { id: 'variationId' } }); + expect(m.prisma.$transaction).toHaveBeenCalledTimes(1); + expect(m.static.deleteImage.mock.calls).toEqual([['image 1'], ['image 2']]); + expect(order).toEqual(['rows', 'file', 'file']); + }); + + it('keeps the files when the rows cannot be deleted', async () => { + const { service, m } = await build(); + m.prisma.$transaction.mockRejectedValueOnce(new Error('db down')); + m.prisma.baseline.findMany.mockResolvedValueOnce(twoBaselines); + + await expect(service.delete('variationId')).rejects.toThrow('db down'); + + expect(m.static.deleteImage).not.toHaveBeenCalled(); + }); + + it('returns the deleted variation', async () => { + const deleted = generateTestVariation({ id: 'variationId' }); + const { service, m } = await build(); + m.prisma.$transaction.mockResolvedValueOnce([{ count: 2 }, deleted]); + m.prisma.baseline.findMany.mockResolvedValueOnce(twoBaselines); + + expect(await service.delete('variationId')).toBe(deleted); + }); + + it('delete runs no raw SQL', async () => { + const { service, m } = await build(); + + await service.delete('id'); + + expect(m.prisma.$executeRaw).not.toHaveBeenCalled(); + }); + }); + + describe('createOrTakeWinner', () => { + const request = { + projectId: 'projectId', + branchName: 'develop', + name: 'Test name', + os: 'OS', + browser: 'browser', + viewport: 'viewport', + device: 'device', + customTags: '', + }; + + it('creates the variation without looking it up first', async () => { + const created = generateTestVariation({ id: 'created' }); + const { service, m } = await build(); + m.prisma.testVariation.create.mockResolvedValueOnce(created); + + expect(await service.createOrTakeWinner(request)).toBe(created); + expect(m.prisma.testVariation.create).toHaveBeenCalledTimes(1); + expect(m.prisma.testVariation.findUnique).not.toHaveBeenCalled(); + }); + + it('returns the variation a concurrent request created', async () => { + const winner = generateTestVariation({ id: 'winner' }); + const { service, m } = await build(); + m.prisma.testVariation.findUnique.mockResolvedValueOnce(winner); + m.prisma.testVariation.create.mockRejectedValueOnce(prismaError('P2002')); + + expect(await service.createOrTakeWinner(request)).toBe(winner); + expect(m.prisma.testVariation.findUnique).toHaveBeenCalledTimes(1); + }); + + it('rethrows a non-P2002 error', async () => { + const { service, m } = await build(); + m.prisma.testVariation.create.mockRejectedValueOnce(new Error('db down')); + + await expect(service.createOrTakeWinner(request)).rejects.toThrow('db down'); + expect(m.prisma.testVariation.findUnique).not.toHaveBeenCalled(); + }); + + it('rethrows P2002 when the winner cannot be found', async () => { + const error = prismaError('P2002'); + const { service, m } = await build(); + m.prisma.testVariation.findUnique.mockResolvedValue(null); + m.prisma.testVariation.create.mockRejectedValueOnce(error); + + await expect(service.createOrTakeWinner(request)).rejects.toBe(error); + }); + }); + + describe('transaction client', () => { + const txWith = (testVariation: Record) => + ({ testVariation }) as unknown as Prisma.TransactionClient; + const idKey = { + projectId: 'p', + branchName: 'b', + name: 'n', + os: '', + device: '', + browser: '', + viewport: '', + customTags: '', + }; + + it('update uses the transaction client it is given', async () => { + const { service, m } = await build(); + const txUpdate = jest.fn(); + + await service.update('id', { comment: 'c' }, 'runId', txWith({ update: txUpdate })); + + expect(txUpdate).toHaveBeenCalledWith(expect.objectContaining({ where: { id: 'id' } })); + expect(m.prisma.testVariation.update).not.toHaveBeenCalled(); }); - it('delete detaches the test runs with a raw update before deleting the variation', async () => { - const $executeRawMock = jest.fn(); - const variationDeleteMock = jest.fn().mockResolvedValueOnce('deleted'); - service = await initModule({ $executeRawMock, variationDeleteMock }); - service.getDetails = jest.fn().mockResolvedValueOnce(generateTestVariation({ id: 'variationId' }, [])); + it('addBaseline uses the transaction client it is given', async () => { + const { service, m } = await build(); + const txUpdate = jest.fn(); - const result = await service.delete('variationId'); + await service.addBaseline({ id: 'id', testRunId: 'runId', baselineName: 'b.png' }, txWith({ update: txUpdate })); - const [strings, ...values] = $executeRawMock.mock.calls[0]; - expect(strings.join('?')).toBe( - 'UPDATE "public"."TestRun" SET "testVariationId" = NULL::text WHERE "testVariationId" = ?' + expect(txUpdate).toHaveBeenCalledWith(expect.objectContaining({ where: { id: 'id' } })); + expect(m.prisma.testVariation.update).not.toHaveBeenCalled(); + }); + + it('findUnique uses the transaction client it is given', async () => { + const { service, m } = await build(); + const txFindUnique = jest.fn(); + + await service.findUnique(idKey, txWith({ findUnique: txFindUnique })); + + expect(txFindUnique).toHaveBeenCalledTimes(1); + expect(m.prisma.testVariation.findUnique).not.toHaveBeenCalled(); + }); + + it('create uses the transaction client it is given', async () => { + const { service, m } = await build(); + const txCreate = jest.fn(); + + await service.create( + { createTestRequestDto: { projectId: 'p', branchName: 'b', name: 'n' } }, + txWith({ create: txCreate }) ); - expect(values).toEqual(['variationId']); - expect(result).toBe('deleted'); + + expect(txCreate).toHaveBeenCalledTimes(1); + expect(m.prisma.testVariation.create).not.toHaveBeenCalled(); }); }); }); diff --git a/src/test-variations/test-variations.service.ts b/src/test-variations/test-variations.service.ts index 69224317..5152dd06 100644 --- a/src/test-variations/test-variations.service.ts +++ b/src/test-variations/test-variations.service.ts @@ -1,16 +1,19 @@ import { Injectable, Inject, forwardRef, Logger, NotFoundException } from '@nestjs/common'; import { PrismaService } from '../prisma/prisma.service'; -import { TestVariation, Baseline, Build, TestRun, User } from '@prisma/client'; +import { isPrismaError } from '../prisma/prisma-errors'; +import { TestVariation, Baseline, Build, Project, TestRun, User, Prisma } from '@prisma/client'; import { StaticService } from '../static/static.service'; import { BuildsService } from '../builds/builds.service'; import { TestRunsService } from '../test-runs/test-runs.service'; import { CreateTestRequestDto } from '../test-runs/dto/create-test-request.dto'; import { BuildDto } from '../builds/dto/build.dto'; -import { getTestVariationUniqueData } from '../utils'; +import { getTestVariationUniqueData, parseIgnoreAreas, variationKey } from '../utils'; import { TestVariationUpdateDto } from './dto/test-variation-update.dto'; import { BaselineDataDto } from '../shared/dto/baseline-data.dto'; import type { WrapperType } from '../shared/wrapper-type'; +export type BaselineRef = Pick; + @Injectable() export class TestVariationsService { private readonly logger = new Logger(TestVariationsService.name); @@ -25,10 +28,9 @@ export class TestVariationsService { ) {} async getDetails(id: string): Promise< - | (TestVariation & { - baselines: (Baseline & { testRun: TestRun | null; user: Omit | null })[]; - }) - | null + TestVariation & { + baselines: (Baseline & { testRun: TestRun | null; user: Omit | null })[]; + } > { const testVariation = await this.prismaService.testVariation.findUnique({ where: { id }, @@ -36,8 +38,6 @@ export class TestVariationsService { baselines: { include: { testRun: true, - // who approved the baseline, for the history list — never their - // credentials: any signed-in user, guests included, can read this user: { omit: { password: true, apiKey: true } }, }, orderBy: { @@ -52,10 +52,37 @@ export class TestVariationsService { return testVariation; } + async findBaselineHistory(testVariationId: string): Promise { + return this.prismaService.baseline.findMany({ + where: { testVariationId }, + orderBy: { createdAt: 'desc' }, + select: { id: true, baselineName: true }, + }); + } + + async findLatestBaselines(testVariationIds: string[]): Promise { + if (testVariationIds.length === 0) { + return []; + } + const baselines = await this.prismaService.baseline.findMany({ + where: { testVariationId: { in: testVariationIds } }, + orderBy: { createdAt: 'desc' }, + select: { id: true, baselineName: true, testVariationId: true }, + }); + const latest = new Map(); + for (const { testVariationId, ...baseline } of baselines) { + if (!latest.has(testVariationId)) { + latest.set(testVariationId, baseline); + } + } + return testVariationIds.flatMap((id) => latest.get(id) ?? []); + } + async findUnique( - data: ReturnType & { projectId: string; branchName: string } + data: ReturnType & { projectId: string; branchName: string }, + tx: Prisma.TransactionClient = this.prismaService ): Promise { - return this.prismaService.testVariation.findUnique({ + return tx.testVariation.findUnique({ where: { projectId_name_browser_device_os_viewport_customTags_branchName: { projectId: data.projectId, @@ -71,8 +98,13 @@ export class TestVariationsService { }); } - async update(id: string, data: TestVariationUpdateDto, testRunId?: string): Promise { - return this.prismaService.testVariation.update({ + async update( + id: string, + data: TestVariationUpdateDto, + testRunId?: string, + tx: Prisma.TransactionClient = this.prismaService + ): Promise { + return tx.testVariation.update({ where: { id }, data: { baselineName: data.baselineName, @@ -83,79 +115,79 @@ export class TestVariationsService { }); } - /** - * Tries to get test variation for the same branch - * Falls back to main branch if not found - * @param projectId - * @param baselineData - * @returns - */ async find( - createTestRequestDto: BaselineDataDto & { projectId: string; sourceBranch?: string } - ): Promise { - const project = (await this.prismaService.project.findUnique({ where: { id: createTestRequestDto.projectId } }))!; - const mainBranch = createTestRequestDto.sourceBranch ?? project.mainBranchName; + createTestRequestDto: BaselineDataDto & { projectId: string; sourceBranch?: string }, + mainBranchName?: string + ): Promise { + const mainBranch = + createTestRequestDto.sourceBranch ?? + mainBranchName ?? + (await this.prismaService.project.findUnique({ where: { id: createTestRequestDto.projectId } }))!.mainBranchName; const [mainBranchTestVariation, currentBranchTestVariation] = await Promise.all([ - // search main branch variation - this.findUnique({ - projectId: createTestRequestDto.projectId, - branchName: mainBranch, - ...getTestVariationUniqueData(createTestRequestDto), - }), - // search current branch variation + this.findUnique(variationKey(createTestRequestDto.projectId, mainBranch, createTestRequestDto)), createTestRequestDto.branchName !== mainBranch && - this.findUnique({ - projectId: createTestRequestDto.projectId, - branchName: createTestRequestDto.branchName, - ...getTestVariationUniqueData(createTestRequestDto), - }), + this.findUnique( + variationKey(createTestRequestDto.projectId, createTestRequestDto.branchName, createTestRequestDto) + ), ]); - if (!!currentBranchTestVariation) { + if (currentBranchTestVariation) { if (mainBranchTestVariation && mainBranchTestVariation.updatedAt > currentBranchTestVariation.updatedAt) { return mainBranchTestVariation; } return currentBranchTestVariation; } - if (!!mainBranchTestVariation) { - return mainBranchTestVariation; - } + return mainBranchTestVariation ?? undefined; } - /** - * Creates empty test variation (no baseline) - */ - create({ - testRunId, - createTestRequestDto, - }: { - testRunId?: string; - createTestRequestDto: Omit; - }): Promise { - return this.prismaService.testVariation.create({ + create( + { createTestRequestDto }: { createTestRequestDto: Omit }, + tx: Prisma.TransactionClient = this.prismaService + ): Promise { + return tx.testVariation.create({ data: { project: { connect: { id: createTestRequestDto.projectId } }, - testRuns: testRunId ? { connect: { id: testRunId } } : undefined, branchName: createTestRequestDto.branchName, ...getTestVariationUniqueData(createTestRequestDto), }, }); } - async addBaseline({ - id, - userId, - testRunId, - baselineName, - }: { - id: string; - userId?: string; - testRunId: string; - baselineName: string; - }): Promise { - return this.prismaService.testVariation.update({ + async createOrTakeWinner( + createTestRequestDto: Omit + ): Promise { + try { + return await this.create({ createTestRequestDto }); + } catch (e) { + if (isPrismaError(e, 'P2002')) { + const winner = await this.findUnique( + variationKey(createTestRequestDto.projectId, createTestRequestDto.branchName, createTestRequestDto) + ); + if (winner) { + return winner; + } + } + throw e; + } + } + + async addBaseline( + { + id, + userId, + testRunId, + baselineName, + }: { + id: string; + userId?: string; + testRunId: string; + baselineName: string; + }, + tx: Prisma.TransactionClient = this.prismaService + ): Promise { + return tx.testVariation.update({ where: { id }, data: { baselineName, @@ -181,95 +213,72 @@ export class TestVariationsService { } async merge(projectId: string, fromBranch: string, toBranch: string): Promise { - // create build const build: Build = await this.buildsService.findOrCreate({ branchName: toBranch, projectId, }); try { - await this.mergeInto(build, projectId, fromBranch, toBranch); + const project = await this.prismaService.project.findUnique({ where: { id: projectId } }); + if (!project) { + throw new NotFoundException(`Project not found: ${projectId}`); + } + await this.mergeInto(build, project, fromBranch, toBranch); } catch (e) { - // the build is stopped either way, so a failed merge does not stay - // "running" in the list forever; the cause still reaches the client await this.buildsService.update(build.id, { isRunning: false }).catch((): void => undefined); throw e; } - // stop build return this.buildsService.update(build.id, { isRunning: false }); } - private async mergeInto(build: Build, projectId: string, fromBranch: string, toBranch: string): Promise { - // find source branch variations + private async mergeInto(build: Build, project: Project, fromBranch: string, toBranch: string): Promise { const testVariations: TestVariation[] = await this.prismaService.testVariation.findMany({ - where: { projectId, branchName: fromBranch }, + where: { projectId: project.id, branchName: fromBranch }, }); - // compare source to destination branch variations for (const sourceBranchTestVariation of testVariations) { const baselineBuffer = await this.staticService.getImageBuffer(sourceBranchTestVariation.baselineName); if (baselineBuffer) { - // get destination branch request const createTestRequestDto: CreateTestRequestDto = { ...sourceBranchTestVariation, branchName: toBranch, buildId: build.id, diffTollerancePercent: 0, merge: true, - ignoreAreas: JSON.parse(sourceBranchTestVariation.ignoreAreas), + ignoreAreas: parseIgnoreAreas(sourceBranchTestVariation.ignoreAreas), }; - // get destination branch variation — on that branch only: find() would - // hand back the main branch's when it is the newer one, and the merge - // would then try to create a second variation on toBranch - let destintionBranchTestVariation = await this.findUnique({ - projectId, - branchName: toBranch, - ...getTestVariationUniqueData(sourceBranchTestVariation), - }); - - if (!destintionBranchTestVariation) { - destintionBranchTestVariation = await this.create({ createTestRequestDto }); - } + // not find(): it may return main's variation and the merge would create a second one on toBranch + const destinationBranchTestVariation = + (await this.findUnique(variationKey(project.id, toBranch, createTestRequestDto))) ?? + (await this.createOrTakeWinner(createTestRequestDto)); const testRun = await this.testRunsService.create({ - testVariation: destintionBranchTestVariation, + testVariation: destinationBranchTestVariation, createTestRequestDto, imageBuffer: baselineBuffer, }); - await this.testRunsService.calculateDiff(projectId, testRun); + await this.testRunsService.calculateDiff(project.id, testRun, baselineBuffer, project); } } } async delete(id: string): Promise { this.logger.debug(`Going to remove TestVariation ${id}`); - const testVariation = (await this.getDetails(id))!; - - // delete Baselines - for (const baseline of testVariation.baselines) { - await this.deleteBaseline(baseline); - } - - // disconnect TestRuns - // workaround due to https://github.com/prisma/prisma/issues/2810 - await this.prismaService - .$executeRaw`UPDATE "public"."TestRun" SET "testVariationId" = NULL::text WHERE "testVariationId" = ${id}`; - - // delete TestVariation - return this.prismaService.testVariation.delete({ - where: { id }, - }); - } - - async deleteBaseline(baseline: Baseline): Promise { - this.logger.debug(`Going to remove Baseline ${baseline.id}`); - - this.staticService.deleteImage(baseline.baselineName); - return this.prismaService.baseline.delete({ - where: { id: baseline.id }, + const baselines = await this.prismaService.baseline.findMany({ + where: { testVariationId: id }, + select: { baselineName: true }, }); + const [, testVariation] = await this.prismaService.$transaction([ + this.prismaService.baseline.deleteMany({ where: { testVariationId: id } }), + this.prismaService.testVariation.delete({ where: { id } }), + ]); + await this.staticService.deleteImages( + baselines.map(({ baselineName }) => baselineName), + 8 + ); + return testVariation; } } diff --git a/src/users/db/dbusers.service.spec.ts b/src/users/db/dbusers.service.spec.ts index cab1d56c..75a3341c 100644 --- a/src/users/db/dbusers.service.spec.ts +++ b/src/users/db/dbusers.service.spec.ts @@ -1,4 +1,4 @@ -import { HttpException, HttpStatus } from '@nestjs/common'; +import { HttpException, HttpStatus, NotFoundException } from '@nestjs/common'; import { Role, User } from '@prisma/client'; import { DbUsersService } from './dbusers.service'; import { PrismaService } from '../../prisma/prisma.service'; @@ -61,18 +61,41 @@ describe('DbUsersService', () => { }); describe('update', () => { - it('updates the profile fields as given and returns a fresh token', async () => { + const current = { ...user, email: 'Test@Example.com' }; + + it('updates the profile fields and returns a fresh token', async () => { + prisma.user.findUnique.mockResolvedValue(current); + prisma.user.update.mockResolvedValue(user); + + const result = await service.update('1', { email: 'test@example.com', firstName: 'J', lastName: 'D' }); + + expect(prisma.user.findUnique).toHaveBeenCalledWith({ where: { id: '1' } }); + expect(auth.signToken).toHaveBeenCalledWith(user); + expect(result).toEqual(new UserLoginResponseDto(user, 'token')); + }); + + it.each([ + ['stores a changed email in lower case', ' New@Example.com ', 'new@example.com'], + ['keeps the stored spelling when only the case differs', 'test@EXAMPLE.com', 'Test@Example.com'], + ])('%s', async (_name, sent, stored) => { + prisma.user.findUnique.mockResolvedValue(current); prisma.user.update.mockResolvedValue(user); - const result = await service.update('1', { email: ' Mixed@Case.com', firstName: 'J', lastName: 'D' }); + await service.update('1', { email: sent, firstName: 'J', lastName: 'D' }); - // NOTE: current behavior — unlike create, the email is stored as given (not trimmed or lower-cased) expect(prisma.user.update).toHaveBeenCalledWith({ where: { id: '1' }, - data: { email: ' Mixed@Case.com', firstName: 'J', lastName: 'D' }, + data: { email: stored, firstName: 'J', lastName: 'D' }, }); - expect(auth.signToken).toHaveBeenCalledWith(user); - expect(result).toEqual(new UserLoginResponseDto(user, 'token')); + }); + + it('answers 404 when the user no longer exists', async () => { + prisma.user.findUnique.mockResolvedValue(null); + + await expect(service.update('1', { email: 'a@b.com', firstName: 'J', lastName: 'D' })).rejects.toBeInstanceOf( + NotFoundException + ); + expect(prisma.user.update).not.toHaveBeenCalled(); }); }); @@ -88,6 +111,8 @@ describe('DbUsersService', () => { }); describe('login', () => { + const invalid = new HttpException('Invalid email or password.', HttpStatus.BAD_REQUEST); + it('returns the user with a token when the password matches', async () => { prisma.user.findUnique.mockResolvedValue(user); auth.compare.mockResolvedValue(true); @@ -102,9 +127,7 @@ describe('DbUsersService', () => { it('rejects an unknown email with the same message as a wrong password', async () => { prisma.user.findUnique.mockResolvedValue(null); - await expect(service.login({ email: 'nobody@example.com', password: 'secret' })).rejects.toThrow( - new HttpException('Invalid email or password.', HttpStatus.BAD_REQUEST) - ); + await expect(service.login({ email: 'nobody@example.com', password: 'secret' })).rejects.toThrow(invalid); expect(auth.compare).not.toHaveBeenCalled(); }); @@ -112,20 +135,37 @@ describe('DbUsersService', () => { prisma.user.findUnique.mockResolvedValue(user); auth.compare.mockResolvedValue(false); - await expect(service.login({ email: 'test@example.com', password: 'wrong' })).rejects.toThrow( - new HttpException('Invalid email or password.', HttpStatus.BAD_REQUEST) - ); + await expect(service.login({ email: 'test@example.com', password: 'wrong' })).rejects.toThrow(invalid); expect(auth.signToken).not.toHaveBeenCalled(); }); - it('looks the email up exactly as typed', async () => { + it('signs in with a differently cased email', async () => { + prisma.user.findUnique.mockResolvedValueOnce(null).mockResolvedValueOnce(user); + auth.compare.mockResolvedValue(true); + + const result = await service.login({ email: ' John@Example.com ', password: 'secret' }); + + expect(prisma.user.findUnique).toHaveBeenNthCalledWith(1, { where: { email: 'John@Example.com' } }); + expect(prisma.user.findUnique).toHaveBeenNthCalledWith(2, { where: { email: 'john@example.com' } }); + expect(result).toEqual(new UserLoginResponseDto(user, 'token')); + }); + + it('keeps signing in a mixed-case row with its exact spelling', async () => { + prisma.user.findUnique.mockResolvedValueOnce({ ...user, email: 'John@Example.com' }); + auth.compare.mockResolvedValue(true); + + await service.login({ email: 'John@Example.com', password: 'secret' }); + + expect(prisma.user.findUnique).toHaveBeenCalledTimes(1); + expect(prisma.user.findUnique).toHaveBeenCalledWith({ where: { email: 'John@Example.com' } }); + }); + + it('looks a lower-case email up once', async () => { prisma.user.findUnique.mockResolvedValue(null); - await expect(service.login({ email: ' Test@Example.com', password: 'secret' })).rejects.toThrow(HttpException); + await expect(service.login({ email: 'nobody@example.com', password: 'secret' })).rejects.toThrow(HttpException); - // NOTE: current behavior — create() lower-cases and trims, login() does not, - // so a user registered as "Test@Example.com" must log in as "test@example.com" - expect(prisma.user.findUnique).toHaveBeenCalledWith({ where: { email: ' Test@Example.com' } }); + expect(prisma.user.findUnique).toHaveBeenCalledTimes(1); }); }); }); diff --git a/src/users/db/dbusers.service.ts b/src/users/db/dbusers.service.ts index da8d4954..e2630dee 100644 --- a/src/users/db/dbusers.service.ts +++ b/src/users/db/dbusers.service.ts @@ -1,4 +1,4 @@ -import { HttpException, HttpStatus, Logger } from '@nestjs/common'; +import { HttpException, HttpStatus, NotFoundException } from '@nestjs/common'; import { CreateUserDto } from '../dto/user-create.dto'; import { UserLoginResponseDto } from '../dto/user-login-response.dto'; import { PrismaService } from '../../prisma/prisma.service'; @@ -9,7 +9,7 @@ import { UserLoginRequestDto } from '../dto/user-login-request.dto'; import { Users } from '../users.interface'; export class DbUsersService implements Users { - private readonly logger: Logger = new Logger(DbUsersService.name); + readonly allowsRegistration = true; constructor( private readonly prismaService: PrismaService, @@ -33,10 +33,17 @@ export class DbUsersService implements Users { } async update(id: string, userDto: UpdateUserDto): Promise { + const current = await this.prismaService.user.findUnique({ where: { id } }); + if (!current) { + throw new NotFoundException(`User not found: ${id}`); + } + const requested = userDto.email.trim(); + // a case-only difference keeps the stored spelling: lower-casing it could collide with another row + const email = requested.toLowerCase() === current.email.toLowerCase() ? current.email : requested.toLowerCase(); const user = await this.prismaService.user.update({ where: { id }, data: { - email: userDto.email, + email, firstName: userDto.firstName, lastName: userDto.lastName, }, @@ -56,9 +63,13 @@ export class DbUsersService implements Users { } async login(userLoginRequestDto: UserLoginRequestDto) { - const user = await this.prismaService.user.findUnique({ - where: { email: userLoginRequestDto.email }, - }); + const email = userLoginRequestDto.email.trim(); + // rows saved before emails were lower-cased keep their spelling, so the exact match goes first + const user = + (await this.prismaService.user.findUnique({ where: { email } })) ?? + (email === email.toLowerCase() + ? null + : await this.prismaService.user.findUnique({ where: { email: email.toLowerCase() } })); if (!user) { throw new HttpException('Invalid email or password.', HttpStatus.BAD_REQUEST); } diff --git a/src/users/dto/change-password.dto.spec.ts b/src/users/dto/change-password.dto.spec.ts new file mode 100644 index 00000000..99c6be2d --- /dev/null +++ b/src/users/dto/change-password.dto.spec.ts @@ -0,0 +1,20 @@ +import { plainToInstance } from 'class-transformer'; +import { validate } from 'class-validator'; +import { ChangePasswordDto } from './change-password.dto'; + +const failedProperties = async (body: object): Promise => + (await validate(plainToInstance(ChangePasswordDto, body))).map((error) => error.property); + +describe('ChangePasswordDto', () => { + it.each([ + ['a missing password', {}], + ['an empty password', { password: '' }], + ['a password that is not a string', { password: 1 }], + ])('rejects %s', async (_name, body) => { + expect(await failedProperties(body)).toEqual(['password']); + }); + + it('accepts a non-empty password', async () => { + expect(await failedProperties({ password: 'x' })).toEqual([]); + }); +}); diff --git a/src/users/dto/change-password.dto.ts b/src/users/dto/change-password.dto.ts new file mode 100644 index 00000000..8ccb3a60 --- /dev/null +++ b/src/users/dto/change-password.dto.ts @@ -0,0 +1,9 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { IsNotEmpty, IsString } from 'class-validator'; + +export class ChangePasswordDto { + @ApiProperty() + @IsString() + @IsNotEmpty() + readonly password!: string; +} diff --git a/src/users/dto/user.dto.ts b/src/users/dto/user.dto.ts index 5f97aa21..6f9af9e7 100644 --- a/src/users/dto/user.dto.ts +++ b/src/users/dto/user.dto.ts @@ -1,4 +1,4 @@ -import { ApiProperty } from '@nestjs/swagger'; +import { ApiProperty, OmitType } from '@nestjs/swagger'; import { Role, User } from '@prisma/client'; export class UserDto { @@ -29,3 +29,13 @@ export class UserDto { this.role = user.role; } } + +export class UserSummaryDto extends OmitType(UserDto, ['apiKey'] as const) {} + +export const toUserSummary = ({ id, email, firstName, lastName, role }: User): UserSummaryDto => ({ + id, + email, + firstName, + lastName, + role, +}); diff --git a/src/users/ldap/ldapusers.concurrency.spec.ts b/src/users/ldap/ldapusers.concurrency.spec.ts new file mode 100644 index 00000000..c7091042 --- /dev/null +++ b/src/users/ldap/ldapusers.concurrency.spec.ts @@ -0,0 +1,100 @@ +import { ConfigService } from '@nestjs/config'; +import { Role, User } from '@prisma/client'; +import { Client as LdapClient } from 'ldapts'; +import { AuthService } from '../../auth/auth.service'; +import { PrismaService } from '../../prisma/prisma.service'; +import { LdapUsersService } from './ldapusers.service'; + +const mockClients: { generation: number; unbind: jest.Mock }[] = []; + +jest.mock('ldapts', () => ({ + Client: jest.fn(() => { + const client = { + generation: 0, + bind: jest.fn(async () => { + const started = client.generation; + await new Promise(setImmediate); + if (client.generation !== started) { + throw new Error('socket closed'); + } + }), + search: jest.fn(async () => { + const started = client.generation; + await new Promise(setImmediate); + if (client.generation !== started) { + throw new Error('socket closed'); + } + return { + searchEntries: [ + { + dn: 'uid=jane,ou=people', + LDAP_ATTRIBUTE_MAIL: 'jane@example.com', + LDAP_ATTRIBUTE_FIRST_NAME: 'Jane', + LDAP_ATTRIBUTE_LAST_NAME: 'Roe', + }, + ], + searchReferences: [], + }; + }), + unbind: jest.fn(async () => { + client.generation++; + }), + }; + mockClients.push(client); + return client; + }), +})); + +const user: User = { + id: '1', + email: 'jane@example.com', + firstName: 'Jane', + lastName: 'Roe', + apiKey: 'key', + password: 'hash', + isActive: true, + role: Role.editor, + updatedAt: new Date(), + createdAt: new Date(), +}; + +describe('LdapUsersService concurrent logins', () => { + let service: LdapUsersService; + + beforeEach(() => { + mockClients.length = 0; + (LdapClient as unknown as jest.Mock).mockClear(); + const configService = { + get: jest.fn((_key: string, defaultValue?: string) => defaultValue), + getOrThrow: jest.fn((key: string) => (key === 'LDAP_USERS_SEARCH_FILTER' ? '(mail={{email}})' : key)), + }; + const prisma = { user: { findUnique: jest.fn().mockResolvedValue(user) } }; + const auth = { signToken: jest.fn().mockReturnValue('token') }; + service = new LdapUsersService( + configService as unknown as ConfigService, + prisma as unknown as PrismaService, + auth as unknown as AuthService + ); + }); + + it("serves two concurrent logins without one closing the other's connection", async () => { + await expect( + Promise.all([ + service.login({ email: 'jane@example.com', password: 'a' }), + service.login({ email: 'jane@example.com', password: 'b' }), + ]) + ).resolves.toHaveLength(2); + }); + + it('opens a connection for the search and one for the user bind, and closes each', async () => { + await Promise.all([ + service.login({ email: 'jane@example.com', password: 'a' }), + service.login({ email: 'jane@example.com', password: 'b' }), + ]); + + expect(LdapClient).toHaveBeenCalledTimes(4); + for (const client of mockClients) { + expect(client.unbind).toHaveBeenCalledTimes(1); + } + }); +}); diff --git a/src/users/ldap/ldapusers.service.spec.ts b/src/users/ldap/ldapusers.service.spec.ts index 7f0b44d6..48fb11fd 100644 --- a/src/users/ldap/ldapusers.service.spec.ts +++ b/src/users/ldap/ldapusers.service.spec.ts @@ -1,41 +1,15 @@ -import { Test, TestingModule } from '@nestjs/testing'; import { LdapUsersService } from './ldapusers.service'; import { AuthService } from '../../auth/auth.service'; import { PrismaService } from '../../prisma/prisma.service'; import { ConfigService } from '@nestjs/config'; import { Role, User } from '@prisma/client'; -import { HttpException, HttpStatus } from '@nestjs/common'; +import { HttpException, HttpStatus, NotFoundException, ServiceUnavailableException } from '@nestjs/common'; import { UserLoginResponseDto } from '../dto/user-login-response.dto'; -// eslint-disable-next-line @typescript-eslint/no-unused-vars -import { Entry as LdapEntry, Client as LdapClient, SearchOptions, SearchResult } from 'ldapts'; +import { Client as LdapClient } from 'ldapts'; jest.mock('ldapts', () => { - const mockLdapClient = { - // eslint-disable-next-line @typescript-eslint/no-unused-vars - search: jest.fn((_searchDN, searchOptions: SearchOptions): Promise => { - if (searchOptions.filter!.toString().includes('exists@example.com')) { - return Promise.resolve({ - searchEntries: [ - { - dn: 'dn', - LDAP_ATTRIBUTE_MAIL: 'exists@example.com', - LDAP_ATTRIBUTE_FIRST_NAME: 'first', - LDAP_ATTRIBUTE_LAST_NAME: 'last', - }, - ], - searchReferences: [], - }); - } else { - return Promise.resolve({ searchEntries: [], searchReferences: [] }); - } - }), - bind: jest.fn(), - unbind: jest.fn(), - }; - - return { - Client: jest.fn(() => mockLdapClient), - }; + const mockLdapClient = { search: jest.fn(), bind: jest.fn(), unbind: jest.fn().mockResolvedValue(undefined) }; + return { Client: jest.fn(() => mockLdapClient) }; }); const user: User = { @@ -51,133 +25,7 @@ const user: User = { createdAt: new Date(), }; -describe('LdapUsersService match from ldap', () => { - let service: LdapUsersService; - let prismaService: PrismaService; - let configService: ConfigService; - let authService: AuthService; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [ - { - provide: ConfigService, - useValue: { - get: jest.fn().mockReturnValue('true'), - getOrThrow: jest.fn((string) => (string === 'LDAP_USERS_SEARCH_FILTER' ? '(&(mail={{email}}))' : string)), - }, - }, - { - provide: PrismaService, - useValue: { - user: { - findMany: jest.fn().mockResolvedValueOnce([]), - findUnique: jest.fn(), - create: jest.fn(), - update: jest.fn(), - }, - }, - }, - { - provide: AuthService, - useValue: { - generateApiKey: jest.fn(() => 'generatedApiKey'), - encryptPassword: jest.fn(() => 'encryptedPassword'), - signToken: jest.fn(() => 'token'), - }, - }, - ], - }).compile(); - configService = module.get(ConfigService); - prismaService = module.get(PrismaService); - authService = module.get(AuthService); - service = new LdapUsersService(configService, prismaService, authService); - }); - - it('should be defined', () => { - expect(service).toBeDefined(); - }); - - it('should create new user on login', async () => { - jest.spyOn(prismaService.user, 'findUnique').mockResolvedValue(null); - const prismaUserCreateMock = jest.spyOn(prismaService.user, 'create').mockResolvedValue(user); - await service.login({ email: 'exists@example.com', password: 'password' }); - expect(prismaUserCreateMock).toHaveBeenCalled(); - }); - - it('should login with user already in db', async () => { - jest.spyOn(prismaService.user, 'findUnique').mockResolvedValue(user); - const prismaUserCreateMock = jest.spyOn(prismaService.user, 'create'); - await service.login({ email: 'exists@example.com', password: 'password' }); - expect(prismaUserCreateMock).not.toHaveBeenCalled(); - }); -}); - -describe('LdapUsersService with no results from ldap', () => { - let service: LdapUsersService; - let prismaService: PrismaService; - let configService: ConfigService; - let authService: AuthService; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [ - { - provide: ConfigService, - useValue: { - get: jest.fn().mockReturnValue('true'), - getOrThrow: jest.fn().mockReturnValue('string'), - }, - }, - { - provide: PrismaService, - useValue: { - user: { - findMany: jest.fn().mockResolvedValueOnce([]), - create: jest.fn(), - update: jest.fn(), - }, - }, - }, - { - provide: AuthService, - useValue: { - generateApiKey: jest.fn(() => 'generatedApiKey'), - encryptPassword: jest.fn(() => 'encryptedPassword'), - }, - }, - ], - }).compile(); - configService = module.get(ConfigService); - prismaService = module.get(PrismaService); - authService = module.get(AuthService); - service = new LdapUsersService(configService, prismaService, authService); - }); - - it('should be defined', () => { - expect(service).toBeDefined(); - }); - - it('should never change a password', async () => { - const prismaUserCreateMock = jest.spyOn(prismaService.user, 'create'); - const prismaUserUpdateMock = jest.spyOn(prismaService.user, 'update'); - const result = await service.changePassword(user, 'newPassword'); - expect(prismaUserCreateMock).not.toHaveBeenCalled(); - expect(prismaUserUpdateMock).not.toHaveBeenCalled(); - expect(result).toBe(true); - }); - - it('should not login when ldap search results are empty', async () => { - expect.assertions(1); - try { - await service.login({ email: 'testÜ*()\\\0@example.com', password: 'password' }); - } catch (e) { - expect((e as Error).message).toBe('Invalid email or password.'); - } - }); -}); - -describe('LdapUsersService characterization', () => { +describe('LdapUsersService', () => { let service: LdapUsersService; let ldapClient: { search: jest.Mock; bind: jest.Mock; unbind: jest.Mock }; let prisma: { user: { findUnique: jest.Mock; create: jest.Mock; update: jest.Mock } }; @@ -213,6 +61,13 @@ describe('LdapUsersService characterization', () => { ); }; + const found = () => { + ldapClient.search.mockResolvedValueOnce({ searchEntries: [entry], searchReferences: [] }); + prisma.user.findUnique.mockResolvedValueOnce(user); + }; + const login = (svc = service, password = 'pw') => svc.login({ email: 'jane', password }); + const invalidLogin = new HttpException('Invalid email or password.', HttpStatus.BAD_REQUEST); + beforeEach(() => { jest.clearAllMocks(); prisma = { user: { findUnique: jest.fn(), create: jest.fn(), update: jest.fn() } }; @@ -222,18 +77,52 @@ describe('LdapUsersService characterization', () => { signToken: jest.fn().mockReturnValue('token'), }; service = createService(); - ldapClient = (service as any).ldapClient; + ldapClient = new (LdapClient as unknown as jest.Mock)(); + (LdapClient as unknown as jest.Mock).mockClear(); }); it.each([ ['false', true], ['TRUE', false], - ])('creates the client with LDAP_TLS_NO_VERIFY=%s -> rejectUnauthorized %s', (tlsNoVerify, rejectUnauthorized) => { - (LdapClient as unknown as jest.Mock).mockClear(); + ])( + 'creates the client with LDAP_TLS_NO_VERIFY=%s -> rejectUnauthorized %s', + async (tlsNoVerify, rejectUnauthorized) => { + found(); + + await login(createService(tlsNoVerify)); + + expect(LdapClient).toHaveBeenCalledWith(expect.objectContaining({ tlsOptions: { rejectUnauthorized } })); + } + ); + + it('passes timeouts to every client', async () => { + found(); + + await login(); + + expect(LdapClient).toHaveBeenCalledTimes(2); + expect(LdapClient).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ url: 'ldaps://ldap.example.com', timeout: 10000, connectTimeout: 5000 }) + ); + expect(LdapClient).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ url: 'ldaps://ldap.example.com', timeout: 10000, connectTimeout: 5000 }) + ); + }); + + it('never changes a password', async () => { + await expect(service.changePassword(user, 'newPassword')).rejects.toThrow( + new HttpException('Password changes are disabled. Change your password in the directory.', HttpStatus.BAD_REQUEST) + ); + expect(prisma.user.create).not.toHaveBeenCalled(); + expect(prisma.user.update).not.toHaveBeenCalled(); + }); - createService(tlsNoVerify); + it('rejects an email the directory does not know', async () => { + ldapClient.search.mockResolvedValueOnce({ searchEntries: [], searchReferences: [] }); - expect(LdapClient).toHaveBeenCalledWith({ url: 'ldaps://ldap.example.com', tlsOptions: { rejectUnauthorized } }); + await expect(service.login({ email: 'testÜ*()\\\0@example.com', password: 'pw' })).rejects.toThrow(invalidLogin); }); it('refuses to create users', async () => { @@ -245,9 +134,17 @@ describe('LdapUsersService characterization', () => { }); describe('login', () => { + it.each([ + ['an empty string', ''], + ['a missing value', undefined as unknown as string], + ])('rejects %s as the password without contacting LDAP', async (_name, password) => { + await expect(service.login({ email: 'exists@example.com', password })).rejects.toThrow(invalidLogin); + expect(ldapClient.bind).not.toHaveBeenCalled(); + expect(ldapClient.search).not.toHaveBeenCalled(); + }); + it('searches as the bind user with the escaped, trimmed email in every placeholder', async () => { - ldapClient.search.mockResolvedValueOnce({ searchEntries: [entry], searchReferences: [] }); - prisma.user.findUnique.mockResolvedValueOnce(user); + found(); await service.login({ email: ' j*ne(x)\\@example.com ', password: 'pw' }); @@ -260,10 +157,9 @@ describe('LdapUsersService characterization', () => { }); it('binds as the found user and signs in the existing VRT user by the lower-cased LDAP email', async () => { - ldapClient.search.mockResolvedValueOnce({ searchEntries: [entry], searchReferences: [] }); - prisma.user.findUnique.mockResolvedValueOnce(user); + found(); - const result = await service.login({ email: 'jane', password: 'pw' }); + const result = await login(); expect(ldapClient.bind).toHaveBeenNthCalledWith(2, 'uid=jane,ou=people', 'pw'); expect(ldapClient.unbind).toHaveBeenCalledTimes(2); @@ -278,7 +174,7 @@ describe('LdapUsersService characterization', () => { prisma.user.findUnique.mockResolvedValueOnce(null); prisma.user.create.mockResolvedValueOnce(user); - await service.login({ email: 'jane', password: 'pw' }); + await login(); expect(auth.encryptPassword).toHaveBeenCalledWith(expect.any(String)); expect(auth.encryptPassword.mock.calls[0][0]).toHaveLength(12); @@ -298,9 +194,7 @@ describe('LdapUsersService characterization', () => { ldapClient.search.mockResolvedValueOnce({ searchEntries: [entry], searchReferences: [] }); ldapClient.bind.mockResolvedValueOnce(undefined).mockRejectedValueOnce(new Error('invalid credentials')); - await expect(service.login({ email: 'jane', password: 'wrong' })).rejects.toThrow( - new HttpException('Invalid email or password.', HttpStatus.BAD_REQUEST) - ); + await expect(login(service, 'wrong')).rejects.toThrow(invalidLogin); expect(ldapClient.unbind).toHaveBeenCalledTimes(2); expect(prisma.user.findUnique).not.toHaveBeenCalled(); }); @@ -311,46 +205,90 @@ describe('LdapUsersService characterization', () => { const incomplete: Partial = { ...entry, [attribute]: undefined }; ldapClient.search.mockResolvedValueOnce({ searchEntries: [incomplete], searchReferences: [] }); - await expect(service.login({ email: 'jane', password: 'pw' })).rejects.toThrow( - new HttpException('Invalid email or password.', HttpStatus.BAD_REQUEST) - ); + await expect(login()).rejects.toThrow(invalidLogin); expect(ldapClient.bind).toHaveBeenCalledTimes(1); expect(ldapClient.unbind).toHaveBeenCalledTimes(1); } ); - it('passes a failing service bind through and still unbinds', async () => { + it('answers 503 when the service bind fails and still unbinds', async () => { ldapClient.bind.mockRejectedValueOnce(new Error('ldap down')); - await expect(service.login({ email: 'jane', password: 'pw' })).rejects.toThrow('ldap down'); + await expect(login()).rejects.toBeInstanceOf(ServiceUnavailableException); expect(ldapClient.search).not.toHaveBeenCalled(); expect(ldapClient.unbind).toHaveBeenCalledTimes(1); }); + + it('answers 503 when the search fails and still unbinds', async () => { + ldapClient.search.mockRejectedValueOnce(new Error('timeout')); + + await expect(login()).rejects.toBeInstanceOf(ServiceUnavailableException); + expect(ldapClient.unbind).toHaveBeenCalledTimes(1); + }); + + it('does not fail the login when unbind throws', async () => { + found(); + ldapClient.unbind.mockRejectedValue(new Error('socket already closed')); + + await expect(login()).resolves.toEqual(new UserLoginResponseDto(user, 'token')); + ldapClient.unbind.mockResolvedValue(undefined); + }); }); describe('update', () => { - it('overwrites the profile with the LDAP entry found for the new email', async () => { + const own = { ...user, email: 'exists@example.com' }; + + it("refreshes the caller's own LDAP entry and ignores the email sent by the client", async () => { + prisma.user.findUnique.mockResolvedValueOnce(own); + ldapClient.search.mockResolvedValueOnce({ searchEntries: [entry], searchReferences: [] }); + prisma.user.update.mockResolvedValueOnce(user); + + const result = await service.update('1', { email: 'cfo@corp.com', firstName: 'x', lastName: 'y' }); + + expect(prisma.user.findUnique).toHaveBeenCalledWith({ where: { id: '1' } }); + const filter = ldapClient.search.mock.calls[0][1].filter as string; + expect(filter).toContain('exists@example.com'); + expect(filter).not.toContain('cfo@corp.com'); + expect(prisma.user.update).toHaveBeenCalledWith({ + where: { id: '1' }, + data: { email: 'jane@example.com', firstName: 'Jane', lastName: 'Roe' }, + }); + expect(result).toEqual(new UserLoginResponseDto(user, 'token')); + }); + + it('reads the configured attribute names', async () => { + prisma.user.findUnique.mockResolvedValueOnce(own); ldapClient.search.mockResolvedValueOnce({ - searchEntries: [{ ...entry, mail: ' Jane.Roe@Example.com', givenName: 'J.', sn: 'R.' }], + searchEntries: [{ ...entry, mail: 'wrong@example.com', givenName: 'wrong', sn: 'wrong' }], searchReferences: [], }); prisma.user.update.mockResolvedValueOnce(user); - const result = await service.update('1', { email: 'jane', firstName: 'ignored', lastName: 'ignored' }); + await service.update('1', { email: 'jane', firstName: 'a', lastName: 'b' }); - // NOTE: current behavior — update reads the hard-coded `mail`, `givenName` - // and `sn` attributes, not the configured LDAP_ATTRIBUTE_* ones login uses expect(prisma.user.update).toHaveBeenCalledWith({ where: { id: '1' }, - data: { email: 'jane.roe@example.com', firstName: 'J.', lastName: 'R.' }, + data: { email: 'jane@example.com', firstName: 'Jane', lastName: 'Roe' }, }); - expect(result).toEqual(new UserLoginResponseDto(user, 'token')); }); - it('fails with a TypeError when the entry has no `mail` attribute', async () => { - ldapClient.search.mockResolvedValueOnce({ searchEntries: [entry], searchReferences: [] }); + it('answers 404 when the caller no longer exists', async () => { + prisma.user.findUnique.mockResolvedValueOnce(null); + + await expect(service.update('1', { email: 'jane', firstName: 'a', lastName: 'b' })).rejects.toBeInstanceOf( + NotFoundException + ); + expect(ldapClient.search).not.toHaveBeenCalled(); + expect(prisma.user.update).not.toHaveBeenCalled(); + }); - await expect(service.update('1', { email: 'jane', firstName: 'a', lastName: 'b' })).rejects.toThrow(TypeError); + it('answers 503 when the directory is not reachable', async () => { + prisma.user.findUnique.mockResolvedValueOnce(own); + ldapClient.bind.mockRejectedValueOnce(new Error('ldap down')); + + await expect(service.update('1', { email: 'jane', firstName: 'a', lastName: 'b' })).rejects.toBeInstanceOf( + ServiceUnavailableException + ); expect(prisma.user.update).not.toHaveBeenCalled(); }); }); diff --git a/src/users/ldap/ldapusers.service.ts b/src/users/ldap/ldapusers.service.ts index 39aaa490..b44f8991 100644 --- a/src/users/ldap/ldapusers.service.ts +++ b/src/users/ldap/ldapusers.service.ts @@ -1,4 +1,4 @@ -import { HttpException, HttpStatus, Logger } from '@nestjs/common'; +import { HttpException, HttpStatus, Logger, NotFoundException, ServiceUnavailableException } from '@nestjs/common'; import { CreateUserDto } from '../dto/user-create.dto'; import { UserLoginResponseDto } from '../dto/user-login-response.dto'; import { PrismaService } from '../../prisma/prisma.service'; @@ -22,8 +22,11 @@ type LDAPConfig = { attributeLastName: string; tlsNoVerify: boolean; }; + +const invalidCredentials = () => new HttpException('Invalid email or password.', HttpStatus.BAD_REQUEST); + export class LdapUsersService implements Users { - private readonly ldapClient: LdapClient; + readonly allowsRegistration = false; private readonly logger: Logger = new Logger(LdapUsersService.name); private readonly ldapConfig: LDAPConfig; @@ -43,20 +46,24 @@ export class LdapUsersService implements Users { attributeLastName: this.configService.getOrThrow('LDAP_ATTRIBUTE_LAST_NAME'), tlsNoVerify: this.configService.get('LDAP_TLS_NO_VERIFY', 'false').toLowerCase() === 'true', }; - this.ldapClient = new LdapClient({ + } + + // one connection per operation: ldapts' unbind() destroys the socket a concurrent login may still be using + private createClient(): LdapClient { + return new LdapClient({ url: this.ldapConfig.url, - tlsOptions: { - rejectUnauthorized: !this.ldapConfig.tlsNoVerify, - }, + timeout: 10_000, + connectTimeout: 5_000, + tlsOptions: { rejectUnauthorized: !this.ldapConfig.tlsNoVerify }, }); } private async findUserInLdap(email: string): Promise { - //escape the email address for LDAP search email = this.escapeLdap(email.trim()); this.logger.verbose(`search '${email}' in LDAP`); + const client = this.createClient(); try { - await this.ldapClient.bind(this.ldapConfig.bindUser, this.ldapConfig.bindPassword); + await client.bind(this.ldapConfig.bindUser, this.ldapConfig.bindPassword); const attributes = [ 'dn', this.ldapConfig.attributeMail, @@ -64,90 +71,95 @@ export class LdapUsersService implements Users { this.ldapConfig.attributeLastName, ]; - const { searchEntries } = await this.ldapClient.search(this.ldapConfig.searchDN, { + const { searchEntries } = await client.search(this.ldapConfig.searchDN, { filter: this.ldapConfig.usersSearchFilter.replaceAll('{{email}}', email), sizeLimit: 1, attributes: attributes, }); if (searchEntries.length === 0) { this.logger.log('User not found in LDAP.'); - throw new HttpException('Invalid email or password.', HttpStatus.BAD_REQUEST); + throw invalidCredentials(); } for (const attribute of attributes) { if (searchEntries[0][attribute] == null) { this.logger.warn(`Attribute '${attribute}' not found in LDAP entry found for '${email}'`); - throw new HttpException('Invalid email or password.', HttpStatus.BAD_REQUEST); + throw invalidCredentials(); } } return searchEntries[0]; + } catch (e) { + if (e instanceof HttpException) { + throw e; + } + this.logger.error(`LDAP search for '${email}' failed: ${e}`); + throw new ServiceUnavailableException('Directory service is not available.'); } finally { - await this.ldapClient.unbind(); + await client.unbind().catch((): undefined => undefined); } } - private async createUserFromLdapEntry(ldapEntry: LdapEntry): Promise { - const userForVRTDb = { - email: ldapEntry[this.ldapConfig.attributeMail].toString().trim().toLowerCase(), - firstName: ldapEntry[this.ldapConfig.attributeFirstName].toString(), - lastName: ldapEntry[this.ldapConfig.attributeLastName].toString(), - apiKey: this.authService.generateApiKey(), - password: await this.authService.encryptPassword(genSaltSync(1).slice(-12)), - role: Role.editor, + private profileOf(entry: LdapEntry) { + return { + email: entry[this.ldapConfig.attributeMail].toString().trim().toLowerCase(), + firstName: entry[this.ldapConfig.attributeFirstName].toString(), + lastName: entry[this.ldapConfig.attributeLastName].toString(), }; + } + private async createUserFromLdapEntry(ldapEntry: LdapEntry): Promise { return this.prismaService.user.create({ - data: userForVRTDb, + data: { + ...this.profileOf(ldapEntry), + apiKey: this.authService.generateApiKey(), + password: await this.authService.encryptPassword(genSaltSync(1).slice(-12)), + role: Role.editor, + }, }); } - // eslint-disable-next-line @typescript-eslint/no-unused-vars - async create(createUserDto: CreateUserDto): Promise { + async create(_createUserDto: CreateUserDto): Promise { throw new HttpException('User creation is disabled. Use your LDAP-Credentials to login.', HttpStatus.BAD_REQUEST); } - async update(id: string, userDto: UpdateUserDto): Promise { - const userFromLdap = await this.findUserInLdap(userDto.email); - const user = await this.prismaService.user.update({ - where: { id }, - data: { - email: userFromLdap.mail.toString().trim().toLowerCase(), - firstName: userFromLdap.givenName.toString(), - lastName: userFromLdap.sn.toString(), - }, - }); - const token = this.authService.signToken(user); - return new UserLoginResponseDto(user, token); + async update(id: string, _userDto: UpdateUserDto): Promise { + // the directory owns the profile: refresh it from the caller's own entry, never from an email the client sent + const current = await this.prismaService.user.findUnique({ where: { id } }); + if (!current) { + throw new NotFoundException(`User not found: ${id}`); + } + const entry = await this.findUserInLdap(current.email); + const user = await this.prismaService.user.update({ where: { id }, data: this.profileOf(entry) }); + return new UserLoginResponseDto(user, this.authService.signToken(user)); } - // eslint-disable-next-line @typescript-eslint/no-unused-vars - async changePassword(user: User, newPassword: string): Promise { - this.logger.warn(`${user.email} tied to change password - this is not supported for LDAP users`); - return true; + changePassword(user: User, _newPassword: string): Promise { + this.logger.warn(`${user.email} tried to change password - this is not supported for LDAP users`); + return Promise.reject( + new HttpException('Password changes are disabled. Change your password in the directory.', HttpStatus.BAD_REQUEST) + ); } async login(userLoginRequestDto: UserLoginRequestDto) { - const userFromLdap = await this.findUserInLdap(userLoginRequestDto.email); - if (!userFromLdap) { - throw new HttpException('Invalid email or password.', HttpStatus.BAD_REQUEST); + // RFC 4513 5.1.2: a DN with an empty password is an unauthenticated bind, which AD answers with success + if (!userLoginRequestDto.password) { + throw invalidCredentials(); } + const userFromLdap = await this.findUserInLdap(userLoginRequestDto.email); - // check if user password is correct using ldap + const client = this.createClient(); try { - await this.ldapClient.bind(userFromLdap.dn, userLoginRequestDto.password); - } catch (e) { - throw new HttpException('Invalid email or password.', HttpStatus.BAD_REQUEST); + await client.bind(userFromLdap.dn, userLoginRequestDto.password); + } catch { + throw invalidCredentials(); } finally { - await this.ldapClient.unbind(); + await client.unbind().catch((): undefined => undefined); } - const userEmailFromLdap = userFromLdap[this.ldapConfig.attributeMail].toString().trim().toLowerCase(); + const userEmailFromLdap = this.profileOf(userFromLdap).email; - let user = await this.prismaService.user.findUnique({ - where: { email: userEmailFromLdap }, - }); + let user = await this.prismaService.user.findUnique({ where: { email: userEmailFromLdap } }); if (!user) { - // create user if not found in VRT database this.logger.log( `'${userLoginRequestDto.email}' (found in ldap as '${userEmailFromLdap}') successfully ` + 'authenticated via LDAP, but not found in VRT database. Creating user.' @@ -159,13 +171,6 @@ export class LdapUsersService implements Users { return new UserLoginResponseDto(user, token); } - /** - * RFC 2254 Escaping of filter strings - * Raw Escaped - * (o=Parens (R Us)) (o=Parens \28R Us\29) - * (cn=star*) (cn=star\2A) - * (filename=C:\MyFile) (filename=C:\5cMyFile) - */ private escapeLdap(input: string): string { let escapedResult = ''; for (const inputChar of input) { diff --git a/src/users/oidc/oidcusers.service.spec.ts b/src/users/oidc/oidcusers.service.spec.ts index 649fa609..3d06f5a5 100644 --- a/src/users/oidc/oidcusers.service.spec.ts +++ b/src/users/oidc/oidcusers.service.spec.ts @@ -73,6 +73,35 @@ const prismaService = { user: { findUnique, create, update }, } as unknown as PrismaService; +const configWith = (overrides: Record = {}): ConfigService => + ({ + get: jest.fn((key: string, defaultValue?: string) => + key in overrides ? overrides[key] : configService.get(key, defaultValue) + ), + getOrThrow: configService.getOrThrow, + }) as unknown as ConfigService; + +const serviceWith = (overrides: Record = {}) => + new OidcUsersService(configWith(overrides), prismaService, authService); + +const idpReturns = (claims: Record, extra: Record = {}) => + mockCallback.mockResolvedValue({ claims: () => claims, ...extra }); + +const noUser = () => { + findUnique.mockResolvedValue(null); + create.mockImplementation(({ data }) => ({ ...user, ...data })); +}; + +const checks = { state: 'mock-state', codeVerifier: 'mock-verifier' }; +const login = (svc: OidcUsersService, params: Record = { code: 'code', state: 'mock-state' }) => + svc.handleCallback(params, checks); +const expectLoginFailed = (svc: OidcUsersService, params?: Record) => + expect(login(svc, params)).rejects.toThrow('OIDC login failed.'); +const expectCreatedWith = (data: object) => + expect(create).toHaveBeenCalledWith(expect.objectContaining({ data: expect.objectContaining(data) })); +const expectRoleUpdate = (role: Role) => + expect(update).toHaveBeenCalledWith({ where: { id: user.id }, data: { role } }); + describe('OidcUsersService', () => { let service: OidcUsersService; @@ -86,7 +115,7 @@ describe('OidcUsersService', () => { }, }); mockAuthorizationUrl.mockReturnValue('https://idp.example.com/authorize'); - service = new OidcUsersService(configService, prismaService, authService); + service = serviceWith(); }); describe('getAuthorizationUrl', () => { @@ -124,15 +153,7 @@ describe('OidcUsersService', () => { }); it('applies the insecure TLS agent only to the issuer origin', async () => { - const tlsConfigService = { - get: jest.fn((key: string, defaultValue?: string) => - key === 'OIDC_TLS_NO_VERIFY' ? 'true' : configService.get(key, defaultValue) - ), - getOrThrow: configService.getOrThrow, - } as unknown as ConfigService; - const tlsService = new OidcUsersService(tlsConfigService, prismaService, authService); - - await tlsService.getAuthorizationUrl(); + await serviceWith({ OIDC_TLS_NO_VERIFY: 'true' }).getAuthorizationUrl(); const provider = (Issuer as unknown as Record Record>)[ custom.http_options as unknown as string @@ -148,15 +169,8 @@ describe('OidcUsersService', () => { }, }; mockDiscover.mockResolvedValue(issuer); - const tlsConfigService = { - get: jest.fn((key: string, defaultValue?: string) => - key === 'OIDC_TLS_NO_VERIFY' ? 'true' : configService.get(key, defaultValue) - ), - getOrThrow: configService.getOrThrow, - } as unknown as ConfigService; - const tlsService = new OidcUsersService(tlsConfigService, prismaService, authService); - await tlsService.getAuthorizationUrl(); + await serviceWith({ OIDC_TLS_NO_VERIFY: 'true' }).getAuthorizationUrl(); const provider = issuer[custom.http_options as unknown as string] as ( url: URL, @@ -179,160 +193,121 @@ describe('OidcUsersService', () => { }); describe('handleCallback', () => { - const checks = { state: 'mock-state', codeVerifier: 'mock-verifier' }; - it('creates a new user and derives the admin role from the claims', async () => { - mockCallback.mockResolvedValue({ - access_token: 'access-token', - claims: () => ({ + idpReturns( + { email: 'new.user@example.com', email_verified: true, given_name: 'New', family_name: 'User', groups: ['vrt-admin'], - }), - }); - findUnique.mockResolvedValue(null); - create.mockImplementation(({ data }) => ({ ...user, ...data })); + }, + { access_token: 'access-token' } + ); + noUser(); - const result = await service.handleCallback({ code: 'code', state: 'mock-state' }, checks); + const result = await login(service); - expect(create).toHaveBeenCalledWith( - expect.objectContaining({ - data: expect.objectContaining({ - email: 'new.user@example.com', - firstName: 'New', - lastName: 'User', - role: Role.admin, - }), - }) - ); + expectCreatedWith({ email: 'new.user@example.com', firstName: 'New', lastName: 'User', role: Role.admin }); expect(result.token).toBe('vrt-jwt'); }); it('falls back to the editor role when no admin claim matches', async () => { - mockCallback.mockResolvedValue({ - claims: () => ({ - email: 'plain@example.com', - email_verified: true, - given_name: 'Plain', - family_name: 'User', - groups: ['other'], - }), + idpReturns({ + email: 'plain@example.com', + email_verified: true, + given_name: 'Plain', + family_name: 'User', + groups: ['other'], }); - findUnique.mockResolvedValue(null); - create.mockImplementation(({ data }) => ({ ...user, ...data })); + noUser(); - await service.handleCallback({ code: 'code', state: 'mock-state' }, checks); + await login(service); - expect(create).toHaveBeenCalledWith( - expect.objectContaining({ data: expect.objectContaining({ role: Role.editor }) }) - ); + expectCreatedWith({ role: Role.editor }); }); it('updates the role of an existing user from the claims', async () => { - mockCallback.mockResolvedValue({ - claims: () => ({ email: 'test@example.com', email_verified: true, groups: ['admin'] }), - }); + idpReturns({ email: 'test@example.com', email_verified: true, groups: ['admin'] }); findUnique.mockResolvedValue(user); update.mockResolvedValue({ ...user, role: Role.admin }); - const result = await service.handleCallback({ code: 'code', state: 'mock-state' }, checks); + const result = await login(service); - expect(update).toHaveBeenCalledWith({ where: { id: user.id }, data: { role: Role.admin } }); + expectRoleUpdate(Role.admin); expect(result.token).toBe('vrt-jwt'); }); it('throws when the email claim is missing', async () => { - mockCallback.mockResolvedValue({ - claims: () => ({ given_name: 'No', family_name: 'Mail' }), - }); + idpReturns({ given_name: 'No', family_name: 'Mail' }); - await expect(service.handleCallback({ code: 'code', state: 'mock-state' }, checks)).rejects.toThrow( - 'OIDC login failed.' - ); + await expectLoginFailed(service); }); it('throws when the callback validation fails', async () => { mockCallback.mockRejectedValue(new Error('state mismatch')); - await expect(service.handleCallback({ code: 'code', state: 'wrong' }, checks)).rejects.toThrow( - 'OIDC login failed.' - ); + await expectLoginFailed(service, { code: 'code', state: 'wrong' }); }); it('falls back to the userinfo endpoint when the email claim is missing', async () => { - mockCallback.mockResolvedValue({ access_token: 'access-token', claims: () => ({ sub: 'user-1' }) }); + idpReturns({ sub: 'user-1' }, { access_token: 'access-token' }); mockUserinfo.mockResolvedValue({ email: 'from.userinfo@example.com', email_verified: true, given_name: 'Info', family_name: 'User', }); - findUnique.mockResolvedValue(null); - create.mockImplementation(({ data }) => ({ ...user, ...data })); + noUser(); - await service.handleCallback({ code: 'code', state: 'mock-state' }, checks); + await login(service); expect(mockUserinfo).toHaveBeenCalledWith(expect.objectContaining({ access_token: 'access-token' })); - expect(create).toHaveBeenCalledWith( - expect.objectContaining({ data: expect.objectContaining({ email: 'from.userinfo@example.com' }) }) - ); + expectCreatedWith({ email: 'from.userinfo@example.com' }); }); it('still fails when the userinfo request fails and no email is present', async () => { - mockCallback.mockResolvedValue({ access_token: 'access-token', claims: () => ({ sub: 'user-1' }) }); + idpReturns({ sub: 'user-1' }, { access_token: 'access-token' }); mockUserinfo.mockRejectedValue(new Error('userinfo down')); - await expect(service.handleCallback({ code: 'code', state: 'mock-state' }, checks)).rejects.toThrow( - 'OIDC login failed.' - ); + await expectLoginFailed(service); }); it('rejects an email the provider marks as unverified', async () => { - mockCallback.mockResolvedValue({ - claims: () => ({ email: 'unverified@example.com', email_verified: false }), - }); + idpReturns({ email: 'unverified@example.com', email_verified: false }); - await expect(service.handleCallback({ code: 'code', state: 'mock-state' }, checks)).rejects.toThrow( - 'OIDC login failed.' - ); + await expectLoginFailed(service); expect(findUnique).not.toHaveBeenCalled(); }); it('accepts an unverified email when the check is disabled', async () => { - const relaxedConfigService = { - get: jest.fn((key: string, defaultValue?: string) => - key === 'OIDC_REQUIRE_VERIFIED_EMAIL' ? 'false' : configService.get(key, defaultValue) - ), - getOrThrow: configService.getOrThrow, - } as unknown as ConfigService; - const relaxedService = new OidcUsersService(relaxedConfigService, prismaService, authService); - mockCallback.mockResolvedValue({ - claims: () => ({ email: 'unverified@example.com', email_verified: false }), - }); + idpReturns({ email: 'unverified@example.com', email_verified: false }); findUnique.mockResolvedValue(user); - const result = await relaxedService.handleCallback({ code: 'code', state: 'mock-state' }, checks); + const result = await login(serviceWith({ OIDC_REQUIRE_VERIFIED_EMAIL: 'false' })); expect(result.token).toBe('vrt-jwt'); }); + it('rejects a missing email even when unverified emails are allowed', async () => { + idpReturns({ sub: 'user-1' }); + + await expectLoginFailed(serviceWith({ OIDC_REQUIRE_VERIFIED_EMAIL: 'false' })); + + expect(findUnique).not.toHaveBeenCalled(); + }); + it('rejects an empty email claim array', async () => { - mockCallback.mockResolvedValue({ claims: () => ({ email: [] as string[] }) }); + idpReturns({ email: [] as string[] }); - await expect(service.handleCallback({ code: 'code', state: 'mock-state' }, checks)).rejects.toThrow( - 'OIDC login failed.' - ); + await expectLoginFailed(service); }); it('accepts an email_verified claim serialized as a string', async () => { - mockCallback.mockResolvedValue({ - claims: () => ({ email: 'test@example.com', email_verified: 'true', groups: ['editor'] }), - }); + idpReturns({ email: 'test@example.com', email_verified: 'true', groups: ['editor'] }); findUnique.mockResolvedValue(user); - const result = await service.handleCallback({ code: 'code', state: 'mock-state' }, checks); + const result = await login(service); expect(result.token).toBe('vrt-jwt'); }); @@ -345,183 +320,61 @@ describe('OidcUsersService', () => { { 'https://vrt.example.com/roles': ['vrt-admin'] }, ], ])('reads the role claim via %s', async (_label, roleClaim, roleClaims) => { - const claimConfigService = { - get: jest.fn((key: string, defaultValue?: string) => - key === 'OIDC_ROLE_CLAIM' ? roleClaim : configService.get(key, defaultValue) - ), - getOrThrow: configService.getOrThrow, - } as unknown as ConfigService; - const claimService = new OidcUsersService(claimConfigService, prismaService, authService); - mockCallback.mockResolvedValue({ - claims: () => ({ email: 'test@example.com', email_verified: true, ...roleClaims }), - }); + idpReturns({ email: 'test@example.com', email_verified: true, ...roleClaims }); findUnique.mockResolvedValue(user); update.mockResolvedValue({ ...user, role: Role.admin }); - await claimService.handleCallback({ code: 'code', state: 'mock-state' }, checks); + await login(serviceWith({ OIDC_ROLE_CLAIM: roleClaim })); - expect(update).toHaveBeenCalledWith({ where: { id: user.id }, data: { role: Role.admin } }); + expectRoleUpdate(Role.admin); }); it('uses the user a concurrent first login created', async () => { - mockCallback.mockResolvedValue({ - claims: () => ({ email: 'race@example.com', email_verified: true }), - }); + idpReturns({ email: 'race@example.com', email_verified: true }); findUnique.mockResolvedValueOnce(null).mockResolvedValueOnce({ ...user, email: 'race@example.com' }); create.mockRejectedValue(new PrismaClientKnownRequestError('unique', { code: 'P2002', clientVersion: '5' })); - const result = await service.handleCallback({ code: 'code', state: 'mock-state' }, checks); + const result = await login(service); expect(findUnique).toHaveBeenCalledTimes(2); expect(result.token).toBe('vrt-jwt'); }); it('rethrows a unique violation that is not a concurrent login', async () => { - mockCallback.mockResolvedValue({ - claims: () => ({ email: 'race@example.com', email_verified: true }), - }); + idpReturns({ email: 'race@example.com', email_verified: true }); findUnique.mockResolvedValue(null); const error = new PrismaClientKnownRequestError('unique', { code: 'P2002', clientVersion: '5' }); create.mockRejectedValue(error); - await expect(service.handleCallback({ code: 'code', state: 'mock-state' }, checks)).rejects.toBe(error); + await expect(login(service)).rejects.toBe(error); }); it('keeps the role of an existing user when the role claim is missing', async () => { - mockCallback.mockResolvedValue({ - claims: () => ({ email: 'test@example.com', email_verified: true }), - }); + idpReturns({ email: 'test@example.com', email_verified: true }); findUnique.mockResolvedValue({ ...user, role: Role.admin }); - const result = await service.handleCallback({ code: 'code', state: 'mock-state' }, checks); + const result = await login(service); expect(update).not.toHaveBeenCalled(); expect(result.token).toBe('vrt-jwt'); }); it('leaves an existing user unchanged when the role already matches', async () => { - mockCallback.mockResolvedValue({ - claims: () => ({ email: 'test@example.com', email_verified: true, groups: ['editor'] }), - }); + idpReturns({ email: 'test@example.com', email_verified: true, groups: ['editor'] }); findUnique.mockResolvedValue(user); - const result = await service.handleCallback({ code: 'code', state: 'mock-state' }, checks); + const result = await login(service); expect(update).not.toHaveBeenCalled(); expect(result.token).toBe('vrt-jwt'); }); - }); - - describe('configuration', () => { - const configWith = (overrides: Record): ConfigService => - ({ - get: jest.fn((key: string, defaultValue?: string) => - key in overrides ? overrides[key] : configService.get(key, defaultValue) - ), - getOrThrow: configService.getOrThrow, - }) as unknown as ConfigService; - const checks = { state: 'mock-state', codeVerifier: 'mock-verifier' }; - - it.each([ - ['a single admin value', { groups: 'admin' }, Role.admin], - ['values separated by spaces', { groups: 'viewer vrt-admin' }, Role.admin], - ['values separated by commas', { groups: 'viewer,admin' }, Role.admin], - ['admin values in another case', { groups: ['VRT-Admin'] }, Role.admin], - ['values that are only separators', { groups: ' , ' }, Role.editor], - ['object values, which are ignored', { groups: [{ name: 'admin' }] }, Role.editor], - ])('derives the role from %s', async (_case, roleClaims, expected) => { - mockCallback.mockResolvedValue({ - claims: () => ({ email: 'new@example.com', email_verified: true, ...roleClaims }), - }); - findUnique.mockResolvedValue(null); - create.mockImplementation(({ data }) => ({ ...user, ...data })); - - await service.handleCallback({ code: 'code' }, checks); - - expect(create).toHaveBeenCalledWith( - expect.objectContaining({ data: expect.objectContaining({ role: expected }) }) - ); - }); - - it('trims and lower-cases the configured admin values', async () => { - const custom = new OidcUsersService( - configWith({ OIDC_ADMIN_VALUES: ' Ops-Team , ,42' }), - prismaService, - authService - ); - mockCallback.mockResolvedValue({ - claims: () => ({ email: 'new@example.com', email_verified: true, groups: ['ops-team'] }), - }); - findUnique.mockResolvedValue(null); - create.mockImplementation(({ data }) => ({ ...user, ...data })); - - await custom.handleCallback({ code: 'code' }, checks); - - expect(create).toHaveBeenCalledWith( - expect.objectContaining({ data: expect.objectContaining({ role: Role.admin }) }) - ); - }); - - it.each([ - ['a number', 42], - ['a boolean', true], - ])('compares a role claim holding %s as a string', async (_case, value) => { - const custom = new OidcUsersService(configWith({ OIDC_ADMIN_VALUES: String(value) }), prismaService, authService); - mockCallback.mockResolvedValue({ - claims: () => ({ email: 'new@example.com', email_verified: true, groups: [value] }), - }); - findUnique.mockResolvedValue(null); - create.mockImplementation(({ data }) => ({ ...user, ...data })); - - await custom.handleCallback({ code: 'code' }, checks); - - expect(create).toHaveBeenCalledWith( - expect.objectContaining({ data: expect.objectContaining({ role: Role.admin }) }) - ); - }); - - it('gives new users without an admin claim the configured default role', async () => { - const custom = new OidcUsersService(configWith({ OIDC_DEFAULT_ROLE: 'Admin' }), prismaService, authService); - mockCallback.mockResolvedValue({ claims: () => ({ email: 'new@example.com', email_verified: true }) }); - findUnique.mockResolvedValue(null); - create.mockImplementation(({ data }) => ({ ...user, ...data })); - - await custom.handleCallback({ code: 'code' }, checks); - - expect(create).toHaveBeenCalledWith( - expect.objectContaining({ data: expect.objectContaining({ role: Role.admin }) }) - ); - }); - }); - - describe('handleCallback claim sources', () => { - const checks = { state: 'mock-state', codeVerifier: 'mock-verifier' }; - const relaxed = () => - new OidcUsersService( - { - get: jest.fn((key: string, defaultValue?: string) => - key === 'OIDC_REQUIRE_VERIFIED_EMAIL' ? 'false' : configService.get(key, defaultValue) - ), - getOrThrow: configService.getOrThrow, - } as unknown as ConfigService, - prismaService, - authService - ); - - it('rejects a missing email even when unverified emails are allowed', async () => { - mockCallback.mockResolvedValue({ claims: () => ({ sub: 'user-1' }) }); - - await expect(relaxed().handleCallback({ code: 'code' }, checks)).rejects.toThrow('OIDC login failed.'); - - expect(findUnique).not.toHaveBeenCalled(); - }); it('warns and keeps the role of an existing user when the role claim is missing', async () => { - mockCallback.mockResolvedValue({ claims: () => ({ email: 'test@example.com', email_verified: true }) }); + idpReturns({ email: 'test@example.com', email_verified: true }); findUnique.mockResolvedValue({ ...user, role: Role.admin }); const warn = jest.spyOn(service['logger'], 'warn').mockImplementation(() => undefined); - await service.handleCallback({ code: 'code' }, checks); + await login(service); expect(warn).toHaveBeenCalledWith("Claim 'groups' not found for 'test@example.com'. Keeping role 'admin'."); expect(update).not.toHaveBeenCalled(); @@ -532,22 +385,10 @@ describe('OidcUsersService', () => { ['a null parent claim', { realm_access: null }], ['a parent claim that is not an object', { realm_access: 'admin' }], ])('treats a dotted role claim with %s as missing', async (_case, parent) => { - const nested = new OidcUsersService( - { - get: jest.fn((key: string, defaultValue?: string) => - key === 'OIDC_ROLE_CLAIM' ? 'realm_access.length' : configService.get(key, defaultValue) - ), - getOrThrow: configService.getOrThrow, - } as unknown as ConfigService, - prismaService, - authService - ); - mockCallback.mockResolvedValue({ - claims: () => ({ email: 'test@example.com', email_verified: true, ...parent }), - }); + idpReturns({ email: 'test@example.com', email_verified: true, ...parent }); findUnique.mockResolvedValue({ ...user, role: Role.admin }); - await nested.handleCallback({ code: 'code' }, checks); + await login(serviceWith({ OIDC_ROLE_CLAIM: 'realm_access.length' })); expect(update).not.toHaveBeenCalled(); }); @@ -556,19 +397,60 @@ describe('OidcUsersService', () => { ['another prisma error', new PrismaClientKnownRequestError('fk', { code: 'P2003', clientVersion: '5' })], ['a non-prisma error', new Error('db down')], ])('rethrows %s from creating the user without looking it up again', async (_case, error) => { - mockCallback.mockResolvedValue({ claims: () => ({ email: 'race@example.com', email_verified: true }) }); - // a lookup after the failed insert would find the user + idpReturns({ email: 'race@example.com', email_verified: true }); findUnique .mockReset() .mockResolvedValueOnce(null) .mockResolvedValue({ ...user, email: 'race@example.com' }); create.mockRejectedValue(error); - await expect(service.handleCallback({ code: 'code' }, checks)).rejects.toBe(error); + await expect(login(service)).rejects.toBe(error); expect(findUnique).toHaveBeenCalledTimes(1); }); }); + describe('configuration', () => { + const newUserWithClaims = (svc: OidcUsersService, roleClaims: Record) => { + idpReturns({ email: 'new@example.com', email_verified: true, ...roleClaims }); + noUser(); + return login(svc); + }; + + it.each([ + ['a single admin value', { groups: 'admin' }, Role.admin], + ['values separated by spaces', { groups: 'viewer vrt-admin' }, Role.admin], + ['values separated by commas', { groups: 'viewer,admin' }, Role.admin], + ['admin values in another case', { groups: ['VRT-Admin'] }, Role.admin], + ['values that are only separators', { groups: ' , ' }, Role.editor], + ['object values, which are ignored', { groups: [{ name: 'admin' }] }, Role.editor], + ])('derives the role from %s', async (_case, roleClaims, expected) => { + await newUserWithClaims(service, roleClaims); + + expectCreatedWith({ role: expected }); + }); + + it('trims and lower-cases the configured admin values', async () => { + await newUserWithClaims(serviceWith({ OIDC_ADMIN_VALUES: ' Ops-Team , ,42' }), { groups: ['ops-team'] }); + + expectCreatedWith({ role: Role.admin }); + }); + + it.each([ + ['a number', 42], + ['a boolean', true], + ])('compares a role claim holding %s as a string', async (_case, value) => { + await newUserWithClaims(serviceWith({ OIDC_ADMIN_VALUES: String(value) }), { groups: [value] }); + + expectCreatedWith({ role: Role.admin }); + }); + + it('gives new users without an admin claim the configured default role', async () => { + await newUserWithClaims(serviceWith({ OIDC_DEFAULT_ROLE: 'Admin' }), {}); + + expectCreatedWith({ role: Role.admin }); + }); + }); + describe('non-SSO operations', () => { it('updates the name but never the email of an OIDC user', async () => { update.mockResolvedValue(user); @@ -588,6 +470,7 @@ describe('OidcUsersService', () => { 'Password changes are disabled. Use SSO to change your password.' ); }); + it('rejects password login', async () => { await expect(service.login({ email: 'a@b.com', password: 'x' })).rejects.toThrow('Password login is disabled.'); }); diff --git a/src/users/oidc/oidcusers.service.ts b/src/users/oidc/oidcusers.service.ts index b63f94cf..9f68e3e0 100644 --- a/src/users/oidc/oidcusers.service.ts +++ b/src/users/oidc/oidcusers.service.ts @@ -3,7 +3,8 @@ import { Client as OidcClient, Issuer, generators, custom, TokenSet } from 'open import { CreateUserDto } from '../dto/user-create.dto'; import { UserLoginResponseDto } from '../dto/user-login-response.dto'; import { PrismaService } from '../../prisma/prisma.service'; -import { Prisma, Role, User } from '@prisma/client'; +import { isPrismaError } from '../../prisma/prisma-errors'; +import { Role, User } from '@prisma/client'; import { UpdateUserDto } from '../dto/user-update.dto'; import { AuthService } from '../../auth/auth.service'; import { UserLoginRequestDto } from '../dto/user-login-request.dto'; @@ -29,6 +30,7 @@ type OIDCConfig = { }; export class OidcUsersService implements Users { + readonly allowsRegistration = false; private readonly logger: Logger = new Logger(OidcUsersService.name); private readonly oidcConfig: OIDCConfig; private readonly appFrontendUrl: string; @@ -73,10 +75,6 @@ export class OidcUsersService implements Users { } } - /** - * Restricts the insecure HTTPS agent to the configured issuer, so a TLS - * bypass never applies to unrelated openid-client requests. - */ private getHttpOptions(): ((url: URL, options: Record) => Record) | null { if (!this.oidcConfig.tlsNoVerify) { return null; @@ -89,16 +87,13 @@ export class OidcUsersService implements Users { private async discoverClient(): Promise { const httpOptions = this.getHttpOptions(); if (httpOptions) { - // Static discovery cannot be scoped per instance; the provider itself - // only bypasses verification for the configured issuer origin. - Issuer[custom.http_options] = httpOptions as never; + // Discovery only takes a static option; httpOptions still checks the origin + Issuer[custom.http_options] = httpOptions; } const issuer = await Issuer.discover(this.oidcConfig.issuerUrl); if (httpOptions) { - // The JWKS used to validate the ID token is fetched through the issuer - // instance, which does not inherit the static option set above. - issuer[custom.http_options] = httpOptions as never; + issuer[custom.http_options] = httpOptions; } const client = new issuer.Client({ client_id: this.oidcConfig.clientId, @@ -108,7 +103,7 @@ export class OidcUsersService implements Users { }); if (httpOptions) { - client[custom.http_options] = httpOptions as never; + client[custom.http_options] = httpOptions; } return client; } @@ -116,7 +111,6 @@ export class OidcUsersService implements Users { private getClient(): Promise { if (!this.clientPromise) { this.clientPromise = this.discoverClient().catch((error) => { - // Allow a later request to retry discovery after a transient failure. this.clientPromise = null; throw error; }); @@ -124,10 +118,6 @@ export class OidcUsersService implements Users { return this.clientPromise; } - /** - * Builds the IdP authorization URL together with the PKCE/state values that - * must be kept (e.g. in a short-lived cookie) until the callback returns. - */ async getAuthorizationUrl(): Promise<{ url: string; state: string; codeVerifier: string }> { const client = await this.getClient(); const state = generators.state(); @@ -141,10 +131,6 @@ export class OidcUsersService implements Users { return { url, state, codeVerifier }; } - /** - * URL the browser is redirected to after OIDC finished (success or error). - * The token travels in the URL fragment so it never reaches server logs. - */ buildFrontendRedirect(token?: string | null, error?: string): string { const base = `${this.appFrontendUrl}/login`; if (token) { @@ -173,8 +159,6 @@ export class OidcUsersService implements Users { const claims: Record = { ...tokenSet.claims() }; if (tokenSet.access_token && !this.claimToString(claims, this.oidcConfig.attributeMail)) { try { - // Passing the whole token set makes openid-client verify that the - // userinfo `sub` matches the ID token before the claims are merged. Object.assign(claims, await client.userinfo(tokenSet)); } catch (e) { this.logger.warn(`OIDC userinfo request failed: ${e}`); @@ -187,10 +171,7 @@ export class OidcUsersService implements Users { throw new HttpException('OIDC login failed.', HttpStatus.BAD_REQUEST); } - // Never trust an email address the identity provider marks as unverified: - // it could otherwise be used to take over an existing VRT account. Can be - // disabled for providers that do not emit a reliable email_verified claim. - // Some providers serialize the claim as the string "true". + // An unverified email could take over an existing VRT account const emailVerified = claims.email_verified === true || claims.email_verified === 'true'; if (this.oidcConfig.requireVerifiedEmail && !emailVerified) { this.logger.warn(`OIDC login rejected: email '${email}' is not verified by the identity provider`); @@ -198,9 +179,7 @@ export class OidcUsersService implements Users { } const role = this.resolveRole(claims); - // Without the role claim (e.g. a missing scope or mapper) the provider says - // nothing about the role, so keep existing users' roles instead of - // demoting every admin to the default role. + // Without the claim the provider says nothing about roles: keep existing ones rather than demote every admin const hasRoleClaim = this.readRoleClaim(claims) != null; let user = await this.prismaService.user.findUnique({ where: { email } }); @@ -219,12 +198,6 @@ export class OidcUsersService implements Users { return new UserLoginResponseDto(user, token); } - /** - * Two first logins for the same email (e.g. two tabs) can both miss the - * lookup; the losing insert hits the unique email constraint, so use the - * user the other request created. Other unique violations (apiKey) still - * surface because the lookup then finds nothing. - */ private async createUserFromClaimsOrFindConcurrent( claims: Record, email: string, @@ -233,7 +206,7 @@ export class OidcUsersService implements Users { try { return await this.createUserFromClaims(claims, email, role); } catch (e) { - if (!(e instanceof Prisma.PrismaClientKnownRequestError) || e.code !== 'P2002') { + if (!isPrismaError(e, 'P2002')) { throw e; } const user = await this.prismaService.user.findUnique({ where: { email } }); @@ -268,7 +241,6 @@ export class OidcUsersService implements Users { return value == null ? [] : [value]; } - // Objects are ignored rather than stringified to '[object Object]'. private scalarToString(value: unknown): string | null { if (typeof value === 'string') { return value; @@ -279,11 +251,6 @@ export class OidcUsersService implements Users { return null; } - /** - * Reads the role claim either by its exact name (which may itself contain - * dots, e.g. namespaced claims like 'https://example.com/roles') or as a - * dotted path into nested claims (e.g. Keycloak's 'realm_access.roles'). - */ private readRoleClaim(claims: Record): unknown { const name = this.oidcConfig.roleClaim; if (claims[name] != null) { @@ -308,13 +275,11 @@ export class OidcUsersService implements Users { return isAdmin ? Role.admin : this.oidcConfig.defaultRole; } - create(createUserDto: CreateUserDto): Promise { + create(_createUserDto: CreateUserDto): Promise { return Promise.reject(new HttpException('User creation is disabled. Use SSO to login.', HttpStatus.BAD_REQUEST)); } async update(id: string, userDto: UpdateUserDto): Promise { - // The email is the account key for OIDC users, so it is owned by the - // identity provider and must not be changed here. const user = await this.prismaService.user.update({ where: { id }, data: { @@ -326,14 +291,14 @@ export class OidcUsersService implements Users { return new UserLoginResponseDto(user, token); } - changePassword(user: User, newPassword: string): Promise { + changePassword(user: User, _newPassword: string): Promise { this.logger.warn(`${user.email} tried to change password - this is not supported for OIDC users`); return Promise.reject( new HttpException('Password changes are disabled. Use SSO to change your password.', HttpStatus.BAD_REQUEST) ); } - login(userLoginRequestDto: UserLoginRequestDto): Promise { + login(_userLoginRequestDto: UserLoginRequestDto): Promise { return Promise.reject(new HttpException('Password login is disabled. Use SSO to login.', HttpStatus.BAD_REQUEST)); } } diff --git a/src/users/users.controller.spec.ts b/src/users/users.controller.spec.ts index 1976bf40..b29ef06e 100644 --- a/src/users/users.controller.spec.ts +++ b/src/users/users.controller.spec.ts @@ -4,12 +4,15 @@ import { UsersController } from './users.controller'; import { UsersService } from './users.service'; import { ConfigService } from '@nestjs/config'; import { Role, User } from '@prisma/client'; -import { UserDto } from './dto/user.dto'; +import { UserDto, toUserSummary } from './dto/user.dto'; +import { ForbiddenException } from '@nestjs/common'; describe('Users Controller', () => { let controller: UsersController; let usersService: { isOidcEnabled: jest.Mock; + isRegistrationAllowedByConfig: jest.Mock; + isRegistrationEnabled: jest.Mock; oidcGetAuthorizationUrl: jest.Mock; oidcBuildFrontendRedirect: jest.Mock; oidcHandleCallback: jest.Mock; @@ -40,6 +43,8 @@ describe('Users Controller', () => { beforeEach(async () => { usersService = { isOidcEnabled: jest.fn(), + isRegistrationAllowedByConfig: jest.fn().mockReturnValue(true), + isRegistrationEnabled: jest.fn(), oidcGetAuthorizationUrl: jest.fn(), oidcBuildFrontendRedirect: jest.fn(), oidcHandleCallback: jest.fn(), @@ -66,10 +71,6 @@ describe('Users Controller', () => { controller = module.get(UsersController); }); - it('should be defined', () => { - expect(controller).toBeDefined(); - }); - it('returns the current user', () => { const result = controller.me(user); expect(result).toBeInstanceOf(UserDto); @@ -81,165 +82,127 @@ describe('Users Controller', () => { expect(controller.oidcEnabled()).toBe(true); }); - it('redirects to the frontend with an error when the state cookie is missing', async () => { - usersService.isOidcEnabled.mockReturnValue(true); - usersService.oidcBuildFrontendRedirect.mockReturnValue('http://localhost:8080/login#error=oidc_login_failed'); - const res = { clearCookie: jest.fn(), redirect: jest.fn(), status: jest.fn().mockReturnThis(), send: jest.fn() }; - - await controller.oidcCallback({ headers: {}, query: {} } as never, res as never); - - expect(usersService.oidcHandleCallback).not.toHaveBeenCalled(); - expect(res.redirect).toHaveBeenCalledWith('http://localhost:8080/login#error=oidc_login_failed'); - }); - - it('redirects to the frontend with an error when the state cookie is malformed', async () => { - usersService.isOidcEnabled.mockReturnValue(true); - usersService.oidcBuildFrontendRedirect.mockReturnValue('http://localhost:8080/login#error=oidc_login_failed'); - const res = { clearCookie: jest.fn(), redirect: jest.fn(), status: jest.fn().mockReturnThis(), send: jest.fn() }; - - await controller.oidcCallback({ headers: { cookie: 'oidc_state=%E0%A4%A' }, query: {} } as never, res as never); - - expect(usersService.oidcHandleCallback).not.toHaveBeenCalled(); - expect(res.redirect).toHaveBeenCalledWith('http://localhost:8080/login#error=oidc_login_failed'); - }); - - it('redirects to the frontend with an error when the identity provider is unreachable', async () => { - usersService.isOidcEnabled.mockReturnValue(true); - usersService.oidcGetAuthorizationUrl.mockRejectedValue(new Error('discovery down')); - usersService.oidcBuildFrontendRedirect.mockReturnValue('http://localhost:8080/login#error=oidc_login_failed'); - const res = { cookie: jest.fn(), redirect: jest.fn() }; - - await controller.oidcLogin(res as never); - - expect(res.cookie).not.toHaveBeenCalled(); - expect(res.redirect).toHaveBeenCalledWith('http://localhost:8080/login#error=oidc_login_failed'); - }); + describe('OIDC', () => { + const mockRes = () => ({ + cookie: jest.fn(), + clearCookie: jest.fn(), + redirect: jest.fn(), + status: jest.fn().mockReturnThis(), + send: jest.fn(), + }); + const callback = (res: ReturnType, headers: object = {}, query: object = {}) => + controller.oidcCallback({ headers, query } as never, res as never); + const stateCookie = { cookie: 'oidc_state=state.verifier' }; - it('returns 404 on login when OIDC is disabled', async () => { - usersService.isOidcEnabled.mockReturnValue(false); - const res = { cookie: jest.fn(), redirect: jest.fn(), status: jest.fn().mockReturnThis(), send: jest.fn() }; + beforeEach(() => { + usersService.isOidcEnabled.mockReturnValue(true); + usersService.oidcBuildFrontendRedirect.mockImplementation((token?: string, error?: string) => + token ? `front#token=${token}` : `front#error=${error}` + ); + }); - await controller.oidcLogin(res as never); + it.each([ + ['is missing', {}], + ['is missing among other cookies', { cookie: 'a=1; b=2' }], + ['is malformed', { cookie: 'oidc_state=%E0%A4%A' }], + ])('redirects to the frontend with an error when the state cookie %s', async (_case, headers) => { + const res = mockRes(); - expect(res.status).toHaveBeenCalledWith(404); - expect(usersService.oidcGetAuthorizationUrl).not.toHaveBeenCalled(); - }); + await callback(res, headers); - it('redirects to the identity provider and stores the PKCE state cookie', async () => { - usersService.isOidcEnabled.mockReturnValue(true); - usersService.oidcGetAuthorizationUrl.mockResolvedValue({ - url: 'https://idp.example.com/authorize', - state: 'state', - codeVerifier: 'verifier', + expect(usersService.oidcHandleCallback).not.toHaveBeenCalled(); + expect(usersService.oidcBuildFrontendRedirect).toHaveBeenCalledWith(undefined, 'oidc_login_failed'); + expect(res.clearCookie).toHaveBeenCalledWith('oidc_state', { path: '/' }); + expect(res.redirect).toHaveBeenCalledWith('front#error=oidc_login_failed'); }); - const res = { cookie: jest.fn(), redirect: jest.fn() }; - - await controller.oidcLogin(res as never); - - expect(res.cookie).toHaveBeenCalledWith( - 'oidc_state', - 'state.verifier', - expect.objectContaining({ httpOnly: true, sameSite: 'lax' }) - ); - expect(res.redirect).toHaveBeenCalledWith('https://idp.example.com/authorize'); - }); - it('redirects to the frontend with the token after a successful callback', async () => { - usersService.isOidcEnabled.mockReturnValue(true); - usersService.oidcHandleCallback.mockResolvedValue({ token: 'jwt' }); - usersService.oidcBuildFrontendRedirect.mockReturnValue('http://localhost:8080/login#token=jwt'); - const res = { clearCookie: jest.fn(), redirect: jest.fn(), status: jest.fn().mockReturnThis(), send: jest.fn() }; - const req = { - headers: { cookie: 'oidc_state=state.verifier' }, - query: { code: 'code', state: 'state', repeated: ['a', 'b'] }, - }; + it.each([new Error('discovery down'), 'plain failure'])( + 'redirects to the frontend with an error when the authorization url rejects with %j', + async (failure) => { + usersService.oidcGetAuthorizationUrl.mockRejectedValue(failure); + const res = mockRes(); - await controller.oidcCallback(req as never, res as never); + await controller.oidcLogin(res as never); - expect(usersService.oidcHandleCallback).toHaveBeenCalledWith( - { code: 'code', state: 'state' }, - { state: 'state', codeVerifier: 'verifier' } + expect(res.cookie).not.toHaveBeenCalled(); + expect(res.redirect).toHaveBeenCalledWith('front#error=oidc_login_failed'); + } ); - expect(res.redirect).toHaveBeenCalledWith('http://localhost:8080/login#token=jwt'); - }); - it('returns 404 on the callback when OIDC is disabled', async () => { - usersService.isOidcEnabled.mockReturnValue(false); - const res = { clearCookie: jest.fn(), redirect: jest.fn(), status: jest.fn().mockReturnThis(), send: jest.fn() }; + it('returns 404 on login when OIDC is disabled', async () => { + usersService.isOidcEnabled.mockReturnValue(false); + const res = mockRes(); - await controller.oidcCallback({ headers: {}, query: {} } as never, res as never); - - expect(res.status).toHaveBeenCalledWith(404); - expect(res.send).toHaveBeenCalled(); - }); + await controller.oidcLogin(res as never); - describe('OIDC cookie handling', () => { - const okResponse = () => ({ - clearCookie: jest.fn(), - redirect: jest.fn(), - status: jest.fn().mockReturnThis(), - send: jest.fn(), + expect(res.status).toHaveBeenCalledWith(404); + expect(usersService.oidcGetAuthorizationUrl).not.toHaveBeenCalled(); }); - beforeEach(() => { - usersService.isOidcEnabled.mockReturnValue(true); - usersService.oidcBuildFrontendRedirect.mockImplementation((token?: string, error?: string) => - token ? `front#token=${token}` : `front#error=${error}` + it('redirects to the identity provider and stores the PKCE state cookie', async () => { + usersService.oidcGetAuthorizationUrl.mockResolvedValue({ + url: 'https://idp.example.com/authorize', + state: 'state', + codeVerifier: 'verifier', + }); + const res = mockRes(); + + await controller.oidcLogin(res as never); + + expect(res.cookie).toHaveBeenCalledWith( + 'oidc_state', + 'state.verifier', + expect.objectContaining({ httpOnly: true, sameSite: 'lax' }) ); + expect(res.redirect).toHaveBeenCalledWith('https://idp.example.com/authorize'); }); - it('finds the state cookie among other cookies and decodes it', async () => { + it('redirects to the frontend with the token after a successful callback', async () => { usersService.oidcHandleCallback.mockResolvedValue({ token: 'jwt' }); - const res = okResponse(); + const res = mockRes(); - await controller.oidcCallback( - { headers: { cookie: 'a=1; oidc_state=st%2Ea.ver=ifier; b=2' }, query: {} } as never, - res as never - ); + await callback(res, stateCookie, { code: 'code', state: 'state', repeated: ['a', 'b'] }); - // "=" inside the value is kept; after decoding it is split on every "." and - // only the first two parts are used. - // NOTE: current behavior — a verifier containing "." (allowed by RFC 7636) would be truncated - expect(usersService.oidcHandleCallback).toHaveBeenCalledWith({}, { state: 'st', codeVerifier: 'a' }); - expect(res.clearCookie).toHaveBeenCalledWith('oidc_state', { path: '/' }); + expect(usersService.oidcHandleCallback).toHaveBeenCalledWith( + { code: 'code', state: 'state' }, + { state: 'state', codeVerifier: 'verifier' } + ); expect(res.redirect).toHaveBeenCalledWith('front#token=jwt'); }); - it('fails the callback when other cookies are present but not the state cookie', async () => { - const res = okResponse(); + it('returns 404 on the callback when OIDC is disabled', async () => { + usersService.isOidcEnabled.mockReturnValue(false); + const res = mockRes(); - await controller.oidcCallback({ headers: { cookie: 'a=1; b=2' }, query: {} } as never, res as never); + await callback(res); - expect(usersService.oidcHandleCallback).not.toHaveBeenCalled(); - expect(usersService.oidcBuildFrontendRedirect).toHaveBeenCalledWith(undefined, 'oidc_login_failed'); - expect(res.clearCookie).toHaveBeenCalledWith('oidc_state', { path: '/' }); - expect(res.redirect).toHaveBeenCalledWith('front#error=oidc_login_failed'); + expect(res.status).toHaveBeenCalledWith(404); + expect(res.send).toHaveBeenCalled(); }); - it('fails the callback when the identity provider rejects the code', async () => { - usersService.oidcHandleCallback.mockRejectedValue(new Error('invalid_grant')); - const res = okResponse(); + it('finds the state cookie among other cookies and splits it at the first dot (generated verifiers are base64url, without dots)', async () => { + usersService.oidcHandleCallback.mockResolvedValue({ token: 'jwt' }); + const res = mockRes(); - await controller.oidcCallback( - { headers: { cookie: 'oidc_state=state.verifier' }, query: { code: 'c' } } as never, - res as never - ); + await callback(res, { cookie: 'a=1; oidc_state=st%2Ea.ver=ifier; b=2' }); + expect(usersService.oidcHandleCallback).toHaveBeenCalledWith({}, { state: 'st', codeVerifier: 'a' }); expect(res.clearCookie).toHaveBeenCalledWith('oidc_state', { path: '/' }); - expect(res.redirect).toHaveBeenCalledWith('front#error=oidc_login_failed'); + expect(res.redirect).toHaveBeenCalledWith('front#token=jwt'); }); - it('fails the callback when the identity provider rejects with a non-Error', async () => { - usersService.oidcHandleCallback.mockRejectedValue('plain failure'); - const res = okResponse(); + it.each([new Error('invalid_grant'), 'plain failure'])( + 'fails the callback when the identity provider rejects with %j', + async (failure) => { + usersService.oidcHandleCallback.mockRejectedValue(failure); + const res = mockRes(); - await controller.oidcCallback( - { headers: { cookie: 'oidc_state=state.verifier' }, query: {} } as never, - res as never - ); + await callback(res, stateCookie, { code: 'c' }); - expect(res.redirect).toHaveBeenCalledWith('front#error=oidc_login_failed'); - }); + expect(res.clearCookie).toHaveBeenCalledWith('oidc_state', { path: '/' }); + expect(res.redirect).toHaveBeenCalledWith('front#error=oidc_login_failed'); + } + ); it.each([ ['https://vrt.example.com/callback', true], @@ -249,7 +212,7 @@ describe('Users Controller', () => { key === 'OIDC_REDIRECT_URI' ? redirectUri : defaultValue ); usersService.oidcGetAuthorizationUrl.mockResolvedValue({ url: 'idp', state: 's', codeVerifier: 'v' }); - const res = { cookie: jest.fn(), redirect: jest.fn() }; + const res = mockRes(); await controller.oidcLogin(res as never); @@ -262,15 +225,6 @@ describe('Users Controller', () => { path: '/', }); }); - - it('redirects to the login page when the authorization url rejects with a non-Error', async () => { - usersService.oidcGetAuthorizationUrl.mockRejectedValue('plain failure'); - const res = { cookie: jest.fn(), redirect: jest.fn() }; - - await controller.oidcLogin(res as never); - - expect(res.redirect).toHaveBeenCalledWith('front#error=oidc_login_failed'); - }); }); describe('account handlers', () => { @@ -282,6 +236,21 @@ describe('Users Controller', () => { expect(usersService.create).toHaveBeenCalledWith(dto); }); + it('refuses to register a user when registration is switched off', () => { + usersService.isRegistrationAllowedByConfig.mockReturnValue(false); + + expect(() => + controller.register({ email: 'new@example.com', firstName: 'N', lastName: 'U', password: 'secret' }) + ).toThrow(ForbiddenException); + expect(usersService.create).not.toHaveBeenCalled(); + }); + + it('reports whether registration is enabled', () => { + usersService.isRegistrationEnabled.mockReturnValue(false); + + expect(controller.registrationEnabled()).toBe(false); + }); + it('logs a user in', async () => { const dto = { email: 'test@example.com', password: 'secret' }; usersService.login.mockResolvedValue('logged in'); @@ -300,7 +269,7 @@ describe('Users Controller', () => { it('changes the password of the current user', async () => { usersService.changePassword.mockResolvedValue(true); - await expect(controller.changePassword(user, 'new password')).resolves.toBe(true); + await expect(controller.changePassword(user, { password: 'new password' })).resolves.toBe(true); expect(usersService.changePassword).toHaveBeenCalledWith(user, 'new password'); }); @@ -314,14 +283,18 @@ describe('Users Controller', () => { }); describe('admin handlers', () => { - it('lists every user as a UserDto without the password', async () => { - prismaService.user.findMany.mockResolvedValue([user, { ...user, id: '2', email: 'other@example.com' }]); + it('lists every user as a summary without the password and the api key', async () => { + const other = { ...user, id: '2', email: 'other@example.com' }; + prismaService.user.findMany.mockResolvedValue([user, other]); const result = await controller.userList(); expect(prismaService.user.findMany).toHaveBeenCalledWith(); - expect(result).toEqual([new UserDto(user), new UserDto({ ...user, id: '2', email: 'other@example.com' })]); - expect(result[0]).not.toHaveProperty('password'); + expect(result).toEqual([toUserSummary(user), toUserSummary(other)]); + for (const summary of result) { + expect(summary).not.toHaveProperty('password'); + expect(summary).not.toHaveProperty('apiKey'); + } }); it('deletes the users in one go', async () => { diff --git a/src/users/users.controller.ts b/src/users/users.controller.ts index 9d89ea37..40998683 100644 --- a/src/users/users.controller.ts +++ b/src/users/users.controller.ts @@ -11,6 +11,8 @@ import { Res, HttpStatus, Header, + Logger, + ForbiddenException, } from '@nestjs/common'; import { Request, Response } from 'express'; import { UsersService } from './users.service'; @@ -18,7 +20,7 @@ import { ApiOkResponse, ApiBearerAuth, ApiTags } from '@nestjs/swagger'; import { UserLoginResponseDto } from './dto/user-login-response.dto'; import { CreateUserDto } from './dto/user-create.dto'; import { JwtAuthGuard } from '../auth/guards/auth.guard'; -import { UserDto } from './dto/user.dto'; +import { UserDto, UserSummaryDto, toUserSummary } from './dto/user.dto'; import { UpdateUserDto } from './dto/user-update.dto'; import { UserLoginRequestDto } from './dto/user-login-request.dto'; import { CurrentUser } from '../shared/current-user.decorator'; @@ -27,7 +29,7 @@ import { RoleGuard } from '../auth/guards/role.guard'; import { Roles } from '../shared/roles.decorator'; import { PrismaService } from '../prisma/prisma.service'; import { AssignRoleDto } from './dto/assign-role.dto'; -import { Logger } from '@nestjs/common'; +import { ChangePasswordDto } from './dto/change-password.dto'; import { ConfigService } from '@nestjs/config'; const OIDC_STATE_COOKIE = 'oidc_state'; @@ -60,9 +62,19 @@ export class UsersController { @Post('register') @ApiOkResponse({ type: UserLoginResponseDto }) register(@Body() createUserDto: CreateUserDto): Promise { + if (!this.usersService.isRegistrationAllowedByConfig()) { + throw new ForbiddenException('User registration is disabled.'); + } return this.usersService.create(createUserDto); } + @Get('registration/enabled') + @ApiOkResponse({ type: Boolean }) + @Header('Content-Type', 'application/json') + registrationEnabled(): boolean { + return this.usersService.isRegistrationEnabled(); + } + @Post('login') @ApiOkResponse({ type: UserLoginResponseDto }) async login(@Body() userLoginRequestDto: UserLoginRequestDto): Promise { @@ -88,8 +100,6 @@ export class UsersController { try { authorization = await this.usersService.oidcGetAuthorizationUrl(); } catch (e) { - // The browser navigated here directly, so send it back to the login page - // instead of showing a raw JSON error (e.g. when discovery fails). this.logger.warn(`OIDC login failed: ${(e as Error)?.message ?? e}`); res.redirect(this.usersService.oidcBuildFrontendRedirect(undefined, 'oidc_login_failed')); return; @@ -114,13 +124,10 @@ export class UsersController { let target = this.usersService.oidcBuildFrontendRedirect(undefined, 'oidc_login_failed'); try { - // Inside the try: decodeURIComponent throws on a malformed cookie value. const [state, codeVerifier] = (readCookie(req, OIDC_STATE_COOKIE) ?? '').split('.'); if (!state || !codeVerifier) { throw new Error('Missing or malformed OIDC state cookie.'); } - // Repeated query keys produce arrays; keep only scalar strings so the - // callback sees a plain parameter map. const params: Record = {}; for (const [key, value] of Object.entries(req.query)) { if (typeof value === 'string') { @@ -160,8 +167,8 @@ export class UsersController { @ApiOkResponse({ type: Boolean }) @ApiBearerAuth() @UseGuards(JwtAuthGuard) - changePassword(@CurrentUser() user: User, @Body('password') password: string): Promise { - return this.usersService.changePassword(user, password); + changePassword(@CurrentUser() user: User, @Body() body: ChangePasswordDto): Promise { + return this.usersService.changePassword(user, body.password); } @Put() @@ -173,13 +180,13 @@ export class UsersController { } @Get('all') - @ApiOkResponse({ type: UserDto, isArray: true }) + @ApiOkResponse({ type: UserSummaryDto, isArray: true }) @ApiBearerAuth() @UseGuards(JwtAuthGuard, RoleGuard) @Roles(Role.admin) - async userList(): Promise { + async userList(): Promise { const users = await this.prismaService.user.findMany(); - return users.map((user) => new UserDto(user)); + return users.map(toUserSummary); } @Delete() @@ -193,11 +200,11 @@ export class UsersController { } @Patch('assignRole') - @ApiOkResponse({ type: UserDto }) + @ApiOkResponse({ type: UserSummaryDto }) @ApiBearerAuth() @UseGuards(JwtAuthGuard, RoleGuard) @Roles(Role.admin) - async assignRole(@Body() data: AssignRoleDto): Promise { + async assignRole(@Body() data: AssignRoleDto): Promise { this.logger.debug(`Going to assign role ${data.role} to User: ${data.id}`); return this.usersService.assignRole(data); } diff --git a/src/users/users.factory.spec.ts b/src/users/users.factory.spec.ts index 40a9ffce..65cda525 100644 --- a/src/users/users.factory.spec.ts +++ b/src/users/users.factory.spec.ts @@ -48,31 +48,14 @@ describe('UsersFactoryService', () => { configService = module.get(ConfigService); }); - it('should be defined', () => { - expect(service).toBeDefined(); - }); - - it('should return DbUsersService when neither OIDC_ENABLED nor LDAP_ENABLED is true', () => { - withFlags({}); - const result = service.getUsersService(); - expect(result).toBeInstanceOf(DbUsersService); - }); - - it('should return LdapUsersService when LDAP_ENABLED is true', () => { - withFlags({ ldap: true }); - const result = service.getUsersService(); - expect(result).toBeInstanceOf(LdapUsersService); - }); - - it('should return OidcUsersService when OIDC_ENABLED is true', () => { - withFlags({ oidc: true }); - const result = service.getUsersService(); - expect(result).toBeInstanceOf(OidcUsersService); - }); - - it('should prefer OidcUsersService when both OIDC_ENABLED and LDAP_ENABLED are true', () => { - withFlags({ oidc: true, ldap: true }); - const result = service.getUsersService(); - expect(result).toBeInstanceOf(OidcUsersService); + it.each([ + ['neither OIDC_ENABLED nor LDAP_ENABLED is true', {}, DbUsersService], + ['LDAP_ENABLED is true', { ldap: true }, LdapUsersService], + ['OIDC_ENABLED is true', { oidc: true }, OidcUsersService], + ['both OIDC_ENABLED and LDAP_ENABLED are true, preferring OIDC', { oidc: true, ldap: true }, OidcUsersService], + ])('returns the matching implementation when %s', (_case, flags, implementation) => { + withFlags(flags); + + expect(service.getUsersService()).toBeInstanceOf(implementation); }); }); diff --git a/src/users/users.factory.ts b/src/users/users.factory.ts index 5ec1cf2a..4c05fa4a 100644 --- a/src/users/users.factory.ts +++ b/src/users/users.factory.ts @@ -30,14 +30,12 @@ export class UsersFactoryService { return new OidcUsersService(this.configService, this.prismaService, this.authService); } - switch (ldapEnabled) { - case true: - this.logger.debug('users service type: LDAP'); - return new LdapUsersService(this.configService, this.prismaService, this.authService); - case false: - default: - this.logger.debug('users service type: DB'); - return new DbUsersService(this.prismaService, this.authService); + if (ldapEnabled) { + this.logger.debug('users service type: LDAP'); + return new LdapUsersService(this.configService, this.prismaService, this.authService); } + + this.logger.debug('users service type: DB'); + return new DbUsersService(this.prismaService, this.authService); } } diff --git a/src/users/users.interface.ts b/src/users/users.interface.ts index 4ac84267..0d9362f9 100644 --- a/src/users/users.interface.ts +++ b/src/users/users.interface.ts @@ -5,6 +5,7 @@ import { UpdateUserDto } from './dto/user-update.dto'; import { UserLoginRequestDto } from './dto/user-login-request.dto'; export interface Users { + readonly allowsRegistration: boolean; create(createUserDto: CreateUserDto): Promise; update(id: string, userDto: UpdateUserDto): Promise; changePassword(user: User, newPassword: string): Promise; diff --git a/src/users/users.service.spec.ts b/src/users/users.service.spec.ts index 3ebc4228..b7574a48 100644 --- a/src/users/users.service.spec.ts +++ b/src/users/users.service.spec.ts @@ -9,9 +9,10 @@ import { Role, User } from '@prisma/client'; import { UserLoginResponseDto } from './dto/user-login-response.dto'; import { AssignRoleDto } from './dto/assign-role.dto'; import { randomUUID } from 'crypto'; -import { UserDto } from './dto/user.dto'; +import { toUserSummary } from './dto/user.dto'; import { UpdateUserDto } from './dto/user-update.dto'; import { OidcUsersService } from './oidc/oidcusers.service'; +import { LdapUsersService } from './ldap/ldapusers.service'; const user: User = { id: '1', @@ -29,17 +30,17 @@ const user: User = { describe('UsersService with DbUserService implementation', () => { let service: UsersService; let prismaService: PrismaService; + let configService: { get: jest.Mock }; beforeEach(async () => { + configService = { get: jest.fn() }; const module: TestingModule = await Test.createTestingModule({ providers: [ UsersService, UsersFactoryService, { provide: ConfigService, - useValue: { - get: jest.fn(), - }, + useValue: configService, }, { provide: PrismaService, @@ -69,25 +70,17 @@ describe('UsersService with DbUserService implementation', () => { prismaService = module.get(PrismaService); }); - it('should be defined', () => { - expect(service).toBeDefined(); - }); - it('should generate new api key', async () => { - // Arrange const prismaUserUpdateMock = jest.spyOn(prismaService.user, 'update'); prismaUserUpdateMock.mockResolvedValueOnce(user); - // Act const result = await service.generateNewApiKey(user); - // Assert expect(prismaUserUpdateMock).toHaveBeenCalled(); expect(typeof result).toBe('string'); }); it('should create a new user', async () => { - // Arrange const createUserDto = { email: 'test@example.com', firstName: 'John', @@ -98,10 +91,8 @@ describe('UsersService with DbUserService implementation', () => { const prismaUserCreateMock = jest.spyOn(prismaService.user, 'create'); prismaUserCreateMock.mockResolvedValueOnce(user); - // Act const result = await service.create(createUserDto); - // Assert expect(prismaUserCreateMock).toHaveBeenCalledWith({ data: { email: 'test@example.com', @@ -115,7 +106,6 @@ describe('UsersService with DbUserService implementation', () => { }); it('should update a user', async () => { - // Arrange const updateUserDto: UpdateUserDto = { email: 'test@example.com', firstName: 'John', @@ -125,43 +115,33 @@ describe('UsersService with DbUserService implementation', () => { const prismaUserUpdateMock = jest.spyOn(prismaService.user, 'update'); prismaUserUpdateMock.mockResolvedValueOnce(user); - // Act const result = await service.update(user.id, updateUserDto); - // Assert expect(prismaUserUpdateMock).toHaveBeenCalled(); expect(result).toEqual(expect.any(UserLoginResponseDto)); }); it('should change password', async () => { - // Arrange const prismaUserUpdateMock = jest.spyOn(prismaService.user, 'update'); prismaUserUpdateMock.mockResolvedValueOnce(user); - // Act const result = await service.changePassword(user, 'newPassword'); - // Assert expect(result).toEqual(true); }); it('should login existing user', async () => { - // Act const result = await service.login({ email: 'test@example.com', password: 'doesntmatter' }); - // Assert expect(result).toEqual(expect.any(UserLoginResponseDto)); }); it('should delete existing user', async () => { - // Arrange const prismaUserDeleteMock = jest.spyOn(prismaService.user, 'delete'); prismaUserDeleteMock.mockResolvedValueOnce(user); - // Act await service.delete(user.id); - // Assert expect(prismaUserDeleteMock).toHaveBeenCalledWith({ where: { id: user.id } }); }); @@ -190,15 +170,6 @@ describe('UsersService with DbUserService implementation', () => { }); }); - it('should get existing user', async () => { - // Act - const result = await service.get(user.id); - - // Assert - expect(result).toBeInstanceOf(UserDto); - expect(prismaService.user.findUnique).toHaveBeenCalledWith({ where: { id: user.id } }); - }); - it('looks a user up by id', async () => { const found = await service.findOne(user.id); @@ -207,7 +178,6 @@ describe('UsersService with DbUserService implementation', () => { }); it('should assign role to user', async () => { - // Arrange const assignRoleDto: AssignRoleDto = { id: randomUUID(), role: Role.editor, @@ -216,12 +186,28 @@ describe('UsersService with DbUserService implementation', () => { const prismaUserCreateMock = jest.spyOn(prismaService.user, 'update'); prismaUserCreateMock.mockResolvedValueOnce(user); - // Act const result = await service.assignRole(assignRoleDto); - // Assert expect(prismaUserCreateMock).toHaveBeenCalled(); - expect(result).toBeInstanceOf(UserDto); + expect(result).toEqual(toUserSummary(user)); + expect(result).not.toHaveProperty('apiKey'); + }); + + it.each([ + [undefined, true], + ['true', true], + ['TRUE', true], + ['yes', true], + ['', true], + ['false', false], + ['0', false], + [' OFF ', false], + ['No', false], + ])('with REGISTRATION_ENABLED %j registration is allowed: %s', (value, allowed) => { + configService.get.mockImplementation((key: string) => (key === 'REGISTRATION_ENABLED' ? value : undefined)); + + expect(service.isRegistrationAllowedByConfig()).toBe(allowed); + expect(service.isRegistrationEnabled()).toBe(allowed); }); it('reports OIDC as disabled for the DB implementation', () => { @@ -233,34 +219,44 @@ describe('UsersService with DbUserService implementation', () => { }); }); -describe('UsersService with OidcUsersService implementation', () => { - let service: UsersService; - let oidcUsersService: OidcUsersService; +const serviceWith = async (implementation: object, get = jest.fn()) => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ + UsersService, + { provide: UsersFactoryService, useValue: { getUsersService: () => implementation } }, + { provide: PrismaService, useValue: {} }, + { provide: AuthService, useValue: {} }, + { provide: ConfigService, useValue: { get } }, + ], + }).compile(); + return module.get(UsersService); +}; - beforeEach(async () => { - oidcUsersService = Object.assign(Object.create(OidcUsersService.prototype), { +describe('UsersService with an identity-provider implementation', () => { + const withoutRegistration = (prototype: object, members: object = {}) => + Object.assign(Object.create(prototype), { allowsRegistration: false, ...members }); + + it.each([ + ['OIDC', OidcUsersService.prototype, true], + ['LDAP', LdapUsersService.prototype, false], + ])('%s: registration is disabled whatever the flag says', async (_name, prototype, oidc) => { + const service = await serviceWith( + withoutRegistration(prototype), + jest.fn(() => 'true') + ); + + expect(service.isRegistrationAllowedByConfig()).toBe(true); + expect(service.isRegistrationEnabled()).toBe(false); + expect(service.isOidcEnabled()).toBe(oidc); + }); + + it('delegates authorization url, callback and redirect building to OIDC', async () => { + const oidcUsersService = withoutRegistration(OidcUsersService.prototype, { getAuthorizationUrl: jest.fn(), handleCallback: jest.fn(), buildFrontendRedirect: jest.fn(), }) as OidcUsersService; - - const module: TestingModule = await Test.createTestingModule({ - providers: [ - UsersService, - { provide: UsersFactoryService, useValue: { getUsersService: () => oidcUsersService } }, - { provide: PrismaService, useValue: {} }, - { provide: AuthService, useValue: {} }, - ], - }).compile(); - - service = module.get(UsersService); - }); - - it('reports OIDC as enabled', () => { - expect(service.isOidcEnabled()).toBe(true); - }); - - it('delegates authorization url, callback and redirect building', async () => { + const service = await serviceWith(oidcUsersService); const authorization = { url: 'https://idp/authorize', state: 'state', codeVerifier: 'verifier' }; (oidcUsersService.getAuthorizationUrl as jest.Mock).mockResolvedValue(authorization); (oidcUsersService.handleCallback as jest.Mock).mockResolvedValue({ token: 'jwt' }); diff --git a/src/users/users.service.ts b/src/users/users.service.ts index 63cd9f38..2d6c6cc5 100644 --- a/src/users/users.service.ts +++ b/src/users/users.service.ts @@ -1,17 +1,17 @@ -import { Injectable } from '@nestjs/common'; import { CreateUserDto } from './dto/user-create.dto'; import { UserLoginResponseDto } from './dto/user-login-response.dto'; import { PrismaService } from '../prisma/prisma.service'; import { User } from '@prisma/client'; -import { UserDto } from './dto/user.dto'; +import { UserSummaryDto, toUserSummary } from './dto/user.dto'; import { UpdateUserDto } from './dto/user-update.dto'; import { AuthService } from '../auth/auth.service'; import { UserLoginRequestDto } from './dto/user-login-request.dto'; import { AssignRoleDto } from './dto/assign-role.dto'; -import { HttpException, HttpStatus, Logger, NotFoundException } from '@nestjs/common'; +import { HttpException, HttpStatus, Injectable, Logger, NotFoundException } from '@nestjs/common'; import { UsersFactoryService } from './users.factory'; import { Users } from './users.interface'; import { OidcUsersService } from './oidc/oidcusers.service'; +import { ConfigService } from '@nestjs/config'; @Injectable() export class UsersService { @@ -21,7 +21,8 @@ export class UsersService { constructor( private readonly usersFactoryService: UsersFactoryService, private prismaService: PrismaService, - private authService: AuthService + private authService: AuthService, + private readonly configService: ConfigService ) { this.usersService = this.usersFactoryService.getUsersService(); } @@ -42,6 +43,18 @@ export class UsersService { return this.usersService.login(userLoginRequestDto); } + // default true: switching registration off is the operator's choice, not an upgrade side effect + isRegistrationAllowedByConfig(): boolean { + const value = String(this.configService.get('REGISTRATION_ENABLED') ?? 'true') + .trim() + .toLowerCase(); + return !['false', '0', 'no', 'off'].includes(value); + } + + isRegistrationEnabled(): boolean { + return this.usersService.allowsRegistration && this.isRegistrationAllowedByConfig(); + } + isOidcEnabled(): boolean { return this.usersService instanceof OidcUsersService; } @@ -77,10 +90,6 @@ export class UsersService { return this.prismaService.user.delete({ where: { id } }); } - /** - * All or nothing: an unknown id fails the request before anyone is removed, - * instead of after the users listed ahead of it are already gone. - */ async deleteMany(ids: string[]): Promise { this.logger.debug(`Removing Users: ${ids}`); await this.prismaService.$transaction(async (tx) => { @@ -93,12 +102,7 @@ export class UsersService { }); } - async get(id: string): Promise { - const user = await this.findOne(id); - return new UserDto(user!); - } - - async assignRole(data: AssignRoleDto): Promise { + async assignRole(data: AssignRoleDto): Promise { const { id, role } = data; this.logger.debug(`Assigning role ${role} to User: ${id}`); @@ -106,7 +110,7 @@ export class UsersService { where: { id }, data: { role }, }); - return new UserDto(user); + return toUserSummary(user); } async generateNewApiKey(user: User): Promise { diff --git a/src/utils/bounded-cache.ts b/src/utils/bounded-cache.ts new file mode 100644 index 00000000..dfe8fc6b --- /dev/null +++ b/src/utils/bounded-cache.ts @@ -0,0 +1,26 @@ +export class BoundedCache { + private readonly entries = new Map(); + + constructor(private readonly limit: number) {} + + get(key: string): T | undefined { + const value = this.entries.get(key); + if (value !== undefined) { + this.entries.delete(key); + this.entries.set(key, value); + } + return value; + } + + set(key: string, value: T): void { + this.entries.delete(key); + this.entries.set(key, value); + while (this.entries.size > this.limit) { + this.entries.delete(this.entries.keys().next().value!); + } + } + + delete(key: string): void { + this.entries.delete(key); + } +} diff --git a/src/utils/env.spec.ts b/src/utils/env.spec.ts new file mode 100644 index 00000000..5048560a --- /dev/null +++ b/src/utils/env.spec.ts @@ -0,0 +1,44 @@ +import { nonNegativeNumber, positiveNumber } from './env'; + +describe('nonNegativeNumber', () => { + it.each([ + ['unset', undefined], + ['empty', ''], + ['blank', ' '], + ['not a number', 'abc'], + ['negative', '-1'], + ['infinite', 'Infinity'], + ])('falls back when the value is %s', (_title, raw) => { + expect(nonNegativeNumber(raw, 7)).toBe(7); + }); + + it.each([ + ['0', 0], + ['12', 12], + [' 3.5 ', 3.5], + ])('accepts %j', (raw, expected) => { + expect(nonNegativeNumber(raw, 7)).toBe(expected); + }); +}); + +describe('positiveNumber', () => { + it.each([ + ['unset', undefined], + ['empty', ''], + ['blank', ' '], + ['zero', '0'], + ['negative', '-5'], + ['not a number', 'soon'], + ['infinite', 'Infinity'], + ])('falls back when the value is %s', (_title, raw) => { + expect(positiveNumber(raw, 9)).toBe(9); + }); + + it.each([ + ['1', 1], + ['2500', 2500], + ['0.5', 0.5], + ])('accepts %j', (raw, expected) => { + expect(positiveNumber(raw, 9)).toBe(expected); + }); +}); diff --git a/src/utils/env.ts b/src/utils/env.ts new file mode 100644 index 00000000..519a11ef --- /dev/null +++ b/src/utils/env.ts @@ -0,0 +1,10 @@ +export function nonNegativeNumber(raw: string | undefined, fallback: number): number { + if (!raw?.trim()) return fallback; + const value = Number(raw); + return Number.isFinite(value) && value >= 0 ? value : fallback; +} + +export function positiveNumber(raw: string | undefined, fallback: number): number { + const value = Number(raw); + return Number.isFinite(value) && value > 0 ? value : fallback; +} diff --git a/src/utils/index.spec.ts b/src/utils/index.spec.ts index 003f7971..41d4086f 100644 --- a/src/utils/index.spec.ts +++ b/src/utils/index.spec.ts @@ -1,5 +1,5 @@ import { generateTestRun, generateTestVariation } from '../_data_'; -import { getTestVariationUniqueData } from '.'; +import { getTestVariationUniqueData, parseIgnoreAreas, testRunImageNames, variationKey } from '.'; describe('getTestVariationUniqueData', () => { it('picks the fields that identify a test variation', () => { @@ -53,3 +53,70 @@ describe('getTestVariationUniqueData', () => { }); }); }); + +describe('parseIgnoreAreas', () => { + it('returns the stored areas', () => { + const areas = [{ x: 1, y: 2, width: 3, height: 4 }]; + + expect(parseIgnoreAreas(JSON.stringify(areas))).toEqual(areas); + }); + + it.each([ + ['null', null], + ['undefined', undefined], + ['an empty string', ''], + ['JSON null', 'null'], + ['an object', '{}'], + ['a string', '"str"'], + ['text that is not JSON', 'not json'], + ])('treats %s as no areas', (_case, stored) => { + expect(parseIgnoreAreas(stored)).toEqual([]); + }); + + it('drops entries that are not objects', () => { + expect(parseIgnoreAreas('[1,null,{"x":1}]')).toEqual([{ x: 1 }]); + }); +}); + +describe('testRunImageNames', () => { + it('lists every stored file of a run', () => { + expect( + testRunImageNames({ + imageName: 'i.png', + diffName: 'd.png', + imageThumbnailName: 'it.png', + diffThumbnailName: 'dt.png', + }).sort() + ).toEqual(['d.png', 'dt.png', 'i.png', 'it.png']); + }); + + it('drops missing and empty names', () => { + expect( + testRunImageNames({ imageName: 'i.png', diffName: null, imageThumbnailName: '', diffThumbnailName: null }) + ).toEqual(['i.png']); + }); +}); + +describe('variationKey', () => { + it('names the project and branch next to the screen the source belongs to', () => { + const source = { + name: 'screen', + os: 'linux', + browser: null, + device: undefined, + viewport: '800x600', + customTags: '', + }; + + expect(variationKey('projectId', 'feature', source as never)).toStrictEqual({ + projectId: 'projectId', + branchName: 'feature', + name: 'screen', + os: 'linux', + browser: '', + device: '', + viewport: '800x600', + customTags: '', + }); + }); +}); diff --git a/src/utils/index.ts b/src/utils/index.ts index e3d40068..bc948df0 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -1,5 +1,6 @@ import { TestRun, TestVariation } from '@prisma/client'; import { BaselineDataDto } from '../shared/dto/baseline-data.dto'; +import type { IgnoreAreaDto } from '../test-runs/dto/ignore-area.dto'; export const getTestVariationUniqueData = ( object: TestRun | TestVariation | BaselineDataDto @@ -20,3 +21,32 @@ export const getTestVariationUniqueData = ( customTags: object.customTags ?? '', }; }; + +export const variationKey = ( + projectId: string, + branchName: string, + source: TestRun | TestVariation | BaselineDataDto +): ReturnType & { projectId: string; branchName: string } => ({ + projectId, + branchName, + ...getTestVariationUniqueData(source), +}); + +export const testRunImageNames = ( + run: Pick +): string[] => + [run.imageName, run.diffName, run.imageThumbnailName, run.diffThumbnailName].filter((name): name is string => !!name); + +export const parseIgnoreAreas = (json: string | null | undefined): IgnoreAreaDto[] => { + if (!json) { + return []; + } + try { + const parsed: unknown = JSON.parse(json); + return Array.isArray(parsed) + ? parsed.filter((area): area is IgnoreAreaDto => typeof area === 'object' && area !== null) + : []; + } catch { + return []; + } +}; diff --git a/src/utils/map-with-concurrency.ts b/src/utils/map-with-concurrency.ts new file mode 100644 index 00000000..f688caef --- /dev/null +++ b/src/utils/map-with-concurrency.ts @@ -0,0 +1,12 @@ +export async function mapWithConcurrency(items: T[], limit: number, task: (item: T) => Promise): Promise { + const results = new Array(items.length); + let next = 0; + const worker = async (): Promise => { + while (next < items.length) { + const index = next++; + results[index] = await task(items[index]); + } + }; + await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker)); + return results; +} diff --git a/src/utils/random-key.ts b/src/utils/random-key.ts index d4f9c91d..fb914169 100644 --- a/src/utils/random-key.ts +++ b/src/utils/random-key.ts @@ -1,32 +1,17 @@ import { randomBytes } from 'crypto'; -// Crockford's base32 digits: no I, L, O or U, so a key read aloud or retyped -// is not mistaken. The alphabet of the keys issued before (uuid-apikey). const CROCKFORD_DIGITS = '0123456789ABCDEFGHJKMNPQRSTVWXYZ'; const KEY_LENGTH = 28; -/** - * A random 28-character key in the format API keys and image names have - * always had — Crockford base32, upper case, no dashes — so keys issued - * before keep working and clients see no change. Each character carries 5 - * random bits (140 in all, more than the 122 a v4 UUID gave). - */ export function generateRandomKey(): string { const bytes = randomBytes(KEY_LENGTH); let key = ''; for (const byte of bytes) { - // 256 is a multiple of 32, so taking the low 5 bits keeps every digit equally likely key += CROCKFORD_DIGITS[byte & 31]; } return key; } -/** - * Whether a project reference is an id rather than a name: dashes where a - * UUID has its first three, and 32 characters without them. The check - * uuid-apikey made, so references resolve as they did; an empty string is - * simply not a UUID instead of an exception. - */ export function isUuidShaped(value: string): boolean { return ( !!value && diff --git a/test/api-helpers.ts b/test/api-helpers.ts index b82a6b93..92afe1b3 100644 --- a/test/api-helpers.ts +++ b/test/api-helpers.ts @@ -1,23 +1,98 @@ -import { INestApplication, ValidationPipe } from '@nestjs/common'; +import { INestApplication, Type } from '@nestjs/common'; +import { NestExpressApplication } from '@nestjs/platform-express'; import { Test, TestingModule } from '@nestjs/testing'; +import { Build } from '@prisma/client'; +import { randomUUID } from 'crypto'; import { existsSync } from 'fs'; import path from 'path'; import { PNG } from 'pngjs'; +import { TEST_PROJECT } from '../src/_data_'; import { AppModule } from '../src/app.module'; +import { ProjectsService } from '../src/projects/projects.service'; +import { GlobalValidationPipe } from '../src/shared/validation'; import { HDD_IMAGE_PATH } from '../src/static/hdd/constants'; +import { TestRunsService } from '../src/test-runs/test-runs.service'; +import { LoggedUser, requestWithApiKey, requestWithAuth } from './preconditions'; -/** Boots the whole app the same way the other e2e specs do. */ -export const createE2eApp = async (): Promise<{ app: INestApplication; moduleFixture: TestingModule }> => { - const moduleFixture: TestingModule = await Test.createTestingModule({ - imports: [AppModule], - }).compile(); +interface E2eAppOptions { + overrides?: Array<[Type, unknown]>; + validation?: boolean; + configure?: (app: NestExpressApplication) => void; +} - const app = moduleFixture.createNestApplication(); - app.useGlobalPipes(new ValidationPipe()); +export const createE2eApp = async ({ overrides = [], validation = true, configure }: E2eAppOptions = {}) => { + let builder = Test.createTestingModule({ imports: [AppModule] }); + for (const [token, value] of overrides) { + builder = builder.overrideProvider(token).useValue(value); + } + const moduleFixture: TestingModule = await builder.compile(); + const app = moduleFixture.createNestApplication(); + if (validation) { + app.useGlobalPipes(new GlobalValidationPipe()); + } + configure?.(app); await app.init(); - return { app, moduleFixture }; + return { app, moduleFixture, get: (token: Type): T => moduleFixture.get(token) }; }; +export const haveProject = ( + projectsService: ProjectsService, + label: string, + overrides: Partial[0]> = {} +) => projectsService.create({ ...TEST_PROJECT, ...overrides, name: `${label} ${randomUUID()}` }); + +export const postRunDirect = (testRunsService: TestRunsService, build: Build, image: Buffer, name = 'Screen') => + testRunsService.postTestRun({ + createTestRequestDto: { projectId: build.projectId, buildId: build.id, branchName: build.branchName!, name }, + imageBuffer: image, + }); + +export const postRunBase64 = async ( + app: INestApplication, + user: LoggedUser, + build: Build, + image: Buffer, + extra: Record = {} +): Promise> => + ( + await requestWithApiKey(app, 'post', '/test-runs', user.apiKey) + .send({ + projectId: build.projectId, + buildId: build.id, + branchName: build.branchName, + name: 'Screen', + imageBase64: image.toString('base64'), + ...extra, + }) + .expect(201) + ).body; + +export const setEnv = (vars: Record): (() => void) => { + const previous = Object.fromEntries(Object.keys(vars).map((key) => [key, process.env[key]])); + Object.assign(process.env, vars); + return () => { + for (const [key, value] of Object.entries(previous)) { + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } + } + }; +}; + +export const withEnv = async (vars: Record, fn: () => PromiseLike): Promise => { + const restore = setEnv(vars); + try { + return await fn(); + } finally { + restore(); + } +}; + +export const approveRuns = (app: INestApplication, user: LoggedUser, ids: string[], merge = false) => + requestWithAuth(app, 'post', `/test-runs/approve?merge=${merge}`, user.token).send(ids).expect(201); + export interface Rect { x: number; y: number; @@ -26,10 +101,6 @@ export interface Rect { color: [number, number, number]; } -/** - * A tiny white PNG with solid rectangles painted on it. Synthetic images keep - * the diffs fast and make the change (its colour and its area) exact. - */ export const pngBuffer = (rects: Rect[] = [], width = 60, height = 60): Buffer => { const png = new PNG({ width, height }); png.data.fill(255); @@ -50,12 +121,14 @@ export const pngBuffer = (rects: Rect[] = [], width = 60, height = 60): Buffer = export const RED: [number, number, number] = [255, 0, 0]; export const BLUE: [number, number, number] = [0, 0, 255]; -/** Where the HDD storage keeps a stored image (the default STATIC_SERVICE). */ +export const WHITE = pngBuffer(); +export const redAt = (x: number, y: number, size = 10): Rect => ({ x, y, w: size, h: size, color: RED }); +export const CHANGED = pngBuffer([redAt(5, 5)]); + export const storedImagePath = (imageName: string): string => path.resolve(HDD_IMAGE_PATH, imageName); export const imageExists = (imageName: string | null): boolean => !!imageName && existsSync(storedImagePath(imageName)); -/** Polls until the predicate holds — for image deletions the app fires and does not await. */ export const waitFor = async (predicate: () => boolean | Promise, timeoutMs = 5000): Promise => { const deadline = Date.now() + timeoutMs; while (!(await predicate())) { @@ -66,7 +139,6 @@ export const waitFor = async (predicate: () => boolean | Promise, timeo } }; -/** Keys of TestRunDto — what the API exposes for a test run. */ export const TEST_RUN_DTO_KEYS = [ 'baselineBranchName', 'baselineName', diff --git a/test/body-limit.e2e-spec.ts b/test/body-limit.e2e-spec.ts new file mode 100644 index 00000000..a15475f7 --- /dev/null +++ b/test/body-limit.e2e-spec.ts @@ -0,0 +1,36 @@ +import { NestExpressApplication } from '@nestjs/platform-express'; +import request from 'supertest'; +import { createE2eApp } from './api-helpers'; + +describe('JSON body limit (e2e)', () => { + let app: NestExpressApplication; + + beforeAll(async () => { + ({ app } = await createE2eApp({ configure: (created) => created.useBodyParser('json', { limit: '1kb' }) })); + }); + + afterAll(async () => { + await app.close(); + }); + + it('answers 413 with the usual error body for a JSON body over the limit', async () => { + const res = await request(app.getHttpServer()) + .post('/users/login') + .send({ email: 'someone@example.com', password: 'x'.repeat(2048) }) + .expect(413); + + expect(res.body).toEqual({ + statusCode: 413, + path: '/users/login', + name: 'PayloadTooLargeError', + message: 'request entity too large', + }); + }); + + it('still answers a small body normally', async () => { + await request(app.getHttpServer()) + .post('/users/login') + .send({ email: 'nobody@example.com', password: 'x' }) + .expect((res) => expect(res.status).not.toBe(413)); + }); +}); diff --git a/test/builds-api.e2e-spec.ts b/test/builds-api.e2e-spec.ts index 9c66e54a..f3f5ff79 100644 --- a/test/builds-api.e2e-spec.ts +++ b/test/builds-api.e2e-spec.ts @@ -1,15 +1,14 @@ import { INestApplication } from '@nestjs/common'; import { Build, Project, TestRun, TestStatus } from '@prisma/client'; import { randomUUID } from 'crypto'; -import { TEST_PROJECT } from '../src/_data_'; import { BuildDto } from '../src/builds/dto/build.dto'; import { BuildsService } from '../src/builds/builds.service'; import { PrismaService } from '../src/prisma/prisma.service'; import { ProjectsService } from '../src/projects/projects.service'; import { TestRunsService } from '../src/test-runs/test-runs.service'; import { UsersService } from '../src/users/users.service'; -import { createE2eApp, imageExists, pngBuffer, RED } from './api-helpers'; -import { haveUserLogged, requestWithApiKey, requestWithAuth, LoggedUser } from './preconditions'; +import { CHANGED, createE2eApp, haveProject, imageExists, postRunDirect, WHITE } from './api-helpers'; +import { haveUserLogged, requestWithApiKey, requestWithAuth, LoggedUser, withUser } from './preconditions'; describe('Builds API (e2e)', () => { let app: INestApplication; @@ -21,30 +20,23 @@ describe('Builds API (e2e)', () => { let user: LoggedUser; let project: Project; - const WHITE = pngBuffer(); - const CHANGED = pngBuffer([{ x: 5, y: 5, w: 10, h: 10, color: RED }]); - beforeAll(async () => { const created = await createE2eApp(); app = created.app; - usersService = created.moduleFixture.get(UsersService); - projectsService = created.moduleFixture.get(ProjectsService); - buildsService = created.moduleFixture.get(BuildsService); - testRunsService = created.moduleFixture.get(TestRunsService); - prisma = created.moduleFixture.get(PrismaService); + usersService = created.get(UsersService); + projectsService = created.get(ProjectsService); + buildsService = created.get(BuildsService); + testRunsService = created.get(TestRunsService); + prisma = created.get(PrismaService); }); beforeEach(async () => { user = await haveUserLogged(usersService); - project = await projectsService.create({ - ...TEST_PROJECT, - autoApproveFeature: false, - maxBuildAllowed: 100, - name: `Builds API E2E ${randomUUID()}`, - }); + project = await haveProject(projectsService, 'Builds API E2E', { autoApproveFeature: false, maxBuildAllowed: 100 }); }); afterEach(async () => { + await buildsService.whenRetentionIdle(); await projectsService.remove(project.id); await usersService.delete(user.id); }); @@ -56,13 +48,16 @@ describe('Builds API (e2e)', () => { const newBuild = (ciBuildId?: string, projectId = project.id): Promise => buildsService.findOrCreate({ projectId, branchName: 'master', ciBuildId }); - const postRun = (build: Build, image: Buffer, name = 'Screen') => - testRunsService.postTestRun({ - createTestRequestDto: { projectId: build.projectId, buildId: build.id, branchName: build.branchName!, name }, - imageBuffer: image, - }); + const postRun = (build: Build, image: Buffer, name?: string) => postRunDirect(testRunsService, build, image, name); + + const createBuild = async (ciBuildId?: string) => { + const res = await requestWithApiKey(app, 'post', '/builds', user.apiKey) + .send({ project: project.id, ciBuildId }) + .expect(201); + await buildsService.whenRetentionIdle(); + return res; + }; - /** The project's builds in the order the list endpoint promises: newest first, id breaking ties. */ const listOrder = async (): Promise => ( await prisma.build.findMany({ @@ -106,7 +101,7 @@ describe('Builds API (e2e)', () => { }); it('aroundBuildId that holds no page falls back to skip', async () => { - const otherProject = await projectsService.create({ ...TEST_PROJECT, name: `Other ${randomUUID()}` }); + const otherProject = await haveProject(projectsService, 'Other'); try { const foreign = await newBuild(undefined, otherProject.id); @@ -143,7 +138,6 @@ describe('Builds API (e2e)', () => { expect(res.body).toMatchObject({ total: 2, take: 1, skip: 1 }); expect(res.body.data.map((build: BuildDto) => build.id)).toEqual([oldestNightly]); - // a build the filter leaves out holds no page const excluded = await list(`take=1&skip=0&ciBuildId=nightly&aroundBuildId=${ordered[4]}`).expect(200); expect(excluded.body.skip).toBe(0); }); @@ -165,6 +159,7 @@ describe('Builds API (e2e)', () => { await list('take=2').expect(400); await list('take=two&skip=0').expect(400); await list('take=2&skip=0&aroundBuildId=nope').expect(400); + await list('take=10&skip=-1').expect(400); }); }); @@ -211,7 +206,6 @@ describe('Builds API (e2e)', () => { const res = await requestWithAuth(app, 'delete', `/builds/${build.id}`, user.token).expect(200); - // the raw build row with the runs it had expect(res.body.id).toBe(build.id); expect(res.body.testRuns.map((testRun: TestRun) => testRun.id)).toEqual([run.id]); expect(await prisma.build.findUnique({ where: { id: build.id } })).toBeNull(); @@ -239,12 +233,9 @@ describe('Builds API (e2e)', () => { it('403 for a guest', async () => { const build = await newBuild(); - const guest = await haveUserLogged(usersService, 'guest'); - try { - await requestWithAuth(app, 'delete', `/builds/${build.id}`, guest.token).expect(403); - } finally { - await usersService.delete(guest.id); - } + await withUser(usersService, 'guest', (guest) => + requestWithAuth(app, 'delete', `/builds/${build.id}`, guest.token).expect(403) + ); expect(await prisma.build.findUnique({ where: { id: build.id } })).not.toBeNull(); }); }); @@ -258,7 +249,7 @@ describe('Builds API (e2e)', () => { const unresolved = await postRun(build, CHANGED, 'Unresolved'); const fresh = await postRun(build, WHITE, 'New'); const failed = await postRun(build, WHITE, 'Failed'); - await testRunsService.setStatus(failed.id, TestStatus.failed); + await prisma.testRun.update({ where: { id: failed.id }, data: { status: TestStatus.failed } }); await requestWithAuth(app, 'patch', `/builds/${build.id}/approve?merge=false`, user.token).expect(200); @@ -270,7 +261,6 @@ describe('Builds API (e2e)', () => { [fresh.id]: TestStatus.approved, [failed.id]: TestStatus.failed, }); - // like POST /test-runs/approve, each new baseline records who approved it const baselines = await prisma.baseline.findMany({ where: { testRunId: { in: [unresolved.id, fresh.id] } } }); expect(baselines.map((baseline) => baseline.userId)).toEqual([user.id, user.id]); }); @@ -287,12 +277,9 @@ describe('Builds API (e2e)', () => { it('403 for a guest', async () => { const build = await newBuild(); - const guest = await haveUserLogged(usersService, 'guest'); - try { - await requestWithAuth(app, 'patch', `/builds/${build.id}/approve?merge=false`, guest.token).expect(403); - } finally { - await usersService.delete(guest.id); - } + await withUser(usersService, 'guest', (guest) => + requestWithAuth(app, 'patch', `/builds/${build.id}/approve?merge=false`, guest.token).expect(403) + ); }); }); @@ -347,14 +334,18 @@ describe('Builds API (e2e)', () => { }); describe('POST /builds retention (maxBuildAllowed)', () => { - const createBuild = () => - requestWithApiKey(app, 'post', '/builds', user.apiKey).send({ project: project.id }).expect(201); + const idleForTwoHours = async () => { + const then = new Date(Date.now() - 2 * 60 * 60 * 1000); + await prisma.$executeRaw`UPDATE "Build" SET "updatedAt" = ${then} WHERE "projectId" = ${project.id}`; + await prisma.testRun.updateMany({ where: { build: { projectId: project.id } }, data: { createdAt: then } }); + }; it('keeps at most maxBuildAllowed builds and removes the files of the dropped ones', async () => { await prisma.project.update({ where: { id: project.id }, data: { maxBuildAllowed: 2 } }); const oldest = (await createBuild()).body; const run = await postRun({ ...oldest, projectId: project.id } as Build, WHITE); await createBuild(); + await idleForTwoHours(); const newest = (await createBuild()).body; @@ -370,6 +361,7 @@ describe('Builds API (e2e)', () => { await createBuild(); await createBuild(); + await idleForTwoHours(); const newest = (await createBuild()).body; expect(await listOrder()).toEqual([newest.id]); @@ -377,18 +369,40 @@ describe('Builds API (e2e)', () => { it('never prunes the build a CI build id reuses, and counts it as one of those kept', async () => { await prisma.project.update({ where: { id: project.id }, data: { maxBuildAllowed: 2 } }); - const joinShared = () => - requestWithApiKey(app, 'post', '/builds', user.apiKey) - .send({ project: project.id, ciBuildId: 'shared' }) - .expect(201); + const joinShared = () => createBuild('shared'); const shared = (await joinShared()).body; const newer = (await createBuild()).body; + await idleForTwoHours(); - // a parallel CI job joins the shared build, which is now the older of the two const joined = (await joinShared()).body; expect(joined.id).toBe(shared.id); expect((await listOrder()).sort()).toEqual([shared.id, newer.id].sort()); }); }); + + describe('POST /builds retention of builds in use', () => { + it('spares a build that is still receiving test runs', async () => { + await prisma.project.update({ where: { id: project.id }, data: { maxBuildAllowed: 1 } }); + const live = (await createBuild()).body; + await postRun({ ...live, projectId: project.id } as Build, WHITE); + + const newest = (await createBuild()).body; + + expect((await listOrder()).sort()).toEqual([live.id, newest.id].sort()); + }); + + it('removes a build idle for longer than the grace period', async () => { + await prisma.project.update({ where: { id: project.id }, data: { maxBuildAllowed: 1 } }); + const idle = (await createBuild()).body; + await postRun({ ...idle, projectId: project.id } as Build, WHITE); + const then = new Date(Date.now() - 2 * 60 * 60 * 1000); + await prisma.$executeRaw`UPDATE "Build" SET "updatedAt" = ${then} WHERE "id" = ${idle.id}`; + await prisma.testRun.updateMany({ where: { buildId: idle.id }, data: { createdAt: then } }); + + const newest = (await createBuild()).body; + + expect(await listOrder()).toEqual([newest.id]); + }); + }); }); diff --git a/test/builds.e2e-spec.ts b/test/builds.e2e-spec.ts index d4883764..6d279819 100644 --- a/test/builds.e2e-spec.ts +++ b/test/builds.e2e-spec.ts @@ -1,53 +1,43 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { INestApplication, ValidationPipe } from '@nestjs/common'; -import { AppModule } from '../src/app.module'; -import { UsersService } from '../src/users/users.service'; -import { haveTestRunCreated, haveUserLogged, requestWithApiKey, requestWithAuth, LoggedUser } from './preconditions'; -import { BuildsService } from '../src/builds/builds.service'; -import { CreateBuildDto } from '../src/builds/dto/build-create.dto'; +import { INestApplication } from '@nestjs/common'; import { Project } from '@prisma/client'; +import { BuildsController } from '../src/builds/builds.controller'; +import { CreateBuildDto } from '../src/builds/dto/build-create.dto'; +import { BuildsService } from '../src/builds/builds.service'; import { ProjectsService } from '../src/projects/projects.service'; import { TestRunsService } from '../src/test-runs/test-runs.service'; -import { BuildsController } from '../src/builds/builds.controller'; +import { UsersService } from '../src/users/users.service'; import { TEST_PROJECT } from '../src/_data_'; -import { randomUUID } from 'crypto'; +import { createE2eApp, haveProject } from './api-helpers'; +import { haveTestRunCreated, haveUserLogged, requestWithApiKey, requestWithAuth, LoggedUser } from './preconditions'; describe('Builds (e2e)', () => { let app: INestApplication; let buildsService: BuildsService; let buildsController: BuildsController; - let projecstService: ProjectsService; + let projectsService: ProjectsService; let usersService: UsersService; let testRunsService: TestRunsService; let user: LoggedUser; let project: Project; beforeAll(async () => { - const moduleFixture: TestingModule = await Test.createTestingModule({ - imports: [AppModule], - }).compile(); - - app = moduleFixture.createNestApplication(); - app.useGlobalPipes(new ValidationPipe()); - buildsService = moduleFixture.get(BuildsService); - buildsController = moduleFixture.get(BuildsController); - usersService = moduleFixture.get(UsersService); - projecstService = moduleFixture.get(ProjectsService); - testRunsService = moduleFixture.get(TestRunsService); - - await app.init(); + const created = await createE2eApp(); + app = created.app; + buildsService = created.get(BuildsService); + buildsController = created.get(BuildsController); + usersService = created.get(UsersService); + projectsService = created.get(ProjectsService); + testRunsService = created.get(TestRunsService); }); beforeEach(async () => { user = await haveUserLogged(usersService); - project = await projecstService.create({ - ...TEST_PROJECT, - name: `Builds E2E test${randomUUID()}`, - }); + project = await haveProject(projectsService, 'Builds E2E test'); }); afterEach(async () => { - await projecstService.remove(project.id); + await buildsService.whenRetentionIdle(); + await projectsService.remove(project.id); await usersService.delete(user.id); }); @@ -55,169 +45,85 @@ describe('Builds (e2e)', () => { await app.close(); }); - describe('POST /', () => { - it('201 by id', () => { - const createBuildDto: CreateBuildDto = { - branchName: 'branchName', - project: project.id, - }; - return requestWithApiKey(app, 'post', '/builds', user.apiKey) - .send(createBuildDto) - .expect(201) - .expect((res) => { - expect(res.body.projectId).toBe(project.id); - expect(res.body.branchName).toBe(createBuildDto.branchName); - expect(res.body.failedCount).toBe(0); - expect(res.body.passedCount).toBe(0); - expect(res.body.unresolvedCount).toBe(0); - expect(res.body.isRunning).toBe(true); - }); - }); + const haveBuild = () => buildsController.create({ project: project.id, branchName: 'develop' }); + const haveRun = (merge: boolean) => + haveTestRunCreated(buildsService, testRunsService, project.id, project.mainBranchName, './test/image.png', merge); - it('201 by null branchname', () => { - // what an older SDK sends; the DTO type does not allow it - const createBuildDto = { - branchName: null, - project: project.id, - }; - return requestWithApiKey(app, 'post', '/builds', user.apiKey) - .send(createBuildDto) - .expect(201) - .expect((res) => { - expect(res.body.projectId).toBe(project.id); - expect(res.body.branchName).toBe(TEST_PROJECT.mainBranchName); - expect(res.body.failedCount).toBe(0); - expect(res.body.passedCount).toBe(0); - expect(res.body.unresolvedCount).toBe(0); - expect(res.body.isRunning).toBe(true); - }); - }); - - it('201 with no branchname and no ciBuildId', () => { - const createBuildDto: CreateBuildDto = { - project: project.id, - }; - return requestWithApiKey(app, 'post', '/builds', user.apiKey) - .send(createBuildDto) - .expect(201) - .expect((res) => { - expect(res.body.projectId).toBe(project.id); - expect(res.body.branchName).toBe(TEST_PROJECT.mainBranchName); - expect(res.body.failedCount).toBe(0); - expect(res.body.passedCount).toBe(0); - expect(res.body.unresolvedCount).toBe(0); - expect(res.body.isRunning).toBe(true); - }); - }); - - it('201 by name', () => { - const createBuildDto: CreateBuildDto = { - branchName: 'branchName', - project: project.name, - }; - return requestWithApiKey(app, 'post', '/builds', user.apiKey) + describe('POST /', () => { + it.each([ + ['by id', { branchName: 'branchName' }, 'branchName', false], + ['by null branchname', { branchName: null }, TEST_PROJECT.mainBranchName, false], + ['with no branchname and no ciBuildId', {}, TEST_PROJECT.mainBranchName, false], + ['by name', { branchName: 'branchName' }, 'branchName', true], + ['by ciBuildId', { ciBuildId: 'ciBuildId', branchName: 'branchName' }, 'branchName', true], + ])('201 %s', async (_name, dto, branchName, byName) => { + const createBuildDto = { ...dto, project: byName ? project.name : project.id } as CreateBuildDto; + const existing = createBuildDto.ciBuildId ? await buildsController.create(createBuildDto) : undefined; + + await requestWithApiKey(app, 'post', '/builds', user.apiKey) .send(createBuildDto) .expect(201) .expect((res) => { - expect(res.body.projectId).toBe(project.id); - expect(res.body.branchName).toBe(createBuildDto.branchName); - expect(res.body.failedCount).toBe(0); - expect(res.body.passedCount).toBe(0); - expect(res.body.unresolvedCount).toBe(0); - expect(res.body.isRunning).toBe(true); + expect(res.body).toMatchObject({ + projectId: project.id, + branchName, + failedCount: 0, + passedCount: 0, + unresolvedCount: 0, + isRunning: true, + }); + if (existing) { + expect(res.body.id).toBe(existing.id); + expect(res.body.ciBuildId).toBe(createBuildDto.ciBuildId); + } }); }); - it('201 by ciBuildId', async () => { + it('201 cuncurrent', async () => { const createBuildDto: CreateBuildDto = { ciBuildId: 'ciBuildId', branchName: 'branchName', project: project.name, }; - const build = await buildsController.create(createBuildDto); + await haveRun(false); + await haveRun(false); - return requestWithApiKey(app, 'post', '/builds', user.apiKey) - .send(createBuildDto) - .expect(201) - .expect((res) => { - expect(res.body.id).toBe(build.id); - expect(res.body.ciBuildId).toBe(createBuildDto.ciBuildId); - expect(res.body.projectId).toBe(project.id); - expect(res.body.branchName).toBe(createBuildDto.branchName); - expect(res.body.failedCount).toBe(0); - expect(res.body.passedCount).toBe(0); - expect(res.body.unresolvedCount).toBe(0); - expect(res.body.isRunning).toBe(true); - }); + const builds = await Promise.all(Array.from({ length: 5 }, () => buildsController.create(createBuildDto))); + + expect(new Set(builds.map((build) => build.id)).size).toBe(1); }); - it('201 cuncurrent', async () => { - const createBuildDto: CreateBuildDto = { - ciBuildId: 'ciBuildId', - branchName: 'branchName', - project: project.name, - }; - await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - project.mainBranchName, - './test/image.png', - false - ); - await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - project.mainBranchName, - './test/image.png', - false - ); - - const builds = await Promise.all([ - buildsController.create(createBuildDto), - buildsController.create(createBuildDto), - buildsController.create(createBuildDto), - buildsController.create(createBuildDto), - buildsController.create(createBuildDto), - ]); + it('parallel creates of one new ciBuildId number it once', async () => { + const dto: CreateBuildDto = { ciBuildId: 'fresh', project: project.name }; - expect(new Set(builds.map((build) => build.id)).size).toBe(1); + const builds = await Promise.all(Array.from({ length: 8 }, () => buildsController.create(dto))); + + expect(new Set(builds.map((build) => build.number))).toEqual(new Set([1])); + expect((await projectsService.findOne(project.id)).buildsCounter).toBe(1); }); it('404', () => { - const createBuildDto: CreateBuildDto = { - branchName: 'branchName', - project: 'random', - }; - return requestWithApiKey(app, 'post', '/builds', user.apiKey).send(createBuildDto).expect(404); + return requestWithApiKey(app, 'post', '/builds', user.apiKey) + .send({ branchName: 'branchName', project: 'random' }) + .expect(404); }); it('403', () => { - const createBuildDto: CreateBuildDto = { - branchName: 'branchName', - project: project.id, - }; - return requestWithApiKey(app, 'post', '/builds', '').send(createBuildDto).expect(403); + return requestWithApiKey(app, 'post', '/builds', '') + .send({ branchName: 'branchName', project: project.id }) + .expect(403); }); }); describe('GET /', () => { it('200', async () => { - const build = await buildsController.create({ project: project.id, branchName: 'develop' }); + const build = await haveBuild(); return requestWithAuth(app, 'get', `/builds?projectId=${project.id}&take=${5}&skip=${0}`, user.token) .send() .expect(200) .expect((res) => { - expect(JSON.stringify(res.body)).toEqual( - JSON.stringify({ - data: [build], - total: 1, - take: 5, - skip: 0, - }) - ); + expect(JSON.stringify(res.body)).toEqual(JSON.stringify({ data: [build], total: 1, take: 5, skip: 0 })); }); }); @@ -228,7 +134,7 @@ describe('Builds (e2e)', () => { describe('GET /:id', () => { it('200', async () => { - const build = await buildsController.create({ project: project.id, branchName: 'develop' }); + const build = await haveBuild(); return requestWithAuth(app, 'get', `/builds/${build.id}`, user.token) .send() @@ -240,14 +146,8 @@ describe('Builds (e2e)', () => { }); describe('DELETE /', () => { - it('200', async () => { - const build = await buildsController.create({ project: project.id, branchName: 'develop' }); - - return requestWithAuth(app, 'delete', `/builds/${build.id}`, user.token).send().expect(200); - }); - it('401', async () => { - const build = await buildsController.create({ project: project.id, branchName: 'develop' }); + const build = await haveBuild(); return requestWithAuth(app, 'delete', `/builds/${build.id}`, '').send().expect(401); }); @@ -255,100 +155,49 @@ describe('Builds (e2e)', () => { describe('PATCH /', () => { it('200 jwt', async () => { - const build = await buildsController.create({ project: project.id, branchName: 'develop' }); + const build = await haveBuild(); - return requestWithAuth(app, 'patch', `/builds/${build.id}`, user.token) + const res = await requestWithAuth(app, 'patch', `/builds/${build.id}`, user.token) .send({ isRunning: false }) - .expect(200) - .expect((res) => { - expect(res.body.isRunning).toBe(false); - }); - }); - - it('200 api', async () => { - const build = await buildsController.create({ project: project.id, branchName: 'develop' }); + .expect(200); - return requestWithApiKey(app, 'patch', `/builds/${build.id}`, user.apiKey) - .send({ isRunning: false }) - .expect(200) - .expect((res) => { - expect(res.body.isRunning).toBe(false); - }); + expect(res.body.isRunning).toBe(false); }); - it('200 api without body stops build', async () => { - const build = await buildsController.create({ project: project.id, branchName: 'develop' }); - - return requestWithApiKey(app, 'patch', `/builds/${build.id}`, user.apiKey) - .send() - .expect(200) - .expect((res) => { - expect(res.body.isRunning).toBe(false); - }); - }); + it.each([ + ['without body', undefined], + ['with empty body', {}], + ])('200 api %s stops build', async (_name, body) => { + const build = await haveBuild(); - it('200 api with empty body stops build', async () => { - const build = await buildsController.create({ project: project.id, branchName: 'develop' }); + const res = await requestWithApiKey(app, 'patch', `/builds/${build.id}`, user.apiKey).send(body).expect(200); - return requestWithApiKey(app, 'patch', `/builds/${build.id}`, user.apiKey) - .send({}) - .expect(200) - .expect((res) => { - expect(res.body.isRunning).toBe(false); - }); + expect(res.body.isRunning).toBe(false); }); it('403', async () => { - const build = await buildsController.create({ project: project.id, branchName: 'develop' }); + const build = await haveBuild(); return requestWithAuth(app, 'patch', `/builds/${build.id}`, '').send().expect(403); }); }); describe('PATCH /:id/approve', () => { - it('200 default', async () => { - const { build } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - project.mainBranchName, - './test/image.png', - false - ); - - await buildsService.approve(build.id, false); - - const result = await buildsService.findOne(build.id); - expect(result).toEqual({ - ...build, - status: 'passed', - passedCount: 1, - failedCount: 0, - unresolvedCount: 0, - merge: false, - }); - }); - - it('200 with merge', async () => { - const { build } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - project.mainBranchName, - './test/image.png', - true - ); + it.each([ + ['default', false], + ['with merge', true], + ])('200 %s', async (_name, merge) => { + const { build } = await haveRun(merge); - await buildsService.approve(build.id, true); + await buildsService.approve(build.id, merge); - const result = await buildsService.findOne(build.id); - expect(result).toEqual({ + expect(await buildsService.findOne(build.id)).toEqual({ ...build, status: 'passed', passedCount: 1, failedCount: 0, unresolvedCount: 0, - merge: true, + merge, }); }); }); diff --git a/test/comparators.e2e-spec.ts b/test/comparators.e2e-spec.ts index 05df6494..903250ab 100644 --- a/test/comparators.e2e-spec.ts +++ b/test/comparators.e2e-spec.ts @@ -1,11 +1,9 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { INestApplication, ValidationPipe } from '@nestjs/common'; +import { INestApplication } from '@nestjs/common'; import { ImageComparison, Project, TestStatus } from '@prisma/client'; import { existsSync, readFileSync, promises as fs } from 'fs'; import { createServer } from 'net'; import { PNG } from 'pngjs'; import { randomUUID } from 'crypto'; -import { AppModule } from '../src/app.module'; import { UsersService } from '../src/users/users.service'; import { ProjectsService } from '../src/projects/projects.service'; import { BuildsService } from '../src/builds/builds.service'; @@ -18,20 +16,9 @@ import { DEFAULT_CONFIG as LOOKS_SAME_DEFAULT_CONFIG } from '../src/compare/libs import { DEFAULT_CONFIG as ODIFF_DEFAULT_CONFIG } from '../src/compare/libs/odiff/odiff.service'; import { DEFAULT_CONFIG as VLM_DEFAULT_CONFIG } from '../src/compare/libs/vlm/vlm.service'; import { Response } from 'supertest'; +import { createE2eApp, withEnv } from './api-helpers'; import { haveUserLogged, requestWithApiKey, requestWithAuth, LoggedUser } from './preconditions'; -/** - * Every comparator end to end: a real project with the comparator selected, - * test runs posted over HTTP (multipart, like the SDKs), the real looks-same - * library, the native odiff-bin binary and the HDD storage underneath. - * - * VLM: no Ollama or Gemini is reachable from the test machine. The real - * OllamaService is used for the "provider unreachable" case (it is pointed at - * a closed local port), the real GeminiService for the "no API key" case, and - * the model's verdict is stubbed with jest.spyOn on the OllamaService instance - * the app resolved, so everything around the provider call stays real. - */ - const SIZE = 360; const TOTAL_PIXELS = SIZE * SIZE; const SLOW_TEST = 90000; @@ -40,7 +27,6 @@ const image = readFileSync('./test/image.png'); const imageEdited = readFileSync('./test/image_edited.png'); const imageEdited2 = readFileSync('./test/image_edited_2.png'); -// The same picture with its last 60 columns cut off (300x360): a layout change const cropWidth = (buffer: Buffer, width: number): Buffer => { const source = PNG.sync.read(buffer); const target = new PNG({ width, height: source.height }); @@ -49,6 +35,11 @@ const cropWidth = (buffer: Buffer, width: number): Buffer => { }; const imageNarrow = cropWidth(image, 300); +const slow = (name: string, fn: () => Promise) => it(name, fn, SLOW_TEST); + +const NO_DIFF = { status: TestStatus.ok, diffPercent: 0, pixelMisMatchCount: 0, diffName: null }; +const NO_NUMBERS = { status: TestStatus.unresolved, diffName: null, diffPercent: null, pixelMisMatchCount: null }; + const WHOLE_IMAGE: IgnoreAreaDto[] = [{ x: 0, y: 0, width: SIZE, height: SIZE }]; const closedPort = (): Promise => @@ -74,21 +65,15 @@ describe('Comparators (e2e)', () => { const projectIds: string[] = []; beforeAll(async () => { - const moduleFixture: TestingModule = await Test.createTestingModule({ - imports: [AppModule], - }).compile(); - - app = moduleFixture.createNestApplication(); - app.useGlobalPipes(new ValidationPipe()); - usersService = moduleFixture.get(UsersService); - projectsService = moduleFixture.get(ProjectsService); - buildsService = moduleFixture.get(BuildsService); - testRunsService = moduleFixture.get(TestRunsService); - hddService = moduleFixture.get(HddService); - prismaService = moduleFixture.get(PrismaService); - ollamaService = moduleFixture.get(OllamaService); - - await app.init(); + const created = await createE2eApp(); + app = created.app; + usersService = created.get(UsersService); + projectsService = created.get(ProjectsService); + buildsService = created.get(BuildsService); + testRunsService = created.get(TestRunsService); + hddService = created.get(HddService); + prismaService = created.get(PrismaService); + ollamaService = created.get(OllamaService); user = await haveUserLogged(usersService); }); @@ -116,7 +101,6 @@ describe('Comparators (e2e)', () => { bulkApproveGroupBy: 'customTags', }); - // created through the projects API, the way the UI does it const DEFAULT_CONFIGS: Partial> = { [ImageComparison.lookSame]: LOOKS_SAME_DEFAULT_CONFIG, [ImageComparison.odiff]: ODIFF_DEFAULT_CONFIG, @@ -134,7 +118,6 @@ describe('Comparators (e2e)', () => { return res.body; }; - // resolves with the response: an async helper cannot hand back the chainable supertest request const postTestRun = async ( project: Project, imageBuffer: Buffer, @@ -164,407 +147,180 @@ describe('Comparators (e2e)', () => { const diffExists = (imageName: string) => existsSync(hddService.getImagePath(imageName)); - // posts the first image and approves it, so the next run of the same screen has a baseline - // and resolves with the name of the baseline image file const haveBaseline = async (project: Project, imageBuffer: Buffer = image): Promise => { const res = await postTestRun(project, imageBuffer); expect(res.body.status).toBe(TestStatus.new); const approved = await testRunsService.approve(res.body.id); - // approving gave the run a variation and the variation a baseline const baselineName = (await prismaService.testVariation.findUnique({ where: { id: approved.testVariationId! } }))! .baselineName!; expect(diffExists(baselineName)).toBe(true); return baselineName; }; - describe('lookSame', () => { - it( - 'new without a baseline, ok for the same image', - async () => { - const project = await haveProject(ImageComparison.lookSame); - await haveBaseline(project); - - const res = await postTestRun(project, image); - - expect(res.body).toMatchObject({ - status: TestStatus.ok, - diffPercent: 0, - pixelMisMatchCount: 0, - diffName: null, - }); - }, - SLOW_TEST - ); - - it( - 'unresolved with a diff file for a changed image', - async () => { - const project = await haveProject(ImageComparison.lookSame); - await haveBaseline(project); - - const res = await postTestRun(project, imageEdited); - - expect(res.body).toMatchObject({ - status: TestStatus.unresolved, - pixelMisMatchCount: 31068, - diffPercent: (31068 * 100) / TOTAL_PIXELS, - // thumbnails and change signatures are pixelmatch-only - imageThumbnailName: null, - diffThumbnailName: null, - }); - expect(res.body.diffName).toMatch(/\.diff\.png$/); - expect(diffExists(res.body.diffName)).toBe(true); - // looks-same writes a full-size diff image - const diff = PNG.sync.read(await fs.readFile(hddService.getImagePath(res.body.diffName))); - expect([diff.width, diff.height]).toEqual([SIZE, SIZE]); - }, - SLOW_TEST - ); - - it( - 'ok, without a diff, when the change is within diffTollerancePercent', - async () => { - const project = await haveProject(ImageComparison.lookSame); - await haveBaseline(project); + const compare = async ( + imageComparison: ImageComparison, + imageComparisonConfig: string | undefined, + candidate: Buffer, + extra: Parameters[2] = {} + ) => { + const project = await haveProject(imageComparison, imageComparisonConfig); + await haveBaseline(project); + return { project, res: await postTestRun(project, candidate, extra) }; + }; - const res = await postTestRun(project, imageEdited, { diffTollerancePercent: 100 }); + describe.each([ + { + comparator: ImageComparison.lookSame, + changedCount: 31068, + changedPercent: (31068 * 100) / TOTAL_PIXELS, + resizedConfig: { allowDiffDimensions: true }, + resizedPercent: (60 * SIZE * 100) / TOTAL_PIXELS, + lenientConfig: { tolerance: 100 }, + }, + { + comparator: ImageComparison.odiff, + changedCount: 31145, + changedPercent: 24.03, + resizedConfig: { outputDiffMask: true, failOnLayoutDiff: false, threshold: 0, antialiasing: true }, + resizedPercent: 16.67, + lenientConfig: { threshold: 1 }, + }, + ])('$comparator', ({ comparator, changedCount, changedPercent, resizedConfig, resizedPercent, lenientConfig }) => { + slow('ok for the same image', async () => { + const { res } = await compare(comparator, undefined, image); + + expect(res.body).toMatchObject(NO_DIFF); + }); - expect(res.body).toMatchObject({ diffTollerancePercent: 100, status: TestStatus.ok, diffName: null }); - expect(res.body.pixelMisMatchCount).toBeGreaterThan(0); - }, - SLOW_TEST - ); + slow('unresolved with a diff file for a changed image', async () => { + const { res } = await compare(comparator, undefined, imageEdited); + + expect(res.body).toMatchObject({ + status: TestStatus.unresolved, + pixelMisMatchCount: changedCount, + diffPercent: changedPercent, + imageThumbnailName: null, + diffThumbnailName: null, + }); + expect(res.body.diffName).toMatch(/\.diff\.png$/); + expect(diffExists(res.body.diffName)).toBe(true); + const diff = PNG.sync.read(await fs.readFile(hddService.getImagePath(res.body.diffName))); + expect([diff.width, diff.height]).toEqual([SIZE, SIZE]); + }); - it( - 'unresolved when the change is over diffTollerancePercent', - async () => { - const project = await haveProject(ImageComparison.lookSame); - await haveBaseline(project); + slow('ok, without a diff, when the change is within diffTollerancePercent', async () => { + const { res } = await compare(comparator, undefined, imageEdited, { diffTollerancePercent: 100 }); - const res = await postTestRun(project, imageEdited, { diffTollerancePercent: 0.01 }); + expect(res.body).toMatchObject({ + diffTollerancePercent: 100, + status: TestStatus.ok, + diffName: null, + pixelMisMatchCount: changedCount, + diffPercent: changedPercent, + }); + }); - expect(res.body.status).toBe(TestStatus.unresolved); - expect(diffExists(res.body.diffName)).toBe(true); - }, - SLOW_TEST - ); + slow('unresolved when the change is over diffTollerancePercent', async () => { + const { res } = await compare(comparator, undefined, imageEdited, { diffTollerancePercent: 0.01 }); - it( - 'ok when an ignore area covers the whole change', - async () => { - const project = await haveProject(ImageComparison.lookSame); - await haveBaseline(project); + expect(res.body.status).toBe(TestStatus.unresolved); + expect(diffExists(res.body.diffName)).toBe(true); + }); - const res = await postTestRun(project, imageEdited, { ignoreAreas: WHOLE_IMAGE }); + slow('ok when an ignore area covers the whole change', async () => { + const { res } = await compare(comparator, undefined, imageEdited, { ignoreAreas: WHOLE_IMAGE }); - expect(res.body).toMatchObject({ - status: TestStatus.ok, - diffPercent: 0, - pixelMisMatchCount: 0, - diffName: null, - }); - }, - SLOW_TEST - ); + expect(res.body).toMatchObject(NO_DIFF); + }); - it( - 'unresolved without numbers for a different size', - async () => { - const project = await haveProject(ImageComparison.lookSame); - await haveBaseline(project); + slow('unresolved without numbers for a different size', async () => { + const { res } = await compare(comparator, undefined, imageNarrow); - const res = await postTestRun(project, imageNarrow); + expect(res.body).toMatchObject(NO_NUMBERS); + }); - expect(res.body).toMatchObject({ - status: TestStatus.unresolved, - diffName: null, - diffPercent: null, - pixelMisMatchCount: null, - }); - }, - SLOW_TEST - ); + slow('compares a different size when the project config allows it', async () => { + const { res } = await compare(comparator, JSON.stringify(resizedConfig), imageNarrow); - it( - 'compares a different size when the project config allows it', - async () => { - const project = await haveProject(ImageComparison.lookSame, JSON.stringify({ allowDiffDimensions: true })); - await haveBaseline(project); + expect(res.body).toMatchObject({ + status: TestStatus.unresolved, + pixelMisMatchCount: 60 * SIZE, + diffPercent: resizedPercent, + }); + expect(diffExists(res.body.diffName)).toBe(true); + }); - const res = await postTestRun(project, imageNarrow); + slow('reads the tolerance from the project config', async () => { + const { res } = await compare(comparator, JSON.stringify(lenientConfig), imageEdited); - // the 60 missing columns are the difference, counted against the baseline's area - expect(res.body).toMatchObject({ - status: TestStatus.unresolved, - pixelMisMatchCount: 60 * SIZE, - diffPercent: (60 * SIZE * 100) / TOTAL_PIXELS, - }); - expect(diffExists(res.body.diffName)).toBe(true); - }, - SLOW_TEST - ); + expect(res.body).toMatchObject(NO_DIFF); + }); - it( - 'reads tolerance from the project config', - async () => { - // a partial config replaces the defaults instead of being merged into them - const project = await haveProject(ImageComparison.lookSame, JSON.stringify({ tolerance: 100 })); - await haveBaseline(project); + slow('treats a missing baseline file as no baseline', async () => { + const project = await haveProject(comparator); + await fs.unlink(hddService.getImagePath(await haveBaseline(project))); - const res = await postTestRun(project, imageEdited); + const res = await postTestRun(project, imageEdited); - // a ΔE tolerance of 100 accepts every colour change in this picture - expect(res.body).toMatchObject({ - status: TestStatus.ok, - diffPercent: 0, - pixelMisMatchCount: 0, - diffName: null, - }); - }, - SLOW_TEST - ); + expect(res.body).toMatchObject({ status: TestStatus.new, diffName: null, diffPercent: null }); + }); + }); + describe('lookSame', () => { it('the projects API rejects a config that is not JSON', async () => { await requestWithAuth(app, 'post', '/projects', user.token) .send(projectBody(ImageComparison.lookSame, 'not json')) .expect(400) .expect((res) => { expect(res.body.message).toBe('imageComparisonConfig must be a json string'); - expect(res.body.exception.response.message).toEqual(['imageComparisonConfig must be a json string']); }); }); - it( - 'falls back to the default config when the stored config is not JSON', - async () => { - const created = await haveProject(ImageComparison.lookSame); - // only reachable by writing the column directly: the API validates the JSON - const project = await prismaService.project.update({ - where: { id: created.id }, - data: { imageComparisonConfig: 'not json' }, - }); - await haveBaseline(project); - - const res = await postTestRun(project, imageEdited); - - // same numbers as with DEFAULT_CONFIG - expect(res.body).toMatchObject({ status: TestStatus.unresolved, pixelMisMatchCount: 31068 }); - expect(diffExists(res.body.diffName)).toBe(true); - }, - SLOW_TEST - ); - - it( - 'falls back to the default config when looks-same rejects the config', - async () => { - // looks-same refuses strict together with tolerance - const project = await haveProject(ImageComparison.lookSame, JSON.stringify({ strict: true, tolerance: 5 })); - await haveBaseline(project); + slow('falls back to the default config when the stored config is not JSON', async () => { + const created = await haveProject(ImageComparison.lookSame); + const project = await prismaService.project.update({ + where: { id: created.id }, + data: { imageComparisonConfig: 'not json' }, + }); + await haveBaseline(project); - const res = await postTestRun(project, imageEdited); + const res = await postTestRun(project, imageEdited); - // the same numbers as with the default config (see the config that is not JSON) - expect(res.body).toMatchObject({ status: TestStatus.unresolved, pixelMisMatchCount: 31068 }); - expect(diffExists(res.body.diffName)).toBe(true); - }, - SLOW_TEST - ); - - it( - 'treats a missing baseline file as no baseline', - async () => { - const project = await haveProject(ImageComparison.lookSame); - const baselineName = await haveBaseline(project); - await fs.unlink(hddService.getImagePath(baselineName)); + expect(res.body).toMatchObject({ status: TestStatus.unresolved, pixelMisMatchCount: 31068 }); + expect(diffExists(res.body.diffName)).toBe(true); + }); - const res = await postTestRun(project, imageEdited); + slow('falls back to the default config when looks-same rejects the config', async () => { + const { res } = await compare( + ImageComparison.lookSame, + JSON.stringify({ strict: true, tolerance: 5 }), + imageEdited + ); - expect(res.body).toMatchObject({ status: TestStatus.new, diffName: null, diffPercent: null }); - }, - SLOW_TEST - ); + expect(res.body).toMatchObject({ status: TestStatus.unresolved, pixelMisMatchCount: 31068 }); + expect(diffExists(res.body.diffName)).toBe(true); + }); }); describe('odiff', () => { - it( - 'ok for the same image', - async () => { - const project = await haveProject(ImageComparison.odiff); - await haveBaseline(project); - - const res = await postTestRun(project, image); - - expect(res.body).toMatchObject({ - status: TestStatus.ok, - diffPercent: 0, - pixelMisMatchCount: 0, - diffName: null, - }); - }, - SLOW_TEST - ); - - it( - 'unresolved with a diff file for a changed image', - async () => { - const project = await haveProject(ImageComparison.odiff); - await haveBaseline(project); - - const res = await postTestRun(project, imageEdited); - - // odiff rounds the percentage to two decimals - expect(res.body).toMatchObject({ - status: TestStatus.unresolved, - pixelMisMatchCount: 31145, - diffPercent: 24.03, - imageThumbnailName: null, - diffThumbnailName: null, - }); - expect(res.body.diffName).toMatch(/\.diff\.png$/); - expect(diffExists(res.body.diffName)).toBe(true); - }, - SLOW_TEST - ); - - it( - 'ok within diffTollerancePercent, without keeping a diff', - async () => { - const project = await haveProject(ImageComparison.odiff); - await haveBaseline(project); - - const res = await postTestRun(project, imageEdited, { diffTollerancePercent: 100 }); - - // the numbers are kept, the diff image odiff wrote is deleted - expect(res.body).toMatchObject({ - status: TestStatus.ok, - diffName: null, - pixelMisMatchCount: 31145, - diffPercent: 24.03, - }); - }, - SLOW_TEST - ); - - it( - 'ok when an ignore region covers the whole change', - async () => { - const project = await haveProject(ImageComparison.odiff); - await haveBaseline(project); - - const res = await postTestRun(project, imageEdited, { ignoreAreas: WHOLE_IMAGE }); - - expect(res.body).toMatchObject({ - status: TestStatus.ok, - diffPercent: 0, - pixelMisMatchCount: 0, - diffName: null, - }); - }, - SLOW_TEST - ); - - it( - 'unresolved without numbers for a different size', - async () => { - const project = await haveProject(ImageComparison.odiff); - await haveBaseline(project); - - const res = await postTestRun(project, imageNarrow); - - expect(res.body).toMatchObject({ - status: TestStatus.unresolved, - diffName: null, - diffPercent: null, - pixelMisMatchCount: null, - }); - }, - SLOW_TEST - ); - - it( - 'compares a different size when failOnLayoutDiff is off', - async () => { - const project = await haveProject( - ImageComparison.odiff, - JSON.stringify({ outputDiffMask: true, failOnLayoutDiff: false, threshold: 0, antialiasing: true }) + slow('fails the request when storage is not HDD', async () => { + const project = await haveProject(ImageComparison.odiff); + await haveBaseline(project); + + await withEnv({ STATIC_SERVICE: 's3' }, async () => { + const res = await postTestRun(project, imageEdited, {}, 500); + expect(res.body.message).toBe( + 'Odiff can only be used with HDD static service. Please use another image comparison lib in project settings or switch STATIC_SERVICE envitonmental variable to HDD.' ); - await haveBaseline(project); - - const res = await postTestRun(project, imageNarrow); - - expect(res.body).toMatchObject({ - status: TestStatus.unresolved, - pixelMisMatchCount: 60 * SIZE, - diffPercent: 16.67, - }); - expect(diffExists(res.body.diffName)).toBe(true); - }, - SLOW_TEST - ); - - it( - 'reads threshold from the project config', - async () => { - // a partial config replaces the defaults instead of being merged into them - const project = await haveProject(ImageComparison.odiff, JSON.stringify({ threshold: 1 })); - await haveBaseline(project); - - const res = await postTestRun(project, imageEdited); - - // the maximum colour threshold treats every changed pixel as equal - expect(res.body).toMatchObject({ - status: TestStatus.ok, - diffPercent: 0, - pixelMisMatchCount: 0, - diffName: null, - }); - }, - SLOW_TEST - ); - - it( - 'treats a missing baseline file as no baseline', - async () => { - const project = await haveProject(ImageComparison.odiff); - const baselineName = await haveBaseline(project); - await fs.unlink(hddService.getImagePath(baselineName)); - - const res = await postTestRun(project, imageEdited); + }); - expect(res.body).toMatchObject({ status: TestStatus.new, diffName: null, diffPercent: null }); - }, - SLOW_TEST - ); - - it( - 'fails the request when storage is not HDD', - async () => { - const project = await haveProject(ImageComparison.odiff); - await haveBaseline(project); - const previous = process.env.STATIC_SERVICE; - process.env.STATIC_SERVICE = 's3'; - try { - const res = await postTestRun(project, imageEdited, {}, 500); - expect(res.body.message).toBe( - 'Odiff can only be used with HDD static service. Please use another image comparison lib in project settings or switch STATIC_SERVICE envitonmental variable to HDD.' - ); - } finally { - if (previous === undefined) { - delete process.env.STATIC_SERVICE; - } else { - process.env.STATIC_SERVICE = previous; - } - } - // refused before the run was stored: only the baseline's run is there - const runs = await prismaService.testRun.findMany({ where: { projectId: project.id } }); - expect(runs.map((run) => run.status)).toEqual([TestStatus.approved]); - }, - SLOW_TEST - ); + const runs = await prismaService.testRun.findMany({ where: { projectId: project.id } }); + expect(runs.map((run) => run.status)).toEqual([TestStatus.approved]); + }); }); describe.each([ImageComparison.lookSame, ImageComparison.odiff])('%s on a feature branch', (imageComparison) => { - it( + slow( 'auto-approves a run that matches an older main-branch baseline, and compares without keeping diffs', async () => { const created = await haveProject(imageComparison); @@ -573,41 +329,32 @@ describe('Comparators (e2e)', () => { data: { autoApproveFeature: true }, }); await haveBaseline(project, image); - // a second approval on main: the variation's baseline history is now [edited, image] const res = await postTestRun(project, imageEdited); expect(res.body.status).toBe(TestStatus.unresolved); await testRunsService.approve(res.body.id); const matching = await postTestRun(project, image, { branchName: 'feature' }); - // compared with the older baseline with saveDiffAsFile off, then auto-approved expect(matching.body.status).toBe(TestStatus.autoApproved); const other = await postTestRun(project, imageEdited2, { branchName: 'feature' }); expect(other.body.status).toBe(TestStatus.unresolved); - // the diff kept is the one against the current baseline expect(diffExists(other.body.diffName)).toBe(true); - }, - SLOW_TEST + } ); }); describe('vlm', () => { const vlmConfig = JSON.stringify({ provider: 'ollama', model: 'llava:7b', prompt: 'compare', temperature: 0 }); + const answer = (verdict: object) => ({ content: JSON.stringify(verdict) }); - it( - 'ok for the same image without asking the model', - async () => { - const generate = jest.spyOn(ollamaService, 'generate'); - const project = await haveProject(ImageComparison.vlm, vlmConfig); - await haveBaseline(project); + slow('ok for the same image without asking the model', async () => { + const generate = jest.spyOn(ollamaService, 'generate'); - const res = await postTestRun(project, image); + const { res } = await compare(ImageComparison.vlm, vlmConfig, image); - expect(res.body).toMatchObject({ status: TestStatus.ok, diffName: null, vlmDescription: null }); - expect(generate).not.toHaveBeenCalled(); - }, - SLOW_TEST - ); + expect(res.body).toMatchObject({ status: TestStatus.ok, diffName: null, vlmDescription: null }); + expect(generate).not.toHaveBeenCalled(); + }); it.each([ [true, TestStatus.ok], @@ -617,11 +364,9 @@ describe('Comparators (e2e)', () => { async (identical, status) => { const generate = jest .spyOn(ollamaService, 'generate') - .mockResolvedValue({ content: JSON.stringify({ identical, description: 'Button moved' }) }); - const project = await haveProject(ImageComparison.vlm, vlmConfig); - await haveBaseline(project); + .mockResolvedValue(answer({ identical, description: 'Button moved' })); - const res = await postTestRun(project, imageEdited); + const { res } = await compare(ImageComparison.vlm, vlmConfig, imageEdited); expect(res.body.status).toBe(status); expect(res.body.vlmDescription).toBe('Button moved'); @@ -636,24 +381,19 @@ describe('Comparators (e2e)', () => { SLOW_TEST ); - it( - 'reads the verdict from thinking when useThinking is set', - async () => { - jest - .spyOn(ollamaService, 'generate') - .mockResolvedValue({ content: '', thinking: JSON.stringify({ identical: true, description: 'Same' }) }); - const project = await haveProject( - ImageComparison.vlm, - JSON.stringify({ ...JSON.parse(vlmConfig), useThinking: true }) - ); - await haveBaseline(project); + slow('reads the verdict from thinking when useThinking is set', async () => { + jest + .spyOn(ollamaService, 'generate') + .mockResolvedValue({ content: '', thinking: JSON.stringify({ identical: true, description: 'Same' }) }); - const res = await postTestRun(project, imageEdited); + const { res } = await compare( + ImageComparison.vlm, + JSON.stringify({ ...JSON.parse(vlmConfig), useThinking: true }), + imageEdited + ); - expect(res.body).toMatchObject({ status: TestStatus.ok, vlmDescription: 'Same' }); - }, - SLOW_TEST - ); + expect(res.body).toMatchObject({ status: TestStatus.ok, vlmDescription: 'Same' }); + }); it.each([ [ @@ -663,12 +403,10 @@ describe('Comparators (e2e)', () => { [{ content: '', thinking: '' }, 'VLM analysis failed: Empty response from model'], ])( 'keeps the pixelmatch verdict when the model answers %j', - async (answer, description) => { - jest.spyOn(ollamaService, 'generate').mockResolvedValue(answer); - const project = await haveProject(ImageComparison.vlm, vlmConfig); - await haveBaseline(project); + async (response, description) => { + jest.spyOn(ollamaService, 'generate').mockResolvedValue(response); - const res = await postTestRun(project, imageEdited); + const { res } = await compare(ImageComparison.vlm, vlmConfig, imageEdited); expect(res.body.status).toBe(TestStatus.unresolved); expect(res.body.vlmDescription).toBe(description); @@ -676,74 +414,44 @@ describe('Comparators (e2e)', () => { SLOW_TEST ); - it( - 'keeps the pixelmatch verdict when Ollama is unreachable', - async () => { - // the Ollama client is created lazily from OLLAMA_BASE_URL on first use - const previous = process.env.OLLAMA_BASE_URL; - process.env.OLLAMA_BASE_URL = `http://127.0.0.1:${await closedPort()}`; - try { - const project = await haveProject(ImageComparison.vlm, vlmConfig); - await haveBaseline(project); - - const res = await postTestRun(project, imageEdited); - - expect(res.body.status).toBe(TestStatus.unresolved); - expect(res.body.vlmDescription).toBe('VLM analysis failed: fetch failed'); - expect(diffExists(res.body.diffName)).toBe(true); - } finally { - if (previous === undefined) { - delete process.env.OLLAMA_BASE_URL; - } else { - process.env.OLLAMA_BASE_URL = previous; - } - } - }, - SLOW_TEST - ); - - it( - 'keeps the pixelmatch verdict when the Gemini config has no API key', - async () => { - const project = await haveProject( - ImageComparison.vlm, - JSON.stringify({ provider: 'gemini', model: 'gemini-2.0-flash', prompt: 'compare', temperature: 0 }) - ); - await haveBaseline(project); - - const res = await postTestRun(project, imageEdited); + slow('keeps the pixelmatch verdict when Ollama is unreachable', async () => { + await withEnv({ OLLAMA_BASE_URL: `http://127.0.0.1:${await closedPort()}` }, async () => { + const { res } = await compare(ImageComparison.vlm, vlmConfig, imageEdited); expect(res.body.status).toBe(TestStatus.unresolved); - expect(res.body.vlmDescription).toBe('VLM analysis failed: Gemini API key is required'); - }, - SLOW_TEST - ); + expect(res.body.vlmDescription).toBe('VLM analysis failed: fetch failed'); + expect(diffExists(res.body.diffName)).toBe(true); + }); + }); - it( - 'sends the project config as is when it is not a VLM config', - async () => { - const generate = jest - .spyOn(ollamaService, 'generate') - .mockResolvedValue({ content: JSON.stringify({ identical: false, description: 'Changed' }) }); - // no imageComparisonConfig: the column default is a pixelmatch config - const project = await projectsService.create({ - ...projectBody(ImageComparison.vlm, undefined as unknown as string), - }); - projectIds.push(project.id); - await haveBaseline(project); + slow('keeps the pixelmatch verdict when the Gemini config has no API key', async () => { + const { res } = await compare( + ImageComparison.vlm, + JSON.stringify({ provider: 'gemini', model: 'gemini-2.0-flash', prompt: 'compare', temperature: 0 }), + imageEdited + ); - const res = await postTestRun(project, imageEdited); + expect(res.body.status).toBe(TestStatus.unresolved); + expect(res.body.vlmDescription).toBe('VLM analysis failed: Gemini API key is required'); + }); - expect(res.body).toMatchObject({ status: TestStatus.unresolved, vlmDescription: 'Changed' }); - // the column default is pixelmatch's, laid over the VLM defaults: Ollama still gets a model and a prompt - expect(generate).toHaveBeenCalledTimes(1); - expect(generate.mock.calls[0][0]).toMatchObject({ - model: VLM_DEFAULT_CONFIG.model, - prompt: VLM_DEFAULT_CONFIG.prompt, - temperature: VLM_DEFAULT_CONFIG.temperature, - }); - }, - SLOW_TEST - ); + slow('sends the project config as is when it is not a VLM config', async () => { + const generate = jest + .spyOn(ollamaService, 'generate') + .mockResolvedValue(answer({ identical: false, description: 'Changed' })); + const project = await projectsService.create(projectBody(ImageComparison.vlm, undefined as unknown as string)); + projectIds.push(project.id); + await haveBaseline(project); + + const res = await postTestRun(project, imageEdited); + + expect(res.body).toMatchObject({ status: TestStatus.unresolved, vlmDescription: 'Changed' }); + expect(generate).toHaveBeenCalledTimes(1); + expect(generate.mock.calls[0][0]).toMatchObject({ + model: VLM_DEFAULT_CONFIG.model, + prompt: VLM_DEFAULT_CONFIG.prompt, + temperature: VLM_DEFAULT_CONFIG.temperature, + }); + }); }); }); diff --git a/test/diff-worker-pool.e2e-spec.ts b/test/diff-worker-pool.e2e-spec.ts new file mode 100644 index 00000000..0e3e9c7c --- /dev/null +++ b/test/diff-worker-pool.e2e-spec.ts @@ -0,0 +1,71 @@ +import { execFileSync } from 'child_process'; +import { existsSync, readFileSync } from 'fs'; +import { join } from 'path'; + +const ROOT = join(__dirname, '..'); +const DIST = join(ROOT, 'dist', 'compare'); + +type PoolModule = typeof import('../src/compare/diff-worker-pool'); +type JobModule = typeof import('../src/compare/libs/pixelmatch/worker-job'); + +describe('DiffWorkerPool on real worker threads (compiled build)', () => { + const baseline = readFileSync(join(__dirname, 'image.png')); + const image = readFileSync(join(__dirname, 'image_edited.png')); + const workersCount = process.env.DIFF_WORKERS_COUNT; + let pool: InstanceType; + let runWorkerJob: JobModule['runWorkerJob']; + + beforeAll(() => { + execFileSync('npm', ['run', 'build'], { cwd: ROOT, stdio: 'pipe' }); + const { DiffWorkerPool } = require(join(DIST, 'diff-worker-pool.js')) as PoolModule; + ({ runWorkerJob } = require(join(DIST, 'libs', 'pixelmatch', 'worker-job.js')) as JobModule); + process.env.DIFF_WORKERS_COUNT = '2'; + pool = new DiffWorkerPool(); + }, 120000); + + afterAll(async () => { + await pool?.onApplicationShutdown(); + if (workersCount === undefined) delete process.env.DIFF_WORKERS_COUNT; + else process.env.DIFF_WORKERS_COUNT = workersCount; + }); + + const internals = () => pool as unknown as { inline: boolean; workers: unknown[] }; + + it('finds the worker file where the build puts it', () => { + expect(existsSync(join(DIST, 'libs', 'pixelmatch', 'pixelmatch.worker.js'))).toBe(true); + expect(internals().inline).toBe(false); + }); + + it.each([ + [ + 'diff', + { + kind: 'diff', + ignoreAreas: [], + threshold: 0.1, + includeAA: false, + diffTolerancePercent: 0, + saveDiff: true, + }, + ], + ['signature', { kind: 'signature', ignoreAreas: [], threshold: 0.1, includeAA: false }], + ['blur-amplitude', { kind: 'blur-amplitude', ignoreAreas: [], gateIfNoise: { threshold: 80 } }], + [ + 'looks-same', + { + kind: 'looks-same', + ignoreAreas: [], + config: { strict: false, ignoreAntialiasing: true, ignoreCaret: true, allowDiffDimensions: false }, + saveDiff: true, + }, + ], + ['vlm-inputs', { kind: 'vlm-inputs', ignoreAreas: [] }], + ])('runs a %s job in a worker with the same result as inline', async (_kind, params) => { + const input = { ...params, baseline, image } as Parameters[0]; + + const output = await pool.run(input); + + expect(internals().workers.length).toBeGreaterThan(0); + expect(output).toEqual(await runWorkerJob(input)); + }); +}); diff --git a/test/jest.config.ts b/test/jest.config.ts index 013478ae..86301f1e 100644 --- a/test/jest.config.ts +++ b/test/jest.config.ts @@ -1,4 +1,3 @@ -/** @returns {Promise} */ module.exports = async () => { return { displayName: 'E2E Tests', @@ -6,7 +5,7 @@ module.exports = async () => { testTimeout: 30000, testRegex: '.e2e-spec.ts$', transform: { - '^.+\\.tsx?$': ['ts-jest', { tsconfig: '/../tsconfig.jest.json' }], + '^.+\\.(t|j)s$': '@swc/jest', }, coverageDirectory: '../coverage', testEnvironment: 'node', diff --git a/test/ollama.e2e-spec.ts b/test/ollama.e2e-spec.ts index 32191aba..b4187c90 100644 --- a/test/ollama.e2e-spec.ts +++ b/test/ollama.e2e-spec.ts @@ -1,10 +1,9 @@ -import { Test, TestingModule } from '@nestjs/testing'; import { INestApplication } from '@nestjs/common'; import { Role } from '@prisma/client'; import request from 'supertest'; -import { AppModule } from '../src/app.module'; import { UsersService } from '../src/users/users.service'; import { OllamaService } from '../src/compare/libs/vlm/providers/ollama/ollama.service'; +import { createE2eApp } from './api-helpers'; import { haveUserLogged, requestWithApiKey, requestWithAuth } from './preconditions'; const image = './test/image.png'; @@ -26,17 +25,9 @@ describe('Ollama (e2e)', () => { }; beforeAll(async () => { - const moduleFixture: TestingModule = await Test.createTestingModule({ - imports: [AppModule], - }) - .overrideProvider(OllamaService) - .useValue(ollamaServiceMock) - .compile(); - - app = moduleFixture.createNestApplication(); - usersService = moduleFixture.get(UsersService); - - await app.init(); + const created = await createE2eApp({ validation: false, overrides: [[OllamaService, ollamaServiceMock]] }); + app = created.app; + usersService = created.get(UsersService); }); beforeEach(() => { @@ -79,8 +70,6 @@ describe('Ollama (e2e)', () => { }); describe('POST /ollama/compare', () => { - // Guards reject before the upload is read, so rejected requests are sent without files: - // streaming a multipart body into an already closed connection fails with EPIPE. it('401 without authentication', async () => { await request(app.getHttpServer()).post(compareUrl).expect(401); expect(ollamaServiceMock.generate).not.toHaveBeenCalled(); diff --git a/test/preconditions.ts b/test/preconditions.ts index 26babde6..1a3e0b2d 100644 --- a/test/preconditions.ts +++ b/test/preconditions.ts @@ -10,7 +10,6 @@ import { Build, Role } from '@prisma/client'; import { CreateUserDto } from 'src/users/dto/user-create.dto'; import { UserLoginResponseDto } from 'src/users/dto/user-login-response.dto'; -/** A signed-in user: login always issues a token, only creating a user answers without one. */ export type LoggedUser = UserLoginResponseDto & { token: string }; export const generateUser = (password: string): CreateUserDto => ({ @@ -49,6 +48,19 @@ export const haveUserLogged = async (usersService: UsersService, role: Role = 'a }) as Promise; }; +export const withUser = async ( + usersService: UsersService, + role: Role, + fn: (user: LoggedUser) => PromiseLike +): Promise => { + const user = await haveUserLogged(usersService, role); + try { + return await fn(user); + } finally { + await usersService.delete(user.id); + } +}; + export const haveTestRunCreated = async ( buildsService: BuildsService, testRunsService: TestRunsService, diff --git a/test/project-favorites.e2e-spec.ts b/test/project-favorites.e2e-spec.ts index 2e748249..fb3a32df 100644 --- a/test/project-favorites.e2e-spec.ts +++ b/test/project-favorites.e2e-spec.ts @@ -1,13 +1,11 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { INestApplication, ValidationPipe } from '@nestjs/common'; +import { INestApplication } from '@nestjs/common'; import { Project, Role } from '@prisma/client'; import request from 'supertest'; import { randomUUID } from 'crypto'; -import { AppModule } from '../src/app.module'; import { UsersService } from '../src/users/users.service'; import { PrismaService } from '../src/prisma/prisma.service'; import { ProjectsService } from '../src/projects/projects.service'; -import { TEST_PROJECT } from '../src/_data_'; +import { createE2eApp, haveProject as createProject } from './api-helpers'; import { haveUserLogged, requestWithAuth, LoggedUser } from './preconditions'; describe('Project favourites (e2e)', () => { @@ -25,7 +23,7 @@ describe('Project favourites (e2e)', () => { }; const haveProject = async (): Promise => { - const project = await projectsService.create({ ...TEST_PROJECT, name: `Favourite ${randomUUID()}` }); + const project = await createProject(projectsService, 'Favourite'); projectIds.push(project.id); return project; }; @@ -34,20 +32,14 @@ describe('Project favourites (e2e)', () => { (await requestWithAuth(app, 'get', '/projects/favorites', user.token).expect(200)).body; beforeAll(async () => { - const moduleFixture: TestingModule = await Test.createTestingModule({ - imports: [AppModule], - }).compile(); - - app = moduleFixture.createNestApplication(); - app.useGlobalPipes(new ValidationPipe()); - usersService = moduleFixture.get(UsersService); - projectsService = moduleFixture.get(ProjectsService); - prismaService = moduleFixture.get(PrismaService); - await app.init(); + const created = await createE2eApp(); + app = created.app; + usersService = created.get(UsersService); + projectsService = created.get(ProjectsService); + prismaService = created.get(PrismaService); }); afterEach(async () => { - // some tests delete their project or user already await prismaService.project.deleteMany({ where: { id: { in: projectIds.splice(0) } } }); await prismaService.user.deleteMany({ where: { id: { in: userIds.splice(0) } } }); }); diff --git a/test/projects-remove.e2e-spec.ts b/test/projects-remove.e2e-spec.ts new file mode 100644 index 00000000..48054183 --- /dev/null +++ b/test/projects-remove.e2e-spec.ts @@ -0,0 +1,71 @@ +import { HttpException, HttpStatus, INestApplication } from '@nestjs/common'; +import { Project } from '@prisma/client'; +import { BuildsService } from '../src/builds/builds.service'; +import { PrismaService } from '../src/prisma/prisma.service'; +import { ProjectsService } from '../src/projects/projects.service'; +import { TestRunsService } from '../src/test-runs/test-runs.service'; +import { createE2eApp, haveProject, imageExists } from './api-helpers'; +import { haveTestRunCreated } from './preconditions'; + +describe('Project removal (e2e)', () => { + let app: INestApplication; + let projectsService: ProjectsService; + let buildsService: BuildsService; + let testRunsService: TestRunsService; + let prisma: PrismaService; + let project: Project; + + beforeAll(async () => { + const created = await createE2eApp(); + app = created.app; + projectsService = created.get(ProjectsService); + buildsService = created.get(BuildsService); + testRunsService = created.get(TestRunsService); + prisma = created.get(PrismaService); + }); + + beforeEach(async () => { + project = await haveProject(projectsService, 'Remove E2E'); + }); + + afterEach(async () => { + await prisma.project.deleteMany({ where: { id: project.id } }); + }); + + afterAll(async () => { + await app.close(); + }); + + it('removes builds, runs, variations, baselines and their files in one go', async () => { + const { testRun } = await haveTestRunCreated( + buildsService, + testRunsService, + project.id, + project.mainBranchName, + './test/image.png' + ); + const approved = await testRunsService.approve(testRun.id, false); + const baseline = await prisma.baseline.findFirstOrThrow({ where: { testRunId: approved.id } }); + const { imageName } = await prisma.testRun.findUniqueOrThrow({ where: { id: testRun.id } }); + expect(imageExists(imageName)).toBe(true); + expect(imageExists(baseline.baselineName)).toBe(true); + + await projectsService.remove(project.id); + + expect(await prisma.project.count({ where: { id: project.id } })).toBe(0); + expect(await prisma.build.count({ where: { projectId: project.id } })).toBe(0); + expect(await prisma.testVariation.count({ where: { projectId: project.id } })).toBe(0); + expect(await prisma.testRun.count({ where: { id: testRun.id } })).toBe(0); + expect(await prisma.baseline.count({ where: { id: baseline.id } })).toBe(0); + expect(imageExists(imageName)).toBe(false); + expect(imageExists(baseline.baselineName)).toBe(false); + }); + + it('answers 404 for a project removed already', async () => { + await projectsService.remove(project.id); + + await expect(projectsService.remove(project.id)).rejects.toThrow( + new HttpException('Project not found', HttpStatus.NOT_FOUND) + ); + }); +}); diff --git a/test/projects.e2e-spec.ts b/test/projects.e2e-spec.ts index 24d9762e..3d2a409c 100644 --- a/test/projects.e2e-spec.ts +++ b/test/projects.e2e-spec.ts @@ -1,7 +1,6 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { INestApplication, ValidationPipe } from '@nestjs/common'; -import { AppModule } from '../src/app.module'; +import { INestApplication } from '@nestjs/common'; import { UsersService } from '../src/users/users.service'; +import { createE2eApp } from './api-helpers'; import { requestWithAuth, haveUserLogged, LoggedUser } from './preconditions'; import { ProjectsService } from '../src/projects/projects.service'; import { randomUUID } from 'crypto'; @@ -36,18 +35,9 @@ describe('Projects (e2e)', () => { let loggedUser: LoggedUser; beforeAll(async () => { - const moduleFixture: TestingModule = await Test.createTestingModule({ - imports: [AppModule], - }) - .overrideProvider(ProjectsService) - .useValue(projectServiceMock) - .compile(); - - app = moduleFixture.createNestApplication(); - app.useGlobalPipes(new ValidationPipe()); - usersService = moduleFixture.get(UsersService); - - await app.init(); + const created = await createE2eApp({ overrides: [[ProjectsService, projectServiceMock]] }); + app = created.app; + usersService = created.get(UsersService); }); beforeEach(async () => { @@ -62,72 +52,59 @@ describe('Projects (e2e)', () => { await app.close(); }); - describe('POST /', () => { - it('200', () => { - return requestWithAuth(app, 'post', '/projects', loggedUser.token) - .send(project) - .expect(201) - .expect((res) => { - expect(res.body.name).toBe(project.name); - expect(res.body.mainBranchName).toBe(project.mainBranchName); - expect(res.body.autoApproveFeature).toBe(project.autoApproveFeature); - expect(res.body.imageComparison).toBe(project.imageComparison); - expect(res.body.imageComparisonConfig).toBe(project.imageComparisonConfig); - expect(res.body.maxBuildAllowed).toBe(project.maxBuildAllowed); - expect(res.body.maxBranchLifetime).toBe(project.maxBranchLifetime); + it.each([ + ['post', '/projects', ''], + ['get', '/projects', ''], + ['delete', `/projects/${project.id}`, 'asd'], + ['put', '/projects', 'asd'], + ] as const)('%s %s answers 401 for token "%s"', (method, url, token) => + requestWithAuth(app, method, url, token).send(project).expect(401) + ); + + it('POST / creates a project', () => { + return requestWithAuth(app, 'post', '/projects', loggedUser.token) + .send(project) + .expect(201) + .expect((res) => { + expect(res.body).toMatchObject({ + name: project.name, + mainBranchName: project.mainBranchName, + autoApproveFeature: project.autoApproveFeature, + imageComparison: project.imageComparison, + imageComparisonConfig: project.imageComparisonConfig, + maxBuildAllowed: project.maxBuildAllowed, + maxBranchLifetime: project.maxBranchLifetime, }); - }); - - it('401', () => { - return requestWithAuth(app, 'post', '/projects', '').send(project).expect(401); - }); + }); }); - describe('GET /', () => { - it('200', async () => { - const res = await requestWithAuth(app, 'get', '/projects', loggedUser.token).send().expect(200); + it('GET / lists the projects', async () => { + const res = await requestWithAuth(app, 'get', '/projects', loggedUser.token).send().expect(200); - expect(res.body).toEqual(expect.arrayContaining(projectServiceMock.findAll())); - }); - - it('401', async () => { - await requestWithAuth(app, 'get', '/projects', '').send().expect(401); - }); + expect(res.body).toEqual(expect.arrayContaining(projectServiceMock.findAll())); }); - describe('DELETE /', () => { - it('can delete', async () => { - const res = await requestWithAuth(app, 'delete', `/projects/${project.id}`, loggedUser.token).send().expect(200); + it('DELETE /:id removes the project', async () => { + const res = await requestWithAuth(app, 'delete', `/projects/${project.id}`, loggedUser.token).send().expect(200); - expect(res.body).toStrictEqual({ - ...projectServiceMock.remove(), - createdAt: expect.any(String), - updatedAt: expect.any(String), - }); - }); - - it('not valid UUID', async () => { - await requestWithAuth(app, 'delete', `/projects/123`, loggedUser.token).send().expect(400); - }); - - it('not valid token', async () => { - await requestWithAuth(app, 'delete', `/projects/${project.id}`, 'asd').send().expect(401); + expect(res.body).toStrictEqual({ + ...projectServiceMock.remove(), + createdAt: expect.any(String), + updatedAt: expect.any(String), }); }); - describe('PUT /', () => { - it('can edit', async () => { - const res = await requestWithAuth(app, 'put', `/projects`, loggedUser.token).send(project).expect(200); + it('DELETE /:id answers 400 for an id that is not a UUID', async () => { + await requestWithAuth(app, 'delete', `/projects/123`, loggedUser.token).send().expect(400); + }); - expect(res.body).toStrictEqual({ - ...projectServiceMock.update(), - createdAt: expect.any(String), - updatedAt: expect.any(String), - }); - }); + it('PUT / edits the project', async () => { + const res = await requestWithAuth(app, 'put', `/projects`, loggedUser.token).send(project).expect(200); - it('not valid token', async () => { - await requestWithAuth(app, 'put', `/projects`, 'asd').send(project).expect(401); + expect(res.body).toStrictEqual({ + ...projectServiceMock.update(), + createdAt: expect.any(String), + updatedAt: expect.any(String), }); }); }); diff --git a/test/static.e2e-spec.ts b/test/static.e2e-spec.ts index 28b064ef..91e0633a 100644 --- a/test/static.e2e-spec.ts +++ b/test/static.e2e-spec.ts @@ -3,14 +3,12 @@ import { Project, TestStatus } from '@prisma/client'; import { readFileSync } from 'fs'; import { PNG } from 'pngjs'; import request from 'supertest'; -import { randomUUID } from 'crypto'; -import { TEST_PROJECT } from '../src/_data_'; import { BuildsService } from '../src/builds/builds.service'; import { PrismaService } from '../src/prisma/prisma.service'; import { ProjectsService } from '../src/projects/projects.service'; import { StaticService } from '../src/static/static.service'; import { TestRunsService } from '../src/test-runs/test-runs.service'; -import { createE2eApp, pngBuffer, RED, storedImagePath } from './api-helpers'; +import { CHANGED, createE2eApp, haveProject, pngBuffer, postRunDirect, storedImagePath } from './api-helpers'; describe('Images (e2e)', () => { let app: INestApplication; @@ -21,12 +19,9 @@ describe('Images (e2e)', () => { let prisma: PrismaService; let project: Project; - // supertest does not follow redirects, so the redirect itself is what is asserted const get = (path: string) => request(app.getHttpServer()).get(path); - /** Binary response bodies as a Buffer. */ const asBuffer = (res: request.Response, callback: (err: Error | null, body: Buffer) => void) => { - // in Node the parser is handed the raw response stream, which superagent's types do not model const stream = res as unknown as NodeJS.ReadableStream; const chunks: Buffer[] = []; stream.on('data', (chunk: Buffer) => chunks.push(chunk)); @@ -36,19 +31,15 @@ describe('Images (e2e)', () => { beforeAll(async () => { const created = await createE2eApp(); app = created.app; - projectsService = created.moduleFixture.get(ProjectsService); - buildsService = created.moduleFixture.get(BuildsService); - testRunsService = created.moduleFixture.get(TestRunsService); - staticService = created.moduleFixture.get(StaticService); - prisma = created.moduleFixture.get(PrismaService); + projectsService = created.get(ProjectsService); + buildsService = created.get(BuildsService); + testRunsService = created.get(TestRunsService); + staticService = created.get(StaticService); + prisma = created.get(PrismaService); }); beforeEach(async () => { - project = await projectsService.create({ - ...TEST_PROJECT, - autoApproveFeature: false, - name: `Images E2E ${randomUUID()}`, - }); + project = await haveProject(projectsService, 'Images E2E', { autoApproveFeature: false }); }); afterEach(async () => { @@ -60,13 +51,12 @@ describe('Images (e2e)', () => { await app.close(); }); - const postRun = async (image: Buffer) => { - const build = await buildsService.findOrCreate({ projectId: project.id, branchName: 'master' }); - return testRunsService.postTestRun({ - createTestRequestDto: { projectId: project.id, buildId: build.id, branchName: 'master', name: 'Screen' }, - imageBuffer: image, - }); - }; + const postRun = async (image: Buffer) => + postRunDirect( + testRunsService, + await buildsService.findOrCreate({ projectId: project.id, branchName: 'master' }), + image + ); describe('GET /images/:fileName', () => { it('redirects to the stored file, cacheable privately, without authentication', async () => { @@ -78,16 +68,13 @@ describe('Images (e2e)', () => { expect(res.headers['cache-control']).toBe('private, max-age=3000'); }); - it('a name that is not stored is still redirected', async () => { - // NOTE: current behavior — the HDD backend does not check the file, the static handler 404s after the redirect + it('redirects a well-formed name without checking that it is stored', async () => { const res = await get('/images/missing.screenshot.png').expect(302); expect(res.headers.location).toBe('/missing.screenshot.png'); }); - // a leading slash used to come back as a protocol-relative Location - // ('//example.com'): an open redirect to another host - it.each(['%2Fexample.com', '%2F%2Fexample.com', 'nested%2Fimage.png', '..'])( + it.each(['%2Fexample.com', '%2F%2Fexample.com', 'nested%2Fimage.png', '..', '%5Cevil.com'])( 'refuses %s as an image name instead of redirecting', async (name) => { const res = await get(`/images/${name}`).expect(400); @@ -100,7 +87,7 @@ describe('Images (e2e)', () => { it('500 when storage cannot produce a URL', async () => { jest.spyOn(staticService, 'getImageUrl').mockRejectedValueOnce(new Error('storage down')); - const res = await get('/images/any.png').expect(500); + const res = await get('/images/any.screenshot.png').expect(500); expect(res.text).toBe('Error occurred while getting the file.'); expect(res.headers['cache-control']).toBeUndefined(); @@ -115,13 +102,14 @@ describe('Images (e2e)', () => { expect(res.headers['content-type']).toBe('image/png'); expect(res.headers['content-disposition']).toBe(`attachment; filename="${run.imageName}"`); + expect(res.headers['cache-control']).toBe('private, max-age=31536000, immutable'); expect(Buffer.compare(res.body, readFileSync(storedImagePath(run.imageName)))).toBe(0); }); it('serves the diff and thumbnails of an unresolved run', async () => { const first = await postRun(pngBuffer()); await testRunsService.approve(first.id); - const run = await postRun(pngBuffer([{ x: 5, y: 5, w: 10, h: 10, color: RED }])); + const run = await postRun(CHANGED); expect(run.status).toBe(TestStatus.unresolved); const stored = (await prisma.testRun.findUnique({ where: { id: run.id } }))!; @@ -138,8 +126,8 @@ describe('Images (e2e)', () => { expect(res.body.message).toBe('Image not found: missing.screenshot.png'); }); - it('a name that merely starts with dots stays inside the image directory', async () => { - await get('/images/..missing.png/download').expect(404); + it('400 for a name that merely starts with dots, it is not a generated name', async () => { + await get('/images/..missing.png/download').expect(400); }); it.each([ diff --git a/test/tasks.e2e-spec.ts b/test/tasks.e2e-spec.ts index 7d2778f8..77d556ea 100644 --- a/test/tasks.e2e-spec.ts +++ b/test/tasks.e2e-spec.ts @@ -1,9 +1,6 @@ -import { Test, TestingModule } from '@nestjs/testing'; import { INestApplication } from '@nestjs/common'; import { Project, TestStatus } from '@prisma/client'; import { existsSync } from 'fs'; -import { randomUUID } from 'crypto'; -import { AppModule } from '../src/app.module'; import { CLEANUP_LOCK_KEY, TasksService } from '../src/shared/tasks/tasks.service'; import { PrismaService } from '../src/prisma/prisma.service'; import { ProjectsService } from '../src/projects/projects.service'; @@ -11,18 +8,12 @@ import { BuildsService } from '../src/builds/builds.service'; import { TestRunsService } from '../src/test-runs/test-runs.service'; import { HddService } from '../src/static/hdd/hdd.service'; import { TestVariationsService } from '../src/test-variations/test-variations.service'; -import { TEST_PROJECT } from '../src/_data_'; +import { createE2eApp, haveProject as createProject, waitFor } from './api-helpers'; import { haveTestRunCreated } from './preconditions'; const DAY = 24 * 60 * 60 * 1000; const image = './test/image.png'; -/** - * The nightly cleanup (@Cron at midnight) called directly on the TasksService - * instance of the app. It walks every project in the database, so the other - * e2e files running in parallel are affected only if they keep non-main - * variations untouched for longer than their project's maxBranchLifetime. - */ describe('Tasks: cleanOldTestVariations (e2e)', () => { let app: INestApplication; let tasksService: TasksService; @@ -35,19 +26,15 @@ describe('Tasks: cleanOldTestVariations (e2e)', () => { const projectIds: string[] = []; beforeAll(async () => { - const moduleFixture: TestingModule = await Test.createTestingModule({ - imports: [AppModule], - }).compile(); - - app = moduleFixture.createNestApplication(); - tasksService = moduleFixture.get(TasksService); - prismaService = moduleFixture.get(PrismaService); - projectsService = moduleFixture.get(ProjectsService); - buildsService = moduleFixture.get(BuildsService); - testRunsService = moduleFixture.get(TestRunsService); - hddService = moduleFixture.get(HddService); - testVariationsService = moduleFixture.get(TestVariationsService); - await app.init(); + const created = await createE2eApp({ validation: false }); + app = created.app; + tasksService = created.get(TasksService); + prismaService = created.get(PrismaService); + projectsService = created.get(ProjectsService); + buildsService = created.get(BuildsService); + testRunsService = created.get(TestRunsService); + hddService = created.get(HddService); + testVariationsService = created.get(TestVariationsService); }); afterEach(async () => { @@ -62,9 +49,7 @@ describe('Tasks: cleanOldTestVariations (e2e)', () => { }); const haveProject = async (maxBranchLifetime: number): Promise => { - const project = await projectsService.create({ - ...TEST_PROJECT, - name: `Tasks E2E ${randomUUID()}`, + const project = await createProject(projectsService, 'Tasks E2E', { maxBuildAllowed: 100, maxBranchLifetime, autoApproveFeature: false, @@ -73,10 +58,8 @@ describe('Tasks: cleanOldTestVariations (e2e)', () => { return project; }; - // an approved run on the branch: a variation with a baseline, a test run and their images const haveApprovedVariation = async (project: Project, branchName: string) => { const { testRun } = await haveTestRunCreated(buildsService, testRunsService, project.id, branchName, image); - // merge keeps the variation on this branch instead of the main one const approved = await testRunsService.approve(testRun.id, true); const variation = (await prismaService.testVariation.findUnique({ where: { id: approved.testVariationId! }, @@ -87,24 +70,12 @@ describe('Tasks: cleanOldTestVariations (e2e)', () => { return { variation, testRun: approved }; }; - // the variation and the runs compared against it, as if last used days ago const ageBy = async (variationId: string, days: number) => { const then = new Date(Date.now() - days * DAY); - // @updatedAt would be overwritten by an update through Prisma await prismaService.$executeRaw`UPDATE "TestVariation" SET "updatedAt" = ${then} WHERE "id" = ${variationId}`; await prismaService.testRun.updateMany({ where: { testVariationId: variationId }, data: { createdAt: then } }); }; - const waitUntil = async (condition: () => boolean, timeout = 5000) => { - const start = Date.now(); - while (!condition()) { - if (Date.now() - start > timeout) { - throw new Error('condition not met in time'); - } - await new Promise((resolve) => setTimeout(resolve, 50)); - } - }; - it('removes feature-branch variations not used within maxBranchLifetime, with their baselines', async () => { const project = await haveProject(5); const old = await haveApprovedVariation(project, 'feature/old'); @@ -121,14 +92,11 @@ describe('Tasks: cleanOldTestVariations (e2e)', () => { const variations = await prismaService.testVariation.findMany({ where: { projectId: project.id } }); expect(variations.map((variation) => variation.id).sort()).toEqual([fresh.variation.id, main.variation.id].sort()); expect(await prismaService.baseline.findMany({ where: { testVariationId: old.variation.id } })).toEqual([]); - // the baseline image is deleted without being awaited - await waitUntil(() => !existsSync(oldBaselineFile)); + await waitFor(() => !existsSync(oldBaselineFile)); - // the test runs stay in their builds, detached from the removed variation const oldRun = (await prismaService.testRun.findUnique({ where: { id: old.testRun.id } }))!; expect(oldRun).toMatchObject({ testVariationId: null, status: TestStatus.approved, branchName: 'feature/old' }); expect(existsSync(hddService.getImagePath(oldRun.imageName))).toBe(true); - // the kept variations keep their baselines expect(await prismaService.baseline.count({ where: { testVariationId: fresh.variation.id } })).toBe(1); expect(await prismaService.baseline.count({ where: { testVariationId: main.variation.id } })).toBe(1); }); @@ -184,7 +152,6 @@ describe('Tasks: cleanOldTestVariations (e2e)', () => { await ageBy(feature.variation.id, 10); const { testRun } = await haveTestRunCreated(buildsService, testRunsService, project.id, 'feature/active', image); expect(testRun.status).toBe(TestStatus.ok); - // a run that matches the baseline leaves the variation as it was const { updatedAt } = (await prismaService.testVariation.findUnique({ where: { id: feature.variation.id } }))!; expect(updatedAt.getTime()).toBeLessThan(Date.now() - 5 * DAY); @@ -206,7 +173,6 @@ describe('Tasks: cleanOldTestVariations (e2e)', () => { } const deleteSpy = jest.spyOn(testVariationsService, 'delete'); - // each run opens its own lock connection, as on two instances await Promise.all([tasksService.cleanOldTestVariations(), tasksService.cleanOldTestVariations()]); const deleted = deleteSpy.mock.calls.map(([id]) => id).filter((id) => stale.includes(id)); @@ -232,7 +198,6 @@ describe('Tasks: cleanOldTestVariations (e2e)', () => { await otherInstance.end(); } - // released with the other instance's connection await tasksService.cleanOldTestVariations(); expect(await prismaService.testVariation.findUnique({ where: { id: variation.id } })).toBeNull(); diff --git a/test/test-runs-review.e2e-spec.ts b/test/test-runs-review.e2e-spec.ts index 81cbc18b..8ef5030c 100644 --- a/test/test-runs-review.e2e-spec.ts +++ b/test/test-runs-review.e2e-spec.ts @@ -1,14 +1,25 @@ import { INestApplication } from '@nestjs/common'; import { Build, Project, TestStatus } from '@prisma/client'; import { randomUUID } from 'crypto'; -import { TEST_PROJECT } from '../src/_data_'; import { BuildsService } from '../src/builds/builds.service'; import { PrismaService } from '../src/prisma/prisma.service'; import { ProjectsService } from '../src/projects/projects.service'; import { TestRunDto } from '../src/test-runs/dto/testRun.dto'; import { UsersService } from '../src/users/users.service'; -import { BLUE, createE2eApp, imageExists, pngBuffer, RED, Rect, TEST_RUN_DTO_KEYS, waitFor } from './api-helpers'; -import { haveUserLogged, requestWithApiKey, requestWithAuth, LoggedUser } from './preconditions'; +import { + approveRuns, + BLUE, + createE2eApp, + haveProject, + imageExists, + pngBuffer, + postRunBase64, + redAt, + TEST_RUN_DTO_KEYS, + waitFor, + WHITE, +} from './api-helpers'; +import { haveUserLogged, requestWithApiKey, requestWithAuth, LoggedUser, withUser } from './preconditions'; describe('TestRuns review API (e2e)', () => { let app: INestApplication; @@ -19,25 +30,20 @@ describe('TestRuns review API (e2e)', () => { let user: LoggedUser; let project: Project; - const WHITE = pngBuffer(); - const redAt = (x: number, y: number, size = 10): Rect => ({ x, y, w: size, h: size, color: RED }); - beforeAll(async () => { const created = await createE2eApp(); app = created.app; - usersService = created.moduleFixture.get(UsersService); - projectsService = created.moduleFixture.get(ProjectsService); - buildsService = created.moduleFixture.get(BuildsService); - prisma = created.moduleFixture.get(PrismaService); + usersService = created.get(UsersService); + projectsService = created.get(ProjectsService); + buildsService = created.get(BuildsService); + prisma = created.get(PrismaService); }); beforeEach(async () => { user = await haveUserLogged(usersService); - project = await projectsService.create({ - ...TEST_PROJECT, + project = await haveProject(projectsService, 'TestRuns review E2E', { autoApproveFeature: false, maxBuildAllowed: 100, - name: `TestRuns review E2E ${randomUUID()}`, }); }); @@ -53,29 +59,14 @@ describe('TestRuns review API (e2e)', () => { const newBuild = (branchName = project.mainBranchName): Promise => buildsService.findOrCreate({ projectId: project.id, branchName }); - /** Posts a screenshot through the public base64 endpoint and returns the response body. */ - const postRun = async ( - build: Build, - image: Buffer, - extra: Record = {} - ): Promise> => { - const res = await requestWithApiKey(app, 'post', '/test-runs', user.apiKey) - .send({ - projectId: project.id, - buildId: build.id, - branchName: build.branchName, - name: 'Screen', - imageBase64: image.toString('base64'), - ...extra, - }) - .expect(201); - return res.body; - }; + const postRun = (build: Build, image: Buffer, extra: Record = {}) => + postRunBase64(app, user, build, image, extra); - const approve = (ids: string[], merge = false) => - requestWithAuth(app, 'post', `/test-runs/approve?merge=${merge}`, user.token).send(ids).expect(201); + const variationOf = async (id: string) => (await prisma.testVariation.findUnique({ where: { id } }))!; + const runOf = async (id: string) => (await prisma.testRun.findUnique({ where: { id } }))!; + + const approve = (ids: string[], merge = false) => approveRuns(app, user, ids, merge); - /** Approves `baseline` as the master baseline of the variation, then posts `image` against it in `build`. */ const runAgainstBaseline = async ( build: Build, baseline: Buffer, @@ -88,6 +79,22 @@ describe('TestRuns review API (e2e)', () => { return postRun(build, image, extra); }; + it.each([ + ['approve', 'post', '/test-runs/approve?merge=false', (id: string) => [id]], + ['reject', 'post', '/test-runs/reject', (id: string) => [id]], + ['revert', 'post', '/test-runs/revert', (id: string) => [id]], + ['delete', 'post', '/test-runs/delete', (id: string) => [id]], + ['update', 'patch', '/test-runs/update/:id', () => ({ comment: 'x' })], + ] as const)('403 for a guest on %s', async (_name, method, url, body) => { + const run = await postRun(await newBuild(), WHITE); + + await withUser(usersService, 'guest', (guest) => + requestWithAuth(app, method, url.replace(':id', run.id), guest.token).send(body(run.id)).expect(403) + ); + + expect((await runOf(run.id)).status).toBe(TestStatus.new); + }); + describe('POST /test-runs (base64)', () => { it('creates a new run and stores the image on disk', async () => { const build = await newBuild(); @@ -118,6 +125,14 @@ describe('TestRuns review API (e2e)', () => { it('403 without an API key', async () => { await requestWithApiKey(app, 'post', '/test-runs', '').send({}).expect(403); }); + + it('parallel builds uploading the same brand-new screen all succeed', async () => { + const builds = await Promise.all(Array.from({ length: 8 }, () => newBuild())); + + await Promise.all(builds.map((build) => postRun(build, WHITE, { name: 'Brand new screen' }))); + + expect(await prisma.testVariation.count({ where: { projectId: project.id, name: 'Brand new screen' } })).toBe(1); + }); }); describe('GET /test-runs?buildId=', () => { @@ -131,7 +146,6 @@ describe('TestRuns review API (e2e)', () => { const res = await requestWithAuth(app, 'get', `/test-runs?buildId=${build.id}`, user.token).expect(200); expect(res.body.map((run: TestRunDto) => run.id).sort()).toEqual([a.id, b.id].sort()); - // internal columns (pixelMisMatchCount, changeSignature, timestamps, projectId) are not exposed expect(Object.keys(res.body[0]).sort()).toEqual(TEST_RUN_DTO_KEYS); }); @@ -163,7 +177,6 @@ describe('TestRuns review API (e2e)', () => { expect(res.body.browser).toBe('firefox'); expect(res.body.testVariation.id).toBe(run.testVariationId); expect(res.body.testVariation.browser).toBe('firefox'); - // unlike the list, the details route returns the raw row expect(res.body).toHaveProperty('pixelMisMatchCount'); expect(res.body).toHaveProperty('changeSignature'); expect(res.body.projectId).toBe(project.id); @@ -189,7 +202,7 @@ describe('TestRuns review API (e2e)', () => { await approve([run.id]); - const stored = (await prisma.testRun.findUnique({ where: { id: run.id } }))!; + const stored = await runOf(run.id); expect(stored.status).toBe(TestStatus.approved); const variation = (await prisma.testVariation.findUnique({ where: { id: run.testVariationId }, @@ -213,7 +226,7 @@ describe('TestRuns review API (e2e)', () => { await approve([featureRun.id]); - const stored = (await prisma.testRun.findUnique({ where: { id: featureRun.id } }))!; + const stored = await runOf(featureRun.id); expect(stored.status).toBe(TestStatus.approved); expect(stored.testVariationId).not.toBe(masterRun.testVariationId); const featureVariation = (await prisma.testVariation.findUnique({ @@ -223,7 +236,6 @@ describe('TestRuns review API (e2e)', () => { expect(featureVariation.branchName).toBe('feature'); expect(featureVariation.comment).toBe('feature comment'); expect(featureVariation.baselines).toHaveLength(1); - // master keeps its single baseline expect(await prisma.baseline.count({ where: { testVariationId: masterRun.testVariationId } })).toBe(1); }); @@ -236,7 +248,7 @@ describe('TestRuns review API (e2e)', () => { await approve([featureRun.id], true); - const stored = (await prisma.testRun.findUnique({ where: { id: featureRun.id } }))!; + const stored = await runOf(featureRun.id); expect(stored.status).toBe(TestStatus.approved); expect(stored.merge).toBe(true); expect(stored.testVariationId).toBe(masterRun.testVariationId); @@ -249,30 +261,44 @@ describe('TestRuns review API (e2e)', () => { await approve([featureBaseline.id]); const masterBaseline = await postRun(await newBuild(), WHITE); await approve([masterBaseline.id]); - // master was updated last, so the new feature run is compared against master const featureRun = await postRun(await newBuild('feature'), pngBuffer([redAt(0, 0)])); expect(featureRun.testVariationId).toBe(masterBaseline.testVariationId); expect(featureRun.baselineBranchName).toBe(project.mainBranchName); await approve([featureRun.id]); - const stored = (await prisma.testRun.findUnique({ where: { id: featureRun.id } }))!; + const stored = await runOf(featureRun.id); expect(stored.testVariationId).toBe(featureBaseline.testVariationId); expect(await prisma.baseline.count({ where: { testVariationId: featureBaseline.testVariationId } })).toBe(2); expect(await prisma.baseline.count({ where: { testVariationId: masterBaseline.testVariationId } })).toBe(1); }); + it('approving a feature-branch run twice keeps one baseline', async () => { + const masterRun = await postRun(await newBuild(), WHITE); + await approve([masterRun.id]); + const featureRun = await postRun(await newBuild('feature'), pngBuffer([redAt(0, 0)])); + await approve([featureRun.id]); + const { testVariationId } = await runOf(featureRun.id); + const { baselineName } = await variationOf(testVariationId!); + + await approve([featureRun.id]); + + const baselines = await prisma.baseline.findMany({ where: { testVariationId: testVariationId! } }); + expect(baselines).toHaveLength(1); + expect(baselines[0].testRunId).toBe(featureRun.id); + expect((await variationOf(testVariationId!)).baselineName).toBe(baselineName); + }); + it('a run whose variation was deleted', async () => { const run = await postRun(await newBuild(), WHITE); await requestWithAuth(app, 'delete', `/test-variations/${run.testVariationId}`, user.token).expect(200); - // NOTE: current behavior — a 500 ("No test variation found. Re-create test run"), and the run stays new const res = await requestWithAuth(app, 'post', `/test-runs/approve?merge=false`, user.token) .send([run.id]) - .expect(500); + .expect(409); expect(res.body.message).toBe('No test variation found. Re-create test run'); - expect((await prisma.testRun.findUnique({ where: { id: run.id } }))!.status).toBe(TestStatus.new); + expect((await runOf(run.id)).status).toBe(TestStatus.new); }); it('400 without the merge query parameter', async () => { @@ -281,19 +307,7 @@ describe('TestRuns review API (e2e)', () => { await requestWithAuth(app, 'post', `/test-runs/approve`, user.token).send([run.id]).expect(400); - expect((await prisma.testRun.findUnique({ where: { id: run.id } }))!.status).toBe(TestStatus.new); - }); - - it('403 for a guest', async () => { - const guest = await haveUserLogged(usersService, 'guest'); - try { - const build = await newBuild(); - const run = await postRun(build, WHITE); - - await requestWithAuth(app, 'post', `/test-runs/approve?merge=false`, guest.token).send([run.id]).expect(403); - } finally { - await usersService.delete(guest.id); - } + expect((await runOf(run.id)).status).toBe(TestStatus.new); }); it('404 for an unknown id', async () => { @@ -311,17 +325,115 @@ describe('TestRuns review API (e2e)', () => { const runs = await prisma.testRun.findMany({ where: { buildId: build.id } }); expect(runs.map((run) => run.status)).toEqual([TestStatus.failed, TestStatus.failed]); - // no baseline is created expect(await prisma.baseline.count({ where: { testVariation: { projectId: project.id } } })).toBe(0); }); + }); - it('403 for a guest', async () => { - const guest = await haveUserLogged(usersService, 'guest'); - try { - await requestWithAuth(app, 'post', `/test-runs/reject`, guest.token).send([]).expect(403); - } finally { - await usersService.delete(guest.id); - } + describe('POST /test-runs/revert', () => { + const revert = (ids: string[], token = user.token) => + requestWithAuth(app, 'post', '/test-runs/revert', token).send(ids); + + it('approve, then revert: the variation gets its previous baseline back and the run its status', async () => { + const run = await runAgainstBaseline(await newBuild(), WHITE, pngBuffer([redAt(0, 0)])); + expect(run.status).toBe(TestStatus.unresolved); + const before = await variationOf(run.testVariationId); + await approve([run.id]); + const approvedName = (await variationOf(run.testVariationId)).baselineName!; + expect(approvedName).not.toBe(before.baselineName); + expect(imageExists(approvedName)).toBe(true); + + await revert([run.id]).expect(204); + + expect((await variationOf(run.testVariationId)).baselineName).toBe(before.baselineName); + const stored = await runOf(run.id); + expect(stored.status).toBe(TestStatus.unresolved); + expect(stored.statusBeforeDecision).toBeNull(); + expect(await prisma.baseline.count({ where: { testRunId: run.id } })).toBe(0); + expect(await prisma.baseline.count({ where: { testVariationId: run.testVariationId } })).toBe(1); + expect(imageExists(approvedName)).toBe(false); + expect(imageExists(before.baselineName)).toBe(true); + }); + + it('reject, then revert: the status is restored', async () => { + const run = await runAgainstBaseline(await newBuild(), WHITE, pngBuffer([redAt(0, 0)])); + await requestWithAuth(app, 'post', '/test-runs/reject', user.token).send([run.id]).expect(201); + expect((await runOf(run.id)).status).toBe(TestStatus.failed); + + await revert([run.id]).expect(204); + + expect((await runOf(run.id)).status).toBe(TestStatus.unresolved); + }); + + it('two approvals of one screen: reverting the first is a 409 and nothing changes', async () => { + const first = await postRun(await newBuild(), WHITE); + await approve([first.id]); + const second = await postRun(await newBuild(), pngBuffer([redAt(0, 0)])); + await approve([second.id]); + const variationBefore = await variationOf(first.testVariationId); + + const res = await revert([first.id]).expect(409); + + expect(res.body.message).toContain(first.id); + expect(await variationOf(first.testVariationId)).toEqual(variationBefore); + expect((await runOf(first.id)).status).toBe(TestStatus.approved); + expect((await runOf(second.id)).status).toBe(TestStatus.approved); + expect(await prisma.baseline.count({ where: { testVariationId: first.testVariationId } })).toBe(2); + expect(imageExists(variationBefore.baselineName)).toBe(true); + }); + + it('two approvals of one screen are reverted in one request, latest first', async () => { + const first = await postRun(await newBuild(), WHITE); + await approve([first.id]); + const second = await postRun(await newBuild(), pngBuffer([redAt(0, 0)])); + await approve([second.id]); + + await revert([first.id, second.id]).expect(204); + + expect((await variationOf(first.testVariationId)).baselineName).toBeNull(); + expect(await prisma.baseline.count({ where: { testVariationId: first.testVariationId } })).toBe(0); + expect((await runOf(first.id)).status).toBe(TestStatus.new); + expect((await runOf(second.id)).status).toBe(TestStatus.unresolved); + }); + + it('a feature-branch approval: the feature variation it created is deleted and the run is back on main', async () => { + const masterRun = await postRun(await newBuild(), WHITE); + await approve([masterRun.id]); + const featureRun = await postRun(await newBuild('feature'), pngBuffer([redAt(0, 0)])); + await approve([featureRun.id]); + const featureVariationId = (await runOf(featureRun.id)).testVariationId!; + expect(featureVariationId).not.toBe(masterRun.testVariationId); + const featureBaselineName = (await variationOf(featureVariationId)).baselineName!; + + await revert([featureRun.id]).expect(204); + + const stored = await runOf(featureRun.id); + expect(stored.testVariationId).toBe(masterRun.testVariationId); + expect(stored.status).toBe(TestStatus.unresolved); + expect(await prisma.testVariation.count({ where: { projectId: project.id, branchName: 'feature' } })).toBe(0); + expect(imageExists(featureBaselineName)).toBe(false); + expect(await prisma.baseline.count({ where: { testVariationId: masterRun.testVariationId } })).toBe(1); + }); + + it('409 for a run that was not decided', async () => { + const run = await postRun(await newBuild(), WHITE); + + const res = await revert([run.id]).expect(409); + + expect(res.body.message).toContain(run.id); + }); + + it('409 for an empty list', async () => { + await revert([]).expect(409); + }); + + it('404 for an unknown id', async () => { + await revert([randomUUID()]).expect(404); + }); + + it('400 for an id that is not a uuid', async () => { + const run = await postRun(await newBuild(), WHITE); + + await revert([run.id, 'not-a-uuid']).expect(400); }); }); @@ -330,7 +442,7 @@ describe('TestRuns review API (e2e)', () => { const build = await newBuild(); const run = await runAgainstBaseline(build, WHITE, pngBuffer([redAt(5, 5)])); expect(run.status).toBe(TestStatus.unresolved); - const stored = (await prisma.testRun.findUnique({ where: { id: run.id } }))!; + const stored = await runOf(run.id); const files = [stored.imageName, stored.diffName, stored.imageThumbnailName, stored.diffThumbnailName]; expect(files.every((name) => imageExists(name))).toBe(true); const baselineName = stored.baselineName; @@ -339,42 +451,31 @@ describe('TestRuns review API (e2e)', () => { expect(await prisma.testRun.findUnique({ where: { id: run.id } })).toBeNull(); expect(files.some((name) => imageExists(name))).toBe(false); - // the baseline belongs to the variation and stays expect(imageExists(baselineName)).toBe(true); }); it('ignores ids that do not exist', async () => { await requestWithAuth(app, 'post', `/test-runs/delete`, user.token).send([randomUUID()]).expect(201); }); - - it('403 for a guest', async () => { - const guest = await haveUserLogged(usersService, 'guest'); - try { - await requestWithAuth(app, 'post', `/test-runs/delete`, guest.token).send([]).expect(403); - } finally { - await usersService.delete(guest.id); - } - }); }); describe('POST /test-runs/ignoreAreas/*', () => { const area = { x: 0, y: 0, width: 30, height: 30 }; const otherArea = { x: 30, y: 30, width: 30, height: 30 }; + const setAreas = (action: 'update' | 'add', ids: string[], ignoreAreas: unknown[]) => + requestWithAuth(app, 'post', `/test-runs/ignoreAreas/${action}`, user.token).send({ ids, ignoreAreas }); it('update: stores the areas on run and variation and recomputes the diff', async () => { const build = await newBuild(); const run = await runAgainstBaseline(build, WHITE, pngBuffer([redAt(5, 5)])); expect(run.status).toBe(TestStatus.unresolved); - const before = (await prisma.testRun.findUnique({ where: { id: run.id } }))!; + const before = await runOf(run.id); - await requestWithAuth(app, 'post', `/test-runs/ignoreAreas/update`, user.token) - .send({ ids: [run.id], ignoreAreas: [area] }) - .expect(201); + await setAreas('update', [run.id], [area]).expect(201); const after = (await prisma.testRun.findUnique({ where: { id: run.id }, include: { testVariation: true } }))!; expect(JSON.parse(after.ignoreAreas)).toEqual([area]); expect(JSON.parse(after.testVariation!.ignoreAreas)).toEqual([area]); - // the change is now masked: the run passes and its diff pictures are gone expect(after.status).toBe(TestStatus.ok); expect(after.diffName).toBeNull(); expect(after.imageThumbnailName).toBeNull(); @@ -389,24 +490,17 @@ describe('TestRuns review API (e2e)', () => { const build = await newBuild(); const run = await postRun(build, WHITE); - await requestWithAuth(app, 'post', `/test-runs/ignoreAreas/update`, user.token) - .send({ ids: [run.id], ignoreAreas: [area] }) - .expect(201); - await requestWithAuth(app, 'post', `/test-runs/ignoreAreas/add`, user.token) - .send({ ids: [run.id], ignoreAreas: [otherArea] }) - .expect(201); + await setAreas('update', [run.id], [area]).expect(201); + await setAreas('add', [run.id], [otherArea]).expect(201); const stored = (await prisma.testRun.findUnique({ where: { id: run.id }, include: { testVariation: true } }))!; expect(JSON.parse(stored.ignoreAreas)).toEqual([area, otherArea]); expect(JSON.parse(stored.testVariation!.ignoreAreas)).toEqual([area, otherArea]); - // no baseline: still new after the recompute expect(stored.status).toBe(TestStatus.new); - await requestWithAuth(app, 'post', `/test-runs/ignoreAreas/update`, user.token) - .send({ ids: [run.id], ignoreAreas: [] }) - .expect(201); + await setAreas('update', [run.id], []).expect(201); - const cleared = (await prisma.testRun.findUnique({ where: { id: run.id } }))!; + const cleared = await runOf(run.id); expect(JSON.parse(cleared.ignoreAreas)).toEqual([]); }); @@ -422,7 +516,6 @@ describe('TestRuns review API (e2e)', () => { expect(JSON.parse(stored.tempIgnoreAreas)).toEqual([area]); expect(JSON.parse(stored.ignoreAreas)).toEqual([]); expect(JSON.parse(stored.testVariation!.ignoreAreas)).toEqual([]); - // the previous unresolved run of the same variation in the build was replaced expect(await prisma.testRun.findUnique({ where: { id: run.id } })).toBeNull(); }); @@ -430,15 +523,11 @@ describe('TestRuns review API (e2e)', () => { const build = await newBuild(); const run = await postRun(build, WHITE); - await requestWithAuth(app, 'post', `/test-runs/ignoreAreas/update`, user.token) - .send({ ids: [run.id], ignoreAreas: [{ ...area, x: 'left' }] }) - .expect(400); + await setAreas('update', [run.id], [{ ...area, x: 'left' }]).expect(400); }); it('404 for an unknown id', async () => { - await requestWithAuth(app, 'post', `/test-runs/ignoreAreas/update`, user.token) - .send({ ids: [randomUUID()], ignoreAreas: [area] }) - .expect(404); + await setAreas('update', [randomUUID()], [area]).expect(404); }); }); @@ -452,10 +541,25 @@ describe('TestRuns review API (e2e)', () => { .expect(200); expect(res.body.comment).toBe('looks fine'); - const variation = (await prisma.testVariation.findUnique({ where: { id: run.testVariationId } }))!; + const variation = await variationOf(run.testVariationId); expect(variation.comment).toBe('looks fine'); }); + it('only the comment reaches the variation', async () => { + const run = await postRun(await newBuild(), WHITE); + await approve([run.id]); + const before = await variationOf(run.testVariationId); + + await requestWithAuth(app, 'patch', `/test-runs/update/${run.id}`, user.token) + .send({ comment: 'x', baselineName: 'evil.png', ignoreAreas: 'not json' }) + .expect(200); + + const after = await variationOf(run.testVariationId); + expect(after.comment).toBe('x'); + expect(after.baselineName).toBe(before.baselineName); + expect(after.ignoreAreas).toBe(before.ignoreAreas); + }); + it('the variation comment is the default for the next run', async () => { const build = await newBuild(); const run = await postRun(build, WHITE); @@ -474,20 +578,6 @@ describe('TestRuns review API (e2e)', () => { await requestWithAuth(app, 'patch', `/test-runs/update/${run.id}`, user.token).send({}).expect(400); }); - - it('403 for a guest', async () => { - const guest = await haveUserLogged(usersService, 'guest'); - try { - const build = await newBuild(); - const run = await postRun(build, WHITE); - - await requestWithAuth(app, 'patch', `/test-runs/update/${run.id}`, guest.token) - .send({ comment: 'x' }) - .expect(403); - } finally { - await usersService.delete(guest.id); - } - }); }); describe('GET /test-runs/matchingSiblings/:id', () => { @@ -499,17 +589,10 @@ describe('TestRuns review API (e2e)', () => { }); const byTag = (list: Array<{ customTags: string; reason?: string }>) => [...list].sort((a, b) => a.customTags.localeCompare(b.customTags)); + const idsOf = (res: { body: { variations: TestRunDto[] } }) => res.body.variations.map((run) => run.id); + const skippedOf = (res: { body: { skipped: Array<{ customTags: string; reason?: string }> } }) => + byTag(res.body.skipped).map((run) => [run.customTags, run.reason]); - /** - * One screen in several locales of the same build, each compared against a - * white baseline: - * - en (reviewed) and de: a 10x10 red square, at different positions - * - fr: a 10x10 blue square (same size, other colour) - * - es: a 30x30 red square (9x the change area) - * - it: unchanged (ok — not a review candidate) - * - pt: no baseline yet (new — nothing to compare) - * plus runs that differ on another axis or by name, which are never siblings. - */ const haveLocalesBuild = async () => { const build = await newBuild(); const en = await runAgainstBaseline(build, WHITE, pngBuffer([redAt(5, 5)]), { customTags: 'en' }); @@ -542,7 +625,7 @@ describe('TestRuns review API (e2e)', () => { const res = await matching(en.id).expect(200); - expect(res.body.variations.map((run: TestRunDto) => run.id)).toEqual([en.id]); + expect(idsOf(res)).toEqual([en.id]); expect(res.body.skipped).toEqual([]); expect(Object.keys(res.body.variations[0]).sort()).toEqual(TEST_RUN_DTO_KEYS); }); @@ -553,14 +636,13 @@ describe('TestRuns review API (e2e)', () => { const res = await matching(runs.en.id).expect(200); - expect(res.body.variations.map((run: TestRunDto) => run.id)).toEqual([runs.en.id, runs.de.id]); - expect(byTag(res.body.skipped).map((run) => [run.customTags, run.reason])).toEqual([ + expect(idsOf(res)).toEqual([runs.en.id, runs.de.id]); + expect(skippedOf(res)).toEqual([ ['es', 'different change size'], ['fr', 'different change pattern'], ['pt', 'no diff to match'], ]); expect(Object.keys(res.body.skipped[0]).sort()).toEqual([...TEST_RUN_DTO_KEYS, 'reason'].sort()); - // read-only: nothing changed status const statuses = await prisma.testRun.findMany({ where: { buildId: runs.build.id }, select: { status: true } }); expect(statuses.filter((run) => run.status === TestStatus.approved)).toHaveLength(0); }, 60000); @@ -579,12 +661,11 @@ describe('TestRuns review API (e2e)', () => { await prisma.testRun.update({ where: { id: runs.fr.id }, data: { changeSignature: stampedWithOtherThreshold } }); const first = await matching(runs.en.id).expect(200); - // second request is served from the in-memory signature memo const second = await matching(runs.en.id).expect(200); for (const res of [first, second]) { - expect(res.body.variations.map((run: TestRunDto) => run.id)).toEqual([runs.en.id, runs.de.id]); - expect(byTag(res.body.skipped).map((run) => [run.customTags, run.reason])).toEqual([ + expect(idsOf(res)).toEqual([runs.en.id, runs.de.id]); + expect(skippedOf(res)).toEqual([ ['es', 'different change size'], ['fr', 'different change pattern'], ['pt', 'no diff to match'], @@ -602,8 +683,8 @@ describe('TestRuns review API (e2e)', () => { const res = await matching(runs.en.id).expect(200); - expect(res.body.variations.map((run: TestRunDto) => run.id)).toEqual([runs.en.id]); - expect(byTag(res.body.skipped).map((run) => [run.customTags, run.reason])).toEqual([ + expect(idsOf(res)).toEqual([runs.en.id]); + expect(skippedOf(res)).toEqual([ ['de', 'no diff to match'], ['es', 'different change size'], ['fr', 'different change pattern'], @@ -617,8 +698,8 @@ describe('TestRuns review API (e2e)', () => { const res = await matching(runs.pt.id).expect(200); - expect(res.body.variations.map((run: TestRunDto) => run.id)).toEqual([runs.pt.id]); - expect(byTag(res.body.skipped).map((run) => [run.customTags, run.reason])).toEqual([ + expect(idsOf(res)).toEqual([runs.pt.id]); + expect(skippedOf(res)).toEqual([ ['de', 'no reference diff'], ['en', 'no reference diff'], ['es', 'no reference diff'], @@ -630,13 +711,12 @@ describe('TestRuns review API (e2e)', () => { const build = await newBuild(); const mac = await runAgainstBaseline(build, WHITE, pngBuffer([redAt(5, 5)]), { os: 'mac' }); const win = await runAgainstBaseline(build, WHITE, pngBuffer([redAt(30, 30)]), { os: 'win' }); - // same os as the reviewed run but another locale: not a sibling under groupBy=os await runAgainstBaseline(build, WHITE, pngBuffer([redAt(5, 5)]), { os: 'mac', customTags: 'de' }); await enableBulkApprove('os'); const res = await matching(mac.id).expect(200); - expect(res.body.variations.map((run: TestRunDto) => run.id)).toEqual([mac.id, win.id]); + expect(idsOf(res)).toEqual([mac.id, win.id]); expect(res.body.skipped).toEqual([]); }, 60000); @@ -649,7 +729,7 @@ describe('TestRuns review API (e2e)', () => { const res = await matching(en.id).expect(200); - expect(res.body.variations.map((run: TestRunDto) => run.id)).toEqual([en.id, de.id]); + expect(idsOf(res)).toEqual([en.id, de.id]); }, 60000); it('404 for an unknown id', async () => { diff --git a/test/test-runs-validation.e2e-spec.ts b/test/test-runs-validation.e2e-spec.ts new file mode 100644 index 00000000..c4325083 --- /dev/null +++ b/test/test-runs-validation.e2e-spec.ts @@ -0,0 +1,124 @@ +import { INestApplication } from '@nestjs/common'; +import { randomUUID } from 'crypto'; +import { PNG } from 'pngjs'; +import { Build, Project } from '@prisma/client'; +import { UsersService } from '../src/users/users.service'; +import { ProjectsService } from '../src/projects/projects.service'; +import { BuildsService } from '../src/builds/builds.service'; +import { TestRunsService } from '../src/test-runs/test-runs.service'; +import { createE2eApp, haveProject, withEnv } from './api-helpers'; +import { haveUserLogged, LoggedUser, requestWithApiKey, requestWithAuth } from './preconditions'; + +describe('Test runs input validation (e2e)', () => { + let app: INestApplication; + let usersService: UsersService; + let projectsService: ProjectsService; + let buildsService: BuildsService; + let testRunsService: TestRunsService; + let user: LoggedUser; + let project: Project; + let build: Build; + + const image = './test/image.png'; + + beforeAll(async () => { + const created = await createE2eApp(); + app = created.app; + usersService = created.get(UsersService); + projectsService = created.get(ProjectsService); + buildsService = created.get(BuildsService); + testRunsService = created.get(TestRunsService); + }); + + beforeEach(async () => { + user = await haveUserLogged(usersService); + project = await haveProject(projectsService, 'Validation E2E'); + build = await buildsService.findOrCreate({ projectId: project.id, branchName: project.mainBranchName }); + }); + + afterEach(async () => { + await projectsService.remove(project.id); + await usersService.delete(user.id); + }); + + afterAll(async () => { + await app.close(); + }); + + const multipart = () => + requestWithApiKey(app, 'post', '/test-runs/multipart', user.apiKey) + .set('Content-type', 'multipart/form-data') + .field('name', 'Multipart image') + .field('branchName', project.mainBranchName) + .field('buildId', build.id) + .field('projectId', project.id); + + const postBase64 = (imageBase64: string) => + requestWithApiKey(app, 'post', '/test-runs', user.apiKey).send({ + name: 'Base64 image', + branchName: project.mainBranchName, + buildId: build.id, + projectId: project.id, + imageBase64, + }); + + it('keeps every multipart field, the area id included', async () => { + await multipart() + .field('diffTollerancePercent', '0.12') + .field('merge', 'true') + .field('ignoreAreas', '[{"id":"1692476944110","x":182.5,"y":28,"width":38,"height":30}]') + .attach('image', image) + .expect(201); + + const [testRun] = await testRunsService.findMany(build.id); + expect(testRun.merge).toBe(true); + expect(testRun.diffTollerancePercent).toBe(0.12); + expect(JSON.parse(testRun.tempIgnoreAreas)).toEqual([ + { id: '1692476944110', x: 182.5, y: 28, width: 38, height: 30 }, + ]); + }); + + it('400 for a multipart request without the image, storing nothing', async () => { + await multipart().expect(400); + + expect(await testRunsService.findMany(build.id)).toEqual([]); + }); + + it('400 for multipart ignore areas that are not JSON', async () => { + await multipart().field('ignoreAreas', 'not-json').attach('image', image).expect(400); + + expect(await testRunsService.findMany(build.id)).toEqual([]); + }); + + it('413 for a file over the multipart limit', async () => { + await withEnv({ MULTIPART_FILE_SIZE_LIMIT: '1000' }, () => multipart().attach('image', image).expect(413)); + }); + + it('400 for a base64 image that is not a PNG', async () => { + const res = await postBase64(Buffer.from('GIF89a').toString('base64')).expect(400); + + expect(res.body.message).toBe('Image is not a PNG file'); + expect(await testRunsService.findMany(build.id)).toEqual([]); + }); + + it('accepts a base64 image with line breaks', async () => { + const imageBase64 = PNG.sync + .write(new PNG({ width: 40, height: 40 })) + .toString('base64') + .replace(/(.{76})/g, '$1\r\n'); + + await postBase64(imageBase64).expect(201); + }); + + it.each(['approve?merge=false', 'reject', 'delete'])('400 for a %s body that is not an array', async (path) => { + await requestWithAuth(app, 'post', `/test-runs/${path}`, user.token) + .send({ ids: [randomUUID()] }) + .expect(400); + }); + + it('400 for ignore areas that are an object', async () => { + await requestWithAuth(app, 'post', '/test-runs/ignoreAreas/update', user.token) + .send({ ids: [randomUUID()], ignoreAreas: { x: 1, y: 2, width: 3, height: 4 } }) + .expect(400); + }); +}); diff --git a/test/test-runs.e2e-spec.ts b/test/test-runs.e2e-spec.ts index f2125f8e..775cc256 100644 --- a/test/test-runs.e2e-spec.ts +++ b/test/test-runs.e2e-spec.ts @@ -1,22 +1,19 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { INestApplication, ValidationPipe } from '@nestjs/common'; -import { AppModule } from '../src/app.module'; -import { UsersService } from '../src/users/users.service'; -import { haveTestRunCreated, haveUserLogged, requestWithApiKey, requestWithAuth, LoggedUser } from './preconditions'; -import { TestRunsService } from '../src/test-runs/test-runs.service'; -import { ProjectsService } from '../src/projects/projects.service'; +import { INestApplication } from '@nestjs/common'; import { Project, TestStatus } from '@prisma/client'; +import { randomUUID } from 'crypto'; import { BuildsService } from '../src/builds/builds.service'; +import { ProjectsService } from '../src/projects/projects.service'; +import { TestRunsService } from '../src/test-runs/test-runs.service'; import { TestVariationsService } from '../src/test-variations/test-variations.service'; -import { TEST_PROJECT } from '../src/_data_'; -import { randomUUID } from 'crypto'; -import { UpdateIgnoreAreasDto } from 'src/test-runs/dto/update-ignore-area.dto'; +import { UsersService } from '../src/users/users.service'; +import { createE2eApp, haveProject } from './api-helpers'; +import { haveTestRunCreated, haveUserLogged, requestWithApiKey, LoggedUser } from './preconditions'; describe('TestRuns (e2e)', () => { let app: INestApplication; let testRunsService: TestRunsService; let usersService: UsersService; - let projecstService: ProjectsService; + let projectsService: ProjectsService; let buildsService: BuildsService; let testVariationsService: TestVariationsService; let user: LoggedUser; @@ -27,31 +24,22 @@ describe('TestRuns (e2e)', () => { const image_v3 = './test/image_edited_2.png'; beforeAll(async () => { - const moduleFixture: TestingModule = await Test.createTestingModule({ - imports: [AppModule], - }).compile(); - - app = moduleFixture.createNestApplication(); - app.useGlobalPipes(new ValidationPipe()); - testRunsService = moduleFixture.get(TestRunsService); - usersService = moduleFixture.get(UsersService); - projecstService = moduleFixture.get(ProjectsService); - buildsService = moduleFixture.get(BuildsService); - testVariationsService = moduleFixture.get(TestVariationsService); - - await app.init(); + const created = await createE2eApp(); + app = created.app; + testRunsService = created.get(TestRunsService); + usersService = created.get(UsersService); + projectsService = created.get(ProjectsService); + buildsService = created.get(BuildsService); + testVariationsService = created.get(TestVariationsService); }); beforeEach(async () => { user = await haveUserLogged(usersService); - project = await projecstService.create({ - ...TEST_PROJECT, - name: `TestRun E2E test${randomUUID()}`, - }); + project = await haveProject(projectsService, 'TestRun E2E test'); }); afterEach(async () => { - await projecstService.remove(project.id); + await projectsService.remove(project.id); await usersService.delete(user.id); }); @@ -59,194 +47,114 @@ describe('TestRuns (e2e)', () => { await app.close(); }); + const haveRun = async (branchName: string, image: string, merge?: boolean) => + (await haveTestRunCreated(buildsService, testRunsService, project.id, branchName, image, merge)).testRun; + const haveApprovedRun = async (branchName: string, image: string) => { + const testRun = await haveRun(branchName, image); + await testRunsService.approve(testRun.id); + return testRun; + }; + const detailsOf = async (testVariationId: string | null) => testVariationsService.getDetails(testVariationId!); + describe('POST /', () => { it('New if no baseline', async () => { - const { testRun } = await haveTestRunCreated(buildsService, testRunsService, project.id, 'develop', image_v1); + const testRun = await haveRun('develop', image_v1); expect(testRun.status).toBe(TestStatus.new); }); it('Unresolved if diff found', async () => { - const { testRun: testRun1 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - project.mainBranchName, - image_v1 - ); - await testRunsService.approve(testRun1.id); - const { testRun: testRun2 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - project.mainBranchName, - image_v2 - ); - await testRunsService.approve(testRun2.id); - const { testRun: testRun3 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - 'feature', - image_v2 - ); - await testRunsService.approve(testRun3.id); - - const { testRun } = await haveTestRunCreated(buildsService, testRunsService, project.id, 'develop', image_v3); + await haveApprovedRun(project.mainBranchName, image_v1); + await haveApprovedRun(project.mainBranchName, image_v2); + await haveApprovedRun('feature', image_v2); + + const testRun = await haveRun('develop', image_v3); expect(testRun.status).toBe(TestStatus.unresolved); }); it('Ok', async () => { - const { testRun: testRun1 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - project.mainBranchName, - image_v1 - ); - await testRunsService.approve(testRun1.id); + await haveApprovedRun(project.mainBranchName, image_v1); - const { testRun } = await haveTestRunCreated(buildsService, testRunsService, project.id, 'develop', image_v1); + const testRun = await haveRun('develop', image_v1); expect(testRun.status).toBe(TestStatus.ok); }); it('Auto approve not rebased feature branch then Ok after rebase', async () => { - await projecstService.update({ ...project, autoApproveFeature: true }); - const { testRun: testRun1 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - project.mainBranchName, - image_v1 - ); - await testRunsService.approve(testRun1.id); - const { testRun: testRun2 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - project.mainBranchName, - image_v2 - ); - await testRunsService.approve(testRun2.id); - - const { testRun: notRebasedTestRun } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - 'develop', - image_v1 - ); + await projectsService.update({ ...project, autoApproveFeature: true }); + await haveApprovedRun(project.mainBranchName, image_v1); + await haveApprovedRun(project.mainBranchName, image_v2); + + const notRebasedTestRun = await haveRun('develop', image_v1); expect(notRebasedTestRun.status).toBe(TestStatus.autoApproved); - const { testRun: rebasedTestRun } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - 'develop', - image_v2 - ); + const rebasedTestRun = await haveRun('develop', image_v2); expect(rebasedTestRun.status).toBe(TestStatus.ok); }); it('Auto approve merged feature into feature branch', async () => { - await projecstService.update({ ...project, autoApproveFeature: true }); - const { testRun: testRun1 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - 'feature1', - image_v1 - ); - await testRunsService.approve(testRun1.id); - - const { testRun } = await haveTestRunCreated(buildsService, testRunsService, project.id, 'feature2', image_v1); + await projectsService.update({ ...project, autoApproveFeature: true }); + await haveApprovedRun('feature1', image_v1); + + const testRun = await haveRun('feature2', image_v1); expect(testRun.status).toBe(TestStatus.autoApproved); }); it('Auto approve merged feature into main branch', async () => { - await projecstService.update({ ...project, autoApproveFeature: true }); - const { testRun: testRun1 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - project.mainBranchName, - image_v1 - ); - await testRunsService.approve(testRun1.id); - const { testRun: testRun2 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - 'develop', - image_v2 - ); - await testRunsService.approve(testRun2.id); - - const { testRun } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - project.mainBranchName, - image_v2 - ); + await projectsService.update({ ...project, autoApproveFeature: true }); + await haveApprovedRun(project.mainBranchName, image_v1); + await haveApprovedRun('develop', image_v2); + + const testRun = await haveRun(project.mainBranchName, image_v2); expect(testRun.status).toBe(TestStatus.autoApproved); }); }); describe('POST /multipart', () => { - const url = '/test-runs/multipart'; + const post = async (projectId = project.id) => { + const build = await buildsService.findOrCreate({ projectId: project.id, branchName: project.mainBranchName }); + return { + build, + req: requestWithApiKey(app, 'post', '/test-runs/multipart', user.apiKey) + .set('Content-type', 'multipart/form-data') + .field('name', 'Multipart image') + .field('branchName', project.mainBranchName) + .field('buildId', build.id) + .field('projectId', projectId) + .attach('image', image_v1), + }; + }; it('should post multipart all fields', async () => { - const build = await buildsService.findOrCreate({ projectId: project.id, branchName: project.mainBranchName }); + const { req } = await post(); - await requestWithApiKey(app, 'post', url, user.apiKey) - .set('Content-type', 'multipart/form-data') - .field('name', 'Multipart image') + await req .field('os', 'Windows') .field('browser', 'Browser') .field('viewport', '123x456') .field('device', 'Desktop') - .field('branchName', project.mainBranchName) - .field('buildId', build.id) - .field('projectId', project.id) .field('diffTollerancePercent', '0.12') .field('merge', 'true') .field('ignoreAreas', '[{"id":"1692476944110","x":182,"y":28,"width":38,"height":30}]') .field('comment', 'Comment') - .attach('image', image_v1) .expect(201); }); it('should post multipart required fields', async () => { - const build = await buildsService.findOrCreate({ projectId: project.id, branchName: project.mainBranchName }); + const { req } = await post(); - await requestWithApiKey(app, 'post', url, user.apiKey) - .set('Content-type', 'multipart/form-data') - .field('name', 'Multipart image') - .field('branchName', project.mainBranchName) - .field('buildId', build.id) - .field('projectId', project.id) - .attach('image', image_v1) - .expect(201); + await req.expect(201); }); it('404 for a project that does not exist, storing nothing', async () => { - const build = await buildsService.findOrCreate({ projectId: project.id, branchName: project.mainBranchName }); const projectId = randomUUID(); + const { build, req } = await post(projectId); - const res = await requestWithApiKey(app, 'post', url, user.apiKey) - .set('Content-type', 'multipart/form-data') - .field('name', 'Multipart image') - .field('branchName', project.mainBranchName) - .field('buildId', build.id) - .field('projectId', projectId) - .attach('image', image_v1) - .expect(404); + const res = await req.expect(404); expect(res.body.message).toBe(`Project not found: ${projectId}`); expect(await testRunsService.findMany(build.id)).toEqual([]); @@ -255,249 +163,82 @@ describe('TestRuns (e2e)', () => { describe('POST /approve', () => { it('approve changes in new main branch', async () => { - const { testRun: testRun1 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - project.mainBranchName, - image_v1 - ); + const testRun = await haveRun(project.mainBranchName, image_v1); - const result = await testRunsService.approve(testRun1.id); + const result = await testRunsService.approve(testRun.id); expect(result.status).toBe(TestStatus.approved); - const testVariation = (await testVariationsService.getDetails(result.testVariationId!))!; - expect(testVariation.baselines).toHaveLength(1); + expect((await detailsOf(result.testVariationId)).baselines).toHaveLength(1); }); it('approve changes in new feature branch', async () => { - const { testRun: testRun1 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - 'develop', - image_v1 - ); + const testRun = await haveRun('develop', image_v1); - const result = await testRunsService.approve(testRun1.id); + const result = await testRunsService.approve(testRun.id); expect(result.status).toBe(TestStatus.approved); - const testVariation = (await testVariationsService.getDetails(result.testVariationId!))!; + const testVariation = await detailsOf(result.testVariationId); expect(testVariation.baselines).toHaveLength(1); expect(testVariation.branchName).toBe('develop'); }); it('approve changes in main vs feature branch', async () => { - const { testRun: testRun1 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - project.mainBranchName, - image_v1 - ); - await testRunsService.approve(testRun1.id); - const { testRun: testRun2 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - 'develop', - image_v2 - ); - - const featureBranchResult = await testRunsService.approve(testRun2.id); + await haveApprovedRun(project.mainBranchName, image_v1); + const testRun = await haveRun('develop', image_v2); + + const featureBranchResult = await testRunsService.approve(testRun.id); expect(featureBranchResult.status).toBe(TestStatus.approved); - const testVariation = (await testVariationsService.getDetails(featureBranchResult.testVariationId!))!; + const testVariation = await detailsOf(featureBranchResult.testVariationId); expect(testVariation.baselines).toHaveLength(1); expect(testVariation.branchName).toBe('develop'); }); it('approve changes in updated main vs feature branch', async () => { - const { testRun: testRun1 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - 'develop', - image_v1 - ); - await testRunsService.approve(testRun1.id); - const { testRun: testRun2 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - project.mainBranchName, - image_v2 - ); - await testRunsService.approve(testRun2.id); - const { testRun: testRun3 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - 'develop', - image_v1 - ); - - const result = await testRunsService.approve(testRun3.id); + await haveApprovedRun('develop', image_v1); + await haveApprovedRun(project.mainBranchName, image_v2); + const testRun = await haveRun('develop', image_v1); + + const result = await testRunsService.approve(testRun.id); expect(result.status).toBe(TestStatus.approved); expect(result.baselineBranchName).toBe(project.mainBranchName); - const testVariation = (await testVariationsService.getDetails(result.testVariationId!))!; + const testVariation = await detailsOf(result.testVariationId); expect(testVariation.branchName).toBe('develop'); expect(testVariation.baselines).toHaveLength(2); }); it('approve changes with merge', async () => { - const { testRun: testRun1 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - project.mainBranchName, - image_v1 - ); - const mainBranchResult = await testRunsService.approve(testRun1.id); - const { testRun: testRun2 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - 'develop', - image_v2, - true - ); - - const featureBranchResult = await testRunsService.approve(testRun2.id, true); + const mainBranchResult = await testRunsService.approve((await haveRun(project.mainBranchName, image_v1)).id); + const testRun = await haveRun('develop', image_v2, true); + + const featureBranchResult = await testRunsService.approve(testRun.id, true); expect(featureBranchResult.status).toBe(TestStatus.approved); expect(featureBranchResult.merge).toBe(true); - const mainBranchTestVariation = (await testVariationsService.getDetails(mainBranchResult.testVariationId!))!; + const mainBranchTestVariation = await detailsOf(mainBranchResult.testVariationId); expect(mainBranchTestVariation.baselines).toHaveLength(2); expect(mainBranchTestVariation.baselines.shift()!.baselineName).toBe(mainBranchTestVariation.baselineName); }); it('approve feature branch testVariation', async () => { - const { testRun: testRun1 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - project.mainBranchName, - image_v1 - ); - const mainBranchResult = await testRunsService.approve(testRun1.id); - const { testRun: testRun2 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - 'develop', - image_v2 - ); - - const featureBranchResult = await testRunsService.approve(testRun2.id); + const mainBranchResult = await testRunsService.approve((await haveRun(project.mainBranchName, image_v1)).id); + const testRun = await haveRun('develop', image_v2); + + const featureBranchResult = await testRunsService.approve(testRun.id); expect(featureBranchResult.status).toBe(TestStatus.approved); - const mainBranchTestVariation = (await testVariationsService.getDetails(mainBranchResult.testVariationId!))!; - expect(mainBranchTestVariation.baselines).toHaveLength(1); + expect((await detailsOf(mainBranchResult.testVariationId)).baselines).toHaveLength(1); }); it('autoApprove feature branch testVariation', async () => { - const { testRun: testRun1 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - project.mainBranchName, - image_v1 - ); - const mainBranchResult = await testRunsService.approve(testRun1.id); - const { testRun: testRun2 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - 'develop', - image_v2 - ); - - const featureBranchResult = await testRunsService.approve(testRun2.id, false, true); + const mainBranchResult = await testRunsService.approve((await haveRun(project.mainBranchName, image_v1)).id); + const testRun = await haveRun('develop', image_v2); + + const featureBranchResult = await testRunsService.approve(testRun.id, false, true); expect(featureBranchResult.status).toBe(TestStatus.autoApproved); expect(mainBranchResult.testVariationId).toBe(featureBranchResult.testVariationId); }); }); - - describe('POST /delete', () => { - it('Should delete', async () => { - const { testRun: testRun1 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - 'develop', - image_v1 - ); - const { testRun: testRun2 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - 'develop', - image_v1 - ); - - await requestWithAuth(app, 'post', `/test-runs/delete`, user.token).send([testRun1.id, testRun2.id]).expect(201); - }); - }); - - describe('POST /reject', () => { - it('200', async () => { - const { testRun: testRun1 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - 'develop', - image_v1 - ); - const { testRun: testRun2 } = await haveTestRunCreated( - buildsService, - testRunsService, - project.id, - 'develop', - image_v1 - ); - - await requestWithAuth(app, 'post', `/test-runs/reject`, user.token).send([testRun1.id, testRun2.id]).expect(201); - }); - }); - - describe('POST /ignoreAreas/update', () => { - it('200', async () => { - const { testRun } = await haveTestRunCreated(buildsService, testRunsService, project.id, 'develop', image_v1); - - const request: UpdateIgnoreAreasDto = { - ids: [testRun.id], - ignoreAreas: [ - { - x: 182, - y: 28, - width: 38, - height: 30, - }, - ], - }; - await requestWithAuth(app, 'post', `/test-runs/ignoreAreas/update`, user.token).send(request).expect(201); - }); - }); - - describe('POST /ignoreAreas/add', () => { - it('200', async () => { - const { testRun } = await haveTestRunCreated(buildsService, testRunsService, project.id, 'develop', image_v1); - - const request: UpdateIgnoreAreasDto = { - ids: [testRun.id], - ignoreAreas: [ - { - x: 182, - y: 28, - width: 38, - height: 30, - }, - ], - }; - await requestWithAuth(app, 'post', `/test-runs/ignoreAreas/add`, user.token).send(request).expect(201); - }); - }); }); diff --git a/test/test-variations-api.e2e-spec.ts b/test/test-variations-api.e2e-spec.ts index a680b312..69367bed 100644 --- a/test/test-variations-api.e2e-spec.ts +++ b/test/test-variations-api.e2e-spec.ts @@ -64,7 +64,6 @@ describe('TestVariations API (e2e)', () => { }); }; - /** Posts and approves (as the logged-in user), so the variation of that branch gets a baseline. */ const haveBaseline = async (branchName: string, image: Buffer, name = 'Screen') => { const run = await postRun(branchName, image, name); await requestWithAuth(app, 'post', '/test-runs/approve?merge=false', user.token).send([run.id]).expect(201); @@ -131,7 +130,6 @@ describe('TestVariations API (e2e)', () => { expect(res.body.baselineName).toBe(res.body.baselines[0].baselineName); expect(res.body.baselines[0].testRun.id).toBe(second.id); expect(res.body.baselines[0].user.id).toBe(user.id); - // who approved, but never their credentials: any signed-in user can read this expect(res.body.baselines[0].user).toMatchObject({ email: user.email, firstName: user.firstName, @@ -169,7 +167,6 @@ describe('TestVariations API (e2e)', () => { await haveBaseline(project.mainBranchName, WHITE, 'Same'); await haveBaseline('feature', WHITE, 'Same'); await haveBaseline('feature', CHANGED, 'Only on feature'); - // a feature variation that never got a baseline is not merged await postRun('feature', WHITE, 'No baseline'); const res = await merge('feature', project.mainBranchName).expect(200); @@ -189,12 +186,10 @@ describe('TestVariations API (e2e)', () => { expect(byName['Changed']).toMatchObject({ status: TestStatus.unresolved, merge: true, branchName: 'master' }); expect(byName['Same'].status).toBe(TestStatus.ok); expect(byName['Only on feature'].status).toBe(TestStatus.new); - // a master variation was created for the screen that only existed on feature const created = await prisma.testVariation.findUnique({ where: { id: byName['Only on feature'].testVariationId! }, }); expect(created).toMatchObject({ branchName: 'master', baselineName: null }); - // the merged screenshot is a stored copy of the feature baseline expect(imageExists(byName['Changed'].imageName)).toBe(true); }); @@ -212,7 +207,6 @@ describe('TestVariations API (e2e)', () => { const res = await merge('feature', 'release').expect(200); - // compared against release's own baseline, not main's newer one expect(res.body).toMatchObject({ branchName: 'release', isRunning: false, merge: true, unresolvedCount: 1 }); const [release] = await prisma.testVariation.findMany({ where: { projectId: project.id, branchName: 'release' }, @@ -261,7 +255,6 @@ describe('TestVariations API (e2e)', () => { expect(await prisma.testVariation.findUnique({ where: { id: first.testVariationId! } })).toBeNull(); expect(await prisma.baseline.count({ where: { testVariationId: first.testVariationId! } })).toBe(0); await waitFor(() => baselines.every((baseline) => !imageExists(baseline.baselineName))); - // the runs stay, without a variation, and keep their own screenshots const runs = await prisma.testRun.findMany({ where: { id: { in: [first.id, second.id] } } }); expect(runs.map((run) => run.testVariationId)).toEqual([null, null]); expect(runs.every((run) => imageExists(run.imageName))).toBe(true); diff --git a/test/test-variations.e2e-spec.ts b/test/test-variations.e2e-spec.ts deleted file mode 100644 index 9f147aa9..00000000 --- a/test/test-variations.e2e-spec.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { INestApplication, ValidationPipe } from '@nestjs/common'; -import { AppModule } from '../src/app.module'; -import { UsersService } from '../src/users/users.service'; -import { haveTestRunCreated, haveUserLogged, LoggedUser } from './preconditions'; -import { TestRunsService } from '../src/test-runs/test-runs.service'; -import { ProjectsService } from '../src/projects/projects.service'; -import { Project } from '@prisma/client'; -import { BuildsService } from '../src/builds/builds.service'; -import { TestVariationsService } from '../src/test-variations/test-variations.service'; -import { TEST_PROJECT } from '../src/_data_'; -import { randomUUID } from 'crypto'; - -describe('TestVariations (e2e)', () => { - let app: INestApplication; - let testRunsService: TestRunsService; - let usersService: UsersService; - let projecstService: ProjectsService; - let buildsService: BuildsService; - let user: LoggedUser; - let project: Project; - let testVariationsService: TestVariationsService; - - beforeAll(async () => { - const moduleFixture: TestingModule = await Test.createTestingModule({ - imports: [AppModule], - }).compile(); - - app = moduleFixture.createNestApplication(); - app.useGlobalPipes(new ValidationPipe()); - testRunsService = moduleFixture.get(TestRunsService); - usersService = moduleFixture.get(UsersService); - projecstService = moduleFixture.get(ProjectsService); - buildsService = moduleFixture.get(BuildsService); - testVariationsService = moduleFixture.get(TestVariationsService); - - await app.init(); - }); - - beforeEach(async () => { - user = await haveUserLogged(usersService); - project = await projecstService.create({ - ...TEST_PROJECT, - name: `TestVariations E2E test${randomUUID()}`, - }); - }); - - afterEach(async () => { - await projecstService.remove(project.id); - await usersService.delete(user.id); - }); - - afterAll(async () => { - await app.close(); - }); - - describe('DELETE /', () => { - const image_v1 = './test/image.png'; - - it('can delete', async () => { - const { testRun } = await haveTestRunCreated(buildsService, testRunsService, project.id, 'develop', image_v1); - await testRunsService.approve(testRun.id); - - await testVariationsService.delete(testRun.testVariationId!); - - expect((await testRunsService.findOne(testRun.id))!.testVariationId).toBeNull(); - }); - }); -}); diff --git a/test/users-admin.e2e-spec.ts b/test/users-admin.e2e-spec.ts index 51d314bc..3143dff6 100644 --- a/test/users-admin.e2e-spec.ts +++ b/test/users-admin.e2e-spec.ts @@ -1,18 +1,16 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { INestApplication, ValidationPipe } from '@nestjs/common'; +import { INestApplication } from '@nestjs/common'; import { Role, TestStatus } from '@prisma/client'; import request from 'supertest'; import { createHash, generateKeyPairSync, KeyObject, randomUUID, sign } from 'crypto'; import { createServer, IncomingMessage, Server } from 'http'; import { generateRandomKey } from '../src/utils/random-key'; -import { AppModule } from '../src/app.module'; import { UsersService } from '../src/users/users.service'; import { PrismaService } from '../src/prisma/prisma.service'; import { ProjectsService } from '../src/projects/projects.service'; import { BuildsService } from '../src/builds/builds.service'; import { TestRunsService } from '../src/test-runs/test-runs.service'; -import { TEST_PROJECT } from '../src/_data_'; import { UserDto } from '../src/users/dto/user.dto'; +import { approveRuns, createE2eApp, haveProject, setEnv } from './api-helpers'; import { generateUser, haveTestRunCreated, @@ -24,6 +22,13 @@ import { const FORBIDDEN = 'You do not have permission to perform this operation.'; +const expectJsonBoolean = (app: INestApplication, url: string, value: boolean) => + request(app.getHttpServer()) + .get(url) + .expect(200) + .expect('Content-Type', 'application/json; charset=utf-8') + .expect((res) => expect(res.body).toBe(value)); + describe('Users admin and RBAC (e2e)', () => { let app: INestApplication; let usersService: UsersService; @@ -37,19 +42,13 @@ describe('Users admin and RBAC (e2e)', () => { }; beforeAll(async () => { - const moduleFixture: TestingModule = await Test.createTestingModule({ - imports: [AppModule], - }).compile(); - - app = moduleFixture.createNestApplication(); - app.useGlobalPipes(new ValidationPipe()); - usersService = moduleFixture.get(UsersService); - prismaService = moduleFixture.get(PrismaService); - await app.init(); + const created = await createE2eApp(); + app = created.app; + usersService = created.get(UsersService); + prismaService = created.get(PrismaService); }); afterEach(async () => { - // some tests delete their users through the API already await prismaService.user.deleteMany({ where: { id: { in: userIds.splice(0) } } }); }); @@ -57,9 +56,11 @@ describe('Users admin and RBAC (e2e)', () => { await app.close(); }); - describe('GET /users/me', () => { - it('401 without a token', () => request(app.getHttpServer()).get('/users/me').expect(401)); + it.each(['/users/me', '/users/all'])('GET %s answers 401 without a token', (url) => + request(app.getHttpServer()).get(url).expect(401) + ); + describe('GET /users/me', () => { it.each([Role.admin, Role.editor, Role.guest])('returns the current %s without the password', async (role) => { const user = await haveUser(role); @@ -77,8 +78,6 @@ describe('Users admin and RBAC (e2e)', () => { }); describe('GET /users/all', () => { - it('401 without a token', () => request(app.getHttpServer()).get('/users/all').expect(401)); - it('401 with an admin api key only: the list needs a JWT', async () => { const admin = await haveUser(Role.admin); @@ -107,12 +106,12 @@ describe('Users admin and RBAC (e2e)', () => { email: guest.email, firstName: 'fName', lastName: 'lName', - apiKey: guest.apiKey, role: 'guest', }, ]) ); expect(res.body.every((user: UserDto) => !('password' in user))).toBe(true); + expect(res.body.every((user: UserDto) => !('apiKey' in user))).toBe(true); }); }); @@ -138,8 +137,9 @@ describe('Users admin and RBAC (e2e)', () => { .expect(200); expect(res.body).toEqual(expect.objectContaining({ id: target.id, email: target.email, role: Role.admin })); + expect(res.body).not.toHaveProperty('apiKey'); + expect(res.body).not.toHaveProperty('password'); expect((await usersService.findOne(target.id))!.role).toBe(Role.admin); - // the role is read from the database on every request, not from the token await requestWithAuth(app, 'get', '/users/all', target.token).expect(200); }); @@ -222,25 +222,11 @@ describe('Users admin and RBAC (e2e)', () => { expect(await usersService.findOne(last.id)).not.toBeNull(); }); - it('deletes every listed user', async () => { - const admin = await haveUser(Role.admin); - const first = await haveUser(Role.guest); - const last = await haveUser(Role.editor); - - await requestWithAuth(app, 'delete', '/users', admin.token).send([first.id, last.id]).expect(200); - - expect(await usersService.findOne(first.id)).toBeNull(); - expect(await usersService.findOne(last.id)).toBeNull(); - }); - it('keeps the baselines a deleted user approved, without the user', async () => { const admin = await haveUser(Role.admin); const editor = await haveUser(Role.editor); const projectsService = app.get(ProjectsService); - const project = await projectsService.create({ - ...TEST_PROJECT, - name: `Users admin E2E ${randomUUID()}`, - }); + const project = await haveProject(projectsService, 'Users admin E2E'); try { const { testRun } = await haveTestRunCreated( app.get(BuildsService), @@ -249,9 +235,7 @@ describe('Users admin and RBAC (e2e)', () => { project.mainBranchName, './test/image.png' ); - await requestWithAuth(app, 'post', '/test-runs/approve?merge=false', editor.token) - .send([testRun.id]) - .expect(201); + await approveRuns(app, editor, [testRun.id]); const [baseline] = await prismaService.baseline.findMany({ where: { testRunId: testRun.id } }); expect(baseline.userId).toBe(editor.id); @@ -269,12 +253,7 @@ describe('Users admin and RBAC (e2e)', () => { }); describe('OIDC disabled (default DB users)', () => { - it('GET /users/oidc/enabled is false', () => - request(app.getHttpServer()) - .get('/users/oidc/enabled') - .expect(200) - .expect('Content-Type', 'application/json; charset=utf-8') - .expect((res) => expect(res.body).toBe(false))); + it('GET /users/oidc/enabled is false', () => expectJsonBoolean(app, '/users/oidc/enabled', false)); it.each(['/users/oidc/login', '/users/oidc/callback?code=abc&state=xyz'])('GET %s is 404', (url) => request(app.getHttpServer()).get(url).expect(404).expect('OIDC is not enabled.') @@ -282,21 +261,12 @@ describe('Users admin and RBAC (e2e)', () => { }); }); -/** - * OIDC needs an identity provider. Instead of a real one, a minimal in-process - * provider serves discovery, JWKS and the token endpoint on 127.0.0.1, signs ID - * tokens with an RSA key generated for the run and checks PKCE. The app runs - * with the real UsersFactoryService/OidcUsersService and openid-client, - * switched to OIDC through the same environment variables as in production. - * The browser's visit to the provider's authorization page is simulated: the - * test reads state and code_challenge from the redirect and registers a code. - */ describe('Users OIDC (e2e)', () => { const CLIENT_ID = 'vrt-e2e'; const CLIENT_SECRET = 'e2e-secret'; const REDIRECT_URI = 'http://localhost:4200/users/oidc/callback'; const FRONTEND_LOGIN = 'http://localhost:8080/login'; - const OIDC_ENV = ['OIDC_ENABLED', 'OIDC_ISSUER_URL', 'OIDC_CLIENT_ID', 'OIDC_CLIENT_SECRET', 'OIDC_REDIRECT_URI']; + const LOGIN_FAILED = `${FRONTEND_LOGIN}#error=oidc_login_failed`; let app: INestApplication; let prismaService: PrismaService; @@ -307,7 +277,7 @@ describe('Users OIDC (e2e)', () => { const tokenRequests: URLSearchParams[] = []; const codes = new Map }>(); const emails: string[] = []; - const previousEnv: Record = {}; + let restoreEnv: () => void; const base64url = (input: Buffer | string) => Buffer.from(input).toString('base64url'); @@ -381,25 +351,17 @@ describe('Users OIDC (e2e)', () => { await new Promise((resolve) => idp.listen(0, '127.0.0.1', resolve)); issuer = `http://127.0.0.1:${(idp.address() as { port: number }).port}`; - const env: Record = { + restoreEnv = setEnv({ OIDC_ENABLED: 'true', OIDC_ISSUER_URL: issuer, OIDC_CLIENT_ID: CLIENT_ID, OIDC_CLIENT_SECRET: CLIENT_SECRET, OIDC_REDIRECT_URI: REDIRECT_URI, - }; - for (const key of OIDC_ENV) { - previousEnv[key] = process.env[key]; - process.env[key] = env[key]; - } + }); - const moduleFixture: TestingModule = await Test.createTestingModule({ - imports: [AppModule], - }).compile(); - app = moduleFixture.createNestApplication(); - app.useGlobalPipes(new ValidationPipe()); - prismaService = moduleFixture.get(PrismaService); - await app.init(); + const created = await createE2eApp(); + app = created.app; + prismaService = created.get(PrismaService); }); afterEach(async () => { @@ -412,13 +374,7 @@ describe('Users OIDC (e2e)', () => { afterAll(async () => { await app.close(); await new Promise((resolve) => idp.close(resolve)); - for (const key of OIDC_ENV) { - if (previousEnv[key] === undefined) { - delete process.env[key]; - } else { - process.env[key] = previousEnv[key]; - } - } + restoreEnv(); }); const newEmail = () => { @@ -427,7 +383,6 @@ describe('Users OIDC (e2e)', () => { return email; }; - // GET /users/oidc/login, then the "browser" at the provider: returns what the callback needs const startLogin = async (claims: Record) => { const res = await request(app.getHttpServer()).get('/users/oidc/login').expect(302); const location = new URL(res.headers.location); @@ -452,23 +407,19 @@ describe('Users OIDC (e2e)', () => { return decodeURIComponent(location.slice(`${FRONTEND_LOGIN}#token=`.length)); }; - // first on purpose: openid-client's discovery result is cached once it succeeds it('GET /users/oidc/login sends the browser back to the login page when discovery fails', async () => { discoveryDown = true; await request(app.getHttpServer()) .get('/users/oidc/login') .expect(302) - .expect('Location', `${FRONTEND_LOGIN}#error=oidc_login_failed`) + .expect('Location', LOGIN_FAILED) .expect((res) => expect(res.headers['set-cookie']).toBeUndefined()); }); - it('GET /users/oidc/enabled is true', () => - request(app.getHttpServer()) - .get('/users/oidc/enabled') - .expect(200) - .expect('Content-Type', 'application/json; charset=utf-8') - .expect((res) => expect(res.body).toBe(true))); + it('GET /users/registration/enabled is false', () => expectJsonBoolean(app, '/users/registration/enabled', false)); + + it('GET /users/oidc/enabled is true', () => expectJsonBoolean(app, '/users/oidc/enabled', true)); it('GET /users/oidc/login redirects to the provider with PKCE and keeps state in a cookie', async () => { const { state, location, setCookie } = await startLogin({}); @@ -497,7 +448,6 @@ describe('Users OIDC (e2e)', () => { groups: ['staff', 'VRT-Admin'], }); - // other cookies of the site travel along with the state cookie const res = await callback(`code=${code}&state=${state}`, `theme=dark; ${cookie}; lang=en`); const token = tokenFrom(res.headers.location); @@ -524,7 +474,6 @@ describe('Users OIDC (e2e)', () => { const third = await startLogin({ email, email_verified: true, groups: ['developers'] }); tokenFrom((await callback(`code=${third.code}&state=${third.state}`, third.cookie)).headers.location); - // OIDC_DEFAULT_ROLE defaults to editor expect((await prismaService.user.findUnique({ where: { email } }))!.role).toBe(Role.editor); }); @@ -532,15 +481,11 @@ describe('Users OIDC (e2e)', () => { const email = newEmail(); const { state, code, cookie } = await startLogin({ email, email_verified: false, groups: ['admin'] }); - await callback(`code=${code}&state=${state}`, cookie).expect( - 'Location', - `${FRONTEND_LOGIN}#error=oidc_login_failed` - ); + await callback(`code=${code}&state=${state}`, cookie).expect('Location', LOGIN_FAILED); expect(await prismaService.user.findUnique({ where: { email } })).toBeNull(); }); - // cookie: undefined = none, null = the one set by /users/oidc/login it.each<[string, { cookie?: string | null; state?: string }]>([ ['without the state cookie', { cookie: undefined }], ['with only other cookies', { cookie: 'theme=dark; lang=en' }], @@ -554,10 +499,7 @@ describe('Users OIDC (e2e)', () => { const state = arrange.state ?? login.state; const cookie = arrange.cookie === null ? login.cookie : arrange.cookie; - await callback(`code=${code}&state=${state}`, cookie).expect( - 'Location', - `${FRONTEND_LOGIN}#error=oidc_login_failed` - ); + await callback(`code=${code}&state=${state}`, cookie).expect('Location', LOGIN_FAILED); expect(tokenRequests).toHaveLength(0); expect(await prismaService.user.findUnique({ where: { email } })).toBeNull(); @@ -566,10 +508,7 @@ describe('Users OIDC (e2e)', () => { it('fails when the provider refuses the code', async () => { const { state, cookie } = await startLogin({ email: newEmail(), email_verified: true }); - await callback(`code=unknown&state=${state}`, cookie).expect( - 'Location', - `${FRONTEND_LOGIN}#error=oidc_login_failed` - ); + await callback(`code=unknown&state=${state}`, cookie).expect('Location', LOGIN_FAILED); expect(tokenRequests).toHaveLength(1); }); @@ -578,11 +517,7 @@ describe('Users OIDC (e2e)', () => { const email = newEmail(); const { state, code, cookie } = await startLogin({ email, email_verified: true }); - // a repeated code parameter is dropped, so openid-client sees no code at all - await callback(`code=${code}&code=${code}&state=${state}`, cookie).expect( - 'Location', - `${FRONTEND_LOGIN}#error=oidc_login_failed` - ); + await callback(`code=${code}&code=${code}&state=${state}`, cookie).expect('Location', LOGIN_FAILED); expect(tokenRequests).toHaveLength(0); }); diff --git a/test/users.e2e-spec.ts b/test/users.e2e-spec.ts index 92f534a1..0be349a3 100644 --- a/test/users.e2e-spec.ts +++ b/test/users.e2e-spec.ts @@ -1,27 +1,36 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { INestApplication, ValidationPipe } from '@nestjs/common'; +import { INestApplication } from '@nestjs/common'; +import { User } from '@prisma/client'; +import { compareSync } from 'bcryptjs'; import request from 'supertest'; -import { AppModule } from '../src/app.module'; import { UsersService } from '../src/users/users.service'; -import { UserLoginRequestDto } from '../src/users/dto/user-login-request.dto'; -import { compareSync } from 'bcryptjs'; -import { requestWithAuth, generateUser } from './preconditions'; -import { User } from '@prisma/client'; +import { createE2eApp, withEnv } from './api-helpers'; +import { generateUser, requestWithAuth } from './preconditions'; describe('Users (e2e)', () => { let app: INestApplication; let usersService: UsersService; let user: Partial; + const password = '123456'; + const haveUserLoggedIn = async () => { + user = await usersService.create(generateUser(password)); + return usersService.login({ email: user.email!, password }); + }; + const expectProfile = (body: Record, expected: Partial) => { + expect(body).toMatchObject({ + id: expected.id, + email: expected.email, + firstName: expected.firstName, + lastName: expected.lastName, + apiKey: user.apiKey, + }); + expect(body.token).not.toBeNull(); + }; + beforeAll(async () => { - const moduleFixture: TestingModule = await Test.createTestingModule({ - imports: [AppModule], - }).compile(); - - app = moduleFixture.createNestApplication(); - app.useGlobalPipes(new ValidationPipe()); - usersService = moduleFixture.get(UsersService); - await app.init(); + const { app: created, get } = await createE2eApp(); + app = created; + usersService = get(UsersService); }); afterAll(async () => { @@ -33,7 +42,7 @@ describe('Users (e2e)', () => { }); it('POST /register', () => { - user = generateUser('123456'); + user = generateUser(password); return request(app.getHttpServer()) .post('/users/register') .send(user) @@ -47,35 +56,43 @@ describe('Users (e2e)', () => { }); }); + it('GET /registration/enabled is true by default', async () => { + user = await usersService.create(generateUser(password)); + + await request(app.getHttpServer()) + .get('/users/registration/enabled') + .expect(200) + .expect('Content-Type', /application\/json/) + .expect((res) => expect(res.body).toBe(true)); + }); + + it('POST /register answers 403 when REGISTRATION_ENABLED=false', async () => { + user = await usersService.create(generateUser(password)); + await withEnv({ REGISTRATION_ENABLED: 'false' }, async () => { + await request(app.getHttpServer()) + .post('/users/register') + .send(generateUser(password)) + .expect(403) + .expect((res) => expect(res.body.message).toBe('User registration is disabled.')); + await request(app.getHttpServer()) + .get('/users/registration/enabled') + .expect(200) + .expect((res) => expect(res.body).toBe(false)); + }); + }); + it('POST /login', async () => { - const password = '123456'; user = await usersService.create(generateUser(password)); - const loginData: UserLoginRequestDto = { - email: user.email!, - password, - }; - return request(app.getHttpServer()) + await request(app.getHttpServer()) .post('/users/login') - .send(loginData) + .send({ email: user.email!, password }) .expect(201) - .expect((res) => { - expect(res.body.id).toBe(user.id); - expect(res.body.email).toBe(user.email); - expect(res.body.firstName).toBe(user.firstName); - expect(res.body.lastName).toBe(user.lastName); - expect(res.body.apiKey).toBe(user.apiKey); - expect(res.body.token).not.toBeNull(); - }); + .expect((res) => expectProfile(res.body, user)); }); it('GET /newApiKey', async () => { - const password = '123456'; - user = await usersService.create(generateUser(password)); - const loggedUser = await usersService.login({ - email: user.email!, - password, - }); + const loggedUser = await haveUserLoggedIn(); const res = await requestWithAuth(app, 'get', '/users/newApiKey', loggedUser.token!).send().expect(200); @@ -85,41 +102,44 @@ describe('Users (e2e)', () => { }); it('PUT /password', async () => { - const newPassword = 'newPassword'; - const password = '123456'; - user = await usersService.create(generateUser(password)); - const loggedUser = await usersService.login({ - email: user.email!, - password, - }); + const loggedUser = await haveUserLoggedIn(); - await requestWithAuth(app, 'put', '/users/password', loggedUser.token!).send({ password: newPassword }).expect(200); + await requestWithAuth(app, 'put', '/users/password', loggedUser.token!) + .send({ password: 'newPassword' }) + .expect(200); const newUser = (await usersService.findOne(user.id!))!; - expect(compareSync(newPassword, newUser.password)).toBe(true); + expect(compareSync('newPassword', newUser.password)).toBe(true); + }); + + it('PUT /password answers 400 for a body without password', async () => { + const loggedUser = await haveUserLoggedIn(); + + await requestWithAuth(app, 'put', '/users/password', loggedUser.token!).send({}).expect(400); + await requestWithAuth(app, 'put', '/users/password', loggedUser.token!).send({ password: '' }).expect(400); + + const unchanged = (await usersService.findOne(user.id!))!; + expect(compareSync(password, unchanged.password)).toBe(true); }); it('PUT /', async () => { - const password = '123456'; - user = await usersService.create(generateUser(password)); - const editedUser = { - email: 'edited' + user.email, - firstName: 'EDITEDfName', - lastName: 'EDITEDlName', - }; - - const loggedUser = await usersService.login({ - email: user.email!, - password, - }); + const loggedUser = await haveUserLoggedIn(); + const editedUser = { email: 'edited' + user.email, firstName: 'EDITEDfName', lastName: 'EDITEDlName' }; const res = await requestWithAuth(app, 'put', '/users', loggedUser.token!).send(editedUser).expect(200); + expectProfile(res.body, { id: user.id, ...editedUser }); + }); + + it('a token keeps working after the user changes their email', async () => { + const loggedUser = await haveUserLoggedIn(); + + await requestWithAuth(app, 'put', '/users', loggedUser.token!) + .send({ email: 'changed' + user.email, firstName: 'F', lastName: 'L' }) + .expect(200); + + const res = await requestWithAuth(app, 'get', '/users/me', loggedUser.token!).send().expect(200); expect(res.body.id).toBe(user.id); - expect(res.body.email).toBe(editedUser.email); - expect(res.body.firstName).toBe(editedUser.firstName); - expect(res.body.lastName).toBe(editedUser.lastName); - expect(res.body.apiKey).toBe(user.apiKey); - expect(res.body.token).not.toBeNull(); + expect(res.body.email).toBe('changed' + user.email); }); }); diff --git a/test/validation-whitelist.e2e-spec.ts b/test/validation-whitelist.e2e-spec.ts new file mode 100644 index 00000000..b7059de2 --- /dev/null +++ b/test/validation-whitelist.e2e-spec.ts @@ -0,0 +1,117 @@ +import { INestApplication } from '@nestjs/common'; +import { Build, Project, TestStatus } from '@prisma/client'; +import { BuildsService } from '../src/builds/builds.service'; +import { PrismaService } from '../src/prisma/prisma.service'; +import { ProjectsService } from '../src/projects/projects.service'; +import { UsersService } from '../src/users/users.service'; +import { approveRuns, createE2eApp, haveProject, postRunBase64, WHITE } from './api-helpers'; +import { haveUserLogged, LoggedUser, requestWithApiKey, requestWithAuth } from './preconditions'; + +describe('Global whitelist (e2e)', () => { + let app: INestApplication; + let usersService: UsersService; + let projectsService: ProjectsService; + let buildsService: BuildsService; + let prisma: PrismaService; + let user: LoggedUser; + let project: Project; + + beforeAll(async () => { + const created = await createE2eApp(); + app = created.app; + usersService = created.get(UsersService); + projectsService = created.get(ProjectsService); + buildsService = created.get(BuildsService); + prisma = created.get(PrismaService); + }); + + beforeEach(async () => { + user = await haveUserLogged(usersService); + project = await haveProject(projectsService, 'Whitelist E2E'); + }); + + afterEach(async () => { + await projectsService.remove(project.id); + await usersService.delete(user.id); + }); + + afterAll(async () => { + await app.close(); + }); + + const newBuild = (branchName = project.mainBranchName): Promise => + buildsService.findOrCreate({ projectId: project.id, branchName }); + + const postRun = async (build: Build, extra: Record = {}) => + postRunBase64(app, user, build, WHITE, extra); + + it('PATCH /test-runs/update/:id writes the comment and nothing else', async () => { + const run = await postRun(await newBuild()); + await approveRuns(app, user, [run.id]); + const before = (await prisma.testVariation.findUnique({ where: { id: run.testVariationId } }))!; + + await requestWithAuth(app, 'patch', `/test-runs/update/${run.id}`, user.token) + .send({ comment: 'fine', baselineName: 'other.baseline.png' }) + .expect(200); + + const after = (await prisma.testVariation.findUnique({ where: { id: run.testVariationId } }))!; + expect(after.comment).toBe('fine'); + expect(after.baselineName).toBe(before.baselineName); + }); + + it('POST /test-runs/ignoreAreas/update keeps the id of every area', async () => { + const run = await postRun(await newBuild()); + const areas = [ + { id: 'a', x: 1, y: 2, width: 3, height: 4 }, + { id: 'b', x: 5, y: 6, width: 7, height: 8 }, + ]; + + await requestWithAuth(app, 'post', '/test-runs/ignoreAreas/update', user.token) + .send({ ids: [run.id], ignoreAreas: areas }) + .expect(201); + + const stored = (await prisma.testRun.findUnique({ where: { id: run.id }, include: { testVariation: true } }))!; + expect(JSON.parse(stored.ignoreAreas)).toEqual(areas); + expect(JSON.parse(stored.testVariation!.ignoreAreas)).toEqual(areas); + }); + + it('POST /test-runs/multipart keeps the image, the form values and the area ids', async () => { + const build = await newBuild(); + + await requestWithApiKey(app, 'post', '/test-runs/multipart', user.apiKey) + .set('Content-type', 'multipart/form-data') + .field('name', 'Multipart image') + .field('branchName', project.mainBranchName) + .field('buildId', build.id) + .field('projectId', project.id) + .field('merge', 'true') + .field('ignoreAreas', '[{"id":"1692476944110","x":1,"y":2,"width":3,"height":4}]') + .field('unknown', 'dropped') + .attach('image', './test/image.png') + .expect(201); + + const [testRun] = await prisma.testRun.findMany({ where: { buildId: build.id } }); + expect(testRun.merge).toBe(true); + expect(JSON.parse(testRun.tempIgnoreAreas)).toEqual([{ id: '1692476944110', x: 1, y: 2, width: 3, height: 4 }]); + }); + + it('POST /test-runs compares against the sourceBranch variation', async () => { + const release = await postRun(await newBuild('release')); + await approveRuns(app, user, [release.id]); + const feature = await newBuild('feature'); + + const compared = await postRun(feature, { sourceBranch: 'release' }); + + expect(compared.status).toBe(TestStatus.ok); + }); + + it('PATCH /builds/:id still answers 400 for fields it does not declare', async () => { + const build = await newBuild(); + + await requestWithApiKey(app, 'patch', `/builds/${build.id}`, user.apiKey) + .send({ branchName: 'rewritten', number: 4242 }) + .expect(400); + + expect(await prisma.build.findUnique({ where: { id: build.id } })).toMatchObject({ isRunning: true }); + }); +}); diff --git a/test_acceptance/acceptance.spec.ts b/test_acceptance/acceptance.spec.ts index 43b2ab0e..f725923f 100644 --- a/test_acceptance/acceptance.spec.ts +++ b/test_acceptance/acceptance.spec.ts @@ -6,7 +6,7 @@ import { randomUUID } from 'crypto'; axios.defaults.baseURL = 'http://localhost:4200'; -let user: Partial = { +const user: Partial = { email: `${randomUUID()}@example.com`, password: '123456', firstName: 'fName', diff --git a/test_acceptance/jest.config.ts b/test_acceptance/jest.config.ts index c5a70de6..f09f5849 100644 --- a/test_acceptance/jest.config.ts +++ b/test_acceptance/jest.config.ts @@ -1,4 +1,3 @@ -/** @returns {Promise} */ module.exports = async () => { return { displayName: 'Acceptance', @@ -7,7 +6,7 @@ module.exports = async () => { testRegex: '.spec.ts$', moduleFileExtensions: ['js', 'json', 'ts'], transform: { - '^.+\\.(t|j)s$': 'ts-jest', + '^.+\\.(t|j)s$': '@swc/jest', }, testEnvironment: 'node', }; diff --git a/test_ldap/jest.config.ts b/test_ldap/jest.config.ts index a7ec3362..9b9a0900 100644 --- a/test_ldap/jest.config.ts +++ b/test_ldap/jest.config.ts @@ -1,4 +1,3 @@ -/** @returns {Promise} */ module.exports = async () => { return { displayName: 'Ldap', @@ -7,7 +6,7 @@ module.exports = async () => { testRegex: '.spec.ts$', moduleFileExtensions: ['js', 'json', 'ts'], transform: { - '^.+\\.(t|j)s$': 'ts-jest', + '^.+\\.(t|j)s$': '@swc/jest', }, testEnvironment: 'node', }; diff --git a/test_ldap/ldap.spec.ts b/test_ldap/ldap.spec.ts index 475507eb..af115649 100644 --- a/test_ldap/ldap.spec.ts +++ b/test_ldap/ldap.spec.ts @@ -27,4 +27,10 @@ describe('Ldap', () => { expect(error.response.data.message).toContain('Invalid email or password.'); }); }); + + test('Rejects an empty password', async () => { + await expect(axios.post('/users/login', { email: loginData.email, password: '' })).rejects.toMatchObject({ + response: { status: 400, data: { message: expect.stringContaining('Invalid email or password.') } }, + }); + }); }); diff --git a/test_oidc/jest.config.ts b/test_oidc/jest.config.ts index accf320d..841ed8fa 100644 --- a/test_oidc/jest.config.ts +++ b/test_oidc/jest.config.ts @@ -1,4 +1,3 @@ -/** @returns {Promise} */ module.exports = async () => { return { displayName: 'Oidc', @@ -7,7 +6,7 @@ module.exports = async () => { testRegex: '.spec.ts$', moduleFileExtensions: ['js', 'json', 'ts'], transform: { - '^.+\\.(t|j)s$': 'ts-jest', + '^.+\\.(t|j)s$': '@swc/jest', }, testEnvironment: 'node', }; diff --git a/test_oidc/oidc.spec.ts b/test_oidc/oidc.spec.ts index c998d437..aea22e72 100644 --- a/test_oidc/oidc.spec.ts +++ b/test_oidc/oidc.spec.ts @@ -3,9 +3,6 @@ import { UserLoginRequestDto } from 'src/users/dto/user-login-request.dto'; axios.defaults.baseURL = 'http://localhost:4200'; -// These tests run against a running api that was started with OIDC enabled -// (see docker-compose.oidc.yml). The full browser redirect flow cannot be -// automated here, so we verify the pieces the backend owns. describe('Oidc', () => { test('reports OIDC as enabled', async () => { const response = await axios.get('/users/oidc/enabled'); diff --git a/tsconfig.build.json b/tsconfig.build.json index 602ef977..f76cab09 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -4,5 +4,5 @@ "rootDir": "./src" }, "include": ["src"], - "exclude": ["node_modules", "dist", "**/*spec.ts", "src/jest.config.ts"] + "exclude": ["node_modules", "dist", "**/*spec.ts", "**/test-utils.ts", "src/jest.config.ts"] } diff --git a/tsconfig.jest.json b/tsconfig.jest.json deleted file mode 100644 index 48586555..00000000 --- a/tsconfig.jest.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "extends": "./tsconfig.json", - "compilerOptions": { - // transpile each file on its own: ts-jest then skips type-checking, which - // `npm run typecheck` does once for the whole project instead - "isolatedModules": true, - "incremental": false - } -} diff --git a/tsconfig.json b/tsconfig.json index 489d84b4..0c37f237 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -18,6 +18,10 @@ "multer" ], "strict": true, + "noUnusedLocals": true, + "noImplicitReturns": true, + "noImplicitOverride": true, + "noFallthroughCasesInSwitch": true, "paths": { "src/*": [ "./src/*" diff --git a/tslint.json b/tslint.json deleted file mode 100644 index 5601443f..00000000 --- a/tslint.json +++ /dev/null @@ -1,22 +0,0 @@ - -{ - "defaultSeverity": "error", - "extends": ["tslint:recommended"], - "jsRules": { - "no-unused-expression": true - }, - "rules": { - "quotemark": [true, "single"], - "member-access": [false], - "ordered-imports": [false], - "max-line-length": [true, 150], - "member-ordering": [false], - "interface-name": [false], - "arrow-parens": false, - "object-literal-sort-keys": false - }, - "rulesDirectory": [], - "linterOptions": { - "exclude": ["config/**"] - } - } \ No newline at end of file