diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..9109f42 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,14 @@ +.git +.gitignore +.github +.vscode +.editorconfig +.releaserc.json +.renovaterc.json +# .golangci.yaml is needed for the lint stage, so it is not excluded here +bin +dist +*.md +CODEOWNERS +.env +.env.local diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..f337156 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,18 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true + +[*.go] +indent_style = tab +tab_width = 4 + +[*.{yaml,yml}] +indent_style = space +indent_size = 2 + +[*.md] +trim_trailing_whitespace = false diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..91c1781 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +github: [dargmuesli] diff --git a/.github/semantic.yml b/.github/semantic.yml new file mode 100644 index 0000000..40b6927 --- /dev/null +++ b/.github/semantic.yml @@ -0,0 +1,5 @@ +# Always validate the PR title AND all the commits +titleAndCommits: true +# Allows use of Merge commits (eg on github: "Merge branch 'master' into feature/ride-unicorns") +# this is only relevant when using commitsOnly: true (or titleAndCommits: true) +allowMergeCommits: true diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..1018f10 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,48 @@ +name: CI + +on: + pull_request: + push: + branches: + - alpha + - beta + - main + - renovate/** + +permissions: {} + +jobs: + ci-optimization: + name: CI optimization + uses: dargmuesli/github-actions/.github/workflows/ci-optimization.yml@e9c9cf391baeb261cf62c0af6c4af2245bae0662 # 5.9.2 + permissions: + pull-requests: read + release_semantic_dry: + needs: ci-optimization + if: needs.ci-optimization.outputs.continue == 'true' + name: Release (semantic, dry) + uses: dargmuesli/github-actions/.github/workflows/release-semantic.yml@e9c9cf391baeb261cf62c0af6c4af2245bae0662 # 5.9.2 + permissions: + contents: write + id-token: write + secrets: + PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + with: + DRY_RUN: true + build: + name: Build + uses: dargmuesli/github-actions/.github/workflows/docker.yml@e9c9cf391baeb261cf62c0af6c4af2245bae0662 # 5.9.2 + needs: release_semantic_dry + permissions: + packages: write + with: + TAG: ${{ (needs.release_semantic_dry.outputs.new_release_version != null) && needs.release_semantic_dry.outputs.new_release_version || '' }} + release_semantic: + needs: build + name: Release (semantic) + uses: dargmuesli/github-actions/.github/workflows/release-semantic.yml@e9c9cf391baeb261cf62c0af6c4af2245bae0662 # 5.9.2 + permissions: + contents: write + id-token: write + secrets: + PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f706cf3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +# Build output. +/bin/ +/dist/ + +# Local environment, which carries the Postgres/S3/Sentry credentials this worker reads. +.env +.env.local + +# Go test and coverage artifacts. +*.test +*.out diff --git a/.golangci.yaml b/.golangci.yaml new file mode 100644 index 0000000..7ed9fb3 --- /dev/null +++ b/.golangci.yaml @@ -0,0 +1,36 @@ +version: "2" + +run: + timeout: 5m + +linters: + enable: + - errcheck + - govet + - staticcheck + - unused + - ineffassign + - misspell + - unconvert + - gocritic + - nolintlint + settings: + gocritic: + enabled-tags: + - diagnostic + - style + - performance + errcheck: + exclude-functions: + - (io.Closer).Close + +formatters: + enable: + - gofmt + - goimports + settings: + goimports: + local-prefixes: + - github.com/maevsi/temporal + +issues: {} diff --git a/.releaserc.json b/.releaserc.json new file mode 100644 index 0000000..ff090ee --- /dev/null +++ b/.releaserc.json @@ -0,0 +1,21 @@ +{ + "plugins": [ + [ + "@semantic-release/commit-analyzer", + { + "preset": "conventionalcommits", + "presetConfig": {} + } + ], + [ + "@semantic-release/release-notes-generator", + { + "preset": "conventionalcommits", + "presetConfig": {} + } + ], + "@semantic-release/changelog", + "@semantic-release/git", + "@semantic-release/github" + ] +} diff --git a/.renovaterc.json b/.renovaterc.json new file mode 100644 index 0000000..7aaff51 --- /dev/null +++ b/.renovaterc.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": ["github>dargmuesli/renovate-config"] +} diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..5622a1d --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,7 @@ +{ + "recommendations": [ + "esbenp.prettier-vscode", + "github.vscode-github-actions", + "golang.go" + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..5a315f1 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,6 @@ +{ + "editor.defaultFormatter": "esbenp.prettier-vscode", + "[go]": { + "editor.defaultFormatter": "golang.go" + } +} diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000..b5cafd7 --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1 @@ +* @dargmuesli diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..d1edae0 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,25 @@ +# Contributing + +## Prerequisites + +- Go (see `go.mod` for the minimum version) +- Git +- [golangci-lint](https://golangci-lint.run/) v2, for local linting matching CI (`docker build --target lint .` runs the same check without a local install) + +See the [README's Development section](README.md#development) for running the worker locally, and its [Tests section](README.md#tests) for the test suite. + +## Local checks + +```sh +go build ./... +go test -race ./... +golangci-lint run ./... +``` + +CI (`.github/workflows/ci.yaml`) runs the same three checks via the Docker `lint`, `test`, and `build` stages, gating the image build on all three passing. +The `test` stage runs the race detector too, so a data race fails CI rather than only showing up locally. +Lint is enforced at zero issues. + +## Commits + +Commits and PR titles must follow [Conventional Commits](https://www.conventionalcommits.org/) (enforced by the Semantic Pull Requests check, see `.github/semantic.yml`); releases and `CHANGELOG.md` entries are generated from them automatically. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..5ebe990 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,122 @@ +# syntax=docker/dockerfile:1 + +######################## +# Create base. + +FROM golang:1.26.6-alpine AS base + +RUN apk add --no-cache ca-certificates + +ARG GROUP_ID=1000 +ARG USER_ID=1000 +ARG USER_NAME=go + +RUN addgroup -g $GROUP_ID -S $USER_NAME \ + && adduser -u $USER_ID -S -G $USER_NAME -h /home/$USER_NAME $USER_NAME + +WORKDIR /srv/app/ + + +######################## +# Serve development. + +FROM base AS development + +ENV CGO_ENABLED=0 + +RUN mkdir -p /home/$USER_NAME/go/pkg/mod \ + && chown -R $USER_NAME:$USER_NAME /home/$USER_NAME /srv/app + +VOLUME /home/$USER_NAME/go/pkg/mod +VOLUME /srv/app + +USER $USER_NAME +CMD ["go", "run", "./cmd/worker"] +EXPOSE 9090 +# Unlike production, this stage has a shell and busybox wget, so the probe is a plain HTTP request rather than the binary's own -healthcheck flag. +# Running that flag through "go run" would have to compile and link the whole worker, Temporal and AWS SDKs included, before the probe's own timeout, which a cold build cache cannot do; it would also re-link on every interval. +# Shortcut: this hardcodes the default metrics port from config.Metrics.Addr, so a dev container that moves the metrics server has to update this line too. +HEALTHCHECK --interval=30s --start-period=30s --timeout=5s CMD wget --quiet --spider http://127.0.0.1:9090/healthz || exit 1 + + +######################## +# Prepare environment. + +FROM base AS prepare + +COPY go.mod go.sum ./ +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + go mod download + +COPY ./ ./ + + +######################## +# Lint code. + +FROM golangci/golangci-lint:v2.12.2-alpine AS lint + +WORKDIR /srv/app/ +COPY --from=prepare /srv/app/ ./ +RUN golangci-lint run ./... + + +######################## +# Test code. + +FROM prepare AS test + +# The race detector needs cgo, which on Alpine means a C toolchain. +# This is a leaf stage, so nothing it installs reaches the production image. +RUN apk add --no-cache gcc musl-dev + +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + CGO_ENABLED=1 go test -race -count=1 ./... + + +######################## +# Build for production. + +FROM prepare AS build + +ENV CGO_ENABLED=0 + +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + go build -trimpath -ldflags="-s -w" -o /srv/app/bin/worker ./cmd/worker + + +######################## +# Collect results. + +FROM base AS collect + +RUN chown $USER_NAME:$USER_NAME . + +COPY --from=build --chown=$USER_NAME /srv/app/bin/worker ./worker +COPY --from=lint /srv/app/go.mod /dev/null +COPY --from=test /srv/app/go.mod /dev/null + + +######################## +# Serve production. +# +# The final image is FROM scratch: a statically linked Go binary needs nothing else at runtime, which keeps the attack surface and image size minimal. +# There is deliberately no shell, so HEALTHCHECK execs the binary itself with -healthcheck instead of curl/wget (see cmd/worker/main.go). + +FROM scratch AS production + +COPY --from=collect /etc/passwd /etc/passwd +COPY --from=collect /etc/group /etc/group +COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +COPY --from=collect --chown=go /srv/app/worker /srv/app/worker + +WORKDIR /srv/app/ +USER go +ENTRYPOINT ["/srv/app/worker"] +EXPOSE 9090 +HEALTHCHECK --interval=30s --start-period=30s --timeout=5s CMD ["/srv/app/worker", "-healthcheck"] +LABEL org.opencontainers.image.source="https://github.com/maevsi/temporal" +LABEL org.opencontainers.image.description="Temporal worker (Go) running jobs for the Vibetype platform." diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..5cc786a --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2026 maevsi + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..66e355e --- /dev/null +++ b/README.md @@ -0,0 +1,192 @@ +# temporal + +A Go Temporal worker for maevsi's scheduled operations: database backups and outbox purges. +This repo only implements the worker side; the shared self-hosted Temporal Server infrastructure (Temporal Server/UI containers, Prometheus scrape config, Grafana alerting, Docker Swarm secrets) is implemented separately in the `stack` repo and is **not** part of this repo. + +## Workflows + +Two Temporal Workflows, each with one Activity doing the real work: + +| Workflow | Activity | Schedule ID | Cadence | +| --------------------- | -------------- | ------------------------- | ------- | +| `DBBackupWorkflow` | `DBBackup` | `dbbackup-daily` | every 24h | +| `OutboxPurgeWorkflow` | `OutboxPurge` | `outbox-purge-every-2h` | every 2h | + +Both workflows call a shared `SentryCheckIn` activity at start (`in_progress`) and completion (`ok` / `error`). +`OutboxPurge` runs directly against Postgres over `pgx`. + +## Architecture + +``` +cmd/worker/main.go entrypoint: wires config -> clients -> worker, runs until SIGINT/SIGTERM +internal/config env-var configuration, one struct per concern, fails listing every missing var at once +internal/activities DBBackup, OutboxPurge, SentryCheckIn: the actual work, interface-mocked for tests +internal/workflows DBBackupWorkflow, OutboxPurgeWorkflow: orchestration only, no AWS/pgx/HTTP imports +internal/schedule creates the two Temporal Schedules on startup (idempotent, native Schedule API) +internal/metrics Prometheus HTTP handler fed by the Temporal Go SDK's native metrics +internal/postgres pgxpool wiring + adapter to activities.DBExecutor +internal/s3client AWS S3 client construction from this repo's own config +internal/sentrycrons minimal Sentry Crons "ping" check-in client +``` + +Workflows never import `internal/postgres`, `internal/s3client`, or the AWS/pgx SDKs. +Instead they reference activities purely through the `var a *activities.Activities; workflow.ExecuteActivity(ctx, a.DBBackup, ...)` method-expression idiom documented by the Temporal Go SDK. +That keeps activity implementations free to depend on real infrastructure clients while workflow code stays deterministic and easy to unit test with mocked activities. + +## Configuration + +Everything is read from environment variables. +There is no `/run/secrets/*` file-reading convention baked into this repo; that is a production deployment detail, not something the worker binary should know about. +Whatever supplies the environment (a Docker Swarm `secrets:` mapping translated to env vars by the container runtime, a `.env` file in local dev, systemd's `EnvironmentFile=`) is responsible for turning maevsi's `vibetype_role_service_*`-style Postgres credentials, S3 credentials, and Sentry Crons URLs into the variables below. + +The one exception is the outbox table the purge runs against, `vibetype_private.outbox`, which is wired in `cmd/worker/main.go`. +It is a property of the schema this worker is written for rather than of a deployment, so it is not worth an environment variable. + +### Temporal + +| Variable | Default | Notes | +| ----------------------- | --------------------------- | -------------------------------------------------- | +| `TEMPORAL_HOST_PORT` | `temporal-server:7233` | Self-hosted Temporal Server frontend address. | +| `TEMPORAL_NAMESPACE` | `default` | | +| `TEMPORAL_TASK_QUEUE` | `maevsi-jobs` | | + +### Postgres + +| Variable | Default | Notes | +| -------------------- | ----------- | ----------------------------------------------------------------------| +| `POSTGRES_HOST` | `localhost` | | +| `POSTGRES_PORT` | `5432` | | +| `POSTGRES_DATABASE` | *required* | | +| `POSTGRES_USER` | *required* | Should be a dedicated `vibetype_role_service_*` role, per stack convention. | +| `POSTGRES_PASSWORD` | *required* | | +| `POSTGRES_SSLMODE` | `require` | Passed straight through to pgx (`disable`, `require`, `verify-full`, ...). | + +### S3 (DBBackup) + +| Variable | Default | Notes | +| ----------------------- | --------- | ---------------------------------------------------------------| +| `S3_BUCKET` | *required* | | +| `S3_PREFIX` | `backups` | Object key prefix. Set it to the empty string to upload at the bucket root. | +| `S3_REGION` | *required* | | +| `S3_ACCESS_KEY_ID` | *required* | | +| `S3_SECRET_ACCESS_KEY` | *required* | | +| `S3_ENDPOINT` | *(unset)* | Override for S3-compatible storage; leave unset for AWS S3. | +| `S3_USE_PATH_STYLE` | `false` | Typically required by non-AWS S3-compatible endpoints. | +| `BACKUP_SOURCE_DIR` | `/backups` | Local directory synced to S3. | + +The credentials need `s3:PutObject` plus `s3:ListBucket` on the bucket. +`s3:ListBucket` is what makes S3 answer `HeadObject` for a key that does not exist with a 404 instead of a 403; without it the worker cannot tell "object missing" apart from "access denied", and rather than re-uploading everything on every run it fails the sync so the permission problem is visible. + +### Sentry Crons + +| Variable | Default | Notes | +| ----------------------------- | --------- | -------------------------------------------------------------------------| +| `SENTRY_CRONS` | *(unset)* | DBBackup check-in URL. | +| `SENTRY_CRONS_OUTBOX_PURGE` | *(unset)* | OutboxPurge check-in URL. | + +Both accept a Sentry Crons monitor "ping" URL (`https://sentry.io/api/0/cron///`); the worker appends `?status=in_progress|ok|error`. +Leaving either unset makes check-ins for that job a no-op rather than an error. +Setting one to something unusable (no scheme, a scheme other than http/https, no host) fails startup instead, since a monitor that silently never checks in is indistinguishable from one that was never configured. + +### Schedule cadences + +| Variable | Default | Notes | +| -------------------------------- | ------- | ------------------------------------------| +| `DBBACKUP_SCHEDULE_EVERY` | `24h` | Go duration syntax (`24h`, `30m`, ...). Only the cadence: how long a single run may take is derived from the activity's own retry budget, so tightening this does not cut runs short. | +| `OUTBOX_PURGE_SCHEDULE_EVERY` | `2h` | | +| `OUTBOX_PURGE_RETENTION` | `24h` | Rows older than this duration are deleted. | + +### Metrics + +| Variable | Default | Notes | +| ----------------- | ---------- | ------------------------------------------| +| `METRICS_ADDR` | `:9090` | Listen address for the Prometheus HTTP server. | +| `METRICS_PATH` | `/metrics` | Must start with a slash, and be anything but `/healthz`, which the worker serves itself. | + +## Metrics + +The worker exposes the Temporal Go SDK's native metrics (activity/workflow execution counts and latencies, task queue backlog, poller counts, etc.) on a Prometheus HTTP handler at `METRICS_ADDR` `METRICS_PATH` (`:9090/metrics` by default), via `go.temporal.io/sdk/contrib/tally` backed by `github.com/uber-go/tally/v4/prometheus`. +Pointing a Prometheus scrape target at this port is all that is needed on the infrastructure side; wiring that scrape target and any Grafana dashboards/alerts is `stack` repo work happening separately. + +There is also a plain `/healthz` endpoint (200 OK if the HTTP server is up) used by the container `HEALTHCHECK`; see [Docker image](#docker-image). + +## Sentry Crons alerting + +Each workflow calls the `SentryCheckIn` activity twice: once with `in_progress` right after the workflow starts, then once more with `ok` or `error` depending on whether the underlying activity (S3 sync / DELETE) succeeded. +A Sentry Crons outage never fails the workflow itself: check-in failures are logged (`workflow.GetLogger(ctx).Warn(...)`) but not propagated, since alerting must not become a reason the actual backup or purge is marked failed. + +## Retry policy + +Both activities get a `temporal.RetryPolicy` tuned to their shape rather than left at SDK defaults: + +- **DBBackup**: `StartToCloseTimeout: 30m`, `HeartbeatTimeout: 2m`, 3 attempts, exponential backoff from 30s up to a 5-minute cap. + The activity heartbeats once per file and again on every completed upload part, so a dead worker is detected long before the 30-minute timeout while a single large upload still cannot outlast the heartbeat timeout on its own. +- **OutboxPurge**: `StartToCloseTimeout: 2m`, 5 attempts, exponential backoff from 5s up to a 1-minute cap. Short because it's a single `DELETE`, more attempts because transient Postgres connectivity issues are the expected failure mode. +- **SentryCheckIn**: `StartToCloseTimeout: 15s`, 3 attempts. Cheap and fast to retry, but never allowed to block the workflow for long. + +Both DBBackup and OutboxPurge treat a configuration problem (missing bucket/schema/table, unreachable source directory) as a non-retryable `ConfigError` `ApplicationError` (`activities.ErrTypeConfig`), since retrying a bad config can't help. +See `RetryPolicy.NonRetryableErrorTypes` in `internal/workflows`. + +## Schedules + +`internal/schedule.EnsureAll` runs once at worker startup and creates both Temporal Schedules if they don't already exist (`ScheduleClient.Describe` then `Create`, never `Update`). +That's safe to run on every deploy without clobbering a cadence an operator has since changed via `temporal schedule` or the Temporal UI. +Cadences are expressed as `client.ScheduleIntervalSpec` (a native Temporal Schedule concept), not as cron strings interpreted by workflow code, and use `SCHEDULE_OVERLAP_POLICY_SKIP` so a slow run never stacks up concurrent executions. + +## Development + +Requires Go (see `go.mod` for the minimum version) and, for integration testing, a local Temporal Server (e.g. `temporal server start-dev`) and Postgres instance. + +```sh +go build ./... +go vet ./... +go test -race ./... +gofmt -l . # should print nothing +``` + +Run the worker locally against a `temporal server start-dev` instance and a local Postgres: + +```sh +export POSTGRES_HOST=localhost POSTGRES_DATABASE=vibetype \ + POSTGRES_USER=vibetype_role_service_temporal_worker POSTGRES_PASSWORD=... POSTGRES_SSLMODE=disable +export S3_BUCKET=... S3_REGION=... S3_ACCESS_KEY_ID=... S3_SECRET_ACCESS_KEY=... +export TEMPORAL_HOST_PORT=127.0.0.1:7233 +go run ./cmd/worker +``` + +## Tests + +`internal/activities`, `internal/workflows`, `internal/schedule`, `internal/sentrycrons`, `internal/config`, and `internal/metrics` all have unit tests: + +- **Activities** are tested via Temporal's `testsuite.TestActivityEnvironment` against hand-rolled fakes of `activities.S3API` (no real AWS) and `activities.DBExecutor` (no real Postgres), covering the upload/skip diffing logic, S3 key building, the retention-interval formatting, and non-retryable config-error paths. + The S3 fake implements the multipart methods as well as `PutObject`, so the path a real dump file takes is covered rather than assumed, including that progress is reported mid-transfer to keep the activity heartbeating. +- **Workflows** are tested via `testsuite.TestWorkflowEnvironment` with mocked activities, asserting the exact `in_progress` -> `ok`/`error` check-in sequence and that a Sentry Crons failure never fails the workflow. +- **Schedules** are tested as pure functions returning `client.ScheduleOptions`, asserting they use `Intervals` (not `CronExpressions`). + +## Docker image + +The `Dockerfile` builds a multi-stage image: `base` -> `development` / `prepare` -> `lint` / `test` -> `build` -> `collect` -> `production`, following this org's general container conventions (non-root user with matching `GROUP_ID`/`USER_ID` build args, `WORKDIR /srv/app/`, a `HEALTHCHECK`, an `org.opencontainers.image.description` label). + +Unlike this org's other services, the `production` stage is `FROM scratch`: a statically linked (`CGO_ENABLED=0`) Go binary needs nothing else at runtime, so the final image is just the binary, CA certificates (for TLS to Temporal/Postgres/S3), and `/etc/passwd`/`/etc/group` entries for the non-root user, about 44 MB total. +Since `scratch` has no shell, `HEALTHCHECK` execs the binary itself with a `-healthcheck` flag, which loads the same config the running worker would and does a plain HTTP GET against its own `/healthz`. +The `development` stage does have a shell and probes `/healthz` with `wget` instead: running `-healthcheck` there would mean a `go run` compiling and linking the whole worker on every probe. + +```sh +docker build --target test . # go test with the race detector +docker build --target lint . # golangci-lint +docker build --target production -t temporal . +``` + +## Known simplifications + +- **DBBackup's diffing** compares local file size against S3's `HeadObject` `ContentLength` per file, rather than the AWS CLI's default size+mtime heuristic. + Dump files get a fresh mtime on every regeneration regardless of whether their content changed, so an mtime check would force a re-upload on every single run and defeat the point of diffing entirely. + Size-only is close enough for backup files that are either new or fully rewritten, but not byte-for-byte equivalent to `aws s3 sync`. +- **DBBackup uploads through `aws-sdk-go-v2/feature/s3/transfermanager`**, which transparently switches to a multipart upload above its `MultipartUploadThreshold`. + That default is 16 MiB, not S3's 5 GiB single-`PutObject` limit, so in practice every real dump file is uploaded as 8 MiB parts. +- **Schedules are bootstrapped once and never updated** by this worker. + Changing a cadence via `DBBACKUP_SCHEDULE_EVERY`/`OUTBOX_PURGE_SCHEDULE_EVERY` after the Schedule already exists requires deleting it first (`temporal schedule delete`) or updating it out-of-band; this is a deliberate choice so an operator's manual Schedule edits are never silently overwritten on redeploy. + +## License + +[Apache-2.0](LICENSE). diff --git a/cmd/worker/main.go b/cmd/worker/main.go new file mode 100644 index 0000000..6aed404 --- /dev/null +++ b/cmd/worker/main.go @@ -0,0 +1,198 @@ +// Command worker runs the maevsi Temporal worker for scheduled +// DBBackup and OutboxPurge operations. +// It: +// +// 1. loads configuration from the environment (see internal/config), +// 2. connects to the self-hosted Temporal Server, Postgres, and S3, +// 3. starts a Prometheus metrics HTTP server fed by the Temporal SDK's +// native metrics, +// 4. ensures both Temporal Schedules exist (idempotent bootstrap), +// 5. registers the DBBackup/OutboxPurge workflows and activities, and +// 6. runs the worker until it receives SIGINT/SIGTERM. +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "log/slog" + "net/http" + "os" + "strings" + "time" + + "go.temporal.io/sdk/client" + sdklog "go.temporal.io/sdk/log" + "go.temporal.io/sdk/worker" + + "github.com/maevsi/temporal/internal/activities" + "github.com/maevsi/temporal/internal/config" + "github.com/maevsi/temporal/internal/metrics" + "github.com/maevsi/temporal/internal/postgres" + "github.com/maevsi/temporal/internal/s3client" + "github.com/maevsi/temporal/internal/schedule" + "github.com/maevsi/temporal/internal/sentrycrons" + "github.com/maevsi/temporal/internal/workflows" +) + +func main() { + healthcheck := flag.Bool("healthcheck", false, "probe the running worker's /healthz endpoint and exit 0/1 instead of starting the worker") + flag.Parse() + + logger := slog.New(slog.NewJSONHandler(os.Stdout, nil)) + + if *healthcheck { + // The production image ships FROM scratch (no shell, no curl/wget), + // so a container HEALTHCHECK has to be an exec of this same binary + // rather than a shell command. + // See Dockerfile. + os.Exit(runHealthcheck()) + } + + if err := run(logger); err != nil { + logger.Error("worker exited with error", "error", err) + os.Exit(1) + } +} + +// defaultMetricsAddr mirrors the METRICS_ADDR default in internal/config, for the case where the probe cannot read the configuration at all. +const defaultMetricsAddr = ":9090" + +// runHealthcheck probes the running worker's health endpoint and returns the process exit code. +func runHealthcheck() int { + return probeHealth(healthcheckAddr()) +} + +// healthcheckAddr resolves the address the probe should connect to. +// It prefers the configured METRICS_ADDR so a non-default port is respected, but falls back to the documented default rather than failing outright, because the container HEALTHCHECK exec may not have all of the worker's env vars available. +// A host-less listen address such as ":9090" means "every interface", which is not something a client can dial, so it is resolved to loopback. +func healthcheckAddr() string { + addr := defaultMetricsAddr + if cfg, err := config.Load(); err == nil { + addr = cfg.Metrics.Addr + } + + if strings.HasPrefix(addr, ":") { + addr = "127.0.0.1" + addr + } + return addr +} + +// probeHealth requests the worker's health endpoint at addr and returns the process exit code. +func probeHealth(addr string) int { + httpClient := &http.Client{Timeout: 5 * time.Second} + resp, err := httpClient.Get(fmt.Sprintf("http://%s%s", addr, metrics.HealthPath)) + if err != nil { + fmt.Fprintln(os.Stderr, "healthcheck: request failed:", err) + return 1 + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + fmt.Fprintf(os.Stderr, "healthcheck: unexpected status %d\n", resp.StatusCode) + return 1 + } + return 0 +} + +func run(logger *slog.Logger) error { + ctx := context.Background() + + cfg, err := config.Load() + if err != nil { + return fmt.Errorf("load config: %w", err) + } + + metricsHandler := metrics.New("temporal_worker") + defer func() { + if err := metricsHandler.Close(); err != nil { + logger.Error("close metrics handler", "error", err) + } + }() + + metricsCtx, stopMetrics := context.WithCancel(ctx) + defer stopMetrics() + metricsErrCh := make(chan error, 1) + go func() { + logger.Info("starting metrics server", "addr", cfg.Metrics.Addr, "path", cfg.Metrics.Path) + metricsErrCh <- metrics.Serve(metricsCtx, cfg.Metrics.Addr, cfg.Metrics.Path, metricsHandler) + }() + + temporalClient, err := client.Dial(client.Options{ + HostPort: cfg.Temporal.HostPort, + Namespace: cfg.Temporal.Namespace, + MetricsHandler: metricsHandler.Temporal, + Logger: sdklog.NewStructuredLogger(logger), + }) + if err != nil { + return fmt.Errorf("connect to temporal at %q: %w", cfg.Temporal.HostPort, err) + } + defer temporalClient.Close() + + pgPool, err := postgres.NewPool(ctx, &cfg.Postgres) + if err != nil { + return fmt.Errorf("connect to postgres: %w", err) + } + defer pgPool.Close() + + s3Client, err := s3client.New(ctx, &cfg.S3) + if err != nil { + return fmt.Errorf("build s3 client: %w", err) + } + + sentryDBBackup, err := sentrycrons.New(cfg.Sentry.DBBackupCheckInURL) + if err != nil { + return fmt.Errorf("configure sentry crons for dbbackup (SENTRY_CRONS): %w", err) + } + sentryOutboxPurge, err := sentrycrons.New(cfg.Sentry.OutboxPurgeCheckInURL) + if err != nil { + return fmt.Errorf("configure sentry crons for outbox purge (SENTRY_CRONS_OUTBOX_PURGE): %w", err) + } + + a := &activities.Activities{ + S3: s3Client, + Bucket: cfg.S3.Bucket, + Prefix: cfg.S3.Prefix, + SourceDir: cfg.S3.SourceDir, + + DB: &postgres.Executor{Pool: pgPool}, + OutboxSchema: "vibetype_private", + OutboxTable: "outbox", + DefaultRetention: cfg.Schedule.OutboxPurgeRetention, + + SentryDBBackup: sentryDBBackup, + SentryOutboxPurge: sentryOutboxPurge, + } + + if err := schedule.EnsureAll(ctx, temporalClient, &cfg); err != nil { + return fmt.Errorf("ensure schedules: %w", err) + } + + w := worker.New(temporalClient, cfg.Temporal.TaskQueue, worker.Options{}) + w.RegisterWorkflow(workflows.DBBackupWorkflow) + w.RegisterWorkflow(workflows.OutboxPurgeWorkflow) + w.RegisterActivity(a) + + logger.Info("starting worker", + "temporalHostPort", cfg.Temporal.HostPort, + "namespace", cfg.Temporal.Namespace, + "taskQueue", cfg.Temporal.TaskQueue, + ) + + runErrCh := make(chan error, 1) + go func() { runErrCh <- w.Run(worker.InterruptCh()) }() + + select { + case err := <-runErrCh: + if err != nil { + return fmt.Errorf("worker run: %w", err) + } + return nil + case err := <-metricsErrCh: + if err != nil { + return fmt.Errorf("metrics server: %w", err) + } + return errors.New("metrics server stopped unexpectedly") + } +} diff --git a/cmd/worker/main_test.go b/cmd/worker/main_test.go new file mode 100644 index 0000000..3e94b45 --- /dev/null +++ b/cmd/worker/main_test.go @@ -0,0 +1,92 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" +) + +// requiredEnv lists the variables config.Load refuses to start without. +// Tests set them via t.Setenv so the values are restored afterwards; setting one to the empty string still trips the "notEmpty" tag and makes Load fail, which is how the config-error path is exercised without leaking an unset variable into later tests. +var requiredEnv = map[string]string{ + "POSTGRES_DATABASE": "test", + "POSTGRES_USER": "test", + "POSTGRES_PASSWORD": "test", + "S3_BUCKET": "test", + "S3_REGION": "us-east-1", + "S3_ACCESS_KEY_ID": "test", + "S3_SECRET_ACCESS_KEY": "test", +} + +func setRequiredEnv(t *testing.T, value func(string) string) { + t.Helper() + for k, v := range requiredEnv { + t.Setenv(k, value(v)) + } +} + +func valid(v string) string { return v } +func blank(string) string { return "" } + +func TestHealthcheckAddr_UsesConfiguredAddr(t *testing.T) { + setRequiredEnv(t, valid) + t.Setenv("METRICS_ADDR", "10.0.0.5:9999") + + assert.Equal(t, "10.0.0.5:9999", healthcheckAddr()) +} + +// TestHealthcheckAddr_ResolvesHostlessAddrToLoopback covers the default listen address form, which names a port but no host and so cannot be dialed as-is. +func TestHealthcheckAddr_ResolvesHostlessAddrToLoopback(t *testing.T) { + setRequiredEnv(t, valid) + t.Setenv("METRICS_ADDR", ":9191") + + assert.Equal(t, "127.0.0.1:9191", healthcheckAddr()) +} + +// TestHealthcheckAddr_FallsBackOnConfigError covers the container HEALTHCHECK exec, which does not necessarily inherit the worker's environment. +// A missing config must not stop the probe from reaching a worker listening on the default address. +func TestHealthcheckAddr_FallsBackOnConfigError(t *testing.T) { + setRequiredEnv(t, blank) + + assert.Equal(t, "127.0.0.1:9090", healthcheckAddr()) +} + +// TestHealthcheckAddr_FallsBackOnEmptyAddr covers METRICS_ADDR being present but blank. +// The fallback comes from the envDefault on Metrics.Addr, which env applies to a set-but-empty variable just as it does to an unset one, so Load can never hand back the blank address that would produce the unusable URL "http:///healthz". +func TestHealthcheckAddr_FallsBackOnEmptyAddr(t *testing.T) { + setRequiredEnv(t, valid) + t.Setenv("METRICS_ADDR", "") + + assert.Equal(t, "127.0.0.1:9090", healthcheckAddr()) +} + +func TestProbeHealth_ReturnsZeroOnHealthy(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/healthz" { + w.WriteHeader(http.StatusOK) + } + })) + defer srv.Close() + + assert.Equal(t, 0, probeHealth(srv.Listener.Addr().String()), "healthcheck should return 0 when the health endpoint responds 200") +} + +func TestProbeHealth_ReturnsOneOnUnhealthy(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + + assert.Equal(t, 1, probeHealth(srv.Listener.Addr().String()), "healthcheck should return 1 when the health endpoint responds non-200") +} + +// TestProbeHealth_ReturnsOneWhenUnreachable covers the case the container HEALTHCHECK actually exists for: the worker is not serving at all. +func TestProbeHealth_ReturnsOneWhenUnreachable(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + addr := srv.Listener.Addr().String() + srv.Close() + + assert.Equal(t, 1, probeHealth(addr), "healthcheck should return 1 when nothing is listening") +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..bc3c57f --- /dev/null +++ b/go.mod @@ -0,0 +1,68 @@ +module github.com/maevsi/temporal + +go 1.26 + +require ( + github.com/aws/aws-sdk-go-v2 v1.43.6 + github.com/aws/aws-sdk-go-v2/config v1.32.37 + github.com/aws/aws-sdk-go-v2/credentials v1.19.36 + github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.3.14 + github.com/aws/aws-sdk-go-v2/service/s3 v1.107.2 + github.com/aws/smithy-go v1.27.8 + github.com/caarlos0/env/v11 v11.4.1 + github.com/jackc/pgx/v5 v5.10.0 + github.com/prometheus/client_golang v1.24.1 + github.com/stretchr/testify v1.12.0 + github.com/uber-go/tally/v4 v4.1.17 + go.temporal.io/api v1.63.4 + go.temporal.io/sdk v1.48.0 + go.temporal.io/sdk/contrib/tally v0.2.0 +) + +require ( + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.18 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.37 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.37 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.37 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.38 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.17 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.30 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.37 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.38 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.5.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.33.6 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.45.6 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/mock v1.6.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.2 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/nexus-rpc/nexus-proto-annotations v0.1.0 // indirect + github.com/nexus-rpc/sdk-go v0.7.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.70.1 // indirect + github.com/prometheus/procfs v0.21.1 // indirect + github.com/robfig/cron v1.2.0 // indirect + github.com/stretchr/objx v0.5.3 // indirect + github.com/twmb/murmur3 v1.1.8 // indirect + go.uber.org/atomic v1.11.0 // indirect + golang.org/x/net v0.57.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.40.0 // indirect + golang.org/x/time v0.3.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260818201246-1b0934165a6f // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260818201246-1b0934165a6f // indirect + google.golang.org/grpc v1.82.1 // indirect + google.golang.org/protobuf v1.36.11 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..f90a03f --- /dev/null +++ b/go.sum @@ -0,0 +1,426 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/aws/aws-sdk-go-v2 v1.43.6 h1:RrmFcqCBxkJuf7g1axVo5krB4jM/AO8r5e5oujrgdoQ= +github.com/aws/aws-sdk-go-v2 v1.43.6/go.mod h1:tXpPM+v0D1lndmga+HqqLDIzUFJlEeR21aspVklHF00= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.18 h1:LAfOuhAH331fmOjTQpAaOlH+Ftn7RzSDJ2VFwjdMMy4= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.18/go.mod h1:4e5xhuXHx1e4U9EthvbPP1r/DIMp5c2823OL8karzcM= +github.com/aws/aws-sdk-go-v2/config v1.32.37 h1:Ljl7LOJB6ym0liuEl0+TZ3d7f5I8MEZN1Cj9PINlj/g= +github.com/aws/aws-sdk-go-v2/config v1.32.37/go.mod h1:WJ7pe7ZPpmG8Q5kKS53zeypIV4FBGACxmte8Uc6SgUc= +github.com/aws/aws-sdk-go-v2/credentials v1.19.36 h1:84s5xMme6ENYEdKG8rsbSFFg/8+lbHBeM9QYSO0gnDk= +github.com/aws/aws-sdk-go-v2/credentials v1.19.36/go.mod h1:c46BLdagDLIswjgt+GeQOslXgeS0E6wCacs5yZbxPGk= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.37 h1:b5tb+CZItBkydC7r3hTNdSO3pszG1R2EtnA+7TePQPk= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.37/go.mod h1:ZQ+6SU9X0oz6+7MUCSswv9Mjci4eaqZr21HI2RVy/yA= +github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.3.14 h1:ZH5pitzSx4Q7UUVVfTAJTxlbhmELBI8aafgF0wWF5l4= +github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.3.14/go.mod h1:Wl+WygckBndyBhVf1kOVUCYBtS4KI2pHgcN8jGsYhwE= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.37 h1:lznzIOvvbqjfe8UAaciCRJgBgJsxuTROKlhZuXQWfv8= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.37/go.mod h1:otfkzyfQeMMLZAqX59GSXTL3o22BR/l6HFaRzzbWSqA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.37 h1:zCEORWo0eU0gDjG+IyApE/2B+ZGG1m+GU7B263XV8ds= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.37/go.mod h1:i6c0PEl3TNOWxRbQ++KQcVenPWS/GoQeiklKhNuqzJ8= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.38 h1:A3UAuCmx7LyUcrixBTzKJYYIUZ2yTvn6ZhT8PB+7APk= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.38/go.mod h1:1PDUYG9Z+JrbbsobsAZHjWOm9QBT/djiK3QbykTL5Z4= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.17 h1:OvYZOB3qA6zvfdRFiRFRzVSiElMYrz3GdntkXZxlp1o= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.17/go.mod h1:JgR/2Ew50ACfIWau1oeMRX59tMtC0kM+PYQGEaT04cY= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.30 h1:5437eMoOwqqQpZn2XJy74mlDCuPYL81texMT3mXqgtU= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.30/go.mod h1:xfu2m3dOpvW8lj98wQYa8V9ku/Rta59hsbireGzhh3A= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.37 h1:a3D4AjrOrTrP8+d9ILBthqrElf0z1JNol09Xvnwcys8= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.37/go.mod h1:ky0gTu+ukvUTuUKFIpp6Wid4oninrkCyvbFkVs0kpHM= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.38 h1:gX8B8y3Ho30B1LPxefDKMi/HZqWEb47U9ogs3DtSG0M= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.38/go.mod h1:l5WblZlcmGPe4/O7JY2HO25Z+xqTBvyfTyFbRMf8gYw= +github.com/aws/aws-sdk-go-v2/service/s3 v1.107.2 h1:GNU0/xtPEXMKilJZ/a8BedeuQnvu+Usi6qVm9EFfncc= +github.com/aws/aws-sdk-go-v2/service/s3 v1.107.2/go.mod h1:4jYWUecEsQtE73jPl7p3jrbYXH5ffcR4gegyCygagfg= +github.com/aws/aws-sdk-go-v2/service/signin v1.5.6 h1:i68sFvXidKlkiSvI7d7Ilc1/UvW4CtBOaivH7jhG4fs= +github.com/aws/aws-sdk-go-v2/service/signin v1.5.6/go.mod h1:/h7Obr9WTtzbjTHGASRQwLN7Bupw+TC3x8x7fyx39hE= +github.com/aws/aws-sdk-go-v2/service/sso v1.33.6 h1:tpfGChmjUmv3W9WlRvy+stwKDTbFFdq8Zk9DbFPrfMU= +github.com/aws/aws-sdk-go-v2/service/sso v1.33.6/go.mod h1:CSjiDzmG/lsKkTOYjbkM+duLmRlW+LOxD64Na44ijnI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.6 h1:49BBtY68A+KJCQ3a2F3eUe6ROsKucxUdfHKoqorc0wI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.6/go.mod h1:ptG2hbs7QltE1GcQY0MpS4bfrc51KCnBXUr7OT1EEfE= +github.com/aws/aws-sdk-go-v2/service/sts v1.45.6 h1:JvExZWabChDM0qJAirQYGfOYo0ndT3edXj+fqSPNjkE= +github.com/aws/aws-sdk-go-v2/service/sts v1.45.6/go.mod h1:XZcaQkV2cItp6yEkrwljyaPOf22RuX7T43jxap/FOmM= +github.com/aws/smithy-go v1.27.8 h1:FR0dxZfIlV7Z8eh2iHfIofdunw382XsDV3Mxt9nUvRY= +github.com/aws/smithy-go v1.27.8/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/caarlos0/env/v11 v11.4.1 h1:fYwH0sWEsBSMPG7t4e/PEfTFzrWrpjyygXyUnWiSwEw= +github.com/caarlos0/env/v11 v11.4.1/go.mod h1:qupehSf/Y0TUTsxKywqRt/vJjN5nz6vauiYEUUr8P4U= +github.com/cactus/go-statsd-client/statsd v0.0.0-20200423205355-cb0885a1018c/go.mod h1:l/bIBLeOl9eX+wxJAzxS4TveKRtAqlyDpHjhkfO0MEI= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yih2JAcL6Ls/kVkSE8GFACTGVnMPruw= +github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= +github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.2 h1:sGm2vDRFUrQJO/Veii4h4zG2vvqG6uWNkBHSTqXOZk0= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.2/go.mod h1:wd1YpapPLivG6nQgbf7ZkG1hhSOXDhhn4MLTknx2aAc= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 h1:asbCHRVmodnJTuQ3qamDwqVOIjwqUPTYmYuemVOx+Ys= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0/go.mod h1:ggCgvZ2r7uOoQjOyu2Y1NhHmEPPzzuhWgcza5M1Ji1I= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0= +github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= +github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nexus-rpc/nexus-proto-annotations v0.1.0 h1:2fELd+9sqUtNu6Fg//pw8YFsxOvp8vZ8hfP0nHhNI80= +github.com/nexus-rpc/nexus-proto-annotations v0.1.0/go.mod h1:n3UjF1bPCW8llR8tHvbxJ+27yPWrhpo8w/Yg1IOuY0Y= +github.com/nexus-rpc/sdk-go v0.7.0 h1:38NrfY5rLnZAiMMs2ZfCKI/CSDzdfJG+27iAgfA8bUI= +github.com/nexus-rpc/sdk-go v0.7.0/go.mod h1:FHdPfVQwRuJFZFTF0Y2GOAxCrbIBNrcPna9slkGKPYk= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU= +github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY= +github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= +github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= +github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= +github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.12.0 h1:K6Mr6jO9JICuend/5xzTM03ydSV3vdNRYAdPSukj8uI= +github.com/stretchr/testify v1.12.0/go.mod h1:bOYBZb5qJ00vPzWfIqBUZPaxK8jWiXc6d3ErP4Ca9Gw= +github.com/twmb/murmur3 v1.1.5/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= +github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= +github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= +github.com/uber-go/tally/v4 v4.1.1/go.mod h1:aXeSTDMl4tNosyf6rdU8jlgScHyjEGGtfJ/uwCIf/vM= +github.com/uber-go/tally/v4 v4.1.17 h1:C+U4BKtVDXTszuzU+WH8JVQvRVnaVKxzZrROFyDrvS8= +github.com/uber-go/tally/v4 v4.1.17/go.mod h1:ZdpiHRGSa3z4NIAc1VlEH4SiknR885fOIF08xmS0gaU= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.temporal.io/api v1.5.0/go.mod h1:BqKxEJJYdxb5dqf0ODfzfMxh8UEQ5L3zKS51FiIYYkA= +go.temporal.io/api v1.63.4 h1:p4dVIAP3dJop0MfcyH9QSzjU7+V/ttLDhxFhSRUar58= +go.temporal.io/api v1.63.4/go.mod h1:SrlW2JMwVlDP4nRWSNznUFqnSHd+YeMDS1BkYo63HCQ= +go.temporal.io/sdk v1.12.0/go.mod h1:lSp3lH1lI0TyOsus0arnO3FYvjVXBZGi/G7DjnAnm6o= +go.temporal.io/sdk v1.48.0 h1:WDctKDVuh0Z8Nf7euAyqs/EwcPg1JTIIq1Fut8Tq118= +go.temporal.io/sdk v1.48.0/go.mod h1:SHv3+fLzD0GGZAwf0xNSvu8UmO1nFgG9WBSYoowApIk= +go.temporal.io/sdk/contrib/tally v0.2.0 h1:XnTJIQcjOv+WuCJ1u8Ve2nq+s2H4i/fys34MnWDRrOo= +go.temporal.io/sdk/contrib/tally v0.2.0/go.mod h1:1kpSuCms/tHeJQDPuuKkaBsMqfHnIIRnCtUYlPNXxuE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210913180222-943fd674d43e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto/googleapis/api v0.0.0-20260818201246-1b0934165a6f h1:zRL8hmUGXSGBJ2D+r85c0S3xhrg6BtfQ3V6KKbR/I9M= +google.golang.org/genproto/googleapis/api v0.0.0-20260818201246-1b0934165a6f/go.mod h1:q/3oV3jAi5vwelxsVAprMBC8BcM2zmNe+IjRGd+9/ks= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260818201246-1b0934165a6f h1:kMQMi+2r0XRQ/Ad2/tgd+5S7JYSBGYO4pwkLTE8F2y0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260818201246-1b0934165a6f/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/validator.v2 v2.0.0-20200605151824-2b28d334fa05/go.mod h1:o4V0GXN9/CAmCsvJ0oXYZvrZOe7syiDZSN1GWGZTGzc= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/internal/activities/activities.go b/internal/activities/activities.go new file mode 100644 index 0000000..5613cc9 --- /dev/null +++ b/internal/activities/activities.go @@ -0,0 +1,32 @@ +// Package activities implements the Temporal Activities for DBBackup, OutboxPurge, and SentryCheckIn. +// +// Activities are methods on Activities so their real dependencies (an S3 client, a Postgres executor, Sentry Crons clients) are injected once at worker startup instead of being read from globals. +// Workflows never import this package's dependencies directly; they reference activities via method-expressions on a zero-value *Activities and the Input/Result types in types.go (see internal/workflows). +package activities + +import ( + "time" + + "github.com/maevsi/temporal/internal/sentrycrons" +) + +// Activities bundles the dependencies needed by every activity in this package. +// A zero-value *Activities is also used, untyped, purely as a method-expression source when workflows register activity references (see internal/workflows), and that usage never calls through the pointer. +type Activities struct { + // S3 / DBBackup dependencies. + S3 S3API + Bucket string + Prefix string + SourceDir string + + // Postgres / OutboxPurge dependencies. + DB DBExecutor + OutboxSchema string + OutboxTable string + DefaultRetention time.Duration + + // Sentry Crons dependencies, one client per job so each keeps its own + // monitor slug (mirrors SENTRY_CRONS / SENTRY_CRONS_OUTBOX_PURGE). + SentryDBBackup *sentrycrons.Client + SentryOutboxPurge *sentrycrons.Client +} diff --git a/internal/activities/dbbackup.go b/internal/activities/dbbackup.go new file mode 100644 index 0000000..4a89a8d --- /dev/null +++ b/internal/activities/dbbackup.go @@ -0,0 +1,178 @@ +package activities + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "path" + "path/filepath" + "strings" + "time" + + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" + "github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager" + "github.com/aws/aws-sdk-go-v2/service/s3" + "go.temporal.io/sdk/activity" + "go.temporal.io/sdk/temporal" +) + +// DBBackup walks SourceDir and uploads every file that is missing from the bucket or whose size differs from what is already there, mirroring the default (no --delete) behavior of `aws s3 sync`. +// +// Matching is done by comparing file size via HeadObject rather than the CLI's size+mtime heuristic: DB dump files get a fresh mtime on every regeneration regardless of whether their content changed, so an mtime check would force a re-upload on every single run and defeat the point of diffing. This is a deliberate simplification documented in the README. +// Uploads go through the AWS SDK's S3 transfer manager (feature/s3/transfermanager), which automatically switches to a multipart upload for files above its MultipartUploadThreshold (16 MiB by default, so in practice every real dump file takes the multipart path). +func (a *Activities) DBBackup(ctx context.Context, _ DBBackupInput) (DBBackupResult, error) { + logger := activity.GetLogger(ctx) + start := time.Now() + + if a.S3 == nil || a.Bucket == "" || a.SourceDir == "" { + return DBBackupResult{}, temporal.NewApplicationError( + "DBBackup activity is missing required configuration (S3 client, bucket, or source directory)", + ErrTypeConfig, + ) + } + + info, err := os.Stat(a.SourceDir) + if err != nil { + return DBBackupResult{}, temporal.NewApplicationError( + fmt.Sprintf("backup source directory %q is not accessible: %v", a.SourceDir, err), + ErrTypeConfig, + ) + } + if !info.IsDir() { + return DBBackupResult{}, temporal.NewApplicationError( + fmt.Sprintf("backup source %q is not a directory", a.SourceDir), + ErrTypeConfig, + ) + } + + result := DBBackupResult{} + uploader := newUploader(a.S3, func(bytesTransferred, totalBytes int64) { + activity.RecordHeartbeat(ctx, bytesTransferred, totalBytes) + }) + + err = filepath.WalkDir(a.SourceDir, func(filePath string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if d.IsDir() { + return nil + } + + // Long syncs must heartbeat so the Temporal server can detect a dead worker via HeartbeatTimeout instead of waiting out the full StartToCloseTimeout. + // This marks progress through the directory; newUploader keeps the heartbeats coming during a single file's transfer. + activity.RecordHeartbeat(ctx, filePath) + + rel, err := filepath.Rel(a.SourceDir, filePath) + if err != nil { + return fmt.Errorf("resolve relative path for %q: %w", filePath, err) + } + // path.Join collapses the prefix's own slashes, so a stray "backups/" or "/backups" in S3_PREFIX cannot produce a doubled or leading separator in the key. + key := strings.TrimPrefix(path.Join(a.Prefix, filepath.ToSlash(rel)), "/") + + fileInfo, err := d.Info() + if err != nil { + return fmt.Errorf("stat %q: %w", filePath, err) + } + + upload, err := a.needsUpload(ctx, key, fileInfo.Size()) + if err != nil { + return err + } + if !upload { + result.FilesSkipped++ + return nil + } + + f, err := os.Open(filePath) + if err != nil { + return fmt.Errorf("open %q: %w", filePath, err) + } + // This defer runs when the walk callback returns for this file, not at the end of the whole walk, so descriptors do not pile up across a large source directory. + // The close error is dropped because the file is only ever read from, where closing cannot lose data. + defer func() { _ = f.Close() }() + + if _, err := uploader.UploadObject(ctx, &transfermanager.UploadObjectInput{ + Bucket: &a.Bucket, + Key: &key, + Body: f, + }); err != nil { + return fmt.Errorf("upload %q to s3://%s/%s: %w", filePath, a.Bucket, key, err) + } + + logger.Debug("uploaded backup file", "path", filePath, "key", key, "bytes", fileInfo.Size()) + result.FilesUploaded++ + result.BytesUploaded += fileInfo.Size() + return nil + }) + if err != nil { + return result, fmt.Errorf("dbbackup sync failed after uploading %d file(s): %w", result.FilesUploaded, err) + } + + result.Duration = time.Since(start) + logger.Info("dbbackup sync complete", + "filesUploaded", result.FilesUploaded, + "filesSkipped", result.FilesSkipped, + "bytesUploaded", result.BytesUploaded, + "duration", result.Duration, + ) + return result, nil +} + +// progressHeartbeat adapts the transfer manager's byte-progress hook to a plain callback. +// The transfer manager invokes progress hooks synchronously from the goroutines running the transfer, so beat must be cheap and safe to call concurrently. +type progressHeartbeat struct { + beat func(bytesTransferred, totalBytes int64) +} + +// OnObjectBytesTransferred implements transfermanager.ObjectBytesTransferredListener. +func (h progressHeartbeat) OnObjectBytesTransferred(_ context.Context, event *transfermanager.ObjectBytesTransferredEvent) { + h.beat(event.BytesTransferred, event.TotalBytes) +} + +// newUploader builds the transfer manager DBBackup uploads through, wired so transfer progress reaches beat. +// Heartbeating once per file before its upload starts is not enough on its own: transferring a single large dump easily outlasts the workflow's HeartbeatTimeout, and the Temporal server would then time the activity out mid-upload and retry it from scratch on every attempt, so the file would never finish uploading no matter how many attempts it got. +// On the multipart path the hook fires as each part completes; below the multipart threshold it fires only once, after the upload, which is fine because a file that small transfers well inside the timeout. +func newUploader(client S3API, beat func(bytesTransferred, totalBytes int64)) *transfermanager.Client { + return transfermanager.New(client, func(o *transfermanager.Options) { + o.ObjectProgressListeners.Register(progressHeartbeat{beat: beat}) + o.FailTimeout = uploadFailTimeout + }) +} + +// uploadFailTimeout bounds the cleanup the transfer manager does after a failed upload, chiefly the AbortMultipartUpload that releases the parts already sent. +// It has to be set: left at zero the transfer manager runs that cleanup on the caller's context, so when Temporal cancels the activity on a heartbeat or start-to-close timeout mid-upload the abort is issued on an already-cancelled context, fails, and leaves billable incomplete parts behind on every timed-out run. +const uploadFailTimeout = 30 * time.Second + +// needsUpload reports whether the local file at the given size should be +// uploaded to key, based on whether the object exists in S3 and, if so, +// whether its size differs from the local file. +func (a *Activities) needsUpload(ctx context.Context, key string, localSize int64) (bool, error) { + head, err := a.S3.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &a.Bucket, + Key: &key, + }) + if err != nil { + if isNotFound(err) { + return true, nil + } + return false, fmt.Errorf("head s3://%s/%s: %w", a.Bucket, key, err) + } + if head.ContentLength == nil || *head.ContentLength != localSize { + return true, nil + } + return false, nil +} + +// isNotFound reports whether err represents an S3 404 response, whether or +// not the SDK managed to decode it into a typed NotFound error (HeadObject +// responses have no body to decode a typed error from, so this commonly +// surfaces only as an HTTP status). +func isNotFound(err error) bool { + var respErr *awshttp.ResponseError + if errors.As(err, &respErr) { + return respErr.HTTPStatusCode() == 404 + } + return false +} diff --git a/internal/activities/dbbackup_test.go b/internal/activities/dbbackup_test.go new file mode 100644 index 0000000..afaa142 --- /dev/null +++ b/internal/activities/dbbackup_test.go @@ -0,0 +1,333 @@ +package activities + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "slices" + "sync" + "testing" + + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" + "github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager" + "github.com/aws/aws-sdk-go-v2/service/s3" + smithyhttp "github.com/aws/smithy-go/transport/http" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.temporal.io/sdk/testsuite" +) + +// multipartUploadThreshold mirrors transfermanager's unexported defaultMultipartUploadThreshold. +// It is 16 MiB, not S3's 5 GiB single-PutObject limit, so any fixture meant to exercise the path real dump files take has to clear it. +const multipartUploadThreshold = 16 * 1024 * 1024 + +// fakeS3 is a hand-rolled fake of the S3API interface, keyed by S3 object key, that lets tests assert on exactly what the activity would have uploaded without touching real AWS infrastructure. +// +// Both upload paths are implemented, because both are reachable: transfermanager switches to a multipart upload above 16 MiB, which is well under the size of a real database dump, so the multipart path is the one production almost always takes. +// The parts are reassembled the way S3 itself does, in the order the caller lists them at CompleteMultipartUpload, so a test can compare the resulting object against the bytes on disk. +type fakeS3 struct { + // mu guards every field below: the transfer manager uploads parts from several goroutines at once. + mu sync.Mutex + + // existing simulates objects already present in the bucket, keyed by + // object key, valued by their content length. + existing map[string]int64 + // uploaded records the final body of every completed upload, keyed by key, whether it arrived as one PutObject or as reassembled multipart parts. + uploaded map[string][]byte + // parts holds the parts of in-flight multipart uploads, keyed by upload ID and then part number. + parts map[string]map[int32][]byte + // multipartUploads counts CreateMultipartUpload calls, so a test can assert which upload path a fixture actually took. + multipartUploads int + // aborted counts AbortMultipartUpload calls, which the transfer manager issues when a multipart upload fails partway through. + aborted int + + headErr error + putErr error + partErr error +} + +func newFakeS3() *fakeS3 { + return &fakeS3{ + existing: map[string]int64{}, + uploaded: map[string][]byte{}, + parts: map[string]map[int32][]byte{}, + } +} + +func (f *fakeS3) CreateMultipartUpload(_ context.Context, params *s3.CreateMultipartUploadInput, _ ...func(*s3.Options)) (*s3.CreateMultipartUploadOutput, error) { + f.mu.Lock() + defer f.mu.Unlock() + + f.multipartUploads++ + uploadID := fmt.Sprintf("upload-%d", f.multipartUploads) + f.parts[uploadID] = map[int32][]byte{} + return &s3.CreateMultipartUploadOutput{Bucket: params.Bucket, Key: params.Key, UploadId: &uploadID}, nil +} + +func (f *fakeS3) UploadPart(_ context.Context, params *s3.UploadPartInput, _ ...func(*s3.Options)) (*s3.UploadPartOutput, error) { + if f.partErr != nil { + return nil, f.partErr + } + body, err := io.ReadAll(params.Body) + if err != nil { + return nil, err + } + + f.mu.Lock() + defer f.mu.Unlock() + + f.parts[*params.UploadId][*params.PartNumber] = body + etag := fmt.Sprintf("etag-%d", *params.PartNumber) + return &s3.UploadPartOutput{ETag: &etag}, nil +} + +func (f *fakeS3) CompleteMultipartUpload(_ context.Context, params *s3.CompleteMultipartUploadInput, _ ...func(*s3.Options)) (*s3.CompleteMultipartUploadOutput, error) { + f.mu.Lock() + defer f.mu.Unlock() + + var object []byte + for _, part := range params.MultipartUpload.Parts { + object = append(object, f.parts[*params.UploadId][*part.PartNumber]...) + } + delete(f.parts, *params.UploadId) + f.uploaded[*params.Key] = object + return &s3.CompleteMultipartUploadOutput{Bucket: params.Bucket, Key: params.Key}, nil +} + +func (f *fakeS3) AbortMultipartUpload(_ context.Context, params *s3.AbortMultipartUploadInput, _ ...func(*s3.Options)) (*s3.AbortMultipartUploadOutput, error) { + f.mu.Lock() + defer f.mu.Unlock() + + f.aborted++ + delete(f.parts, *params.UploadId) + return &s3.AbortMultipartUploadOutput{}, nil +} + +func (f *fakeS3) GetObject(_ context.Context, _ *s3.GetObjectInput, _ ...func(*s3.Options)) (*s3.GetObjectOutput, error) { + panic("not expected: DBBackup never reads objects back") +} + +func (f *fakeS3) ListObjectsV2(_ context.Context, _ *s3.ListObjectsV2Input, _ ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) { + panic("not expected: DBBackup never lists objects") +} + +func (f *fakeS3) HeadObject(_ context.Context, params *s3.HeadObjectInput, _ ...func(*s3.Options)) (*s3.HeadObjectOutput, error) { + if f.headErr != nil { + return nil, f.headErr + } + + f.mu.Lock() + defer f.mu.Unlock() + + size, ok := f.existing[*params.Key] + if !ok { + return nil, &awshttp.ResponseError{ + ResponseError: &smithyhttp.ResponseError{ + Response: &smithyhttp.Response{Response: &http.Response{StatusCode: 404}}, + }, + } + } + return &s3.HeadObjectOutput{ContentLength: &size}, nil +} + +func (f *fakeS3) PutObject(_ context.Context, params *s3.PutObjectInput, _ ...func(*s3.Options)) (*s3.PutObjectOutput, error) { + if f.putErr != nil { + return nil, f.putErr + } + body, err := io.ReadAll(params.Body) + if err != nil { + return nil, err + } + + f.mu.Lock() + defer f.mu.Unlock() + + f.uploaded[*params.Key] = body + return &s3.PutObjectOutput{}, nil +} + +func writeFile(t *testing.T, dir, rel, content string) { + t.Helper() + path := filepath.Join(dir, rel) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) +} + +func runActivity[TIn, TOut any](t *testing.T, fn func(context.Context, TIn) (TOut, error), in TIn) (TOut, error) { + t.Helper() + var ts testsuite.WorkflowTestSuite + env := ts.NewTestActivityEnvironment() + env.RegisterActivity(fn) + val, err := env.ExecuteActivity(fn, in) + var out TOut + // Only the success path carries a result worth decoding, and a decode failure there would otherwise surface as a confusing assertion against a zero value. + if err == nil && val != nil { + require.NoError(t, val.Get(&out), "decode activity result") + } + return out, err +} + +func TestDBBackup_UploadsNewAndChangedFiles(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "dump.sql", "new file, not in bucket yet") + writeFile(t, dir, "nested/dump2.sql", "changed content") + writeFile(t, dir, "unchanged.sql", "same everywhere") + + fake := newFakeS3() + fake.existing["backups/nested/dump2.sql"] = 3 // wrong size -> re-upload + fake.existing["backups/unchanged.sql"] = int64(len("same everywhere")) + + a := &Activities{ + S3: fake, + Bucket: "maevsi-backups", + Prefix: "backups", + SourceDir: dir, + } + + result, err := runActivity(t, a.DBBackup, DBBackupInput{}) + require.NoError(t, err) + + assert.Equal(t, 2, result.FilesUploaded) + assert.Equal(t, 1, result.FilesSkipped) + assert.Equal(t, []byte("new file, not in bucket yet"), fake.uploaded["backups/dump.sql"]) + assert.Equal(t, []byte("changed content"), fake.uploaded["backups/nested/dump2.sql"]) + assert.NotContains(t, fake.uploaded, "backups/unchanged.sql") +} + +func TestDBBackup_SkipsUnchangedFiles(t *testing.T) { + dir := t.TempDir() + content := "identical content" + writeFile(t, dir, "dump.sql", content) + + fake := newFakeS3() + fake.existing["backups/dump.sql"] = int64(len(content)) + + a := &Activities{S3: fake, Bucket: "b", Prefix: "backups", SourceDir: dir} + + result, err := runActivity(t, a.DBBackup, DBBackupInput{}) + require.NoError(t, err) + assert.Equal(t, 0, result.FilesUploaded) + assert.Equal(t, 1, result.FilesSkipped) +} + +func TestDBBackup_MissingSourceDirIsNonRetryableConfigError(t *testing.T) { + a := &Activities{S3: newFakeS3(), Bucket: "b", Prefix: "backups", SourceDir: "/does/not/exist"} + + _, err := runActivity(t, a.DBBackup, DBBackupInput{}) + require.Error(t, err) + assertApplicationErrorType(t, err, ErrTypeConfig) +} + +func TestDBBackup_MissingConfigIsNonRetryableConfigError(t *testing.T) { + a := &Activities{} + _, err := runActivity(t, a.DBBackup, DBBackupInput{}) + require.Error(t, err) + assertApplicationErrorType(t, err, ErrTypeConfig) +} + +func TestDBBackup_PutObjectErrorFailsActivity(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "dump.sql", "content") + + fake := newFakeS3() + fake.putErr = assert.AnError + + a := &Activities{S3: fake, Bucket: "b", Prefix: "backups", SourceDir: dir} + _, err := runActivity(t, a.DBBackup, DBBackupInput{}) + require.Error(t, err) +} + +// writeLargeFile writes a file of the given size and returns its content, so a test can compare what ended up in the bucket against what was on disk. +// The byte pattern uses a prime stride so parts reassembled in the wrong order show up as a mismatch instead of lining up by coincidence. +func writeLargeFile(t *testing.T, dir, rel string, size int) []byte { + t.Helper() + content := make([]byte, size) + for i := range content { + content[i] = byte(i % 251) + } + path := filepath.Join(dir, rel) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, content, 0o644)) + return content +} + +// TestDBBackup_UploadsLargeFileViaMultipart covers the upload path a real dump file takes. +// Anything over 16 MiB is split into parts and reassembled by S3 rather than sent as one PutObject, so this asserts the object that ends up in the bucket is byte-for-byte the file on disk. +func TestDBBackup_UploadsLargeFileViaMultipart(t *testing.T) { + dir := t.TempDir() + content := writeLargeFile(t, dir, "dump.sql", multipartUploadThreshold+1) + + fake := newFakeS3() + a := &Activities{S3: fake, Bucket: "b", Prefix: "backups", SourceDir: dir} + + result, err := runActivity(t, a.DBBackup, DBBackupInput{}) + require.NoError(t, err) + + assert.Equal(t, 1, result.FilesUploaded) + assert.Equal(t, int64(len(content)), result.BytesUploaded) + assert.Equal(t, 1, fake.multipartUploads, "a file over the threshold must go through a multipart upload") + assert.Zero(t, fake.aborted) + assert.True(t, bytes.Equal(content, fake.uploaded["backups/dump.sql"]), "the reassembled object must match the file on disk") +} + +// TestNewUploader_ReportsProgressDuringTransfer pins the reason DBBackup wires a progress listener into the transfer manager at all. +// Heartbeating once per file before its upload starts leaves the whole transfer of a large dump silent, so the Temporal server would time the activity out mid-upload and retry it from scratch forever; progress has to arrive while the transfer is still running, not only once it has finished. +func TestNewUploader_ReportsProgressDuringTransfer(t *testing.T) { + content := make([]byte, multipartUploadThreshold+1) + fake := newFakeS3() + + var mu sync.Mutex + var progress []int64 + uploader := newUploader(fake, func(bytesTransferred, _ int64) { + mu.Lock() + defer mu.Unlock() + progress = append(progress, bytesTransferred) + }) + + bucket, key := "b", "backups/dump.sql" + _, err := uploader.UploadObject(t.Context(), &transfermanager.UploadObjectInput{ + Bucket: &bucket, + Key: &key, + Body: bytes.NewReader(content), + }) + require.NoError(t, err) + + mu.Lock() + defer mu.Unlock() + require.Greater(t, len(progress), 1, "progress must be reported per part, not once at the end of the transfer") + assert.Equal(t, int64(len(content)), slices.Max(progress), "the final progress report must account for the whole object") +} + +// TestDBBackup_KeyBuildingNormalizesPrefix guards the S3 key layout against a prefix that carries its own separators, which would otherwise produce keys like "backups//dump.sql". +func TestDBBackup_KeyBuildingNormalizesPrefix(t *testing.T) { + for _, prefix := range []string{"backups", "backups/", "/backups", "backups//"} { + t.Run(prefix, func(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "nested/dump.sql", "content") + + fake := newFakeS3() + a := &Activities{S3: fake, Bucket: "b", Prefix: prefix, SourceDir: dir} + + _, err := runActivity(t, a.DBBackup, DBBackupInput{}) + require.NoError(t, err) + assert.Contains(t, fake.uploaded, "backups/nested/dump.sql") + }) + } +} + +// TestDBBackup_EmptyPrefixUploadsAtBucketRoot covers an explicitly empty S3_PREFIX, where keys must not pick up a leading separator. +func TestDBBackup_EmptyPrefixUploadsAtBucketRoot(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "dump.sql", "content") + + fake := newFakeS3() + a := &Activities{S3: fake, Bucket: "b", SourceDir: dir} + + _, err := runActivity(t, a.DBBackup, DBBackupInput{}) + require.NoError(t, err) + assert.Contains(t, fake.uploaded, "dump.sql") +} diff --git a/internal/activities/helpers_test.go b/internal/activities/helpers_test.go new file mode 100644 index 0000000..12687a4 --- /dev/null +++ b/internal/activities/helpers_test.go @@ -0,0 +1,20 @@ +package activities + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.temporal.io/sdk/temporal" +) + +// assertApplicationErrorType asserts that err is (or wraps) a Temporal +// *temporal.ApplicationError with the given Type, which is what +// RetryPolicy.NonRetryableErrorTypes matches against. +func assertApplicationErrorType(t *testing.T, err error, wantType string) { + t.Helper() + var appErr *temporal.ApplicationError + require.True(t, errors.As(err, &appErr), "expected a *temporal.ApplicationError, got %T: %v", err, err) + assert.Equal(t, wantType, appErr.Type()) +} diff --git a/internal/activities/interfaces.go b/internal/activities/interfaces.go new file mode 100644 index 0000000..95ab264 --- /dev/null +++ b/internal/activities/interfaces.go @@ -0,0 +1,19 @@ +package activities + +import ( + "context" + + "github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager" +) + +// S3API is the subset of the AWS S3 client used by the DBBackup activity: transfermanager.S3APIClient already covers both the HeadObject existence/size check and everything needed to upload a file, transparently using a multipart upload above S3's single-PutObject limit. +// It is satisfied directly by *s3.Client (see internal/s3client), and is small enough to fake by hand in tests without pulling in a generated mock of the whole S3 SDK surface. +type S3API interface { + transfermanager.S3APIClient +} + +// DBExecutor is the subset of a Postgres connection used by the OutboxPurge activity: a single parameterized statement that reports the number of affected rows. +// It is satisfied by *postgres.Executor (see internal/postgres) and is trivial to fake in tests without spinning up a real Postgres instance. +type DBExecutor interface { + Exec(ctx context.Context, sql string, args ...any) (rowsAffected int64, err error) +} diff --git a/internal/activities/outboxpurge.go b/internal/activities/outboxpurge.go new file mode 100644 index 0000000..acacb99 --- /dev/null +++ b/internal/activities/outboxpurge.go @@ -0,0 +1,77 @@ +package activities + +import ( + "context" + "fmt" + "strings" + "time" + + "go.temporal.io/sdk/activity" + "go.temporal.io/sdk/temporal" +) + +// OutboxPurge deletes rows older than the configured retention period from the outbox table: +// +// DELETE FROM vibetype_private.outbox WHERE created_at < now() - interval '24 hours' +// +// against the org's central Postgres instance, via the dedicated vibetype_role_service_* role configured for this worker. +// The retention window comes from input.Retention if set, otherwise from the Activities' configured default (OUTBOX_PURGE_RETENTION, defaulting to 24h). +// +// The interval is passed as a bound parameter cast to ::interval rather than interpolated into the query text, so it is always valid regardless of the configured duration's units. +func (a *Activities) OutboxPurge(ctx context.Context, input OutboxPurgeInput) (OutboxPurgeResult, error) { + logger := activity.GetLogger(ctx) + + if a.DB == nil || a.OutboxSchema == "" || a.OutboxTable == "" { + return OutboxPurgeResult{}, temporal.NewApplicationError( + "OutboxPurge activity is missing required configuration (DB executor, schema, or table)", + ErrTypeConfig, + ) + } + + retention := input.Retention + if retention <= 0 { + retention = a.DefaultRetention + } + if retention <= 0 { + return OutboxPurgeResult{}, temporal.NewApplicationError( + "OutboxPurge activity has no positive retention period configured", + ErrTypeConfig, + ) + } + + query := fmt.Sprintf( + `DELETE FROM %s.%s WHERE created_at < now() - $1::interval`, + quoteIdent(a.OutboxSchema), quoteIdent(a.OutboxTable), + ) + + rows, err := a.DB.Exec(ctx, query, pgIntervalLiteral(retention)) + if err != nil { + return OutboxPurgeResult{}, fmt.Errorf("purge outbox rows older than %s: %w", retention, err) + } + + logger.Info("outbox purge complete", "rowsDeleted", rows, "retention", retention) + return OutboxPurgeResult{RowsDeleted: rows}, nil +} + +// pgIntervalLiteral renders a time.Duration as a Postgres interval literal +// (e.g. "3600 seconds") that is always parseable by ::interval regardless +// of the duration's magnitude or the Go string representation's units. +func pgIntervalLiteral(d time.Duration) string { + return fmt.Sprintf("%d seconds", int64(d.Seconds())) +} + +// quoteIdent double-quotes a Postgres identifier and escapes embedded quotes. +// Schema/table names come from this service's own configuration, not user input, but quoting them keeps the query well-formed regardless. +func quoteIdent(ident string) string { + var b strings.Builder + b.WriteByte('"') + for _, r := range ident { + if r == '"' { + b.WriteString(`""`) + continue + } + b.WriteRune(r) + } + b.WriteByte('"') + return b.String() +} diff --git a/internal/activities/outboxpurge_test.go b/internal/activities/outboxpurge_test.go new file mode 100644 index 0000000..09bf566 --- /dev/null +++ b/internal/activities/outboxpurge_test.go @@ -0,0 +1,95 @@ +package activities + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeDB is a hand-rolled fake of the DBExecutor interface that records +// the query and args it was called with and returns a scripted result. +type fakeDB struct { + rowsAffected int64 + err error + + gotQuery string + gotArgs []any +} + +func (f *fakeDB) Exec(_ context.Context, sql string, args ...any) (int64, error) { + f.gotQuery = sql + f.gotArgs = args + if f.err != nil { + return 0, f.err + } + return f.rowsAffected, nil +} + +func TestOutboxPurge_DeletesWithDefaultRetention(t *testing.T) { + db := &fakeDB{rowsAffected: 42} + a := &Activities{ + DB: db, + OutboxSchema: "vibetype_private", + OutboxTable: "outbox", + DefaultRetention: 24 * time.Hour, + } + + result, err := runActivity(t, a.OutboxPurge, OutboxPurgeInput{}) + require.NoError(t, err) + assert.Equal(t, int64(42), result.RowsDeleted) + assert.Equal(t, `DELETE FROM "vibetype_private"."outbox" WHERE created_at < now() - $1::interval`, db.gotQuery) + require.Len(t, db.gotArgs, 1) + assert.Equal(t, "86400 seconds", db.gotArgs[0]) +} + +func TestOutboxPurge_InputRetentionOverridesDefault(t *testing.T) { + db := &fakeDB{rowsAffected: 0} + a := &Activities{ + DB: db, + OutboxSchema: "vibetype_private", + OutboxTable: "outbox", + DefaultRetention: 24 * time.Hour, + } + + _, err := runActivity(t, a.OutboxPurge, OutboxPurgeInput{Retention: 2 * time.Hour}) + require.NoError(t, err) + require.Len(t, db.gotArgs, 1) + assert.Equal(t, "7200 seconds", db.gotArgs[0]) +} + +func TestOutboxPurge_NoRetentionConfiguredIsNonRetryableConfigError(t *testing.T) { + a := &Activities{DB: &fakeDB{}, OutboxSchema: "vibetype_private", OutboxTable: "outbox"} + + _, err := runActivity(t, a.OutboxPurge, OutboxPurgeInput{}) + require.Error(t, err) + assertApplicationErrorType(t, err, ErrTypeConfig) +} + +func TestOutboxPurge_MissingConfigIsNonRetryableConfigError(t *testing.T) { + a := &Activities{} + _, err := runActivity(t, a.OutboxPurge, OutboxPurgeInput{}) + require.Error(t, err) + assertApplicationErrorType(t, err, ErrTypeConfig) +} + +func TestOutboxPurge_DBErrorPropagates(t *testing.T) { + db := &fakeDB{err: assert.AnError} + a := &Activities{DB: db, OutboxSchema: "vibetype_private", OutboxTable: "outbox", DefaultRetention: time.Hour} + + _, err := runActivity(t, a.OutboxPurge, OutboxPurgeInput{}) + require.Error(t, err) +} + +func TestPgIntervalLiteral(t *testing.T) { + assert.Equal(t, "86400 seconds", pgIntervalLiteral(24*time.Hour)) + assert.Equal(t, "7200 seconds", pgIntervalLiteral(2*time.Hour)) + assert.Equal(t, "90 seconds", pgIntervalLiteral(90*time.Second)) +} + +func TestQuoteIdent(t *testing.T) { + assert.Equal(t, `"outbox"`, quoteIdent("outbox")) + assert.Equal(t, `"weird""name"`, quoteIdent(`weird"name`)) +} diff --git a/internal/activities/sentrycheckin.go b/internal/activities/sentrycheckin.go new file mode 100644 index 0000000..047ccae --- /dev/null +++ b/internal/activities/sentrycheckin.go @@ -0,0 +1,44 @@ +package activities + +import ( + "context" + "fmt" + + "go.temporal.io/sdk/activity" + + "github.com/maevsi/temporal/internal/sentrycrons" +) + +// SentryCheckIn sends a single Sentry Crons check-in for the given job and status. +// It is called from both workflows at start (in_progress) and at completion (ok / error). +// +// It is intentionally a thin activity: workflows treat check-in failures +// as best-effort (logged, not propagated) so a Sentry outage never fails +// the underlying backup or purge. +func (a *Activities) SentryCheckIn(ctx context.Context, input SentryCheckInInput) error { + logger := activity.GetLogger(ctx) + + client, err := a.sentryClientFor(input.Job) + if err != nil { + return err + } + + if err := client.CheckIn(ctx, input.Status); err != nil { + return fmt.Errorf("sentry crons check-in for job %q: %w", input.Job, err) + } + logger.Debug("sentry crons check-in sent", "job", input.Job, "status", input.Status) + return nil +} + +// sentryClientFor maps a Job to its configured Sentry Crons client. +// Keeping the mapping in one place means adding a third job only requires a new case here plus a new field on Activities. +func (a *Activities) sentryClientFor(job Job) (*sentrycrons.Client, error) { + switch job { + case JobDBBackup: + return a.SentryDBBackup, nil + case JobOutboxPurge: + return a.SentryOutboxPurge, nil + default: + return nil, fmt.Errorf("sentrycheckin: unknown job %q", job) + } +} diff --git a/internal/activities/sentrycheckin_test.go b/internal/activities/sentrycheckin_test.go new file mode 100644 index 0000000..5e4d28d --- /dev/null +++ b/internal/activities/sentrycheckin_test.go @@ -0,0 +1,63 @@ +package activities + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.temporal.io/sdk/testsuite" + + "github.com/maevsi/temporal/internal/sentrycrons" +) + +// newSentryClient builds a sentrycrons.Client for a test, failing the test rather than returning the error, since these tests only ever pass URLs that are known-good. +func newSentryClient(t *testing.T, checkInURL string) *sentrycrons.Client { + t.Helper() + c, err := sentrycrons.New(checkInURL) + require.NoError(t, err) + return c +} + +// runCheckIn executes the SentryCheckIn activity, which (unlike DBBackup +// and OutboxPurge) returns only an error and so doesn't fit the generic +// runActivity helper used elsewhere in this package. +func runCheckIn(t *testing.T, a *Activities, in SentryCheckInInput) error { + t.Helper() + var ts testsuite.WorkflowTestSuite + env := ts.NewTestActivityEnvironment() + env.RegisterActivity(a.SentryCheckIn) + _, err := env.ExecuteActivity(a.SentryCheckIn, in) + return err +} + +func TestSentryCheckIn_RoutesToConfiguredClient(t *testing.T) { + var gotStatus string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotStatus = r.URL.Query().Get("status") + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + a := &Activities{ + SentryDBBackup: newSentryClient(t, srv.URL), + SentryOutboxPurge: newSentryClient(t, ""), // unconfigured, must not be used + } + + err := runCheckIn(t, a, SentryCheckInInput{Job: JobDBBackup, Status: sentrycrons.StatusOK}) + require.NoError(t, err) + assert.Equal(t, "ok", gotStatus) +} + +func TestSentryCheckIn_UnknownJobErrors(t *testing.T) { + a := &Activities{} + err := runCheckIn(t, a, SentryCheckInInput{Job: "not-a-real-job", Status: sentrycrons.StatusOK}) + require.Error(t, err) +} + +func TestSentryCheckIn_UnconfiguredClientIsNoOp(t *testing.T) { + a := &Activities{SentryOutboxPurge: newSentryClient(t, "")} + err := runCheckIn(t, a, SentryCheckInInput{Job: JobOutboxPurge, Status: sentrycrons.StatusInProgress}) + require.NoError(t, err) +} diff --git a/internal/activities/types.go b/internal/activities/types.go new file mode 100644 index 0000000..4949d03 --- /dev/null +++ b/internal/activities/types.go @@ -0,0 +1,51 @@ +package activities + +import ( + "time" + + "github.com/maevsi/temporal/internal/sentrycrons" +) + +// Job identifies which workflow a Sentry Crons check-in belongs to. +type Job string + +const ( + // JobDBBackup is the DBBackup workflow. + JobDBBackup Job = "dbbackup" + // JobOutboxPurge is the OutboxPurge workflow. + JobOutboxPurge Job = "outbox_purge" +) + +// ErrTypeConfig is the ApplicationError type used for configuration problems (bad bucket name, unreachable source directory, ...) that will never succeed on retry. +// Pair with temporal.RetryPolicy.NonRetryableErrorTypes. +const ErrTypeConfig = "ConfigError" + +// DBBackupInput is the input to the DBBackup activity. +// Empty today; kept as a named struct so new parameters (e.g. an explicit source override) can be added without breaking the activity's serialized signature. +type DBBackupInput struct{} + +// DBBackupResult summarizes the outcome of a DBBackup activity run. +type DBBackupResult struct { + FilesUploaded int + FilesSkipped int + BytesUploaded int64 + Duration time.Duration +} + +// OutboxPurgeInput is the input to the OutboxPurge activity. +type OutboxPurgeInput struct { + // Retention overrides the configured retention period for this run. + // Zero means "use the Activities' configured default". + Retention time.Duration +} + +// OutboxPurgeResult summarizes the outcome of an OutboxPurge activity run. +type OutboxPurgeResult struct { + RowsDeleted int64 +} + +// SentryCheckInInput is the input to the SentryCheckIn activity. +type SentryCheckInInput struct { + Job Job + Status sentrycrons.Status +} diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..e499fdd --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,141 @@ +// Package config loads worker configuration from environment variables. +// +// Every setting is read from the environment rather than from files, so this package intentionally does not know anything about the Docker Swarm secrets convention (files mounted under /run/secrets/*) used by the maevsi stack in production. +// Whatever injects the environment for this process (a compose "secrets" mapping, a systemd EnvironmentFile, a plain .env in local dev) is responsible for turning those secrets into the environment variables documented below and in the README. +package config + +import ( + "fmt" + "os" + "strings" + "time" + + "github.com/caarlos0/env/v11" +) + +// Config is the fully resolved worker configuration. +type Config struct { + Temporal Temporal + Metrics Metrics + Postgres Postgres + S3 S3 + Sentry Sentry + Schedule Schedule +} + +// Temporal holds the connection details for the self-hosted Temporal +// Server (not Temporal Cloud) this worker talks to. +type Temporal struct { + // HostPort is the address of the Temporal frontend service, e.g. "temporal-server:7233". + // Matches client.Options.HostPort. + HostPort string `env:"TEMPORAL_HOST_PORT" envDefault:"temporal-server:7233"` + // Namespace is the Temporal namespace to operate in. + Namespace string `env:"TEMPORAL_NAMESPACE" envDefault:"default"` + // TaskQueue is the task queue this worker polls and that schedules + // dispatch workflow tasks to. + TaskQueue string `env:"TEMPORAL_TASK_QUEUE" envDefault:"maevsi-jobs"` +} + +// Metrics configures the Prometheus metrics HTTP server exposed by the +// worker process, fed by the Temporal Go SDK's native metrics. +type Metrics struct { + // Addr is the listen address for the metrics HTTP server, e.g. ":9090". + Addr string `env:"METRICS_ADDR" envDefault:":9090"` + // Path is the path the Prometheus handler is mounted on, e.g. "/metrics". + Path string `env:"METRICS_PATH" envDefault:"/metrics"` +} + +// Postgres holds connection details for the org's existing central +// Postgres instance, reached via a dedicated role for this service +// (idiomatically named something like vibetype_role_service_temporal_worker +// on the server side; this package has no opinion on the name). +type Postgres struct { + Host string `env:"POSTGRES_HOST" envDefault:"localhost"` + Port int `env:"POSTGRES_PORT" envDefault:"5432"` + Database string `env:"POSTGRES_DATABASE,required,notEmpty"` + User string `env:"POSTGRES_USER,required,notEmpty"` + Password string `env:"POSTGRES_PASSWORD,required,notEmpty"` + // SSLMode is passed through to pgx verbatim (e.g. "disable", "require", "verify-full"). + // Defaults to "require". + SSLMode string `env:"POSTGRES_SSLMODE" envDefault:"require"` +} + +// DSN renders the connection details as a libpq-style connection string suitable for pgxpool.New. +// Every value is single-quoted so passwords and other settings containing spaces, "=", or quotes survive parsing intact. +func (p *Postgres) DSN() string { + return fmt.Sprintf( + "host=%s port=%d dbname=%s user=%s password=%s sslmode=%s", + quoteDSNValue(p.Host), + p.Port, + quoteDSNValue(p.Database), + quoteDSNValue(p.User), + quoteDSNValue(p.Password), + quoteDSNValue(p.SSLMode), + ) +} + +// quoteDSNValue escapes a value for the libpq keyword/value connection string format that DSN produces. +// Backslashes and single quotes are backslash-escaped and the result is wrapped in single quotes, which is the only escaping libpq recognizes here. +// Percent-encoding is deliberately not used: that is the escaping for URL-style DSNs ("postgres://..."), and in keyword/value format pgx passes it through verbatim, so a password like "p@ss w0rd" would be sent as the literal "p%40ss+w0rd" and authentication would fail. +func quoteDSNValue(v string) string { + return "'" + dsnValueEscaper.Replace(v) + "'" +} + +var dsnValueEscaper = strings.NewReplacer(`\`, `\\`, `'`, `\'`) + +// S3 holds the credentials and bucket configuration for database backup uploads. +type S3 struct { + Bucket string `env:"S3_BUCKET,required,notEmpty"` + // Prefix is the key prefix backups are uploaded under, defaulting to defaultS3Prefix when S3_PREFIX is unset. + // It carries no envDefault tag on purpose: env applies a default to a variable that is set but empty too, which would make an explicit "S3_PREFIX=" mean "backups" rather than the bucket root it reads as. + // Load applies the default itself so that distinction survives. + Prefix string `env:"S3_PREFIX"` + Region string `env:"S3_REGION,required,notEmpty"` + AccessKeyID string `env:"S3_ACCESS_KEY_ID,required,notEmpty"` + SecretAccessKey string `env:"S3_SECRET_ACCESS_KEY,required,notEmpty"` + // Endpoint overrides the default AWS endpoint, for S3-compatible object storage. + // Leave empty to use AWS S3 itself. + Endpoint string `env:"S3_ENDPOINT"` + // UsePathStyle forces path-style addressing, typically required by + // non-AWS S3-compatible endpoints. + UsePathStyle bool `env:"S3_USE_PATH_STYLE" envDefault:"false"` + // SourceDir is the local directory synced to S3, e.g. "/backups". + SourceDir string `env:"BACKUP_SOURCE_DIR" envDefault:"/backups"` +} + +// Sentry holds the Sentry Crons check-in URLs used for the two jobs, one +// per job so each keeps its own monitor slug. +// +// Either URL may be left empty, in which case check-ins for that job are +// skipped rather than treated as an error. +type Sentry struct { + DBBackupCheckInURL string `env:"SENTRY_CRONS"` + OutboxPurgeCheckInURL string `env:"SENTRY_CRONS_OUTBOX_PURGE"` +} + +// Schedule holds the cadences the two Temporal Schedules are created with. +// Defaults are daily for DBBackup and every 2 hours for OutboxPurge, but +// they are configurable so they can be tightened for local testing. +type Schedule struct { + DBBackupEvery time.Duration `env:"DBBACKUP_SCHEDULE_EVERY" envDefault:"24h"` + OutboxPurgeEvery time.Duration `env:"OUTBOX_PURGE_SCHEDULE_EVERY" envDefault:"2h"` + OutboxPurgeRetention time.Duration `env:"OUTBOX_PURGE_RETENTION" envDefault:"24h"` +} + +// Load reads configuration from the environment, applying defaults where +// documented above and in the README, and returns an error describing +// every missing or invalid variable at once (via env.Parse's aggregate +// error), rather than failing on the first one encountered. +func Load() (Config, error) { + var cfg Config + if err := env.Parse(&cfg); err != nil { + return Config{}, fmt.Errorf("invalid configuration: %w", err) + } + if _, set := os.LookupEnv("S3_PREFIX"); !set { + cfg.S3.Prefix = defaultS3Prefix + } + return cfg, nil +} + +// defaultS3Prefix is the key prefix used when S3_PREFIX is not set at all. +const defaultS3Prefix = "backups" diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..0812745 --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,154 @@ +package config + +import ( + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// setEnv sets environment variables for the duration of the test and +// restores the previous values afterwards, including unsetting variables +// that were not previously set. +func setEnv(t *testing.T, kv map[string]string) { + t.Helper() + for k, v := range kv { + t.Setenv(k, v) + } +} + +func requiredEnv() map[string]string { + return map[string]string{ + "POSTGRES_DATABASE": "vibetype", + "POSTGRES_USER": "vibetype_role_service_temporal_worker", + "POSTGRES_PASSWORD": "hunter2", + "S3_BUCKET": "maevsi-backups", + "S3_REGION": "eu-central-1", + "S3_ACCESS_KEY_ID": "AKIAEXAMPLE", + "S3_SECRET_ACCESS_KEY": "secret", + } +} + +func TestLoad_Defaults(t *testing.T) { + setEnv(t, requiredEnv()) + + cfg, err := Load() + require.NoError(t, err) + + assert.Equal(t, "temporal-server:7233", cfg.Temporal.HostPort) + assert.Equal(t, "default", cfg.Temporal.Namespace) + assert.Equal(t, "maevsi-jobs", cfg.Temporal.TaskQueue) + assert.Equal(t, ":9090", cfg.Metrics.Addr) + assert.Equal(t, "/metrics", cfg.Metrics.Path) + assert.Equal(t, 5432, cfg.Postgres.Port) + assert.Equal(t, "require", cfg.Postgres.SSLMode) + assert.Equal(t, "/backups", cfg.S3.SourceDir) + assert.Equal(t, "backups", cfg.S3.Prefix) + assert.Equal(t, 24*time.Hour, cfg.Schedule.DBBackupEvery) + assert.Equal(t, 2*time.Hour, cfg.Schedule.OutboxPurgeEvery) + assert.Equal(t, 24*time.Hour, cfg.Schedule.OutboxPurgeRetention) + assert.Empty(t, cfg.Sentry.DBBackupCheckInURL) + assert.Empty(t, cfg.Sentry.OutboxPurgeCheckInURL) +} + +func TestLoad_Overrides(t *testing.T) { + env := requiredEnv() + env["TEMPORAL_HOST_PORT"] = "temporal.internal:7233" + env["TEMPORAL_NAMESPACE"] = "maevsi-prod" + env["OUTBOX_PURGE_SCHEDULE_EVERY"] = "10m" + env["SENTRY_CRONS"] = "https://sentry.example/api/0/cron/dbbackup/token/" + setEnv(t, env) + + cfg, err := Load() + require.NoError(t, err) + + assert.Equal(t, "temporal.internal:7233", cfg.Temporal.HostPort) + assert.Equal(t, "maevsi-prod", cfg.Temporal.Namespace) + assert.Equal(t, 10*time.Minute, cfg.Schedule.OutboxPurgeEvery) + assert.Equal(t, "https://sentry.example/api/0/cron/dbbackup/token/", cfg.Sentry.DBBackupCheckInURL) +} + +func TestLoad_MissingRequired(t *testing.T) { + // Intentionally leave everything unset. + cfg, err := Load() + require.Error(t, err) + assert.Equal(t, Config{}, cfg) + assert.ErrorContains(t, err, "POSTGRES_DATABASE") + assert.ErrorContains(t, err, "S3_BUCKET") +} + +func TestLoad_InvalidDuration(t *testing.T) { + env := requiredEnv() + env["DBBACKUP_SCHEDULE_EVERY"] = "not-a-duration" + setEnv(t, env) + + _, err := Load() + require.Error(t, err) + // The underlying env library's parse-error messages name the Go + // struct field ("DBBackupEvery"), not the env var + // ("DBBACKUP_SCHEDULE_EVERY"); missing-variable errors (see + // TestLoad_MissingRequired) still name the env var itself. + assert.ErrorContains(t, err, "DBBackupEvery") +} + +func TestPostgres_DSN(t *testing.T) { + p := Postgres{ + Host: "pg.internal", + Port: 5432, + Database: "vibetype", + User: "vibetype_role_service_temporal_worker", + Password: "hunter2", + SSLMode: "require", + } + assert.Equal(t, + "host='pg.internal' port=5432 dbname='vibetype' user='vibetype_role_service_temporal_worker' password='hunter2' sslmode='require'", + p.DSN(), + ) +} + +// TestPostgres_DSN_RoundTripsSpecialCharacters is the check that matters here: rendering the DSN is only correct if pgx reads back exactly the password that went in. +// Percent-encoding used to be applied instead of quoting, which pgx passes through verbatim in this connection string format, so every password containing a space, "@", "=", or "+" silently authenticated with the wrong value. +func TestPostgres_DSN_RoundTripsSpecialCharacters(t *testing.T) { + passwords := []string{ + "plain123", + "p@ss w0rd", + "a=b", + "sim+ple", + "has'quote", + `back\slash`, + "100%sure", + } + + for _, password := range passwords { + t.Run(password, func(t *testing.T) { + p := Postgres{ + Host: "pg.internal", + Port: 5432, + Database: "vibetype", + User: "vibetype_role_service_temporal_worker", + Password: password, + SSLMode: "require", + } + + parsed, err := pgx.ParseConfig(p.DSN()) + require.NoError(t, err) + assert.Equal(t, password, parsed.Password) + assert.Equal(t, "pg.internal", parsed.Host) + assert.Equal(t, "vibetype", parsed.Database) + assert.Equal(t, "vibetype_role_service_temporal_worker", parsed.User) + }) + } +} + +// TestLoad_EmptyS3PrefixMeansBucketRoot covers "S3_PREFIX=" being set explicitly, which is how an operator asks for uploads at the bucket root. +// env applies an envDefault to a set-but-empty variable just as it does to an unset one, so the default has to live outside the struct tag for that distinction to survive Load. +func TestLoad_EmptyS3PrefixMeansBucketRoot(t *testing.T) { + setEnv(t, requiredEnv()) + t.Setenv("S3_PREFIX", "") + + cfg, err := Load() + require.NoError(t, err) + assert.Empty(t, cfg.S3.Prefix) +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go new file mode 100644 index 0000000..772f308 --- /dev/null +++ b/internal/metrics/metrics.go @@ -0,0 +1,111 @@ +// Package metrics wires the Temporal Go SDK's native metrics (worker pollers, activity/workflow execution counts and latencies, task queue backlog, etc.) to a Prometheus HTTP handler, so a Prometheus scrape target can be pointed at this process without any custom instrumentation. +// Configuring the actual Prometheus scrape job / Grafana dashboards is stack-repo work happening separately; this package only needs to make the metrics available. +package metrics + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "strings" + "time" + + prom "github.com/prometheus/client_golang/prometheus" + "github.com/uber-go/tally/v4" + promreporter "github.com/uber-go/tally/v4/prometheus" + "go.temporal.io/sdk/client" + sdktally "go.temporal.io/sdk/contrib/tally" +) + +// reportingInterval controls how often tally flushes buffered metrics into the underlying Prometheus registry. +// It only affects timer/histogram bucketing latency, not scrape freshness (Prometheus scrapes read the registry directly). +const reportingInterval = time.Second + +// Handler bundles the Temporal client.MetricsHandler to pass into +// client.Options and the http.Handler to serve on the metrics endpoint. +type Handler struct { + Temporal client.MetricsHandler + HTTP http.Handler + + closer io.Closer +} + +// Close stops the underlying tally scope's background reporting loop. +func (h *Handler) Close() error { + if h.closer == nil { + return nil + } + return h.closer.Close() +} + +// New builds a Prometheus-backed metrics.Handler. +// namePrefix is applied to every emitted metric (e.g. "temporal_worker") to namespace this process's metrics from anything else sharing the same Prometheus instance. +func New(namePrefix string) *Handler { + // Use a dedicated registry rather than prometheus.DefaultRegisterer: + // New may run more than once in a process (e.g. once per test), and + // the default registerer panics on a second registration of the same + // collector. + registry := prom.NewRegistry() + reporter := promreporter.NewReporter(promreporter.Options{ + Registerer: registry, + Gatherer: registry, + }) + scope, closer := tally.NewRootScope(tally.ScopeOptions{ + Prefix: namePrefix, + CachedReporter: reporter, + Separator: promreporter.DefaultSeparator, + SanitizeOptions: &sdktally.PrometheusSanitizeOptions, + }, reportingInterval) + + return &Handler{ + Temporal: sdktally.NewMetricsHandler(sdktally.NewPrometheusNamingScope(scope)), + HTTP: reporter.HTTPHandler(), + closer: closer, + } +} + +// HealthPath is where Serve mounts the health endpoint, and the path the worker's own "-healthcheck" probe requests. +const HealthPath = "/healthz" + +// Serve starts an HTTP server exposing the metrics handler at path, plus a HealthPath endpoint that just confirms this HTTP server itself is up. +// It blocks until ctx is canceled, then shuts the server down. +// +// HealthPath is intentionally trivial (no Temporal/Postgres/S3 reachability checks): it exists so a container HEALTHCHECK can probe over HTTP even though the production image ships FROM scratch with no shell or curl, via the worker binary's own "-healthcheck" flag (see cmd/worker). +func Serve(ctx context.Context, addr, path string, h *Handler) error { + // http.ServeMux panics on a pattern it cannot parse and on a duplicate pattern alike, and either panic would happen on the goroutine this runs on rather than surfacing as a startup error, so both cases have to be rejected up front. + // A pattern without a leading slash is read as a host and rejected ("host/path missing /"), and an empty one is rejected outright. + if !strings.HasPrefix(path, "/") { + return fmt.Errorf("metrics: path %q must start with a slash; set METRICS_PATH to something like %q", path, "/metrics") + } + if path == HealthPath { + return fmt.Errorf("metrics: path %q collides with the health endpoint; pick a different METRICS_PATH", path) + } + + mux := http.NewServeMux() + mux.Handle(path, h.HTTP) + mux.HandleFunc(HealthPath, func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + srv := &http.Server{Addr: addr, Handler: mux} + + errCh := make(chan error, 1) + go func() { + if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + errCh <- fmt.Errorf("metrics: serve: %w", err) + return + } + errCh <- nil + }() + + select { + case <-ctx.Done(): + // Close rather than Shutdown: an in-flight scrape or health probe has nothing to lose by being cut off, and waiting for one would only delay process exit. + _ = srv.Close() + <-errCh + return nil + case err := <-errCh: + return err + } +} diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go new file mode 100644 index 0000000..9bdbbc6 --- /dev/null +++ b/internal/metrics/metrics_test.go @@ -0,0 +1,98 @@ +package metrics + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNew_ExposesTemporalHandlerAndHTTPHandler(t *testing.T) { + h := New("temporal_worker_test") + defer func() { _ = h.Close() }() + + require.NotNil(t, h.Temporal) + require.NotNil(t, h.HTTP) + + // Emit a metric through the Temporal-facing handler and confirm it + // shows up on the Prometheus HTTP handler, proving the two are wired + // to the same underlying registry. + h.Temporal.WithTags(map[string]string{}).Counter("smoke_test_total").Inc(1) + + // tally's reporting loop flushes on its own interval; give it a moment. + time.Sleep(reportingInterval + 200*time.Millisecond) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/metrics", http.NoBody) + h.HTTP.ServeHTTP(rec, req) + + assert.Equal(t, http.StatusOK, rec.Code) + body, err := io.ReadAll(rec.Body) + require.NoError(t, err) + assert.Contains(t, string(body), "smoke_test_total") +} + +func TestServe_ShutsDownOnContextCancel(t *testing.T) { + h := New("temporal_worker_test_serve") + defer func() { _ = h.Close() }() + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + done <- Serve(ctx, "127.0.0.1:0", "/metrics", h) + }() + + // Serve binds an ephemeral port here only to exercise the shutdown + // path; a fixed test port would risk collisions across CI runs. + time.Sleep(50 * time.Millisecond) + cancel() + + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(2 * time.Second): + t.Fatal("Serve did not shut down after context cancellation") + } +} + +func TestServe_RejectsInvalidAddr(t *testing.T) { + h := New("temporal_worker_test_bad_addr") + defer func() { _ = h.Close() }() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + err := Serve(ctx, "not-a-valid-addr", "/metrics", h) + require.Error(t, err) + assert.True(t, strings.Contains(err.Error(), "metrics: serve")) +} + +// TestServe_RejectsHealthPathCollision covers METRICS_PATH being pointed at the health endpoint. +// http.ServeMux panics on a duplicate pattern, and Serve runs on its own goroutine in the worker, so that panic would take the process down instead of surfacing as a startup error. +func TestServe_RejectsHealthPathCollision(t *testing.T) { + h := New("temporal_worker_test_collision") + defer func() { _ = h.Close() }() + + err := Serve(t.Context(), "127.0.0.1:0", HealthPath, h) + require.Error(t, err) + assert.Contains(t, err.Error(), "collides with the health endpoint") +} + +// TestServe_RejectsPathWithoutLeadingSlash covers a METRICS_PATH that http.ServeMux reads as a host rather than a path. +// Like the HealthPath collision, the resulting panic would happen on the goroutine Serve runs on in the worker and take the process down, so it has to be caught before mux.Handle sees it. +func TestServe_RejectsPathWithoutLeadingSlash(t *testing.T) { + h := New("temporal_worker_test_no_slash") + defer func() { _ = h.Close() }() + + for _, path := range []string{"metrics", ""} { + err := Serve(t.Context(), "127.0.0.1:0", path, h) + require.Error(t, err) + assert.Contains(t, err.Error(), "must start with a slash") + } +} diff --git a/internal/postgres/pool.go b/internal/postgres/pool.go new file mode 100644 index 0000000..82857ed --- /dev/null +++ b/internal/postgres/pool.go @@ -0,0 +1,41 @@ +// Package postgres wires up the connection to the org's existing central +// Postgres instance and adapts it to the small activities.DBExecutor +// interface the OutboxPurge activity depends on. +package postgres + +import ( + "context" + "fmt" + + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/maevsi/temporal/internal/config" +) + +// NewPool opens a connection pool using the given configuration. +// Callers own the returned pool and must Close it on shutdown. +func NewPool(ctx context.Context, cfg *config.Postgres) (*pgxpool.Pool, error) { + pool, err := pgxpool.New(ctx, cfg.DSN()) + if err != nil { + return nil, fmt.Errorf("postgres: create pool: %w", err) + } + if err := pool.Ping(ctx); err != nil { + pool.Close() + return nil, fmt.Errorf("postgres: ping: %w", err) + } + return pool, nil +} + +// Executor adapts a *pgxpool.Pool to activities.DBExecutor. +type Executor struct { + Pool *pgxpool.Pool +} + +// Exec runs sql with args and returns the number of affected rows. +func (e *Executor) Exec(ctx context.Context, sql string, args ...any) (int64, error) { + tag, err := e.Pool.Exec(ctx, sql, args...) + if err != nil { + return 0, err + } + return tag.RowsAffected(), nil +} diff --git a/internal/s3client/client.go b/internal/s3client/client.go new file mode 100644 index 0000000..52dbc57 --- /dev/null +++ b/internal/s3client/client.go @@ -0,0 +1,39 @@ +// Package s3client constructs the AWS S3 client used by the DBBackup +// activity from this worker's own config, rather than the default AWS SDK +// credential chain, since credentials are supplied explicitly via env vars +// per this repo's configuration convention (see internal/config). +package s3client + +import ( + "context" + "fmt" + + "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + + "github.com/maevsi/temporal/internal/config" +) + +// New builds an *s3.Client (which satisfies activities.S3API) from the given configuration. +// Endpoint/UsePathStyle support S3-compatible object storage in addition to AWS S3 itself. +func New(ctx context.Context, cfg *config.S3) (*s3.Client, error) { + awsCfg, err := awsconfig.LoadDefaultConfig(ctx, + awsconfig.WithRegion(cfg.Region), + awsconfig.WithCredentialsProvider( + credentials.NewStaticCredentialsProvider(cfg.AccessKeyID, cfg.SecretAccessKey, ""), + ), + ) + if err != nil { + return nil, fmt.Errorf("s3client: load AWS config: %w", err) + } + + client := s3.NewFromConfig(awsCfg, func(o *s3.Options) { + if cfg.Endpoint != "" { + o.BaseEndpoint = aws.String(cfg.Endpoint) + } + o.UsePathStyle = cfg.UsePathStyle + }) + return client, nil +} diff --git a/internal/s3client/client_test.go b/internal/s3client/client_test.go new file mode 100644 index 0000000..b74449b --- /dev/null +++ b/internal/s3client/client_test.go @@ -0,0 +1,36 @@ +package s3client + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/maevsi/temporal/internal/config" +) + +func TestNew_ReturnsClientWithValidConfig(t *testing.T) { + cfg := config.S3{ + Region: "eu-central-1", + AccessKeyID: "AKIAEXAMPLE", + SecretAccessKey: "secret", + } + + client, err := New(context.Background(), &cfg) + require.NoError(t, err) + require.NotNil(t, client) +} + +func TestNew_AcceptsCustomEndpoint(t *testing.T) { + cfg := config.S3{ + Region: "us-east-1", + AccessKeyID: "AKIAEXAMPLE", + SecretAccessKey: "secret", + Endpoint: "https://minio.local:9000", + UsePathStyle: true, + } + + client, err := New(context.Background(), &cfg) + require.NoError(t, err) + require.NotNil(t, client) +} diff --git a/internal/schedule/schedule.go b/internal/schedule/schedule.go new file mode 100644 index 0000000..1c7b43b --- /dev/null +++ b/internal/schedule/schedule.go @@ -0,0 +1,110 @@ +// Package schedule creates the Temporal Schedules for the worker's +// cron jobs: cadences live in Temporal's native Schedule API +// (interval-based, not cron strings baked into workflow code) so they are +// visible and editable via `temporal schedule` / the Temporal UI without a +// worker code change. +package schedule + +import ( + "context" + "errors" + "fmt" + + enumspb "go.temporal.io/api/enums/v1" + "go.temporal.io/api/serviceerror" + "go.temporal.io/sdk/client" + + "github.com/maevsi/temporal/internal/config" + "github.com/maevsi/temporal/internal/workflows" +) + +// Schedule IDs are stable business identifiers; changing them creates new +// Schedules rather than updating existing ones on the next deploy. +const ( + IDDBBackup = "dbbackup-daily" + IDOutboxPurge = "outbox-purge-every-2h" +) + +// EnsureAll idempotently creates both Schedules if they don't already exist. +// It never modifies an existing Schedule (e.g. one paused or re-tuned by an operator via the Temporal CLI/UI), matching the bootstrap-once semantics appropriate for a worker that starts on every deploy. +func EnsureAll(ctx context.Context, c client.Client, cfg *config.Config) error { + specs := []*client.ScheduleOptions{ + dbBackupSchedule(cfg), + outboxPurgeSchedule(cfg), + } + for _, opts := range specs { + if err := ensure(ctx, c, opts); err != nil { + return err + } + } + return nil +} + +func ensure(ctx context.Context, c client.Client, opts *client.ScheduleOptions) error { + handle := c.ScheduleClient().GetHandle(ctx, opts.ID) + if _, err := handle.Describe(ctx); err == nil { + // Already exists; leave whatever an operator has configured alone. + return nil + } else if !isNotFound(err) { + return fmt.Errorf("schedule: describe %q: %w", opts.ID, err) + } + + if _, err := c.ScheduleClient().Create(ctx, *opts); err != nil { + if isAlreadyExists(err) { + // Another replica created it between our Describe and Create; safe to ignore. + return nil + } + return fmt.Errorf("schedule: create %q: %w", opts.ID, err) + } + return nil +} + +func isNotFound(err error) bool { + var nf *serviceerror.NotFound + return errors.As(err, &nf) +} + +// isAlreadyExists reports whether err says the Schedule was created by someone else first, which is what a second worker replica sees when both bootstrap at the same time. +// The check has to go through errors.As on the SDK's own error types rather than grpc's status.Code: the SDK converts gRPC status errors into serviceerror values, and those do not implement GRPCStatus(), so status.Code would report codes.Unknown for every one of them. +func isAlreadyExists(err error) bool { + var ae *serviceerror.AlreadyExists + if errors.As(err, &ae) { + return true + } + var started *serviceerror.WorkflowExecutionAlreadyStarted + return errors.As(err, &started) +} + +func dbBackupSchedule(cfg *config.Config) *client.ScheduleOptions { + return &client.ScheduleOptions{ + ID: IDDBBackup, + Spec: client.ScheduleSpec{ + Intervals: []client.ScheduleIntervalSpec{{Every: cfg.Schedule.DBBackupEvery}}, + }, + Action: &client.ScheduleWorkflowAction{ + ID: IDDBBackup, + Workflow: workflows.DBBackupWorkflow, + TaskQueue: cfg.Temporal.TaskQueue, + WorkflowRunTimeout: workflows.DBBackupRunTimeout, + }, + Overlap: enumspb.SCHEDULE_OVERLAP_POLICY_SKIP, + PauseOnFailure: false, + } +} + +func outboxPurgeSchedule(cfg *config.Config) *client.ScheduleOptions { + return &client.ScheduleOptions{ + ID: IDOutboxPurge, + Spec: client.ScheduleSpec{ + Intervals: []client.ScheduleIntervalSpec{{Every: cfg.Schedule.OutboxPurgeEvery}}, + }, + Action: &client.ScheduleWorkflowAction{ + ID: IDOutboxPurge, + Workflow: workflows.OutboxPurgeWorkflow, + TaskQueue: cfg.Temporal.TaskQueue, + WorkflowRunTimeout: workflows.OutboxPurgeRunTimeout, + }, + Overlap: enumspb.SCHEDULE_OVERLAP_POLICY_SKIP, + PauseOnFailure: false, + } +} diff --git a/internal/schedule/schedule_test.go b/internal/schedule/schedule_test.go new file mode 100644 index 0000000..d5fd109 --- /dev/null +++ b/internal/schedule/schedule_test.go @@ -0,0 +1,99 @@ +package schedule + +import ( + "fmt" + "testing" + "time" + + enumspb "go.temporal.io/api/enums/v1" + "go.temporal.io/api/serviceerror" + "go.temporal.io/sdk/client" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/maevsi/temporal/internal/config" + "github.com/maevsi/temporal/internal/workflows" +) + +func testConfig() config.Config { + return config.Config{ + Temporal: config.Temporal{TaskQueue: "maevsi-jobs"}, + Schedule: config.Schedule{ + DBBackupEvery: 24 * time.Hour, + OutboxPurgeEvery: 2 * time.Hour, + }, + } +} + +func TestDBBackupSchedule_UsesIntervalNotCron(t *testing.T) { + cfg := testConfig() + opts := dbBackupSchedule(&cfg) + + assert.Equal(t, IDDBBackup, opts.ID) + assert.Empty(t, opts.Spec.CronExpressions, "schedule must use native Intervals, not cron strings") + require.Len(t, opts.Spec.Intervals, 1) + assert.Equal(t, 24*time.Hour, opts.Spec.Intervals[0].Every) + assert.Equal(t, enumspb.SCHEDULE_OVERLAP_POLICY_SKIP, opts.Overlap) + + action, ok := opts.Action.(*client.ScheduleWorkflowAction) + require.True(t, ok) + assert.Equal(t, "maevsi-jobs", action.TaskQueue) +} + +func TestOutboxPurgeSchedule_UsesIntervalNotCron(t *testing.T) { + cfg := testConfig() + opts := outboxPurgeSchedule(&cfg) + + assert.Equal(t, IDOutboxPurge, opts.ID) + assert.Empty(t, opts.Spec.CronExpressions) + require.Len(t, opts.Spec.Intervals, 1) + assert.Equal(t, 2*time.Hour, opts.Spec.Intervals[0].Every) + assert.Equal(t, enumspb.SCHEDULE_OVERLAP_POLICY_SKIP, opts.Overlap) +} + +// TestIsAlreadyExists covers the concurrent-bootstrap path in ensure: two replicas starting together race between Describe and Create, and the loser must treat the conflict as success rather than failing startup. +// This previously used grpc's status.Code, which reports codes.Unknown for every serviceerror the SDK returns, so the guard never matched. +func TestIsAlreadyExists(t *testing.T) { + t.Run("matches the SDK's conflict errors", func(t *testing.T) { + assert.True(t, isAlreadyExists(serviceerror.NewAlreadyExists("schedule already exists"))) + assert.True(t, isAlreadyExists(serviceerror.NewWorkflowExecutionAlreadyStarted("already started", "id", "run-id"))) + }) + + t.Run("sees through wrapping", func(t *testing.T) { + wrapped := fmt.Errorf("schedule: create %q: %w", IDDBBackup, serviceerror.NewAlreadyExists("dup")) + assert.True(t, isAlreadyExists(wrapped)) + }) + + t.Run("does not match unrelated errors", func(t *testing.T) { + assert.False(t, isAlreadyExists(serviceerror.NewNotFound("no such schedule"))) + assert.False(t, isAlreadyExists(serviceerror.NewUnavailable("frontend down"))) + assert.False(t, isAlreadyExists(fmt.Errorf("plain error"))) + }) +} + +// TestIsNotFound guards the other branch of ensure: anything other than a NotFound from Describe must surface as a startup error rather than being mistaken for "schedule missing, create it". +func TestIsNotFound(t *testing.T) { + assert.True(t, isNotFound(serviceerror.NewNotFound("no such schedule"))) + assert.True(t, isNotFound(fmt.Errorf("wrapped: %w", serviceerror.NewNotFound("no such schedule")))) + assert.False(t, isNotFound(serviceerror.NewUnavailable("frontend down"))) + assert.False(t, isNotFound(serviceerror.NewAlreadyExists("dup"))) +} + +// TestScheduleRunTimeouts_AreIndependentOfTheCadence guards against deriving WorkflowRunTimeout from the schedule interval. +// The cadences exist to be tightened for local testing, and an interval-derived run timeout would then cut runs short before their activity retry budget was spent: at a one-minute cadence the OutboxPurge activity's own two-minute StartToCloseTimeout could never even complete one attempt. +func TestScheduleRunTimeouts_AreIndependentOfTheCadence(t *testing.T) { + cfg := testConfig() + cfg.Schedule.DBBackupEvery = time.Minute + cfg.Schedule.OutboxPurgeEvery = time.Minute + + dbAction, ok := dbBackupSchedule(&cfg).Action.(*client.ScheduleWorkflowAction) + require.True(t, ok) + purgeAction, ok := outboxPurgeSchedule(&cfg).Action.(*client.ScheduleWorkflowAction) + require.True(t, ok) + + assert.Equal(t, workflows.DBBackupRunTimeout, dbAction.WorkflowRunTimeout) + assert.Equal(t, workflows.OutboxPurgeRunTimeout, purgeAction.WorkflowRunTimeout) + assert.Greater(t, dbAction.WorkflowRunTimeout, cfg.Schedule.DBBackupEvery) + assert.Greater(t, purgeAction.WorkflowRunTimeout, cfg.Schedule.OutboxPurgeEvery) +} diff --git a/internal/sentrycrons/client.go b/internal/sentrycrons/client.go new file mode 100644 index 0000000..e7f5d18 --- /dev/null +++ b/internal/sentrycrons/client.go @@ -0,0 +1,96 @@ +// Package sentrycrons implements the check-in protocol for Sentry Crons +// "ping" style monitors: a plain HTTP GET to a per-monitor URL with a +// status query parameter. +package sentrycrons + +import ( + "context" + "fmt" + "net/http" + "net/url" + "time" +) + +// Status is the check-in status reported to Sentry Crons. +type Status string + +const ( + // StatusInProgress marks the start of a monitored run. + StatusInProgress Status = "in_progress" + // StatusOK marks a monitored run as having completed successfully. + StatusOK Status = "ok" + // StatusError marks a monitored run as having failed. + StatusError Status = "error" +) + +// Client sends check-ins for a single Sentry Crons monitor. +type Client struct { + // parsedURL is the pre-parsed check-in URL. + // Existing query parameters are preserved; "status" is added per call in CheckIn. + parsedURL *url.URL + httpClient *http.Client +} + +// New returns a Client for the given check-in URL. +// An empty URL is valid and produces a Client whose CheckIn calls are no-ops; this matches environments (e.g. local development) that historically relied on email notifications instead of Sentry. +// +// A non-empty URL that cannot be used is an error rather than a silently unconfigured Client. +// Monitoring that quietly disables itself on a typo is worse than none at all: the workflows treat check-in failures as best-effort and only log them, so nothing downstream would ever report the mistake and the monitor would simply never fire. +func New(checkInURL string) (*Client, error) { + if checkInURL == "" { + return &Client{httpClient: &http.Client{Timeout: 10 * time.Second}}, nil + } + + u, err := url.Parse(checkInURL) + if err != nil { + return nil, fmt.Errorf("sentrycrons: parse check-in URL: %w", err) + } + // url.Parse accepts almost anything, so the scheme has to be checked separately. + // Without it a value like "sentry.io/api/0/cron/..." parses fine and only fails much later, inside a check-in that nobody sees fail. + if u.Scheme != "http" && u.Scheme != "https" { + return nil, fmt.Errorf("sentrycrons: check-in URL must be http or https, got %q", checkInURL) + } + if u.Host == "" { + return nil, fmt.Errorf("sentrycrons: check-in URL has no host: %q", checkInURL) + } + + return &Client{ + parsedURL: u, + httpClient: &http.Client{Timeout: 10 * time.Second}, + }, nil +} + +// Configured reports whether this Client has a check-in URL and will +// actually perform requests. +func (c *Client) Configured() bool { + return c != nil && c.parsedURL != nil +} + +// CheckIn reports the given status to Sentry Crons. +// It is a no-op (nil error) if the client has no check-in URL configured, so callers do not need to branch on configuration before calling it. +func (c *Client) CheckIn(ctx context.Context, status Status) error { + if !c.Configured() { + return nil + } + + u := *c.parsedURL + q := u.Query() + q.Set("status", string(status)) + u.RawQuery = q.Encode() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), http.NoBody) + if err != nil { + return fmt.Errorf("sentrycrons: build check-in request: %w", err) + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("sentrycrons: check-in request failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode >= 400 { + return fmt.Errorf("sentrycrons: check-in returned status %d", resp.StatusCode) + } + return nil +} diff --git a/internal/sentrycrons/client_test.go b/internal/sentrycrons/client_test.go new file mode 100644 index 0000000..79c512d --- /dev/null +++ b/internal/sentrycrons/client_test.go @@ -0,0 +1,86 @@ +package sentrycrons + +import ( + "context" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCheckIn_SendsStatus(t *testing.T) { + var gotStatus string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotStatus = r.URL.Query().Get("status") + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + c, err := New(srv.URL) + require.NoError(t, err) + err = c.CheckIn(context.Background(), StatusInProgress) + require.NoError(t, err) + assert.Equal(t, "in_progress", gotStatus) +} + +func TestCheckIn_PreservesExistingQuery(t *testing.T) { + var gotQuery url.Values + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotQuery = r.URL.Query() + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + c, err := New(srv.URL + "?env=production") + require.NoError(t, err) + err = c.CheckIn(context.Background(), StatusOK) + require.NoError(t, err) + assert.Equal(t, "production", gotQuery.Get("env")) + assert.Equal(t, "ok", gotQuery.Get("status")) +} + +func TestCheckIn_NonOKStatusIsError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + + c, err := New(srv.URL) + require.NoError(t, err) + err = c.CheckIn(context.Background(), StatusError) + require.Error(t, err) +} + +func TestCheckIn_UnconfiguredIsNoOp(t *testing.T) { + c, err := New("") + require.NoError(t, err) + err = c.CheckIn(context.Background(), StatusOK) + require.NoError(t, err) + assert.False(t, c.Configured()) +} + +func TestCheckIn_NilClientIsNoOp(t *testing.T) { + var c *Client + err := c.CheckIn(context.Background(), StatusOK) + require.NoError(t, err) +} + +// TestNew_RejectsUnusableURL pins the behavior that a misconfigured check-in URL fails loudly at startup. +// Returning an unconfigured Client here instead would turn every later check-in into a silent no-op, and since workflows only log check-in failures, the monitor would go quiet with nothing anywhere reporting why. +func TestNew_RejectsUnusableURL(t *testing.T) { + for _, checkInURL := range []string{ + "sentry.example/api/0/cron/dbbackup/token/", // no scheme + "ftp://sentry.example/cron/", // wrong scheme + "https://", // no host + "://sentry.example", // unparseable + } { + t.Run(checkInURL, func(t *testing.T) { + c, err := New(checkInURL) + require.Error(t, err) + assert.Nil(t, c) + }) + } +} diff --git a/internal/workflows/dbbackup.go b/internal/workflows/dbbackup.go new file mode 100644 index 0000000..f13fa41 --- /dev/null +++ b/internal/workflows/dbbackup.go @@ -0,0 +1,86 @@ +// Package workflows implements the two Temporal Workflows for DBBackup and OutboxPurge. +// Workflows only orchestrate: all real work (S3 upload, Postgres DELETE, Sentry HTTP calls) lives in internal/activities, reached here purely through method-expression references (see the package doc comment on activities.Activities) so this package never links against the AWS/pgx/HTTP dependencies those activities use. +package workflows + +import ( + "time" + + "go.temporal.io/sdk/temporal" + "go.temporal.io/sdk/workflow" + + "github.com/maevsi/temporal/internal/activities" + "github.com/maevsi/temporal/internal/sentrycrons" +) + +// checkInActivityOptions are deliberately short and cheap to retry: a Sentry Crons check-in should never be the reason a workflow run takes long or ties up worker capacity. +func checkInActivityOptions(ctx workflow.Context) workflow.Context { + return workflow.WithActivityOptions(ctx, workflow.ActivityOptions{ + StartToCloseTimeout: 15 * time.Second, + RetryPolicy: &temporal.RetryPolicy{ + InitialInterval: time.Second, + BackoffCoefficient: 2.0, + MaximumInterval: 10 * time.Second, + MaximumAttempts: 3, + }, + }) +} + +// checkInFinal sends the terminal check-in for a run on a disconnected context so a cancelled run still reports its outcome. +// On the workflow's own context ExecuteActivity short-circuits with a CanceledError without ever scheduling the activity, which would leave the Sentry monitor stuck in StatusInProgress until its own max-runtime alert fired. +func checkInFinal(ctx workflow.Context, job activities.Job, status sentrycrons.Status) { + disconnected, cancel := workflow.NewDisconnectedContext(ctx) + defer cancel() + checkIn(disconnected, job, status) +} + +// checkIn sends a Sentry Crons check-in and treats failure as best-effort: alerting must never fail the underlying backup or purge, so the error is logged rather than returned. +func checkIn(ctx workflow.Context, job activities.Job, status sentrycrons.Status) { + var a *activities.Activities + err := workflow.ExecuteActivity(checkInActivityOptions(ctx), a.SentryCheckIn, activities.SentryCheckInInput{ + Job: job, + Status: status, + }).Get(ctx, nil) + if err != nil { + workflow.GetLogger(ctx).Warn("sentry crons check-in failed", "job", job, "status", status, "error", err) + } +} + +const ( + dbBackupActivityTimeout = 30 * time.Minute + dbBackupMaxAttempts = 3 +) + +// DBBackupRunTimeout bounds a single DBBackupWorkflow run. +// It is derived from the activity's own retry budget (every attempt at its full StartToCloseTimeout, plus headroom for the backoff between attempts) rather than from the schedule cadence: a run timeout tied to the interval would silently truncate runs as soon as the cadence is tightened, which the Schedule config is explicitly meant to allow. +const DBBackupRunTimeout = dbBackupActivityTimeout*dbBackupMaxAttempts + 10*time.Minute + +// DBBackupWorkflow orchestrates the DBBackup activity. +// A single sync of a bucket's worth of database backups can legitimately take a while, so the activity gets a generous StartToCloseTimeout and a HeartbeatTimeout so a dead worker is detected well before that timeout expires; ConfigError failures (bad bucket, missing source dir) are marked non-retryable since retrying them can't help. +// The HeartbeatTimeout has to clear the longest gap the activity can leave between two heartbeats, which on the multipart upload path is one completed 8 MiB part: two minutes leaves room for a slow link without letting a dead worker go unnoticed for long. +func DBBackupWorkflow(ctx workflow.Context) (activities.DBBackupResult, error) { + checkIn(ctx, activities.JobDBBackup, sentrycrons.StatusInProgress) + + ctx = workflow.WithActivityOptions(ctx, workflow.ActivityOptions{ + StartToCloseTimeout: dbBackupActivityTimeout, + HeartbeatTimeout: 2 * time.Minute, + RetryPolicy: &temporal.RetryPolicy{ + InitialInterval: 30 * time.Second, + BackoffCoefficient: 2.0, + MaximumInterval: 5 * time.Minute, + MaximumAttempts: dbBackupMaxAttempts, + NonRetryableErrorTypes: []string{activities.ErrTypeConfig}, + }, + }) + + var a *activities.Activities + var result activities.DBBackupResult + backupErr := workflow.ExecuteActivity(ctx, a.DBBackup, activities.DBBackupInput{}).Get(ctx, &result) + + status := sentrycrons.StatusOK + if backupErr != nil { + status = sentrycrons.StatusError + } + checkInFinal(ctx, activities.JobDBBackup, status) + + return result, backupErr +} diff --git a/internal/workflows/dbbackup_test.go b/internal/workflows/dbbackup_test.go new file mode 100644 index 0000000..daa038c --- /dev/null +++ b/internal/workflows/dbbackup_test.go @@ -0,0 +1,94 @@ +package workflows + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "go.temporal.io/sdk/temporal" + "go.temporal.io/sdk/testsuite" + + "github.com/maevsi/temporal/internal/activities" + "github.com/maevsi/temporal/internal/sentrycrons" +) + +type DBBackupWorkflowSuite struct { + suite.Suite + testsuite.WorkflowTestSuite +} + +func TestDBBackupWorkflowSuite(t *testing.T) { + suite.Run(t, new(DBBackupWorkflowSuite)) +} + +// activityRef mirrors the nil-receiver method-expression the production +// workflow code uses to reference activities (see checkIn / DBBackupWorkflow), +// so OnActivity registers against the exact same function value. +func activityRef() *activities.Activities { + var a *activities.Activities + return a +} + +func (s *DBBackupWorkflowSuite) TestSuccess_ChecksInProgressThenOK() { + env := s.NewTestWorkflowEnvironment() + a := activityRef() + + var statuses []sentrycrons.Status + env.OnActivity(a.SentryCheckIn, mock.Anything, mock.MatchedBy(func(in activities.SentryCheckInInput) bool { + return in.Job == activities.JobDBBackup + })).Return(func(_ context.Context, in activities.SentryCheckInInput) error { + statuses = append(statuses, in.Status) + return nil + }) + + want := activities.DBBackupResult{FilesUploaded: 3, BytesUploaded: 1024} + env.OnActivity(a.DBBackup, mock.Anything, activities.DBBackupInput{}).Return(want, nil) + + env.ExecuteWorkflow(DBBackupWorkflow) + + s.Require().True(env.IsWorkflowCompleted()) + s.Require().NoError(env.GetWorkflowError()) + + var got activities.DBBackupResult + s.Require().NoError(env.GetWorkflowResult(&got)) + s.Equal(want, got) + s.Equal([]sentrycrons.Status{sentrycrons.StatusInProgress, sentrycrons.StatusOK}, statuses) +} + +func (s *DBBackupWorkflowSuite) TestFailure_ChecksInProgressThenError() { + env := s.NewTestWorkflowEnvironment() + a := activityRef() + + var statuses []sentrycrons.Status + env.OnActivity(a.SentryCheckIn, mock.Anything, mock.Anything).Return(func(_ context.Context, in activities.SentryCheckInInput) error { + statuses = append(statuses, in.Status) + return nil + }) + + env.OnActivity(a.DBBackup, mock.Anything, activities.DBBackupInput{}).Return( + activities.DBBackupResult{}, + temporal.NewApplicationError("sync exploded", activities.ErrTypeConfig), + ) + + env.ExecuteWorkflow(DBBackupWorkflow) + + s.Require().True(env.IsWorkflowCompleted()) + s.Require().Error(env.GetWorkflowError()) + s.Equal([]sentrycrons.Status{sentrycrons.StatusInProgress, sentrycrons.StatusError}, statuses) +} + +func (s *DBBackupWorkflowSuite) TestSentryCheckInFailureDoesNotFailWorkflow() { + env := s.NewTestWorkflowEnvironment() + a := activityRef() + + env.OnActivity(a.SentryCheckIn, mock.Anything, mock.Anything).Return(errors.New("sentry is down")) + env.OnActivity(a.DBBackup, mock.Anything, activities.DBBackupInput{}).Return(activities.DBBackupResult{FilesUploaded: 1}, nil) + + env.ExecuteWorkflow(DBBackupWorkflow) + + s.Require().True(env.IsWorkflowCompleted()) + require.NoError(s.T(), env.GetWorkflowError()) +} diff --git a/internal/workflows/outboxpurge.go b/internal/workflows/outboxpurge.go new file mode 100644 index 0000000..2ca8213 --- /dev/null +++ b/internal/workflows/outboxpurge.go @@ -0,0 +1,52 @@ +package workflows + +import ( + "time" + + "go.temporal.io/sdk/temporal" + "go.temporal.io/sdk/workflow" + + "github.com/maevsi/temporal/internal/activities" + "github.com/maevsi/temporal/internal/sentrycrons" +) + +const ( + outboxPurgeActivityTimeout = 2 * time.Minute + outboxPurgeMaxAttempts = 5 +) + +// OutboxPurgeRunTimeout bounds a single OutboxPurgeWorkflow run, derived from the activity's retry budget for the same reason as DBBackupRunTimeout. +const OutboxPurgeRunTimeout = outboxPurgeActivityTimeout*outboxPurgeMaxAttempts + 5*time.Minute + +// OutboxPurgeWorkflow orchestrates the OutboxPurge activity: +// +// DELETE FROM vibetype_private.outbox WHERE created_at < now() - interval '24 hours' +// +// It executes the DELETE directly over a dedicated pgx connection pool. +// The activity itself is short (a single DELETE), so timeouts here are tight; ConfigError failures are marked non-retryable since retrying a misconfigured schema/table name can't help. +func OutboxPurgeWorkflow(ctx workflow.Context) (activities.OutboxPurgeResult, error) { + checkIn(ctx, activities.JobOutboxPurge, sentrycrons.StatusInProgress) + + ctx = workflow.WithActivityOptions(ctx, workflow.ActivityOptions{ + StartToCloseTimeout: outboxPurgeActivityTimeout, + RetryPolicy: &temporal.RetryPolicy{ + InitialInterval: 5 * time.Second, + BackoffCoefficient: 2.0, + MaximumInterval: time.Minute, + MaximumAttempts: outboxPurgeMaxAttempts, + NonRetryableErrorTypes: []string{activities.ErrTypeConfig}, + }, + }) + + var a *activities.Activities + var result activities.OutboxPurgeResult + purgeErr := workflow.ExecuteActivity(ctx, a.OutboxPurge, activities.OutboxPurgeInput{}).Get(ctx, &result) + + status := sentrycrons.StatusOK + if purgeErr != nil { + status = sentrycrons.StatusError + } + checkInFinal(ctx, activities.JobOutboxPurge, status) + + return result, purgeErr +} diff --git a/internal/workflows/outboxpurge_test.go b/internal/workflows/outboxpurge_test.go new file mode 100644 index 0000000..370f8c8 --- /dev/null +++ b/internal/workflows/outboxpurge_test.go @@ -0,0 +1,98 @@ +package workflows + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" + "go.temporal.io/sdk/temporal" + "go.temporal.io/sdk/testsuite" + + "github.com/maevsi/temporal/internal/activities" + "github.com/maevsi/temporal/internal/sentrycrons" +) + +type OutboxPurgeWorkflowSuite struct { + suite.Suite + testsuite.WorkflowTestSuite +} + +func TestOutboxPurgeWorkflowSuite(t *testing.T) { + suite.Run(t, new(OutboxPurgeWorkflowSuite)) +} + +func (s *OutboxPurgeWorkflowSuite) TestSuccess_ChecksInProgressThenOK() { + env := s.NewTestWorkflowEnvironment() + a := activityRef() + + var statuses []sentrycrons.Status + env.OnActivity(a.SentryCheckIn, mock.Anything, mock.MatchedBy(func(in activities.SentryCheckInInput) bool { + return in.Job == activities.JobOutboxPurge + })).Return(func(_ context.Context, in activities.SentryCheckInInput) error { + statuses = append(statuses, in.Status) + return nil + }) + + want := activities.OutboxPurgeResult{RowsDeleted: 7} + env.OnActivity(a.OutboxPurge, mock.Anything, activities.OutboxPurgeInput{}).Return(want, nil) + + env.ExecuteWorkflow(OutboxPurgeWorkflow) + + s.Require().True(env.IsWorkflowCompleted()) + s.Require().NoError(env.GetWorkflowError()) + + var got activities.OutboxPurgeResult + s.Require().NoError(env.GetWorkflowResult(&got)) + s.Equal(want, got) + s.Equal([]sentrycrons.Status{sentrycrons.StatusInProgress, sentrycrons.StatusOK}, statuses) +} + +func (s *OutboxPurgeWorkflowSuite) TestFailure_ChecksInProgressThenError() { + env := s.NewTestWorkflowEnvironment() + a := activityRef() + + var statuses []sentrycrons.Status + env.OnActivity(a.SentryCheckIn, mock.Anything, mock.Anything).Return(func(_ context.Context, in activities.SentryCheckInInput) error { + statuses = append(statuses, in.Status) + return nil + }) + + env.OnActivity(a.OutboxPurge, mock.Anything, activities.OutboxPurgeInput{}).Return( + activities.OutboxPurgeResult{}, + temporal.NewApplicationError("db unreachable", "PostgresError"), + ) + + env.ExecuteWorkflow(OutboxPurgeWorkflow) + + s.Require().True(env.IsWorkflowCompleted()) + s.Require().Error(env.GetWorkflowError()) + s.Equal([]sentrycrons.Status{sentrycrons.StatusInProgress, sentrycrons.StatusError}, statuses) +} + +// TestCancellation_StillChecksInError covers a run cancelled while the activity is in flight. +// The terminal check-in has to go out on a disconnected context: on the workflow's own context ExecuteActivity short-circuits with a CanceledError without scheduling anything, and the Sentry monitor would sit in StatusInProgress until its own max-runtime alert fired. +func (s *OutboxPurgeWorkflowSuite) TestCancellation_StillChecksInError() { + env := s.NewTestWorkflowEnvironment() + a := activityRef() + + var statuses []sentrycrons.Status + env.OnActivity(a.SentryCheckIn, mock.Anything, mock.Anything).Return(func(_ context.Context, in activities.SentryCheckInInput) error { + statuses = append(statuses, in.Status) + return nil + }) + + env.OnActivity(a.OutboxPurge, mock.Anything, activities.OutboxPurgeInput{}).Return( + func(ctx context.Context, _ activities.OutboxPurgeInput) (activities.OutboxPurgeResult, error) { + <-ctx.Done() + return activities.OutboxPurgeResult{}, ctx.Err() + }) + + env.RegisterDelayedCallback(env.CancelWorkflow, time.Second) + env.ExecuteWorkflow(OutboxPurgeWorkflow) + + s.Require().True(env.IsWorkflowCompleted()) + s.Require().Error(env.GetWorkflowError()) + s.Equal([]sentrycrons.Status{sentrycrons.StatusInProgress, sentrycrons.StatusError}, statuses) +}