diff --git a/.github/workflows/admin-sourcemaps.yml b/.github/workflows/admin-sourcemaps.yml deleted file mode 100644 index d34365f7b9d..00000000000 --- a/.github/workflows/admin-sourcemaps.yml +++ /dev/null @@ -1,21 +0,0 @@ -on: - pull_request: - push: - branches: [master] -jobs: - build: - name: "build sourcemaps" - runs-on: ubuntu-latest - env: - SENTRY_AUTH_TOKEN: ${{ secrets.SNUBA_SENTRY_SOURCEMAP_KEY }} - steps: - - uses: actions/checkout@v6.0.2 - name: Checkout code - - uses: actions/setup-python@v6 - with: - python-version: 3.8 - - uses: actions/setup-node@v6 - with: - node-version-file: snuba/admin/package.json - - name: Build admin sourcemaps - run: make build-admin diff --git a/.github/workflows/bump-version.yml b/.github/workflows/bump-version.yml deleted file mode 100644 index 6969349ce1f..00000000000 --- a/.github/workflows/bump-version.yml +++ /dev/null @@ -1,88 +0,0 @@ -name: Bump a dependency -on: - workflow_dispatch: - inputs: - package: - required: true - type: string - description: package name such as `sentry-arroyo` (_ vs - does not matter) - version: - required: true - type: string - description: desired version such as `1.2.3`, or `latest` to pull the latest version from PyPI - - # for use in other (cron/scheduled) workflows to bump specific - # company-internal dependencies on a more aggressive schedule - workflow_call: - inputs: - package: - required: true - type: string - version: - required: true - type: string - -# disable all permissions -- we use the PAT's permissions instead -permissions: {} - -jobs: - bump-version: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6.0.2 - with: - token: ${{ secrets.GETSENTRY_BOT_REVERT_TOKEN }} - - uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1 - - run: uv python install - - run: | - set -euxo pipefail - - if [ "$VERSION" = latest ]; then - VERSION="$(curl -sL https://pypi.org/pypi/$PACKAGE/json | jq -r .info.version)" - fi - - git checkout -b "bot/bump-version/$PACKAGE/$VERSION" - - python3 -S -m tools.bump_version "$PACKAGE" "$VERSION" - - re="$(sed 's/[_-]/[_-]/g' <<< "$PACKAGE")" - - # Update Cargo.toml dependencies (format: package = "version") - sed -i "s/^\($re\) = \"[^\"]*\"/\1 = \"$VERSION\"/g" -- rust_snuba/Cargo.toml - # Also handle dependencies with features (format: package = { version = "version", features = [...] }) - sed -i "s/^\($re\) = { version = \"[^\"]*\"/\1 = { version = \"$VERSION\"/g" -- rust_snuba/Cargo.toml - - # Update Cargo.lock if Cargo.toml was modified - if ! git diff --exit-code -- rust_snuba/Cargo.toml > /dev/null 2>&1; then - cd rust_snuba - # Try updating with underscores (cargo prefers underscores in package names) - CARGO_PACKAGE="$(echo "$PACKAGE" | sed 's/-/_/g')" - if ! cargo update --package "$CARGO_PACKAGE" 2>/dev/null; then - # If that fails, try with the original package name - cargo update --package "$PACKAGE" - fi - cd .. - fi - - if git diff --exit-code; then - exit 0 - fi - - git \ - -c user.name=getsentry-bot \ - -c user.email='10587625+getsentry-bot@users.noreply.github.com' \ - commit \ - --all \ - --message "ref: bump $PACKAGE to $VERSION" \ - --message "Co-Authored-By: $SENDER <$SENDER_ID+$SENDER@users.noreply.github.com>" - - git push origin HEAD --quiet - - gh pr create --fill - env: - # Using this instead of BUMP_SENTRY_TOKEN as per advice from asottile - GH_TOKEN: ${{ secrets.GETSENTRY_BOT_REVERT_TOKEN }} - PACKAGE: ${{ inputs.package }} - VERSION: ${{ inputs.version }} - SENDER: ${{ github.event.sender.login }} - SENDER_ID: ${{ github.event.sender.id }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7423a096819..ae0ebae441e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,10 @@ on: - master pull_request: +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + jobs: files-changed: name: detect what files changed diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index f0de27117f2..00000000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,66 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: "CodeQL" - -on: - schedule: - - cron: "30 3 * * 5" - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write - - strategy: - fail-fast: false - matrix: - language: ["javascript", "python"] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] - # Learn more about CodeQL language support at https://git.io/codeql-language-support - - steps: - - name: Checkout repository - uses: actions/checkout@v6.0.2 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v4 - with: - config-file: ./.github/codeql/codeql-config.yml - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v4 - - # â„šī¸ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl - - # âœī¸ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language - - #- run: | - # make bootstrap - # make release - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v4 diff --git a/.github/workflows/ddl-changes.yml b/.github/workflows/ddl-changes.yml deleted file mode 100644 index d92959c6619..00000000000 --- a/.github/workflows/ddl-changes.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: ddl-changes -on: - pull_request: - types: [opened, synchronize, reopened, labeled, unlabeled] - - -jobs: - post_changes: - name: Post new DDL changes from migrations - runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - uses: actions/checkout@v6.0.2 - name: Checkout master for diffing - with: - ref: master - fetch-depth: 200 - - uses: actions/checkout@v6.0.2 - name: Checkout HEAD of code that may have migration changes - with: - clean: false - fetch-depth: 200 - - - uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1 - with: - # we just cache the venv-dir directly in action-setup-venv - enable-cache: false - - - uses: getsentry/action-setup-venv@5a80476d175edf56cb205b08bc58986fa99d1725 # v3.2.0 - with: - - cache-dependency-path: uv.lock - install-cmd: uv sync --frozen --active - - - name: Run the migration script - run: | - SNUBA_SETTINGS=test_distributed python scripts/ddl-changes.py - - name: Generate SQL for migration - uses: getsentry/action-migrations@v1.2.2 - env: - SNUBA_SETTINGS: test_distributed - with: - githubToken: ${{ secrets.GITHUB_TOKEN }} - migration: "./snuba/migrations/groups.py" - cmd: python scripts/ddl-changes.py - - - name: Check migrations are not coupled with other changes - run: | - # Check that the migration is not coupled with other changes - # If the label skip-check-migrations is present, the check is skipped - SNUBA_SETTINGS=test_distributed python scripts/check-migrations.py --labels ${{join(github.event.pull_request.labels.*.name, '')}} diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml deleted file mode 100644 index 269aea763f1..00000000000 --- a/.github/workflows/dependency-review.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: "Dependency Review" -on: - pull_request: - branches: ["master"] - -permissions: - contents: read - -jobs: - dependency-review: - runs-on: ubuntu-latest - steps: - - name: "Checkout Repository" - uses: actions/checkout@v6.0.2 - - name: Dependency Review - uses: actions/dependency-review-action@v4 - with: - # Possible values: "critical", "high", "moderate", "low" - fail-on-severity: high diff --git a/.github/workflows/docs-pr.yml b/.github/workflows/docs-pr.yml deleted file mode 100644 index 329fd435d01..00000000000 --- a/.github/workflows/docs-pr.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: Snuba Docs on PR's - -on: - pull_request: - -jobs: - docs: - name: Sphinx - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6.0.2 - - - uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1 - with: - # we just cache the venv-dir directly in action-setup-venv - enable-cache: false - - - uses: getsentry/action-setup-venv@5a80476d175edf56cb205b08bc58986fa99d1725 # v3.2.0 - with: - - cache-dependency-path: docs-requirements.txt - install-cmd: echo - - - name: Generate config schema docs - run: | - make generate-config-docs - - - name: Build docs - run: | - make snubadocs diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index 229fd2e97b3..00000000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: Snuba Docs - -on: - push: - branches: - - master - -jobs: - docs: - name: Sphinx - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6.0.2 - - - uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1 - with: - # we just cache the venv-dir directly in action-setup-venv - enable-cache: false - - - uses: getsentry/action-setup-venv@5a80476d175edf56cb205b08bc58986fa99d1725 # v3.2.0 - with: - - cache-dependency-path: docs-requirements.txt - install-cmd: echo - - - name: Generate config schema docs - run: | - make generate-config-docs - - - name: Build docs - run: | - make snubadocs - - - uses: peaceiris/actions-gh-pages@v4.0.0 - name: Publish to GitHub Pages - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: docs/build - force_orphan: true - - - name: Archive Docs - uses: actions/upload-artifact@v7 - with: - name: docs - path: docs/build diff --git a/.github/workflows/enforce-license-compliance.yml b/.github/workflows/enforce-license-compliance.yml deleted file mode 100644 index 8722b90b334..00000000000 --- a/.github/workflows/enforce-license-compliance.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: Enforce License Compliance - -on: - push: - branches: [master, main, release/*] - pull_request: - branches: [master, main] - -jobs: - enforce-license-compliance: - runs-on: ubuntu-latest - steps: - - name: 'Enforce License Compliance' - uses: getsentry/action-enforce-license-compliance@6599a041195852debba3417e069829060d671e76 - with: - fossa_api_key: ${{ secrets.FOSSA_API_KEY }} diff --git a/.github/workflows/fast-revert.yml b/.github/workflows/fast-revert.yml deleted file mode 100644 index 1ffb8c18ab7..00000000000 --- a/.github/workflows/fast-revert.yml +++ /dev/null @@ -1,53 +0,0 @@ -on: - pull_request_target: - types: [labeled] - workflow_dispatch: - inputs: - pr: - required: true - description: pr number - co_authored_by: - required: true - description: '`name ` for triggering user' - -# disable all permissions -- we use the PAT's permissions instead -permissions: {} - -jobs: - revert: - runs-on: ubuntu-latest - if: | - github.event_name == 'workflow_dispatch' || github.event.label.name == 'Trigger: Revert' - steps: - - name: Get auth token - id: token - uses: getsentry/action-github-app-token@d4b5da6c5e37703f8c3b3e43abb5705b46e159cc # v3.0.0 - with: - app_id: ${{ secrets.FAST_REVERT_BOT_APP_ID }} - private_key: ${{ secrets.GH_FAST_REVERT_PRIVATE_KEY }} - - uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 # v3.1.0 - with: - token: ${{ steps.token.outputs.token }} - - uses: getsentry/action-fast-revert@35b4b6c1f8f91b5911159568b3b15e531b5b8174 # v2.0.1 - with: - pr: ${{ github.event.number || github.event.inputs.pr }} - co_authored_by: >- - ${{ github.event.inputs.co_authored_by || format('{0}\n<{1}+{0}@users.noreply.github.com>', github.event.sender.login, github.event.sender.id) }} - committer_name: sentry-snuba-fast-revert-bot[bot] - committer_email: 257653817+sentry-snuba-fast-revert-bot[bot]@users.noreply.github.com - token: ${{ steps.token.outputs.token }} - - name: comment on failure - env: - GITHUB_REPOSITORY: ${{ github.repository }} - GITHUB_RUN_ID: ${{ github.run_id }} - REPOSITORY_ID: ${{ github.event.repository.id }} - PR_NUMBER: ${{ github.event.number || github.event.inputs.pr }} - GITHUB_TOKEN: ${{ secrets.GETSENTRY_BOT_REVERT_TOKEN }} - run: | - curl \ - --silent \ - -X POST \ - -H "Authorization: token $GITHUB_TOKEN" \ - -d"{\"body\": \"revert failed (conflict? already reverted?) -- [check the logs](https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID)\"}" \ - "https://api.github.com/repositories/$REPOSITORY_ID/issues/$PR_NUMBER/comments" - if: failure() diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml index ce6e049679d..13d3e352fb4 100644 --- a/.github/workflows/image.yml +++ b/.github/workflows/image.yml @@ -5,66 +5,15 @@ on: - master - release/** -jobs: - build-multiplatform: - strategy: - matrix: - include: - - os: ubuntu-24.04 - platform: amd64 - - os: ubuntu-24.04-arm - platform: arm64 - runs-on: ${{ matrix.os }} - name: build-${{ matrix.platform }} - permissions: - contents: read - packages: write - if: github.repository_owner == 'getsentry' - steps: - - uses: actions/checkout@v6.0.2 - - uses: getsentry/action-build-and-push-images@8fc75e483c09a68721f2c8951292ee17f8821766 - with: - image_name: 'snuba' - platforms: linux/${{ matrix.platform }} - dockerfile_path: 'Dockerfile' - tag_suffix: -${{ matrix.platform }} - ghcr: true - tag_nightly: false - tag_latest: false +# Cancel in-progress runs for the same branch/PR when a new commit is pushed. +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true - build-production: +jobs: + build: runs-on: ubuntu-24.04 - name: Build and push production image - permissions: - contents: read - id-token: write - if: ${{ github.ref_name == 'master' }} - steps: - - uses: actions/checkout@v6.0.2 - - uses: getsentry/action-build-and-push-images@8fc75e483c09a68721f2c8951292ee17f8821766 - with: - image_name: 'snuba' - platforms: linux/amd64 - dockerfile_path: './Dockerfile' - ghcr: false - tag_nightly: false - tag_latest: false - google_ar: true - google_ar_image_name: us-docker.pkg.dev/sentryio/snuba-mr/image - google_workload_identity_provider: projects/868781662168/locations/global/workloadIdentityPools/prod-github/providers/github-oidc-pool - google_service_account: gha-gcr-push@sac-prod-sa.iam.gserviceaccount.com - - # Distroless image — for testing before switching production - build-distroless-multiplatform: - strategy: - matrix: - include: - - os: ubuntu-24.04 - platform: amd64 - - os: ubuntu-24.04-arm - platform: arm64 - runs-on: ${{ matrix.os }} - name: build-distroless-${{ matrix.platform }} + name: build permissions: contents: read packages: write @@ -74,45 +23,17 @@ jobs: - uses: getsentry/action-build-and-push-images@8fc75e483c09a68721f2c8951292ee17f8821766 with: image_name: 'snuba' - platforms: linux/${{ matrix.platform }} + platforms: linux/amd64 dockerfile_path: 'Dockerfile' - build_target: 'application-distroless' - tag_suffix: -distroless-${{ matrix.platform }} - ghcr: true - tag_nightly: false - tag_latest: false - - # Debug distroless image — with busybox for troubleshooting - build-debug-multiplatform: - strategy: - matrix: - include: - - os: ubuntu-24.04 - platform: amd64 - - os: ubuntu-24.04-arm - platform: arm64 - runs-on: ${{ matrix.os }} - name: build-debug-${{ matrix.platform }} - permissions: - contents: read - packages: write - if: github.repository_owner == 'getsentry' - steps: - - uses: actions/checkout@v6.0.2 - - uses: getsentry/action-build-and-push-images@8fc75e483c09a68721f2c8951292ee17f8821766 - with: - image_name: 'snuba' - platforms: linux/${{ matrix.platform }} - dockerfile_path: 'Dockerfile' - build_target: 'application-distroless-debug' - tag_suffix: -debug-${{ matrix.platform }} + tag_suffix: -amd64 ghcr: true + publish_on_pr: true tag_nightly: false tag_latest: false assemble: - needs: [build-multiplatform] - if: ${{ (github.ref_name == 'master' || startsWith(github.ref_name, 'release/')) && github.event_name != 'pull_request' }} + needs: [build] + if: github.repository_owner == 'getsentry' runs-on: ubuntu-latest permissions: contents: read @@ -126,71 +47,8 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Create multiplatform manifests + - name: Tag SHA image run: | docker buildx imagetools create \ --tag ghcr.io/getsentry/snuba:${{ github.sha }} \ - --tag ghcr.io/getsentry/snuba:nightly \ - ghcr.io/getsentry/snuba:${{ github.sha }}-amd64 \ - ghcr.io/getsentry/snuba:${{ github.sha }}-arm64 - - assemble-distroless: - needs: [build-distroless-multiplatform] - if: ${{ (github.ref_name == 'master' || startsWith(github.ref_name, 'release/')) && github.event_name != 'pull_request' }} - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - name: Docker Login - run: docker login --username '${{ github.actor }}' --password-stdin ghcr.io <<< "$GHCR_TOKEN" - env: - GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Create distroless multiplatform manifests - run: | - docker buildx imagetools create \ - --tag ghcr.io/getsentry/snuba:${{ github.sha }}-distroless \ - --tag ghcr.io/getsentry/snuba:nightly-distroless \ - ghcr.io/getsentry/snuba:${{ github.sha }}-distroless-amd64 \ - ghcr.io/getsentry/snuba:${{ github.sha }}-distroless-arm64 - - assemble-debug: - needs: [build-debug-multiplatform] - if: ${{ (github.ref_name == 'master' || startsWith(github.ref_name, 'release/')) && github.event_name != 'pull_request' }} - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - name: Docker Login - run: docker login --username '${{ github.actor }}' --password-stdin ghcr.io <<< "$GHCR_TOKEN" - env: - GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Create debug multiplatform manifests - run: | - docker buildx imagetools create \ - --tag ghcr.io/getsentry/snuba:${{ github.sha }}-debug \ - --tag ghcr.io/getsentry/snuba:nightly-debug \ - ghcr.io/getsentry/snuba:${{ github.sha }}-debug-amd64 \ - ghcr.io/getsentry/snuba:${{ github.sha }}-debug-arm64 - - self-hosted-end-to-end: - needs: [build-multiplatform, assemble] - runs-on: ubuntu-latest - timeout-minutes: 30 - - steps: - - name: Run Sentry self-hosted e2e CI - uses: getsentry/self-hosted@master - with: - project_name: snuba - image_url: ghcr.io/getsentry/snuba:${{ github.sha }} - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + ghcr.io/getsentry/snuba:${{ github.sha }}-amd64 diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml deleted file mode 100644 index 178ba299670..00000000000 --- a/.github/workflows/labeler.yml +++ /dev/null @@ -1,14 +0,0 @@ -name: "Pull Request Labeler" -on: -- pull_request_target - -jobs: - triage: - permissions: - contents: read - pull-requests: write - runs-on: ubuntu-latest - steps: - - uses: actions/labeler@v6 - with: - repo-token: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 9a528282424..00000000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: release -on: - workflow_dispatch: - inputs: - version: - description: Version to release (or "auto") - required: false - force: - description: Force a release even when there are release-blockers (optional) - required: false - schedule: - # We want the release to be at 9-10am Pacific Time - # We also want it to be 1 hour before the on-prem release - - cron: "0 17 15 * *" -permissions: - contents: write - pull-requests: write - -jobs: - release: - runs-on: ubuntu-latest - name: "Release a new version" - steps: - - name: Get auth token - id: token - uses: actions/create-github-app-token@v2 - with: - app-id: ${{ vars.SENTRY_RELEASE_BOT_CLIENT_ID }} - private-key: ${{ secrets.SENTRY_RELEASE_BOT_PRIVATE_KEY }} - - uses: actions/checkout@v6.0.2 - with: - token: ${{ steps.token.outputs.token }} - fetch-depth: 0 - - name: Prepare release - uses: getsentry/craft@v2 - env: - GITHUB_TOKEN: ${{ steps.token.outputs.token }} - with: - version: ${{ github.event.inputs.version }} - force: ${{ github.event.inputs.force }} diff --git a/.github/workflows/validate-pipelines.yml b/.github/workflows/validate-pipelines.yml deleted file mode 100644 index b396ddbb3e5..00000000000 --- a/.github/workflows/validate-pipelines.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: Validate Deployment Pipelines - -on: - pull_request: - push: - branches: [master, test-me-*] - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - files-changed: - name: files-changed - runs-on: ubuntu-latest - # Map a step output to a job output - outputs: - gocd: ${{ steps.changes.outputs.gocd }} - steps: - - uses: actions/checkout@v6.0.2 - - name: Check for relevant file changes - uses: getsentry/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 - id: changes - with: - filters: | - gocd: - - 'gocd/**' - - - validate: - if: needs.files-changed.outputs.gocd == 'true' - needs: files-changed - name: Validate GoCD Pipelines - runs-on: ubuntu-latest - - # required for google auth - permissions: - contents: "read" - id-token: "write" - - steps: - - uses: actions/checkout@v6.0.2 - - id: 'auth' - uses: google-github-actions/auth@v3 - with: - workload_identity_provider: 'projects/868781662168/locations/global/workloadIdentityPools/prod-github/providers/github-oidc-pool' - service_account: 'gha-gocd-api@sac-prod-sa.iam.gserviceaccount.com' - token_format: 'id_token' - id_token_audience: '610575311308-9bsjtgqg4jm01mt058rncpopujgk3627.apps.googleusercontent.com' - id_token_include_email: true - - uses: getsentry/action-gocd-jsonnet@v1.1.1 - with: - jb-install: true - jsonnet-dir: gocd/templates - generated-dir: gocd/generated-pipelines - - uses: getsentry/action-validate-gocd-pipelines@v1 - with: - configrepo: snuba__master - gocd_access_token: ${{ secrets.GOCD_ACCESS_TOKEN }} - google_oidc_token: ${{ steps.auth.outputs.id_token }} diff --git a/.github/workflows/validate-sentry-options.yml b/.github/workflows/validate-sentry-options.yml deleted file mode 100644 index 50b2d2b9fd0..00000000000 --- a/.github/workflows/validate-sentry-options.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Validate Sentry Options Schema - -on: - pull_request: - merge_group: - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - files-changed: - name: files-changed - runs-on: ubuntu-latest - outputs: - schemas: ${{ steps.changes.outputs.schemas }} - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - name: Check for relevant file changes - uses: getsentry/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 - id: changes - with: - filters: | - schemas: - - 'sentry-options/schemas/**' - - validate-schema: - if: needs.files-changed.outputs.schemas == 'true' - needs: files-changed - name: Validate Schema Evolution - uses: getsentry/sentry-options/.github/workflows/validate-schema.yml@fa066c1d3ef0849153092a0272c17fc0bcce14c8 - secrets: inherit - with: - schemas-path: sentry-options/schemas diff --git a/.gitignore b/.gitignore index ec5476fa210..bb5331a3800 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ gocd/templates/vendor/ gocd/generated-pipelines/ Brewfile.lock.json .zed/ +__pycache__ diff --git a/devservices/config.yml b/devservices/config.yml index 16aa137158b..467270374c9 100644 --- a/devservices/config.yml +++ b/devservices/config.yml @@ -75,7 +75,7 @@ services: restart: unless-stopped snuba: - image: ghcr.io/getsentry/snuba:nightly + image: ghcr.io/getsentry/snuba:48d65526d42231abb4c2d3782a8840bea785cdea-amd64 ports: - 127.0.0.1:1218:1218 - 127.0.0.1:1219:1219 @@ -114,7 +114,7 @@ services: - orchestrator=devservices restart: unless-stopped profiles-consumer: - image: ghcr.io/getsentry/snuba:nightly + image: ghcr.io/getsentry/snuba:48d65526d42231abb4c2d3782a8840bea785cdea-amd64 command: [ rust-consumer, --storage=profiles, @@ -143,7 +143,7 @@ services: - orchestrator=devservices restart: unless-stopped profile-chunks-consumer: - image: ghcr.io/getsentry/snuba:nightly + image: ghcr.io/getsentry/snuba:48d65526d42231abb4c2d3782a8840bea785cdea-amd64 command: [ rust-consumer, --storage=profile_chunks, @@ -172,7 +172,7 @@ services: - orchestrator=devservices restart: unless-stopped functions-consumer: - image: ghcr.io/getsentry/snuba:nightly + image: ghcr.io/getsentry/snuba:48d65526d42231abb4c2d3782a8840bea785cdea-amd64 command: [ rust-consumer, --storage=functions_raw, @@ -201,7 +201,7 @@ services: - orchestrator=devservices restart: unless-stopped metrics-consumer: - image: ghcr.io/getsentry/snuba:nightly + image: ghcr.io/getsentry/snuba:48d65526d42231abb4c2d3782a8840bea785cdea-amd64 command: [ rust-consumer, --storage=metrics_raw, @@ -230,7 +230,7 @@ services: - orchestrator=devservices restart: unless-stopped generic-metrics-distributions-consumer: - image: ghcr.io/getsentry/snuba:nightly + image: ghcr.io/getsentry/snuba:48d65526d42231abb4c2d3782a8840bea785cdea-amd64 command: [ rust-consumer, --storage=generic_metrics_distributions_raw, @@ -259,7 +259,7 @@ services: - orchestrator=devservices restart: unless-stopped generic-metrics-sets-consumer: - image: ghcr.io/getsentry/snuba:nightly + image: ghcr.io/getsentry/snuba:48d65526d42231abb4c2d3782a8840bea785cdea-amd64 command: [ rust-consumer, --storage=generic_metrics_sets_raw, @@ -288,7 +288,7 @@ services: - orchestrator=devservices restart: unless-stopped generic-metrics-counters-consumer: - image: ghcr.io/getsentry/snuba:nightly + image: ghcr.io/getsentry/snuba:48d65526d42231abb4c2d3782a8840bea785cdea-amd64 command: [ rust-consumer, --storage=generic_metrics_counters_raw, @@ -317,7 +317,7 @@ services: - orchestrator=devservices restart: unless-stopped generic-metrics-gauges-consumer: - image: ghcr.io/getsentry/snuba:nightly + image: ghcr.io/getsentry/snuba:48d65526d42231abb4c2d3782a8840bea785cdea-amd64 command: [ rust-consumer, --storage=generic_metrics_gauges_raw, diff --git a/snuba/web/views.py b/snuba/web/views.py index e4661f55ded..86f1325d7cc 100644 --- a/snuba/web/views.py +++ b/snuba/web/views.py @@ -719,6 +719,37 @@ def drop(*, dataset: Dataset) -> RespTuple: return ("ok", 200, {"Content-Type": "text/plain"}) + @application.route("/tests//optimize", methods=["POST"]) + def optimize(*, dataset: Dataset) -> RespTuple: + """ + Force ClickHouse to immediately deduplicate ReplacingMergeTree rows for + every table in this dataset. + + Under normal operation ClickHouse merges parts in the background, so + tombstones and replacement rows (e.g. from merge/unmerge/delete_groups + operations) are not immediately visible in subsequent SELECT queries. + Calling OPTIMIZE TABLE â€Ļ FINAL forces a synchronous deduplication so + test assertions see consistent state right away. + + Usage from tests: + requests.post(f"{SENTRY_SNUBA}/tests/events/optimize") + requests.post(f"{SENTRY_SNUBA}/tests/groupedmessage/optimize") + """ + for entity in dataset.get_all_entities(): + for storage in entity.get_all_storages(): + cluster = storage.get_cluster() + nodes = [*cluster.get_local_nodes(), *cluster.get_distributed_nodes()] + for node in nodes: + clickhouse = cluster.get_node_connection(ClickhouseClientSettings.MIGRATE, node) + database = cluster.get_database() + schema = storage.get_schema() + if not isinstance(schema, TableSchema): + continue + table = schema.get_local_table_name() + clickhouse.execute(f"OPTIMIZE TABLE {database}.{table} FINAL") + + return ("ok", 200, {"Content-Type": "text/plain"}) + @application.route("/tests/error") def error() -> RespTuple: 1 / 0