From 07f94143071c179f417d42f3f7d1897d3e8c38bf Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Sat, 11 Apr 2026 10:12:48 -0700 Subject: [PATCH 1/9] feat(tests): Add /tests/{dataset}/optimize endpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds POST /tests//optimize which calls OPTIMIZE TABLE … FINAL on every ClickHouse table belonging to that dataset. ClickHouse's ReplacingMergeTree deduplicates rows (tombstones from deletions, replacement rows from group merge/unmerge) in the background. Under high parallel load, background merges lag significantly, causing test assertions to see stale data. OPTIMIZE TABLE FINAL forces immediate synchronous deduplication. Usage from Sentry tests: requests.post(f"{settings.SENTRY_SNUBA}/tests/events/optimize") requests.post(f"{settings.SENTRY_SNUBA}/tests/groupedmessage/optimize") --- snuba/web/views.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/snuba/web/views.py b/snuba/web/views.py index e4661f55de..86f1325d7c 100644 --- a/snuba/web/views.py +++ b/snuba/web/views.py @@ -719,6 +719,37 @@ def drop(*, dataset: Dataset) -> RespTuple: return ("ok", 200, {"Content-Type": "text/plain"}) + @application.route("/tests//optimize", methods=["POST"]) + def optimize(*, dataset: Dataset) -> RespTuple: + """ + Force ClickHouse to immediately deduplicate ReplacingMergeTree rows for + every table in this dataset. + + Under normal operation ClickHouse merges parts in the background, so + tombstones and replacement rows (e.g. from merge/unmerge/delete_groups + operations) are not immediately visible in subsequent SELECT queries. + Calling OPTIMIZE TABLE … FINAL forces a synchronous deduplication so + test assertions see consistent state right away. + + Usage from tests: + requests.post(f"{SENTRY_SNUBA}/tests/events/optimize") + requests.post(f"{SENTRY_SNUBA}/tests/groupedmessage/optimize") + """ + for entity in dataset.get_all_entities(): + for storage in entity.get_all_storages(): + cluster = storage.get_cluster() + nodes = [*cluster.get_local_nodes(), *cluster.get_distributed_nodes()] + for node in nodes: + clickhouse = cluster.get_node_connection(ClickhouseClientSettings.MIGRATE, node) + database = cluster.get_database() + schema = storage.get_schema() + if not isinstance(schema, TableSchema): + continue + table = schema.get_local_table_name() + clickhouse.execute(f"OPTIMIZE TABLE {database}.{table} FINAL") + + return ("ok", 200, {"Content-Type": "text/plain"}) + @application.route("/tests/error") def error() -> RespTuple: 1 / 0 From abc32e37c8a3acb731604ac97b23eb83a3a82345 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Sat, 11 Apr 2026 10:27:22 -0700 Subject: [PATCH 2/9] rm all workflows except image --- .github/workflows/admin-sourcemaps.yml | 21 - .github/workflows/bump-version.yml | 88 --- .github/workflows/ci.yml | 577 ------------------ .github/workflows/codeql-analysis.yml | 66 -- .github/workflows/ddl-changes.yml | 51 -- .github/workflows/dependency-review.yml | 19 - .github/workflows/docs-pr.yml | 30 - .github/workflows/docs.yml | 45 -- .../workflows/enforce-license-compliance.yml | 16 - .github/workflows/fast-revert.yml | 53 -- .github/workflows/labeler.yml | 14 - .github/workflows/release.yml | 40 -- .github/workflows/validate-pipelines.yml | 60 -- .github/workflows/validate-sentry-options.yml | 34 -- .gitignore | 1 + 15 files changed, 1 insertion(+), 1114 deletions(-) delete mode 100644 .github/workflows/admin-sourcemaps.yml delete mode 100644 .github/workflows/bump-version.yml delete mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/codeql-analysis.yml delete mode 100644 .github/workflows/ddl-changes.yml delete mode 100644 .github/workflows/dependency-review.yml delete mode 100644 .github/workflows/docs-pr.yml delete mode 100644 .github/workflows/docs.yml delete mode 100644 .github/workflows/enforce-license-compliance.yml delete mode 100644 .github/workflows/fast-revert.yml delete mode 100644 .github/workflows/labeler.yml delete mode 100644 .github/workflows/release.yml delete mode 100644 .github/workflows/validate-pipelines.yml delete mode 100644 .github/workflows/validate-sentry-options.yml diff --git a/.github/workflows/admin-sourcemaps.yml b/.github/workflows/admin-sourcemaps.yml deleted file mode 100644 index d34365f7b9..0000000000 --- a/.github/workflows/admin-sourcemaps.yml +++ /dev/null @@ -1,21 +0,0 @@ -on: - pull_request: - push: - branches: [master] -jobs: - build: - name: "build sourcemaps" - runs-on: ubuntu-latest - env: - SENTRY_AUTH_TOKEN: ${{ secrets.SNUBA_SENTRY_SOURCEMAP_KEY }} - steps: - - uses: actions/checkout@v6.0.2 - name: Checkout code - - uses: actions/setup-python@v6 - with: - python-version: 3.8 - - uses: actions/setup-node@v6 - with: - node-version-file: snuba/admin/package.json - - name: Build admin sourcemaps - run: make build-admin diff --git a/.github/workflows/bump-version.yml b/.github/workflows/bump-version.yml deleted file mode 100644 index 6969349ce1..0000000000 --- a/.github/workflows/bump-version.yml +++ /dev/null @@ -1,88 +0,0 @@ -name: Bump a dependency -on: - workflow_dispatch: - inputs: - package: - required: true - type: string - description: package name such as `sentry-arroyo` (_ vs - does not matter) - version: - required: true - type: string - description: desired version such as `1.2.3`, or `latest` to pull the latest version from PyPI - - # for use in other (cron/scheduled) workflows to bump specific - # company-internal dependencies on a more aggressive schedule - workflow_call: - inputs: - package: - required: true - type: string - version: - required: true - type: string - -# disable all permissions -- we use the PAT's permissions instead -permissions: {} - -jobs: - bump-version: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6.0.2 - with: - token: ${{ secrets.GETSENTRY_BOT_REVERT_TOKEN }} - - uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1 - - run: uv python install - - run: | - set -euxo pipefail - - if [ "$VERSION" = latest ]; then - VERSION="$(curl -sL https://pypi.org/pypi/$PACKAGE/json | jq -r .info.version)" - fi - - git checkout -b "bot/bump-version/$PACKAGE/$VERSION" - - python3 -S -m tools.bump_version "$PACKAGE" "$VERSION" - - re="$(sed 's/[_-]/[_-]/g' <<< "$PACKAGE")" - - # Update Cargo.toml dependencies (format: package = "version") - sed -i "s/^\($re\) = \"[^\"]*\"/\1 = \"$VERSION\"/g" -- rust_snuba/Cargo.toml - # Also handle dependencies with features (format: package = { version = "version", features = [...] }) - sed -i "s/^\($re\) = { version = \"[^\"]*\"/\1 = { version = \"$VERSION\"/g" -- rust_snuba/Cargo.toml - - # Update Cargo.lock if Cargo.toml was modified - if ! git diff --exit-code -- rust_snuba/Cargo.toml > /dev/null 2>&1; then - cd rust_snuba - # Try updating with underscores (cargo prefers underscores in package names) - CARGO_PACKAGE="$(echo "$PACKAGE" | sed 's/-/_/g')" - if ! cargo update --package "$CARGO_PACKAGE" 2>/dev/null; then - # If that fails, try with the original package name - cargo update --package "$PACKAGE" - fi - cd .. - fi - - if git diff --exit-code; then - exit 0 - fi - - git \ - -c user.name=getsentry-bot \ - -c user.email='10587625+getsentry-bot@users.noreply.github.com' \ - commit \ - --all \ - --message "ref: bump $PACKAGE to $VERSION" \ - --message "Co-Authored-By: $SENDER <$SENDER_ID+$SENDER@users.noreply.github.com>" - - git push origin HEAD --quiet - - gh pr create --fill - env: - # Using this instead of BUMP_SENTRY_TOKEN as per advice from asottile - GH_TOKEN: ${{ secrets.GETSENTRY_BOT_REVERT_TOKEN }} - PACKAGE: ${{ inputs.package }} - VERSION: ${{ inputs.version }} - SENDER: ${{ github.event.sender.login }} - SENDER_ID: ${{ github.event.sender.id }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 7423a09681..0000000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,577 +0,0 @@ -name: ci -on: - push: - branches: - - master - pull_request: - -jobs: - files-changed: - name: detect what files changed - runs-on: ubuntu-latest - timeout-minutes: 3 - # Map a step output to a job output - outputs: - api_changes: ${{ steps.changes.outputs.api_changes }} - devservices_changes: ${{ steps.changes.outputs.devservices_changes }} - steps: - - uses: actions/checkout@v6.0.2 - - - name: Check for backend file changes - uses: getsentry/paths-filter@66f7f1844185eb7fb6738ea4ea59d74bb99199e5 # v2 - id: changes - with: - token: ${{ github.token }} - filters: .github/file-filters.yml - - bump-version-test: - name: "Test bump_version.py script" - runs-on: ubuntu-latest - timeout-minutes: 6 - steps: - - uses: actions/checkout@v6.0.2 - name: Checkout code - - - uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1 - with: - enable-cache: false - - - name: Test bump_version.py runs without errors - run: | - # Test that the script can be imported and shows help - uv run --no-project python tools/bump_version.py --help - - # Test with a dry run (should fail with appropriate message for non-existent package) - uv run --no-project python tools/bump_version.py test-package 1.0.0 || echo "Expected failure for non-existent package" - - linting: - name: "pre-commit hooks" # (includes Python formatting + linting) - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - uses: actions/checkout@v6.0.2 - name: Checkout code - - name: Internal github app token - id: token - uses: getsentry/action-github-app-token@d4b5da6c5e37703f8c3b3e43abb5705b46e159cc # v3.0.0 - continue-on-error: true - with: - app_id: ${{ vars.SENTRY_INTERNAL_APP_ID }} - private_key: ${{ secrets.SENTRY_INTERNAL_APP_PRIVATE_KEY }} - - - uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1 - with: - # we just cache the venv-dir directly in action-setup-venv - enable-cache: false - - - uses: getsentry/action-setup-venv@5a80476d175edf56cb205b08bc58986fa99d1725 # v3.2.0 - with: - - cache-dependency-path: uv.lock - # NOTE: can't pass --only-dev yet since we're missing some mypy stub packages - install-cmd: uv sync --frozen --active - - - uses: actions/cache@v5 - with: - path: ~/.cache/pre-commit - key: cache-epoch-1|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml', 'uv.lock') }} - - - name: Setup pre-commit - run: pre-commit install-hooks - - - uses: getsentry/paths-filter@v2 - id: files - with: - # Enable listing of files matching each filter. - # Paths to files will be available in `${FILTER_NAME}_files` output variable. - # Paths will be escaped and space-delimited. - # Output is usable as command line argument list in linux shell - list-files: shell - - # It doesn't make sense to lint deleted files. - # Therefore we specify we are only interested in added or modified files. - filters: | - all: - - added|modified: '**/*' - - - name: Run pre-commit checks - # Run pre-commit to lint and format check files that were changed (but not deleted) compared to master. - # XXX: there is a very small chance that it'll expand to exceed Linux's limits - # `getconf ARG_MAX` - max # bytes of args + environ for exec() - # we skip the `no-commit-to-branch` because in CI we are in fact on master already - # and we have merged to it - run: | - SKIP=no-commit-to-branch pre-commit run --files ${{ steps.files.outputs.all_files }} - - # If working tree is dirty, commit and update if we have a token - - name: Apply any pre-commit fixed files - if: steps.token.outcome == 'success' && github.ref != 'refs/heads/master' && always() - uses: getsentry/action-github-commit@v2.1.0 - with: - github-token: ${{ steps.token.outputs.token }} - - rust-linting: - name: "Linting - Rust" - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - uses: actions/checkout@v6.0.2 - name: Checkout code - - name: Install protoc - uses: arduino/setup-protoc@v3 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: dtolnay/rust-toolchain@stable - with: - components: rustfmt - - name: Run linter - run: | - make lint-rust format-rust-ci - - config-validation: - name: "Dataset Config Validation" - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - uses: actions/checkout@v6.0.2 - name: Checkout code - - - uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1 - with: - # we just cache the venv-dir directly in action-setup-venv - enable-cache: false - - - uses: getsentry/action-setup-venv@5a80476d175edf56cb205b08bc58986fa99d1725 # v3.2.0 - with: - - cache-dependency-path: uv.lock - install-cmd: uv sync --frozen --active - - - name: Validate configs - run: | - make validate-configs - - snuba-image: - name: Build snuba CI image - runs-on: ubuntu-latest - timeout-minutes: 20 - outputs: - branch: ${{ steps.branch.outputs.branch }} - steps: - - name: Checkout code - uses: actions/checkout@v6.0.2 - - - name: Get branch name - id: branch - # strip `refs/heads/` from $GITHUB_REF and replace `/` with `-` so that - # it can be used as a docker tag - run: echo "branch=$(echo ${GITHUB_REF#refs/heads/} | tr / -)" >> "$GITHUB_OUTPUT" - - - uses: getsentry/action-build-and-push-images@8fc75e483c09a68721f2c8951292ee17f8821766 - with: - image_name: 'snuba-ci' - platforms: 'linux/amd64' - dockerfile_path: 'Dockerfile' - ghcr: false - tag_nightly: false - tag_latest: false - tags: 'ghcr.io/getsentry/snuba-ci:${{ github.event.pull_request.head.sha || github.sha }}' - outputs: 'type=docker,dest=/tmp/snuba-ci.tar' - - - name: Publish snuba-ci image to artifacts - # we publish to github artifacts separately so that third-party - # contributions also work, as all the test jobs need this image. - # otherwise third-party contributors would have to provide a working, - # authenticated GHCR, which seems impossible to ensure in the general - # case. - uses: actions/upload-artifact@v7 - with: - name: snuba-ci - path: /tmp/snuba-ci.tar - - - docker-deps: - name: Warm Docker dependency image cache - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - name: Checkout code - uses: actions/checkout@v6.0.2 - - - name: Restore Docker dependency image cache - id: cache-restore - uses: actions/cache/restore@v5 - with: - path: /tmp/docker-deps - key: docker-deps-${{ hashFiles('docker-compose.gcb.yml') }}-${{ github.run_id }} - restore-keys: docker-deps-${{ hashFiles('docker-compose.gcb.yml') }}- - - - name: Load and pull Docker dependency images - id: load-pull - run: | - if [ -f /tmp/docker-deps/deps.tar ]; then - docker load --input /tmp/docker-deps/deps.tar - fi - images=( - "ghcr.io/getsentry/image-mirror-altinity-clickhouse-server:25.3.6.10034.altinitystable" - "ghcr.io/getsentry/image-mirror-confluentinc-cp-kafka:6.2.0" - "ghcr.io/getsentry/image-mirror-confluentinc-cp-zookeeper:6.2.0" - "ghcr.io/getsentry/docker-redis-cluster:7.0.10" - ) - pulled=0 - for image in "${images[@]}"; do - if ! docker image inspect "$image" &>/dev/null; then - docker pull "$image" - pulled=1 - fi - done - if [ "$pulled" -eq 1 ]; then - mkdir -p /tmp/docker-deps - docker save "${images[@]}" -o /tmp/docker-deps/deps.tar - fi - echo "pulled=$pulled" >> $GITHUB_OUTPUT - - - name: Save Docker dependency image cache - if: steps.load-pull.outputs.pulled == '1' - uses: actions/cache/save@v5 - with: - path: /tmp/docker-deps - key: docker-deps-${{ hashFiles('docker-compose.gcb.yml') }}-${{ github.run_id }} - - tests: - needs: [linting, snuba-image, docker-deps] - name: Tests and code coverage - runs-on: ubuntu-latest - timeout-minutes: 40 - strategy: - matrix: - snuba_settings: - [ - "test", - "test_rust", - "test_distributed", - "test_distributed_migrations", - ] - steps: - - name: Checkout code - uses: actions/checkout@v6.0.2 - - - name: Download snuba-ci image from artifacts - uses: actions/download-artifact@v8 - with: - name: snuba-ci - path: /tmp - - - name: Load snuba-ci image - run: | - docker load --input /tmp/snuba-ci.tar - docker image ls -a - - - name: Restore Docker dependency image cache - uses: actions/cache/restore@v5 - with: - path: /tmp/docker-deps - key: docker-deps-${{ hashFiles('docker-compose.gcb.yml') }}-${{ github.run_id }} - restore-keys: docker-deps-${{ hashFiles('docker-compose.gcb.yml') }}- - - - name: Load Docker dependency images - run: | - if [ -f /tmp/docker-deps/deps.tar ]; then - docker load --input /tmp/docker-deps/deps.tar - fi - images=( - "ghcr.io/getsentry/image-mirror-altinity-clickhouse-server:25.3.6.10034.altinitystable" - "ghcr.io/getsentry/image-mirror-confluentinc-cp-kafka:6.2.0" - "ghcr.io/getsentry/image-mirror-confluentinc-cp-zookeeper:6.2.0" - "ghcr.io/getsentry/docker-redis-cluster:7.0.10" - ) - for image in "${images[@]}"; do - if ! docker image inspect "$image" &>/dev/null; then - docker pull "$image" - fi - done - - - name: Docker set up - run: | - docker network create --attachable cloudbuild - - - name: Docker Snuba Rust tests - run: | - SNUBA_IMAGE=ghcr.io/getsentry/snuba-ci:${{ github.event.pull_request.head.sha || github.sha }} SNUBA_SETTINGS=test docker compose -f docker-compose.gcb.yml run --rm snuba-test-rust - if: ${{ matrix.snuba_settings == 'test_rust' }} - - - name: Docker Snuba tests - run: | - SNUBA_IMAGE=ghcr.io/getsentry/snuba-ci:${{ github.event.pull_request.head.sha || github.sha }} SNUBA_SETTINGS=${{ matrix.snuba_settings }} docker compose -f docker-compose.gcb.yml run --rm snuba-test - if: ${{ matrix.snuba_settings == 'test' || matrix.snuba_settings == 'test_distributed' }} - - - name: Docker Snuba Multi-Node Tests - run: | - SNUBA_IMAGE=ghcr.io/getsentry/snuba-ci:${{ github.event.pull_request.head.sha || github.sha }} SNUBA_SETTINGS=test_distributed_migrations docker compose --profile multi_node -f docker-compose.gcb.yml up -d - SNUBA_IMAGE=ghcr.io/getsentry/snuba-ci:${{ github.event.pull_request.head.sha || github.sha }} SNUBA_SETTINGS=test_distributed_migrations TEST_LOCATION=test_distributed_migrations docker compose --profile multi_node -f docker-compose.gcb.yml run --rm snuba-test - if: ${{ matrix.snuba_settings == 'test_distributed_migrations' }} - - - name: Docker Snuba Init Tests - run: | - SNUBA_IMAGE=ghcr.io/getsentry/snuba-ci:${{ github.event.pull_request.head.sha || github.sha }} SNUBA_SETTINGS=test_initialization TEST_LOCATION=test_initialization docker compose -f docker-compose.gcb.yml run --rm snuba-test - if: ${{ matrix.snuba_settings == 'test' }} - - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - - - name: Upload to codecov - run: | - curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov && ./codecov -t ${CODECOV_TOKEN} - - admin-tests: - needs: [linting] - name: Front end tests for snuba admin - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6.0.2 - name: Checkout code - - uses: volta-cli/action@v4 - - name: Set up and run tests through yarn - run: cd snuba/admin && yarn install && yarn run test --coverage - - name: Upload to codecov - run: | - curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov && ./codecov -t ${CODECOV_TOKEN} - - sentry: - needs: [snuba-image, files-changed] - runs-on: ubuntu-latest - timeout-minutes: 60 - strategy: - matrix: - instance: [0, 1, 2, 3] - env: - MIGRATIONS_TEST_MIGRATE: 1 - # XXX: `MATRIX_INSTANCE_TOTAL` must be hardcoded to the length of `strategy.matrix.instance`. - MATRIX_INSTANCE_TOTAL: 4 - TEST_GROUP_STRATEGY: ROUND_ROBIN - - steps: - - name: Checkout code - uses: actions/checkout@v6.0.2 - - - name: Download snuba-ci image from artifacts - uses: actions/download-artifact@v8 - with: - name: snuba-ci - path: /tmp - - - name: Load snuba-ci image - run: | - docker load --input /tmp/snuba-ci.tar - docker image ls -a - - - name: Checkout sentry - uses: actions/checkout@v6.0.2 - with: - repository: getsentry/sentry - path: sentry - - - name: Setup steps - id: setup - run: | - # We cannot execute actions that are not placed under .github of the main repo - mkdir -p .github/actions - cp -r sentry/.github/actions/* .github/actions - - - name: Setup Sentry - id: setup-sentry - uses: ./.github/actions/setup-sentry - with: - workdir: sentry - mode: minimal - - - name: Start snuba - run: | - # TODO(hubertdeng123): New devservices doesn't support running sentry without snuba yet, remove this when it does - docker stop snuba-snuba-1 - docker rm snuba-snuba-1 - docker run -d --rm \ - -p 127.0.0.1:1218:1218 \ - -e PYTHONUNBUFFERED=1 \ - -e SNUBA_SETTINGS=docker \ - -e DEBUG=1 \ - -e DEFAULT_BROKERS=kafka:9093 \ - -e CLICKHOUSE_HOST=clickhouse \ - -e CLICKHOUSE_PORT=9000 \ - -e CLICKHOUSE_HTTP_PORT=8123 \ - -e REDIS_HOST=redis \ - -e REDIS_PORT=6379 \ - -e REDIS_DB=1 \ - --name snuba-snuba-1 \ - --network devservices \ - ghcr.io/getsentry/snuba-ci:${{ github.event.pull_request.head.sha || github.sha }} - docker exec snuba-snuba-1 snuba migrations migrate --force - - - name: Run snuba tests - if: needs.files-changed.outputs.api_changes == 'false' - working-directory: sentry - run: | - pytest -k 'not __In' tests \ - -m snuba_ci \ - -vv --cov . --cov-report="xml:.artifacts/snuba.coverage.xml" - - - name: Run full tests - if: needs.files-changed.outputs.api_changes == 'true' - working-directory: sentry - run: | - pytest -k 'not __In' \ - tests/snuba \ - tests/sentry/snuba \ - tests/sentry/eventstream/kafka \ - tests/sentry/post_process_forwarder \ - tests/sentry/services/eventstore/snuba \ - tests/sentry/search/events \ - tests/sentry/event_manager \ - tests/sentry/api/endpoints/test_organization_profiling_functions.py \ - tests/sentry/integrations/slack/test_unfurl.py \ - tests/sentry/uptime/endpoints/test_project_uptime_alert_check_index.py \ - tests/sentry/uptime/endpoints/test_organization_uptime_stats.py \ - tests/sentry/api/endpoints/test_organization_traces.py \ - tests/sentry/api/endpoints/test_organization_spans_fields.py \ - tests/sentry/api/endpoints/test_organization_spans_fields_stats.py \ - tests/sentry/sentry_metrics/querying \ - -vv --cov . --cov-report="xml:.artifacts/snuba.coverage.xml" - - - name: Run CI module tests - if: needs.files-changed.outputs.api_changes == 'true' - working-directory: sentry - run: pytest -k 'not __In' tests -vv -m snuba_ci - - clickhouse-versions: - needs: [linting, snuba-image, docker-deps] - name: Tests on multiple clickhouse versions - runs-on: ubuntu-latest - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - version: - [ - "25.3.6.10034.altinitystable", - ] - - steps: - - name: Checkout code - uses: actions/checkout@v6.0.2 - - - name: Download snuba-ci image from artifacts - uses: actions/download-artifact@v8 - with: - name: snuba-ci - path: /tmp - - - name: Load snuba-ci image - run: | - docker load --input /tmp/snuba-ci.tar - docker image ls -a - - - name: Restore Docker dependency image cache - uses: actions/cache/restore@v5 - with: - path: /tmp/docker-deps - key: docker-deps-${{ hashFiles('docker-compose.gcb.yml') }}-${{ github.run_id }} - restore-keys: docker-deps-${{ hashFiles('docker-compose.gcb.yml') }}- - - - name: Load Docker dependency images - run: | - if [ -f /tmp/docker-deps/deps.tar ]; then - docker load --input /tmp/docker-deps/deps.tar - fi - images=( - "ghcr.io/getsentry/image-mirror-altinity-clickhouse-server:${{ matrix.version }}" - "ghcr.io/getsentry/image-mirror-confluentinc-cp-kafka:6.2.0" - "ghcr.io/getsentry/image-mirror-confluentinc-cp-zookeeper:6.2.0" - "ghcr.io/getsentry/docker-redis-cluster:7.0.10" - ) - for image in "${images[@]}"; do - if ! docker image inspect "$image" &>/dev/null; then - docker pull "$image" - fi - done - - - name: Docker set up - run: | - docker network create --attachable cloudbuild - - - name: Docker Snuba Test other ClickHouse versions - run: | - export CLICKHOUSE_IMAGE=ghcr.io/getsentry/image-mirror-altinity-clickhouse-server:${{matrix.version}} - SNUBA_IMAGE=ghcr.io/getsentry/snuba-ci:${{ github.event.pull_request.head.sha || github.sha }} SNUBA_SETTINGS=test docker compose -f docker-compose.gcb.yml run --rm snuba-test - - - name: Upload to codecov - run: | - curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov && ./codecov -t ${CODECOV_TOKEN} - - distroless-smoke-test: - name: Distroless image smoke test - needs: [snuba-image] - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - name: Checkout code - uses: actions/checkout@v6.0.2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build distroless image - uses: docker/build-push-action@v6 - with: - context: . - target: application-distroless - load: true - tags: snuba-distroless:test - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Verify --help works - run: docker run --rm snuba-distroless:test --help - - - name: Verify Python imports - run: | - docker run --rm --entrypoint python3 snuba-distroless:test \ - -c "import snuba, rust_snuba, confluent_kafka; print('OK')" - - - name: Verify jemalloc LD_PRELOAD is active - run: | - docker run --rm --entrypoint python3 snuba-distroless:test \ - -c "import ctypes; ctypes.CDLL('libjemalloc.so.2'); print('jemalloc OK')" - - - name: Verify no shell available - run: | - # This should fail — distroless has no /bin/sh - if docker run --rm --entrypoint /bin/sh snuba-distroless:test -c "echo hi" 2>/dev/null; then - echo "ERROR: /bin/sh should not be available in distroless image" - exit 1 - else - echo "OK: /bin/sh is not available" - fi - - validate-devservices-config: - runs-on: ubuntu-24.04 - needs: files-changed - if: ${{ needs.files-changed.outputs.devservices_changes == 'true' }} - steps: - - uses: actions/checkout@v6.0.2 - name: Checkout repository - - - name: Get devservices version - id: get-devservices-version - run: | - awk -F'"' ' - /name/ { pkg = $2 } - /version/ { if (pkg == "devservices") print "version="$2 } - ' uv.lock >> $GITHUB_OUTPUT - - - uses: getsentry/action-validate-devservices-config@711ae7221998ddf81211f25f5e3873ecffd22387 - name: Validate devservices config - with: - devservices-version: ${{ steps.get-devservices-version.outputs.version }} diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index f0de27117f..0000000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,66 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: "CodeQL" - -on: - schedule: - - cron: "30 3 * * 5" - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write - - strategy: - fail-fast: false - matrix: - language: ["javascript", "python"] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] - # Learn more about CodeQL language support at https://git.io/codeql-language-support - - steps: - - name: Checkout repository - uses: actions/checkout@v6.0.2 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v4 - with: - config-file: ./.github/codeql/codeql-config.yml - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v4 - - # ℹ️ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl - - # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language - - #- run: | - # make bootstrap - # make release - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v4 diff --git a/.github/workflows/ddl-changes.yml b/.github/workflows/ddl-changes.yml deleted file mode 100644 index d92959c661..0000000000 --- a/.github/workflows/ddl-changes.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: ddl-changes -on: - pull_request: - types: [opened, synchronize, reopened, labeled, unlabeled] - - -jobs: - post_changes: - name: Post new DDL changes from migrations - runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - uses: actions/checkout@v6.0.2 - name: Checkout master for diffing - with: - ref: master - fetch-depth: 200 - - uses: actions/checkout@v6.0.2 - name: Checkout HEAD of code that may have migration changes - with: - clean: false - fetch-depth: 200 - - - uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1 - with: - # we just cache the venv-dir directly in action-setup-venv - enable-cache: false - - - uses: getsentry/action-setup-venv@5a80476d175edf56cb205b08bc58986fa99d1725 # v3.2.0 - with: - - cache-dependency-path: uv.lock - install-cmd: uv sync --frozen --active - - - name: Run the migration script - run: | - SNUBA_SETTINGS=test_distributed python scripts/ddl-changes.py - - name: Generate SQL for migration - uses: getsentry/action-migrations@v1.2.2 - env: - SNUBA_SETTINGS: test_distributed - with: - githubToken: ${{ secrets.GITHUB_TOKEN }} - migration: "./snuba/migrations/groups.py" - cmd: python scripts/ddl-changes.py - - - name: Check migrations are not coupled with other changes - run: | - # Check that the migration is not coupled with other changes - # If the label skip-check-migrations is present, the check is skipped - SNUBA_SETTINGS=test_distributed python scripts/check-migrations.py --labels ${{join(github.event.pull_request.labels.*.name, '')}} diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml deleted file mode 100644 index 269aea763f..0000000000 --- a/.github/workflows/dependency-review.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: "Dependency Review" -on: - pull_request: - branches: ["master"] - -permissions: - contents: read - -jobs: - dependency-review: - runs-on: ubuntu-latest - steps: - - name: "Checkout Repository" - uses: actions/checkout@v6.0.2 - - name: Dependency Review - uses: actions/dependency-review-action@v4 - with: - # Possible values: "critical", "high", "moderate", "low" - fail-on-severity: high diff --git a/.github/workflows/docs-pr.yml b/.github/workflows/docs-pr.yml deleted file mode 100644 index 329fd435d0..0000000000 --- a/.github/workflows/docs-pr.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: Snuba Docs on PR's - -on: - pull_request: - -jobs: - docs: - name: Sphinx - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6.0.2 - - - uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1 - with: - # we just cache the venv-dir directly in action-setup-venv - enable-cache: false - - - uses: getsentry/action-setup-venv@5a80476d175edf56cb205b08bc58986fa99d1725 # v3.2.0 - with: - - cache-dependency-path: docs-requirements.txt - install-cmd: echo - - - name: Generate config schema docs - run: | - make generate-config-docs - - - name: Build docs - run: | - make snubadocs diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index 229fd2e97b..0000000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: Snuba Docs - -on: - push: - branches: - - master - -jobs: - docs: - name: Sphinx - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6.0.2 - - - uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1 - with: - # we just cache the venv-dir directly in action-setup-venv - enable-cache: false - - - uses: getsentry/action-setup-venv@5a80476d175edf56cb205b08bc58986fa99d1725 # v3.2.0 - with: - - cache-dependency-path: docs-requirements.txt - install-cmd: echo - - - name: Generate config schema docs - run: | - make generate-config-docs - - - name: Build docs - run: | - make snubadocs - - - uses: peaceiris/actions-gh-pages@v4.0.0 - name: Publish to GitHub Pages - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: docs/build - force_orphan: true - - - name: Archive Docs - uses: actions/upload-artifact@v7 - with: - name: docs - path: docs/build diff --git a/.github/workflows/enforce-license-compliance.yml b/.github/workflows/enforce-license-compliance.yml deleted file mode 100644 index 8722b90b33..0000000000 --- a/.github/workflows/enforce-license-compliance.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: Enforce License Compliance - -on: - push: - branches: [master, main, release/*] - pull_request: - branches: [master, main] - -jobs: - enforce-license-compliance: - runs-on: ubuntu-latest - steps: - - name: 'Enforce License Compliance' - uses: getsentry/action-enforce-license-compliance@6599a041195852debba3417e069829060d671e76 - with: - fossa_api_key: ${{ secrets.FOSSA_API_KEY }} diff --git a/.github/workflows/fast-revert.yml b/.github/workflows/fast-revert.yml deleted file mode 100644 index 1ffb8c18ab..0000000000 --- a/.github/workflows/fast-revert.yml +++ /dev/null @@ -1,53 +0,0 @@ -on: - pull_request_target: - types: [labeled] - workflow_dispatch: - inputs: - pr: - required: true - description: pr number - co_authored_by: - required: true - description: '`name ` for triggering user' - -# disable all permissions -- we use the PAT's permissions instead -permissions: {} - -jobs: - revert: - runs-on: ubuntu-latest - if: | - github.event_name == 'workflow_dispatch' || github.event.label.name == 'Trigger: Revert' - steps: - - name: Get auth token - id: token - uses: getsentry/action-github-app-token@d4b5da6c5e37703f8c3b3e43abb5705b46e159cc # v3.0.0 - with: - app_id: ${{ secrets.FAST_REVERT_BOT_APP_ID }} - private_key: ${{ secrets.GH_FAST_REVERT_PRIVATE_KEY }} - - uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 # v3.1.0 - with: - token: ${{ steps.token.outputs.token }} - - uses: getsentry/action-fast-revert@35b4b6c1f8f91b5911159568b3b15e531b5b8174 # v2.0.1 - with: - pr: ${{ github.event.number || github.event.inputs.pr }} - co_authored_by: >- - ${{ github.event.inputs.co_authored_by || format('{0}\n<{1}+{0}@users.noreply.github.com>', github.event.sender.login, github.event.sender.id) }} - committer_name: sentry-snuba-fast-revert-bot[bot] - committer_email: 257653817+sentry-snuba-fast-revert-bot[bot]@users.noreply.github.com - token: ${{ steps.token.outputs.token }} - - name: comment on failure - env: - GITHUB_REPOSITORY: ${{ github.repository }} - GITHUB_RUN_ID: ${{ github.run_id }} - REPOSITORY_ID: ${{ github.event.repository.id }} - PR_NUMBER: ${{ github.event.number || github.event.inputs.pr }} - GITHUB_TOKEN: ${{ secrets.GETSENTRY_BOT_REVERT_TOKEN }} - run: | - curl \ - --silent \ - -X POST \ - -H "Authorization: token $GITHUB_TOKEN" \ - -d"{\"body\": \"revert failed (conflict? already reverted?) -- [check the logs](https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID)\"}" \ - "https://api.github.com/repositories/$REPOSITORY_ID/issues/$PR_NUMBER/comments" - if: failure() diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml deleted file mode 100644 index 178ba29967..0000000000 --- a/.github/workflows/labeler.yml +++ /dev/null @@ -1,14 +0,0 @@ -name: "Pull Request Labeler" -on: -- pull_request_target - -jobs: - triage: - permissions: - contents: read - pull-requests: write - runs-on: ubuntu-latest - steps: - - uses: actions/labeler@v6 - with: - repo-token: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 9a52828242..0000000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: release -on: - workflow_dispatch: - inputs: - version: - description: Version to release (or "auto") - required: false - force: - description: Force a release even when there are release-blockers (optional) - required: false - schedule: - # We want the release to be at 9-10am Pacific Time - # We also want it to be 1 hour before the on-prem release - - cron: "0 17 15 * *" -permissions: - contents: write - pull-requests: write - -jobs: - release: - runs-on: ubuntu-latest - name: "Release a new version" - steps: - - name: Get auth token - id: token - uses: actions/create-github-app-token@v2 - with: - app-id: ${{ vars.SENTRY_RELEASE_BOT_CLIENT_ID }} - private-key: ${{ secrets.SENTRY_RELEASE_BOT_PRIVATE_KEY }} - - uses: actions/checkout@v6.0.2 - with: - token: ${{ steps.token.outputs.token }} - fetch-depth: 0 - - name: Prepare release - uses: getsentry/craft@v2 - env: - GITHUB_TOKEN: ${{ steps.token.outputs.token }} - with: - version: ${{ github.event.inputs.version }} - force: ${{ github.event.inputs.force }} diff --git a/.github/workflows/validate-pipelines.yml b/.github/workflows/validate-pipelines.yml deleted file mode 100644 index b396ddbb3e..0000000000 --- a/.github/workflows/validate-pipelines.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: Validate Deployment Pipelines - -on: - pull_request: - push: - branches: [master, test-me-*] - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - files-changed: - name: files-changed - runs-on: ubuntu-latest - # Map a step output to a job output - outputs: - gocd: ${{ steps.changes.outputs.gocd }} - steps: - - uses: actions/checkout@v6.0.2 - - name: Check for relevant file changes - uses: getsentry/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 - id: changes - with: - filters: | - gocd: - - 'gocd/**' - - - validate: - if: needs.files-changed.outputs.gocd == 'true' - needs: files-changed - name: Validate GoCD Pipelines - runs-on: ubuntu-latest - - # required for google auth - permissions: - contents: "read" - id-token: "write" - - steps: - - uses: actions/checkout@v6.0.2 - - id: 'auth' - uses: google-github-actions/auth@v3 - with: - workload_identity_provider: 'projects/868781662168/locations/global/workloadIdentityPools/prod-github/providers/github-oidc-pool' - service_account: 'gha-gocd-api@sac-prod-sa.iam.gserviceaccount.com' - token_format: 'id_token' - id_token_audience: '610575311308-9bsjtgqg4jm01mt058rncpopujgk3627.apps.googleusercontent.com' - id_token_include_email: true - - uses: getsentry/action-gocd-jsonnet@v1.1.1 - with: - jb-install: true - jsonnet-dir: gocd/templates - generated-dir: gocd/generated-pipelines - - uses: getsentry/action-validate-gocd-pipelines@v1 - with: - configrepo: snuba__master - gocd_access_token: ${{ secrets.GOCD_ACCESS_TOKEN }} - google_oidc_token: ${{ steps.auth.outputs.id_token }} diff --git a/.github/workflows/validate-sentry-options.yml b/.github/workflows/validate-sentry-options.yml deleted file mode 100644 index 50b2d2b9fd..0000000000 --- a/.github/workflows/validate-sentry-options.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Validate Sentry Options Schema - -on: - pull_request: - merge_group: - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - files-changed: - name: files-changed - runs-on: ubuntu-latest - outputs: - schemas: ${{ steps.changes.outputs.schemas }} - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - name: Check for relevant file changes - uses: getsentry/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 - id: changes - with: - filters: | - schemas: - - 'sentry-options/schemas/**' - - validate-schema: - if: needs.files-changed.outputs.schemas == 'true' - needs: files-changed - name: Validate Schema Evolution - uses: getsentry/sentry-options/.github/workflows/validate-schema.yml@fa066c1d3ef0849153092a0272c17fc0bcce14c8 - secrets: inherit - with: - schemas-path: sentry-options/schemas diff --git a/.gitignore b/.gitignore index ec5476fa21..bb5331a380 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ gocd/templates/vendor/ gocd/generated-pipelines/ Brewfile.lock.json .zed/ +__pycache__ From 581001d6db2b9f55101b7308acae5e0642552675 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Sat, 11 Apr 2026 10:29:25 -0700 Subject: [PATCH 3/9] ci(image): Trim to amd64-only and push SHA tag on PRs Changes: - Remove arm64 from all build jobs (amd64-only); CI runners are all amd64 and building arm64 doubles build time with no benefit for CI - Make `assemble` run on PRs (not just master/release) so that ghcr.io/getsentry/snuba:{sha} is available for Sentry's devservices to reference in integration tests and local development overrides - `nightly` tag is still only applied on master pushes - Collapse multi-matrix jobs into single jobs (no matrix needed for one arch) - `self-hosted-end-to-end` now depends on `assemble` so the SHA image is guaranteed to exist before the e2e run --- .github/workflows/image.yml | 86 +++++++++++++++---------------------- 1 file changed, 34 insertions(+), 52 deletions(-) diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml index ce6e049679..50a520ceca 100644 --- a/.github/workflows/image.yml +++ b/.github/workflows/image.yml @@ -6,16 +6,9 @@ on: - release/** jobs: - build-multiplatform: - strategy: - matrix: - include: - - os: ubuntu-24.04 - platform: amd64 - - os: ubuntu-24.04-arm - platform: arm64 - runs-on: ${{ matrix.os }} - name: build-${{ matrix.platform }} + build: + runs-on: ubuntu-24.04 + name: build permissions: contents: read packages: write @@ -25,9 +18,9 @@ jobs: - uses: getsentry/action-build-and-push-images@8fc75e483c09a68721f2c8951292ee17f8821766 with: image_name: 'snuba' - platforms: linux/${{ matrix.platform }} + platforms: linux/amd64 dockerfile_path: 'Dockerfile' - tag_suffix: -${{ matrix.platform }} + tag_suffix: -amd64 ghcr: true tag_nightly: false tag_latest: false @@ -54,17 +47,9 @@ jobs: google_workload_identity_provider: projects/868781662168/locations/global/workloadIdentityPools/prod-github/providers/github-oidc-pool google_service_account: gha-gcr-push@sac-prod-sa.iam.gserviceaccount.com - # Distroless image — for testing before switching production - build-distroless-multiplatform: - strategy: - matrix: - include: - - os: ubuntu-24.04 - platform: amd64 - - os: ubuntu-24.04-arm - platform: arm64 - runs-on: ${{ matrix.os }} - name: build-distroless-${{ matrix.platform }} + build-distroless: + runs-on: ubuntu-24.04 + name: build-distroless permissions: contents: read packages: write @@ -74,25 +59,17 @@ jobs: - uses: getsentry/action-build-and-push-images@8fc75e483c09a68721f2c8951292ee17f8821766 with: image_name: 'snuba' - platforms: linux/${{ matrix.platform }} + platforms: linux/amd64 dockerfile_path: 'Dockerfile' build_target: 'application-distroless' - tag_suffix: -distroless-${{ matrix.platform }} + tag_suffix: -distroless-amd64 ghcr: true tag_nightly: false tag_latest: false - # Debug distroless image — with busybox for troubleshooting - build-debug-multiplatform: - strategy: - matrix: - include: - - os: ubuntu-24.04 - platform: amd64 - - os: ubuntu-24.04-arm - platform: arm64 - runs-on: ${{ matrix.os }} - name: build-debug-${{ matrix.platform }} + build-debug: + runs-on: ubuntu-24.04 + name: build-debug permissions: contents: read packages: write @@ -102,17 +79,19 @@ jobs: - uses: getsentry/action-build-and-push-images@8fc75e483c09a68721f2c8951292ee17f8821766 with: image_name: 'snuba' - platforms: linux/${{ matrix.platform }} + platforms: linux/amd64 dockerfile_path: 'Dockerfile' build_target: 'application-distroless-debug' - tag_suffix: -debug-${{ matrix.platform }} + tag_suffix: -debug-amd64 ghcr: true tag_nightly: false tag_latest: false assemble: - needs: [build-multiplatform] - if: ${{ (github.ref_name == 'master' || startsWith(github.ref_name, 'release/')) && github.event_name != 'pull_request' }} + needs: [build] + # Run on PRs too so that ghcr.io/getsentry/snuba:{sha} is available for + # Sentry's devservices to reference in CI and local development. + if: github.repository_owner == 'getsentry' runs-on: ubuntu-latest permissions: contents: read @@ -126,16 +105,21 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Create multiplatform manifests + - name: Tag SHA image run: | docker buildx imagetools create \ --tag ghcr.io/getsentry/snuba:${{ github.sha }} \ + ghcr.io/getsentry/snuba:${{ github.sha }}-amd64 + + - name: Tag nightly + if: github.ref_name == 'master' && github.event_name != 'pull_request' + run: | + docker buildx imagetools create \ --tag ghcr.io/getsentry/snuba:nightly \ - ghcr.io/getsentry/snuba:${{ github.sha }}-amd64 \ - ghcr.io/getsentry/snuba:${{ github.sha }}-arm64 + ghcr.io/getsentry/snuba:${{ github.sha }}-amd64 assemble-distroless: - needs: [build-distroless-multiplatform] + needs: [build-distroless] if: ${{ (github.ref_name == 'master' || startsWith(github.ref_name, 'release/')) && github.event_name != 'pull_request' }} runs-on: ubuntu-latest permissions: @@ -150,16 +134,15 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Create distroless multiplatform manifests + - name: Create distroless manifest run: | docker buildx imagetools create \ --tag ghcr.io/getsentry/snuba:${{ github.sha }}-distroless \ --tag ghcr.io/getsentry/snuba:nightly-distroless \ - ghcr.io/getsentry/snuba:${{ github.sha }}-distroless-amd64 \ - ghcr.io/getsentry/snuba:${{ github.sha }}-distroless-arm64 + ghcr.io/getsentry/snuba:${{ github.sha }}-distroless-amd64 assemble-debug: - needs: [build-debug-multiplatform] + needs: [build-debug] if: ${{ (github.ref_name == 'master' || startsWith(github.ref_name, 'release/')) && github.event_name != 'pull_request' }} runs-on: ubuntu-latest permissions: @@ -174,16 +157,15 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Create debug multiplatform manifests + - name: Create debug manifest run: | docker buildx imagetools create \ --tag ghcr.io/getsentry/snuba:${{ github.sha }}-debug \ --tag ghcr.io/getsentry/snuba:nightly-debug \ - ghcr.io/getsentry/snuba:${{ github.sha }}-debug-amd64 \ - ghcr.io/getsentry/snuba:${{ github.sha }}-debug-arm64 + ghcr.io/getsentry/snuba:${{ github.sha }}-debug-amd64 self-hosted-end-to-end: - needs: [build-multiplatform, assemble] + needs: [build, assemble] runs-on: ubuntu-latest timeout-minutes: 30 From ae36d5b1c5f8dfdfb3ec8a65df9ee447085d32a5 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Sat, 11 Apr 2026 10:30:41 -0700 Subject: [PATCH 4/9] ci(image): Simplify to just build + assemble --- .github/workflows/image.yml | 123 ------------------------------------ 1 file changed, 123 deletions(-) diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml index 50a520ceca..171de02187 100644 --- a/.github/workflows/image.yml +++ b/.github/workflows/image.yml @@ -25,72 +25,8 @@ jobs: tag_nightly: false tag_latest: false - build-production: - runs-on: ubuntu-24.04 - name: Build and push production image - permissions: - contents: read - id-token: write - if: ${{ github.ref_name == 'master' }} - steps: - - uses: actions/checkout@v6.0.2 - - uses: getsentry/action-build-and-push-images@8fc75e483c09a68721f2c8951292ee17f8821766 - with: - image_name: 'snuba' - platforms: linux/amd64 - dockerfile_path: './Dockerfile' - ghcr: false - tag_nightly: false - tag_latest: false - google_ar: true - google_ar_image_name: us-docker.pkg.dev/sentryio/snuba-mr/image - google_workload_identity_provider: projects/868781662168/locations/global/workloadIdentityPools/prod-github/providers/github-oidc-pool - google_service_account: gha-gcr-push@sac-prod-sa.iam.gserviceaccount.com - - build-distroless: - runs-on: ubuntu-24.04 - name: build-distroless - permissions: - contents: read - packages: write - if: github.repository_owner == 'getsentry' - steps: - - uses: actions/checkout@v6.0.2 - - uses: getsentry/action-build-and-push-images@8fc75e483c09a68721f2c8951292ee17f8821766 - with: - image_name: 'snuba' - platforms: linux/amd64 - dockerfile_path: 'Dockerfile' - build_target: 'application-distroless' - tag_suffix: -distroless-amd64 - ghcr: true - tag_nightly: false - tag_latest: false - - build-debug: - runs-on: ubuntu-24.04 - name: build-debug - permissions: - contents: read - packages: write - if: github.repository_owner == 'getsentry' - steps: - - uses: actions/checkout@v6.0.2 - - uses: getsentry/action-build-and-push-images@8fc75e483c09a68721f2c8951292ee17f8821766 - with: - image_name: 'snuba' - platforms: linux/amd64 - dockerfile_path: 'Dockerfile' - build_target: 'application-distroless-debug' - tag_suffix: -debug-amd64 - ghcr: true - tag_nightly: false - tag_latest: false - assemble: needs: [build] - # Run on PRs too so that ghcr.io/getsentry/snuba:{sha} is available for - # Sentry's devservices to reference in CI and local development. if: github.repository_owner == 'getsentry' runs-on: ubuntu-latest permissions: @@ -117,62 +53,3 @@ jobs: docker buildx imagetools create \ --tag ghcr.io/getsentry/snuba:nightly \ ghcr.io/getsentry/snuba:${{ github.sha }}-amd64 - - assemble-distroless: - needs: [build-distroless] - if: ${{ (github.ref_name == 'master' || startsWith(github.ref_name, 'release/')) && github.event_name != 'pull_request' }} - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - name: Docker Login - run: docker login --username '${{ github.actor }}' --password-stdin ghcr.io <<< "$GHCR_TOKEN" - env: - GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Create distroless manifest - run: | - docker buildx imagetools create \ - --tag ghcr.io/getsentry/snuba:${{ github.sha }}-distroless \ - --tag ghcr.io/getsentry/snuba:nightly-distroless \ - ghcr.io/getsentry/snuba:${{ github.sha }}-distroless-amd64 - - assemble-debug: - needs: [build-debug] - if: ${{ (github.ref_name == 'master' || startsWith(github.ref_name, 'release/')) && github.event_name != 'pull_request' }} - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - name: Docker Login - run: docker login --username '${{ github.actor }}' --password-stdin ghcr.io <<< "$GHCR_TOKEN" - env: - GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Create debug manifest - run: | - docker buildx imagetools create \ - --tag ghcr.io/getsentry/snuba:${{ github.sha }}-debug \ - --tag ghcr.io/getsentry/snuba:nightly-debug \ - ghcr.io/getsentry/snuba:${{ github.sha }}-debug-amd64 - - self-hosted-end-to-end: - needs: [build, assemble] - runs-on: ubuntu-latest - timeout-minutes: 30 - - steps: - - name: Run Sentry self-hosted e2e CI - uses: getsentry/self-hosted@master - with: - project_name: snuba - image_url: ghcr.io/getsentry/snuba:${{ github.sha }} - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} From 57f5bd1775c77c66c14d0537d3d88bccfae032b7 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Sat, 11 Apr 2026 10:31:19 -0700 Subject: [PATCH 5/9] just tag sha --- .github/workflows/image.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml index 171de02187..68ac9c97f5 100644 --- a/.github/workflows/image.yml +++ b/.github/workflows/image.yml @@ -47,9 +47,3 @@ jobs: --tag ghcr.io/getsentry/snuba:${{ github.sha }} \ ghcr.io/getsentry/snuba:${{ github.sha }}-amd64 - - name: Tag nightly - if: github.ref_name == 'master' && github.event_name != 'pull_request' - run: | - docker buildx imagetools create \ - --tag ghcr.io/getsentry/snuba:nightly \ - ghcr.io/getsentry/snuba:${{ github.sha }}-amd64 From a260623c31e844e618c28bf999481877d6eb0374 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Sat, 11 Apr 2026 10:39:27 -0700 Subject: [PATCH 6/9] ci: Cancel in-progress runs on new push --- .github/workflows/ci.yml | 581 ++++++++++++++++++++++++++++++++++++ .github/workflows/image.yml | 5 + 2 files changed, 586 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..ae0ebae441 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,581 @@ +name: ci +on: + push: + branches: + - master + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + files-changed: + name: detect what files changed + runs-on: ubuntu-latest + timeout-minutes: 3 + # Map a step output to a job output + outputs: + api_changes: ${{ steps.changes.outputs.api_changes }} + devservices_changes: ${{ steps.changes.outputs.devservices_changes }} + steps: + - uses: actions/checkout@v6.0.2 + + - name: Check for backend file changes + uses: getsentry/paths-filter@66f7f1844185eb7fb6738ea4ea59d74bb99199e5 # v2 + id: changes + with: + token: ${{ github.token }} + filters: .github/file-filters.yml + + bump-version-test: + name: "Test bump_version.py script" + runs-on: ubuntu-latest + timeout-minutes: 6 + steps: + - uses: actions/checkout@v6.0.2 + name: Checkout code + + - uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1 + with: + enable-cache: false + + - name: Test bump_version.py runs without errors + run: | + # Test that the script can be imported and shows help + uv run --no-project python tools/bump_version.py --help + + # Test with a dry run (should fail with appropriate message for non-existent package) + uv run --no-project python tools/bump_version.py test-package 1.0.0 || echo "Expected failure for non-existent package" + + linting: + name: "pre-commit hooks" # (includes Python formatting + linting) + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v6.0.2 + name: Checkout code + - name: Internal github app token + id: token + uses: getsentry/action-github-app-token@d4b5da6c5e37703f8c3b3e43abb5705b46e159cc # v3.0.0 + continue-on-error: true + with: + app_id: ${{ vars.SENTRY_INTERNAL_APP_ID }} + private_key: ${{ secrets.SENTRY_INTERNAL_APP_PRIVATE_KEY }} + + - uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1 + with: + # we just cache the venv-dir directly in action-setup-venv + enable-cache: false + + - uses: getsentry/action-setup-venv@5a80476d175edf56cb205b08bc58986fa99d1725 # v3.2.0 + with: + + cache-dependency-path: uv.lock + # NOTE: can't pass --only-dev yet since we're missing some mypy stub packages + install-cmd: uv sync --frozen --active + + - uses: actions/cache@v5 + with: + path: ~/.cache/pre-commit + key: cache-epoch-1|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml', 'uv.lock') }} + + - name: Setup pre-commit + run: pre-commit install-hooks + + - uses: getsentry/paths-filter@v2 + id: files + with: + # Enable listing of files matching each filter. + # Paths to files will be available in `${FILTER_NAME}_files` output variable. + # Paths will be escaped and space-delimited. + # Output is usable as command line argument list in linux shell + list-files: shell + + # It doesn't make sense to lint deleted files. + # Therefore we specify we are only interested in added or modified files. + filters: | + all: + - added|modified: '**/*' + + - name: Run pre-commit checks + # Run pre-commit to lint and format check files that were changed (but not deleted) compared to master. + # XXX: there is a very small chance that it'll expand to exceed Linux's limits + # `getconf ARG_MAX` - max # bytes of args + environ for exec() + # we skip the `no-commit-to-branch` because in CI we are in fact on master already + # and we have merged to it + run: | + SKIP=no-commit-to-branch pre-commit run --files ${{ steps.files.outputs.all_files }} + + # If working tree is dirty, commit and update if we have a token + - name: Apply any pre-commit fixed files + if: steps.token.outcome == 'success' && github.ref != 'refs/heads/master' && always() + uses: getsentry/action-github-commit@v2.1.0 + with: + github-token: ${{ steps.token.outputs.token }} + + rust-linting: + name: "Linting - Rust" + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v6.0.2 + name: Checkout code + - name: Install protoc + uses: arduino/setup-protoc@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + - uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt + - name: Run linter + run: | + make lint-rust format-rust-ci + + config-validation: + name: "Dataset Config Validation" + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v6.0.2 + name: Checkout code + + - uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1 + with: + # we just cache the venv-dir directly in action-setup-venv + enable-cache: false + + - uses: getsentry/action-setup-venv@5a80476d175edf56cb205b08bc58986fa99d1725 # v3.2.0 + with: + + cache-dependency-path: uv.lock + install-cmd: uv sync --frozen --active + + - name: Validate configs + run: | + make validate-configs + + snuba-image: + name: Build snuba CI image + runs-on: ubuntu-latest + timeout-minutes: 20 + outputs: + branch: ${{ steps.branch.outputs.branch }} + steps: + - name: Checkout code + uses: actions/checkout@v6.0.2 + + - name: Get branch name + id: branch + # strip `refs/heads/` from $GITHUB_REF and replace `/` with `-` so that + # it can be used as a docker tag + run: echo "branch=$(echo ${GITHUB_REF#refs/heads/} | tr / -)" >> "$GITHUB_OUTPUT" + + - uses: getsentry/action-build-and-push-images@8fc75e483c09a68721f2c8951292ee17f8821766 + with: + image_name: 'snuba-ci' + platforms: 'linux/amd64' + dockerfile_path: 'Dockerfile' + ghcr: false + tag_nightly: false + tag_latest: false + tags: 'ghcr.io/getsentry/snuba-ci:${{ github.event.pull_request.head.sha || github.sha }}' + outputs: 'type=docker,dest=/tmp/snuba-ci.tar' + + - name: Publish snuba-ci image to artifacts + # we publish to github artifacts separately so that third-party + # contributions also work, as all the test jobs need this image. + # otherwise third-party contributors would have to provide a working, + # authenticated GHCR, which seems impossible to ensure in the general + # case. + uses: actions/upload-artifact@v7 + with: + name: snuba-ci + path: /tmp/snuba-ci.tar + + + docker-deps: + name: Warm Docker dependency image cache + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout code + uses: actions/checkout@v6.0.2 + + - name: Restore Docker dependency image cache + id: cache-restore + uses: actions/cache/restore@v5 + with: + path: /tmp/docker-deps + key: docker-deps-${{ hashFiles('docker-compose.gcb.yml') }}-${{ github.run_id }} + restore-keys: docker-deps-${{ hashFiles('docker-compose.gcb.yml') }}- + + - name: Load and pull Docker dependency images + id: load-pull + run: | + if [ -f /tmp/docker-deps/deps.tar ]; then + docker load --input /tmp/docker-deps/deps.tar + fi + images=( + "ghcr.io/getsentry/image-mirror-altinity-clickhouse-server:25.3.6.10034.altinitystable" + "ghcr.io/getsentry/image-mirror-confluentinc-cp-kafka:6.2.0" + "ghcr.io/getsentry/image-mirror-confluentinc-cp-zookeeper:6.2.0" + "ghcr.io/getsentry/docker-redis-cluster:7.0.10" + ) + pulled=0 + for image in "${images[@]}"; do + if ! docker image inspect "$image" &>/dev/null; then + docker pull "$image" + pulled=1 + fi + done + if [ "$pulled" -eq 1 ]; then + mkdir -p /tmp/docker-deps + docker save "${images[@]}" -o /tmp/docker-deps/deps.tar + fi + echo "pulled=$pulled" >> $GITHUB_OUTPUT + + - name: Save Docker dependency image cache + if: steps.load-pull.outputs.pulled == '1' + uses: actions/cache/save@v5 + with: + path: /tmp/docker-deps + key: docker-deps-${{ hashFiles('docker-compose.gcb.yml') }}-${{ github.run_id }} + + tests: + needs: [linting, snuba-image, docker-deps] + name: Tests and code coverage + runs-on: ubuntu-latest + timeout-minutes: 40 + strategy: + matrix: + snuba_settings: + [ + "test", + "test_rust", + "test_distributed", + "test_distributed_migrations", + ] + steps: + - name: Checkout code + uses: actions/checkout@v6.0.2 + + - name: Download snuba-ci image from artifacts + uses: actions/download-artifact@v8 + with: + name: snuba-ci + path: /tmp + + - name: Load snuba-ci image + run: | + docker load --input /tmp/snuba-ci.tar + docker image ls -a + + - name: Restore Docker dependency image cache + uses: actions/cache/restore@v5 + with: + path: /tmp/docker-deps + key: docker-deps-${{ hashFiles('docker-compose.gcb.yml') }}-${{ github.run_id }} + restore-keys: docker-deps-${{ hashFiles('docker-compose.gcb.yml') }}- + + - name: Load Docker dependency images + run: | + if [ -f /tmp/docker-deps/deps.tar ]; then + docker load --input /tmp/docker-deps/deps.tar + fi + images=( + "ghcr.io/getsentry/image-mirror-altinity-clickhouse-server:25.3.6.10034.altinitystable" + "ghcr.io/getsentry/image-mirror-confluentinc-cp-kafka:6.2.0" + "ghcr.io/getsentry/image-mirror-confluentinc-cp-zookeeper:6.2.0" + "ghcr.io/getsentry/docker-redis-cluster:7.0.10" + ) + for image in "${images[@]}"; do + if ! docker image inspect "$image" &>/dev/null; then + docker pull "$image" + fi + done + + - name: Docker set up + run: | + docker network create --attachable cloudbuild + + - name: Docker Snuba Rust tests + run: | + SNUBA_IMAGE=ghcr.io/getsentry/snuba-ci:${{ github.event.pull_request.head.sha || github.sha }} SNUBA_SETTINGS=test docker compose -f docker-compose.gcb.yml run --rm snuba-test-rust + if: ${{ matrix.snuba_settings == 'test_rust' }} + + - name: Docker Snuba tests + run: | + SNUBA_IMAGE=ghcr.io/getsentry/snuba-ci:${{ github.event.pull_request.head.sha || github.sha }} SNUBA_SETTINGS=${{ matrix.snuba_settings }} docker compose -f docker-compose.gcb.yml run --rm snuba-test + if: ${{ matrix.snuba_settings == 'test' || matrix.snuba_settings == 'test_distributed' }} + + - name: Docker Snuba Multi-Node Tests + run: | + SNUBA_IMAGE=ghcr.io/getsentry/snuba-ci:${{ github.event.pull_request.head.sha || github.sha }} SNUBA_SETTINGS=test_distributed_migrations docker compose --profile multi_node -f docker-compose.gcb.yml up -d + SNUBA_IMAGE=ghcr.io/getsentry/snuba-ci:${{ github.event.pull_request.head.sha || github.sha }} SNUBA_SETTINGS=test_distributed_migrations TEST_LOCATION=test_distributed_migrations docker compose --profile multi_node -f docker-compose.gcb.yml run --rm snuba-test + if: ${{ matrix.snuba_settings == 'test_distributed_migrations' }} + + - name: Docker Snuba Init Tests + run: | + SNUBA_IMAGE=ghcr.io/getsentry/snuba-ci:${{ github.event.pull_request.head.sha || github.sha }} SNUBA_SETTINGS=test_initialization TEST_LOCATION=test_initialization docker compose -f docker-compose.gcb.yml run --rm snuba-test + if: ${{ matrix.snuba_settings == 'test' }} + + - name: Upload test results to Codecov + if: ${{ !cancelled() }} + uses: codecov/test-results-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + + - name: Upload to codecov + run: | + curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov && ./codecov -t ${CODECOV_TOKEN} + + admin-tests: + needs: [linting] + name: Front end tests for snuba admin + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6.0.2 + name: Checkout code + - uses: volta-cli/action@v4 + - name: Set up and run tests through yarn + run: cd snuba/admin && yarn install && yarn run test --coverage + - name: Upload to codecov + run: | + curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov && ./codecov -t ${CODECOV_TOKEN} + + sentry: + needs: [snuba-image, files-changed] + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + matrix: + instance: [0, 1, 2, 3] + env: + MIGRATIONS_TEST_MIGRATE: 1 + # XXX: `MATRIX_INSTANCE_TOTAL` must be hardcoded to the length of `strategy.matrix.instance`. + MATRIX_INSTANCE_TOTAL: 4 + TEST_GROUP_STRATEGY: ROUND_ROBIN + + steps: + - name: Checkout code + uses: actions/checkout@v6.0.2 + + - name: Download snuba-ci image from artifacts + uses: actions/download-artifact@v8 + with: + name: snuba-ci + path: /tmp + + - name: Load snuba-ci image + run: | + docker load --input /tmp/snuba-ci.tar + docker image ls -a + + - name: Checkout sentry + uses: actions/checkout@v6.0.2 + with: + repository: getsentry/sentry + path: sentry + + - name: Setup steps + id: setup + run: | + # We cannot execute actions that are not placed under .github of the main repo + mkdir -p .github/actions + cp -r sentry/.github/actions/* .github/actions + + - name: Setup Sentry + id: setup-sentry + uses: ./.github/actions/setup-sentry + with: + workdir: sentry + mode: minimal + + - name: Start snuba + run: | + # TODO(hubertdeng123): New devservices doesn't support running sentry without snuba yet, remove this when it does + docker stop snuba-snuba-1 + docker rm snuba-snuba-1 + docker run -d --rm \ + -p 127.0.0.1:1218:1218 \ + -e PYTHONUNBUFFERED=1 \ + -e SNUBA_SETTINGS=docker \ + -e DEBUG=1 \ + -e DEFAULT_BROKERS=kafka:9093 \ + -e CLICKHOUSE_HOST=clickhouse \ + -e CLICKHOUSE_PORT=9000 \ + -e CLICKHOUSE_HTTP_PORT=8123 \ + -e REDIS_HOST=redis \ + -e REDIS_PORT=6379 \ + -e REDIS_DB=1 \ + --name snuba-snuba-1 \ + --network devservices \ + ghcr.io/getsentry/snuba-ci:${{ github.event.pull_request.head.sha || github.sha }} + docker exec snuba-snuba-1 snuba migrations migrate --force + + - name: Run snuba tests + if: needs.files-changed.outputs.api_changes == 'false' + working-directory: sentry + run: | + pytest -k 'not __In' tests \ + -m snuba_ci \ + -vv --cov . --cov-report="xml:.artifacts/snuba.coverage.xml" + + - name: Run full tests + if: needs.files-changed.outputs.api_changes == 'true' + working-directory: sentry + run: | + pytest -k 'not __In' \ + tests/snuba \ + tests/sentry/snuba \ + tests/sentry/eventstream/kafka \ + tests/sentry/post_process_forwarder \ + tests/sentry/services/eventstore/snuba \ + tests/sentry/search/events \ + tests/sentry/event_manager \ + tests/sentry/api/endpoints/test_organization_profiling_functions.py \ + tests/sentry/integrations/slack/test_unfurl.py \ + tests/sentry/uptime/endpoints/test_project_uptime_alert_check_index.py \ + tests/sentry/uptime/endpoints/test_organization_uptime_stats.py \ + tests/sentry/api/endpoints/test_organization_traces.py \ + tests/sentry/api/endpoints/test_organization_spans_fields.py \ + tests/sentry/api/endpoints/test_organization_spans_fields_stats.py \ + tests/sentry/sentry_metrics/querying \ + -vv --cov . --cov-report="xml:.artifacts/snuba.coverage.xml" + + - name: Run CI module tests + if: needs.files-changed.outputs.api_changes == 'true' + working-directory: sentry + run: pytest -k 'not __In' tests -vv -m snuba_ci + + clickhouse-versions: + needs: [linting, snuba-image, docker-deps] + name: Tests on multiple clickhouse versions + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + version: + [ + "25.3.6.10034.altinitystable", + ] + + steps: + - name: Checkout code + uses: actions/checkout@v6.0.2 + + - name: Download snuba-ci image from artifacts + uses: actions/download-artifact@v8 + with: + name: snuba-ci + path: /tmp + + - name: Load snuba-ci image + run: | + docker load --input /tmp/snuba-ci.tar + docker image ls -a + + - name: Restore Docker dependency image cache + uses: actions/cache/restore@v5 + with: + path: /tmp/docker-deps + key: docker-deps-${{ hashFiles('docker-compose.gcb.yml') }}-${{ github.run_id }} + restore-keys: docker-deps-${{ hashFiles('docker-compose.gcb.yml') }}- + + - name: Load Docker dependency images + run: | + if [ -f /tmp/docker-deps/deps.tar ]; then + docker load --input /tmp/docker-deps/deps.tar + fi + images=( + "ghcr.io/getsentry/image-mirror-altinity-clickhouse-server:${{ matrix.version }}" + "ghcr.io/getsentry/image-mirror-confluentinc-cp-kafka:6.2.0" + "ghcr.io/getsentry/image-mirror-confluentinc-cp-zookeeper:6.2.0" + "ghcr.io/getsentry/docker-redis-cluster:7.0.10" + ) + for image in "${images[@]}"; do + if ! docker image inspect "$image" &>/dev/null; then + docker pull "$image" + fi + done + + - name: Docker set up + run: | + docker network create --attachable cloudbuild + + - name: Docker Snuba Test other ClickHouse versions + run: | + export CLICKHOUSE_IMAGE=ghcr.io/getsentry/image-mirror-altinity-clickhouse-server:${{matrix.version}} + SNUBA_IMAGE=ghcr.io/getsentry/snuba-ci:${{ github.event.pull_request.head.sha || github.sha }} SNUBA_SETTINGS=test docker compose -f docker-compose.gcb.yml run --rm snuba-test + + - name: Upload to codecov + run: | + curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov && ./codecov -t ${CODECOV_TOKEN} + + distroless-smoke-test: + name: Distroless image smoke test + needs: [snuba-image] + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout code + uses: actions/checkout@v6.0.2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build distroless image + uses: docker/build-push-action@v6 + with: + context: . + target: application-distroless + load: true + tags: snuba-distroless:test + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Verify --help works + run: docker run --rm snuba-distroless:test --help + + - name: Verify Python imports + run: | + docker run --rm --entrypoint python3 snuba-distroless:test \ + -c "import snuba, rust_snuba, confluent_kafka; print('OK')" + + - name: Verify jemalloc LD_PRELOAD is active + run: | + docker run --rm --entrypoint python3 snuba-distroless:test \ + -c "import ctypes; ctypes.CDLL('libjemalloc.so.2'); print('jemalloc OK')" + + - name: Verify no shell available + run: | + # This should fail — distroless has no /bin/sh + if docker run --rm --entrypoint /bin/sh snuba-distroless:test -c "echo hi" 2>/dev/null; then + echo "ERROR: /bin/sh should not be available in distroless image" + exit 1 + else + echo "OK: /bin/sh is not available" + fi + + validate-devservices-config: + runs-on: ubuntu-24.04 + needs: files-changed + if: ${{ needs.files-changed.outputs.devservices_changes == 'true' }} + steps: + - uses: actions/checkout@v6.0.2 + name: Checkout repository + + - name: Get devservices version + id: get-devservices-version + run: | + awk -F'"' ' + /name/ { pkg = $2 } + /version/ { if (pkg == "devservices") print "version="$2 } + ' uv.lock >> $GITHUB_OUTPUT + + - uses: getsentry/action-validate-devservices-config@711ae7221998ddf81211f25f5e3873ecffd22387 + name: Validate devservices config + with: + devservices-version: ${{ steps.get-devservices-version.outputs.version }} diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml index 68ac9c97f5..d239054bc2 100644 --- a/.github/workflows/image.yml +++ b/.github/workflows/image.yml @@ -5,6 +5,11 @@ on: - master - release/** +# Cancel in-progress runs for the same branch/PR when a new commit is pushed. +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + jobs: build: runs-on: ubuntu-24.04 From 55907b7be84cefbcb5d1901cb9813d9ee888fc5e Mon Sep 17 00:00:00 2001 From: "getsantry[bot]" <66042841+getsantry[bot]@users.noreply.github.com> Date: Sat, 11 Apr 2026 17:41:39 +0000 Subject: [PATCH 7/9] [getsentry/action-github-commit] Auto commit --- .github/workflows/image.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml index d239054bc2..f112bf99b0 100644 --- a/.github/workflows/image.yml +++ b/.github/workflows/image.yml @@ -51,4 +51,3 @@ jobs: docker buildx imagetools create \ --tag ghcr.io/getsentry/snuba:${{ github.sha }} \ ghcr.io/getsentry/snuba:${{ github.sha }}-amd64 - From 48d65526d42231abb4c2d3782a8840bea785cdea Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Sat, 11 Apr 2026 10:43:54 -0700 Subject: [PATCH 8/9] ci(image): Set publish_on_pr=true so PRs push to ghcr --- .github/workflows/image.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml index f112bf99b0..13d3e352fb 100644 --- a/.github/workflows/image.yml +++ b/.github/workflows/image.yml @@ -27,6 +27,7 @@ jobs: dockerfile_path: 'Dockerfile' tag_suffix: -amd64 ghcr: true + publish_on_pr: true tag_nightly: false tag_latest: false From 70577b40bd92d3e6fb314bdc6c0694b0c9d73bb1 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Sat, 11 Apr 2026 11:13:01 -0700 Subject: [PATCH 9/9] [skip ci] devservices: Pin snuba image to test-optimize-endpoint SHA --- devservices/config.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/devservices/config.yml b/devservices/config.yml index 16aa137158..467270374c 100644 --- a/devservices/config.yml +++ b/devservices/config.yml @@ -75,7 +75,7 @@ services: restart: unless-stopped snuba: - image: ghcr.io/getsentry/snuba:nightly + image: ghcr.io/getsentry/snuba:48d65526d42231abb4c2d3782a8840bea785cdea-amd64 ports: - 127.0.0.1:1218:1218 - 127.0.0.1:1219:1219 @@ -114,7 +114,7 @@ services: - orchestrator=devservices restart: unless-stopped profiles-consumer: - image: ghcr.io/getsentry/snuba:nightly + image: ghcr.io/getsentry/snuba:48d65526d42231abb4c2d3782a8840bea785cdea-amd64 command: [ rust-consumer, --storage=profiles, @@ -143,7 +143,7 @@ services: - orchestrator=devservices restart: unless-stopped profile-chunks-consumer: - image: ghcr.io/getsentry/snuba:nightly + image: ghcr.io/getsentry/snuba:48d65526d42231abb4c2d3782a8840bea785cdea-amd64 command: [ rust-consumer, --storage=profile_chunks, @@ -172,7 +172,7 @@ services: - orchestrator=devservices restart: unless-stopped functions-consumer: - image: ghcr.io/getsentry/snuba:nightly + image: ghcr.io/getsentry/snuba:48d65526d42231abb4c2d3782a8840bea785cdea-amd64 command: [ rust-consumer, --storage=functions_raw, @@ -201,7 +201,7 @@ services: - orchestrator=devservices restart: unless-stopped metrics-consumer: - image: ghcr.io/getsentry/snuba:nightly + image: ghcr.io/getsentry/snuba:48d65526d42231abb4c2d3782a8840bea785cdea-amd64 command: [ rust-consumer, --storage=metrics_raw, @@ -230,7 +230,7 @@ services: - orchestrator=devservices restart: unless-stopped generic-metrics-distributions-consumer: - image: ghcr.io/getsentry/snuba:nightly + image: ghcr.io/getsentry/snuba:48d65526d42231abb4c2d3782a8840bea785cdea-amd64 command: [ rust-consumer, --storage=generic_metrics_distributions_raw, @@ -259,7 +259,7 @@ services: - orchestrator=devservices restart: unless-stopped generic-metrics-sets-consumer: - image: ghcr.io/getsentry/snuba:nightly + image: ghcr.io/getsentry/snuba:48d65526d42231abb4c2d3782a8840bea785cdea-amd64 command: [ rust-consumer, --storage=generic_metrics_sets_raw, @@ -288,7 +288,7 @@ services: - orchestrator=devservices restart: unless-stopped generic-metrics-counters-consumer: - image: ghcr.io/getsentry/snuba:nightly + image: ghcr.io/getsentry/snuba:48d65526d42231abb4c2d3782a8840bea785cdea-amd64 command: [ rust-consumer, --storage=generic_metrics_counters_raw, @@ -317,7 +317,7 @@ services: - orchestrator=devservices restart: unless-stopped generic-metrics-gauges-consumer: - image: ghcr.io/getsentry/snuba:nightly + image: ghcr.io/getsentry/snuba:48d65526d42231abb4c2d3782a8840bea785cdea-amd64 command: [ rust-consumer, --storage=generic_metrics_gauges_raw,