From 4d3e6fd2b150c8dc942ffe6afddf3f42915b7e09 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Wed, 2 Sep 2026 13:17:13 -0400 Subject: [PATCH 1/2] Increase frozen perf compute budget --- .github/workflows/scenario-dev.yml | 4 ++-- tests/mw-dev/README.md | 15 ++++++++------- tests/mw-dev/manifests.trino.tmpl.yaml | 16 ++++++++-------- tests/mw-dev/run.sh | 5 +++-- tests/mw-dev/run_sh_test.go | 12 ++++++++---- tests/mw-dev/scenario/script_test.go | 4 ++-- 6 files changed, 31 insertions(+), 25 deletions(-) diff --git a/.github/workflows/scenario-dev.yml b/.github/workflows/scenario-dev.yml index 2782a040..8e856e07 100644 --- a/.github/workflows/scenario-dev.yml +++ b/.github/workflows/scenario-dev.yml @@ -81,8 +81,8 @@ jobs: DUCKGRES_SCENARIO_MAX_RUNTIME: 4h DUCKGRES_SCENARIO_GO_TEST_TIMEOUT: 4h15m # Add process headroom for repeated full-dataset pgwire aggregates. - DUCKGRES_K8S_WORKER_CPU_REQUEST: "2" - DUCKGRES_K8S_WORKER_MEMORY_REQUEST: 8Gi + DUCKGRES_K8S_WORKER_CPU_REQUEST: "3" + DUCKGRES_K8S_WORKER_MEMORY_REQUEST: 12Gi steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 diff --git a/tests/mw-dev/README.md b/tests/mw-dev/README.md index 4c5a53e9..7aec13ab 100644 --- a/tests/mw-dev/README.md +++ b/tests/mw-dev/README.md @@ -54,7 +54,7 @@ The isolated control plane's default worker request is configurable through `DUCKGRES_K8S_WORKER_CPU_REQUEST` and `DUCKGRES_K8S_WORKER_MEMORY_REQUEST`; `run.sh` defaults them to `750m` and `1536Mi`, respectively, preserving the e2e harness's worker-packing behavior. -`scenario-dev.yml` explicitly overrides them to 2 CPU and 8Gi for the frozen +`scenario-dev.yml` explicitly overrides them to 3 CPU and 12Gi for the frozen perf workload. Direct `run.sh` callers can make the same explicit override. ### Scenario Trino readiness @@ -132,12 +132,13 @@ The lane defaults `TRINO_IMAGE` to the pinned PostHog fork used when this suite was added. That fork contains the DuckLake connector and PostgreSQL dynamic catalog store; upstream `trinodb/trino` is not compatible. Update the default in `run.sh` and `e2e-mw-dev.yml` together when promoting a Trino build. -Each Trino worker has requests and limits of 1 CPU and 4Gi. Together they match -the frozen perf Duckgres worker's aggregate 2 CPU and 8Gi execution budget while -exercising Trino's distributed execution path. Trino permits 2GB of query -memory per worker and 4GB cluster-wide. The coordinator does not execute query -tasks (`node-scheduler.include-coordinator=false`) and is additional Trino -control-plane overhead rather than part of the matched execution budget. +Each Trino worker has requests and limits of 1500m CPU and 6Gi. Together they +match the frozen perf Duckgres worker's aggregate 3 CPU and 12Gi execution +budget while exercising Trino's distributed execution path. Trino permits 3GB +of query memory per worker and 6GB cluster-wide. The coordinator does not +execute query tasks (`node-scheduler.include-coordinator=false`) and is +additional Trino control-plane overhead rather than part of the matched +execution budget. `TRINO_TLS_PASSWORD` defaults to `duckgres-e2e-keystore`; it protects only the random, two-day, per-run PKCS12 file. `run.sh` generates a fresh CA and leaf certificate under `DUCKGRES_CI_SECRET_DIR`, mounts the CA into the PR control diff --git a/tests/mw-dev/manifests.trino.tmpl.yaml b/tests/mw-dev/manifests.trino.tmpl.yaml index 8148a628..ce52b521 100644 --- a/tests/mw-dev/manifests.trino.tmpl.yaml +++ b/tests/mw-dev/manifests.trino.tmpl.yaml @@ -52,9 +52,9 @@ data: http-server.https.keystore.path=/etc/trino/tls/keystore.p12 http-server.https.keystore.key=${ENV:TRINO_TLS_KEYSTORE_PASSWORD} discovery.uri=http://duckgres-trino.${NAMESPACE}.svc:8080 - query.max-memory=4GB + query.max-memory=6GB # The coordinator does not execute tasks and retains headroom in its 2G - # heap; execution workers carry the 2GB per-node allowance below. + # heap; execution workers carry the 3GB per-node allowance below. query.max-memory-per-node=1GB catalog.management=dynamic catalog.store=posthog @@ -94,7 +94,7 @@ data: node.data-dir=/data/trino jvm.config: | -server - -Xmx3G + -Xmx4608M -XX:+UseG1GC -XX:+ExitOnOutOfMemoryError -Dfile.encoding=UTF-8 @@ -102,8 +102,8 @@ data: coordinator=false http-server.http.port=8080 discovery.uri=http://duckgres-trino.${NAMESPACE}.svc:8080 - query.max-memory=4GB - query.max-memory-per-node=2GB + query.max-memory=6GB + query.max-memory-per-node=3GB shutdown.grace-period=10s catalog.management=dynamic internal-communication.shared-secret=${ENV:TRINO_INTERNAL_COMMUNICATION_SHARED_SECRET} @@ -261,11 +261,11 @@ spec: initialDelaySeconds: 20 periodSeconds: 5 resources: - requests: { cpu: "1", memory: 4Gi } - # Two replicas provide the same aggregate 2-CPU/8Gi execution + requests: { cpu: "1500m", memory: 6Gi } + # Two replicas provide the same aggregate 3-CPU/12Gi execution # budget as the frozen-perf Duckgres worker. Keep requests equal # to limits so node contention cannot change the comparison. - limits: { cpu: "1", memory: 4Gi } + limits: { cpu: "1500m", memory: 6Gi } volumeMounts: - { name: config, mountPath: /etc/trino/node.properties, subPath: node.properties } - { name: config, mountPath: /etc/trino/jvm.config, subPath: jvm.config } diff --git a/tests/mw-dev/run.sh b/tests/mw-dev/run.sh index 82bf904d..e86feeec 100755 --- a/tests/mw-dev/run.sh +++ b/tests/mw-dev/run.sh @@ -346,8 +346,9 @@ cmd_deploy() { # Patch the Deployment resources directly. The CI deployer intentionally # cannot patch the deployments/scale subresource, while it already needs # narrowly scoped Deployment patch access for the control-plane config. - # Two 1-CPU/4Gi workers match the frozen-perf Duckgres worker's aggregate - # 2-CPU/8Gi execution budget while exercising Trino's distributed path. + # Two 1500m-CPU/6Gi workers match the frozen-perf Duckgres worker's + # aggregate 3-CPU/12Gi execution budget while exercising Trino's + # distributed path. "${KUBECTL[@]}" -n "$NS" patch deployment duckgres-trino-coordinator \ --type=merge -p '{"spec":{"replicas":1}}' "${KUBECTL[@]}" -n "$NS" patch deployment duckgres-trino-worker \ diff --git a/tests/mw-dev/run_sh_test.go b/tests/mw-dev/run_sh_test.go index 03ce1a21..5a5698e7 100644 --- a/tests/mw-dev/run_sh_test.go +++ b/tests/mw-dev/run_sh_test.go @@ -181,14 +181,18 @@ func TestTrinoWorkersMatchDuckgresAggregateCompute(t *testing.T) { resources := worker["resources"].(map[string]any) for _, field := range []string{"requests", "limits"} { values := resources[field].(map[string]any) - if values["cpu"] != "1" || values["memory"] != "4Gi" { - t.Errorf("Trino worker %s = %v, want 1 CPU and 4Gi", field, values) + if values["cpu"] != "1500m" || values["memory"] != "6Gi" { + t.Errorf("Trino worker %s = %v, want 1500m CPU and 6Gi", field, values) } } + workerData := workerConfig["data"].(map[string]any) + if jvmConfig := workerData["jvm.config"].(string); !strings.Contains(jvmConfig, "-Xmx4608M") { + t.Errorf("Trino worker JVM config does not increase heap by 50%%:\n%s", jvmConfig) + } for name, expectedPerNode := range map[string]string{ "coordinator": "1GB", - "worker": "2GB", + "worker": "3GB", } { manifest := coordinatorConfig if name == "worker" { @@ -196,7 +200,7 @@ func TestTrinoWorkersMatchDuckgresAggregateCompute(t *testing.T) { } data := manifest["data"].(map[string]any) config := data["config.properties"].(string) - for _, want := range []string{"query.max-memory=4GB", "query.max-memory-per-node=" + expectedPerNode} { + for _, want := range []string{"query.max-memory=6GB", "query.max-memory-per-node=" + expectedPerNode} { if !strings.Contains(config, want) { t.Errorf("Trino %s config missing %q:\n%s", name, want, config) } diff --git a/tests/mw-dev/scenario/script_test.go b/tests/mw-dev/scenario/script_test.go index 0c344150..7047ce0f 100644 --- a/tests/mw-dev/scenario/script_test.go +++ b/tests/mw-dev/scenario/script_test.go @@ -90,8 +90,8 @@ func TestDevScenarioWorkflowUsesUnifiedMwDevHarness(t *testing.T) { "TRINO_POD_IDENTITY_ROLE: ${{ secrets.MW_DEV_TRINO_POD_IDENTITY_ROLE }}", "TRINO_IMAGE: ghcr.io/posthog/trino:", "E2E_SUITE: ${{ (github.event_name == 'schedule' || inputs.scenario == 'posthog_frozen_perf') && 'trino' || 'neutral' }}", - "DUCKGRES_K8S_WORKER_CPU_REQUEST: \"2\"", - "DUCKGRES_K8S_WORKER_MEMORY_REQUEST: 8Gi", + "DUCKGRES_K8S_WORKER_CPU_REQUEST: \"3\"", + "DUCKGRES_K8S_WORKER_MEMORY_REQUEST: 12Gi", "role-duration-seconds: 16200", "tests/mw-dev/run.sh deploy", "tests/mw-dev/run.sh test-scenario", From 5213885c7545ea943096ddc3a5f187230587ca61 Mon Sep 17 00:00:00 2001 From: Bill Guowei Yang Date: Wed, 2 Sep 2026 13:24:10 -0400 Subject: [PATCH 2/2] Use three workers for Trino perf --- tests/mw-dev/README.md | 6 +++--- tests/mw-dev/manifests.trino.tmpl.yaml | 12 ++++++------ tests/mw-dev/run.sh | 4 ++-- tests/mw-dev/run_sh_test.go | 12 ++++++------ 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/mw-dev/README.md b/tests/mw-dev/README.md index 7aec13ab..6474d0cc 100644 --- a/tests/mw-dev/README.md +++ b/tests/mw-dev/README.md @@ -121,7 +121,7 @@ Pod Identity associations. ## Isolated Trino lane The Trino lane never uses the shared mw-dev Trino namespace or coordinator. -It deploys one coordinator, two workers, and an OPA sidecar in the lane's +It deploys one coordinator, three workers, and an OPA sidecar in the lane's `duckgres-ci-pr-` namespace. The control plane projects fixed-name auth, tenant-secret, OPA-token, and resource-group objects into that same namespace; its cell id and PostgreSQL catalog store are also PR-local. This isolation is @@ -132,9 +132,9 @@ The lane defaults `TRINO_IMAGE` to the pinned PostHog fork used when this suite was added. That fork contains the DuckLake connector and PostgreSQL dynamic catalog store; upstream `trinodb/trino` is not compatible. Update the default in `run.sh` and `e2e-mw-dev.yml` together when promoting a Trino build. -Each Trino worker has requests and limits of 1500m CPU and 6Gi. Together they +Each Trino worker has requests and limits of 1 CPU and 4Gi. Together they match the frozen perf Duckgres worker's aggregate 3 CPU and 12Gi execution -budget while exercising Trino's distributed execution path. Trino permits 3GB +budget while exercising Trino's distributed execution path. Trino permits 2GB of query memory per worker and 6GB cluster-wide. The coordinator does not execute query tasks (`node-scheduler.include-coordinator=false`) and is additional Trino control-plane overhead rather than part of the matched diff --git a/tests/mw-dev/manifests.trino.tmpl.yaml b/tests/mw-dev/manifests.trino.tmpl.yaml index ce52b521..fcde21bc 100644 --- a/tests/mw-dev/manifests.trino.tmpl.yaml +++ b/tests/mw-dev/manifests.trino.tmpl.yaml @@ -54,7 +54,7 @@ data: discovery.uri=http://duckgres-trino.${NAMESPACE}.svc:8080 query.max-memory=6GB # The coordinator does not execute tasks and retains headroom in its 2G - # heap; execution workers carry the 3GB per-node allowance below. + # heap; execution workers carry the 2GB per-node allowance below. query.max-memory-per-node=1GB catalog.management=dynamic catalog.store=posthog @@ -94,7 +94,7 @@ data: node.data-dir=/data/trino jvm.config: | -server - -Xmx4608M + -Xmx3G -XX:+UseG1GC -XX:+ExitOnOutOfMemoryError -Dfile.encoding=UTF-8 @@ -103,7 +103,7 @@ data: http-server.http.port=8080 discovery.uri=http://duckgres-trino.${NAMESPACE}.svc:8080 query.max-memory=6GB - query.max-memory-per-node=3GB + query.max-memory-per-node=2GB shutdown.grace-period=10s catalog.management=dynamic internal-communication.shared-secret=${ENV:TRINO_INTERNAL_COMMUNICATION_SHARED_SECRET} @@ -261,11 +261,11 @@ spec: initialDelaySeconds: 20 periodSeconds: 5 resources: - requests: { cpu: "1500m", memory: 6Gi } - # Two replicas provide the same aggregate 3-CPU/12Gi execution + requests: { cpu: "1", memory: 4Gi } + # Three replicas provide the same aggregate 3-CPU/12Gi execution # budget as the frozen-perf Duckgres worker. Keep requests equal # to limits so node contention cannot change the comparison. - limits: { cpu: "1500m", memory: 6Gi } + limits: { cpu: "1", memory: 4Gi } volumeMounts: - { name: config, mountPath: /etc/trino/node.properties, subPath: node.properties } - { name: config, mountPath: /etc/trino/jvm.config, subPath: jvm.config } diff --git a/tests/mw-dev/run.sh b/tests/mw-dev/run.sh index e86feeec..dd0e22d4 100755 --- a/tests/mw-dev/run.sh +++ b/tests/mw-dev/run.sh @@ -346,13 +346,13 @@ cmd_deploy() { # Patch the Deployment resources directly. The CI deployer intentionally # cannot patch the deployments/scale subresource, while it already needs # narrowly scoped Deployment patch access for the control-plane config. - # Two 1500m-CPU/6Gi workers match the frozen-perf Duckgres worker's + # Three 1-CPU/4Gi workers match the frozen-perf Duckgres worker's # aggregate 3-CPU/12Gi execution budget while exercising Trino's # distributed path. "${KUBECTL[@]}" -n "$NS" patch deployment duckgres-trino-coordinator \ --type=merge -p '{"spec":{"replicas":1}}' "${KUBECTL[@]}" -n "$NS" patch deployment duckgres-trino-worker \ - --type=merge -p '{"spec":{"replicas":2}}' + --type=merge -p '{"spec":{"replicas":3}}' "${KUBECTL[@]}" -n "$NS" rollout status deploy/duckgres-trino-coordinator --timeout=300s "${KUBECTL[@]}" -n "$NS" rollout status deploy/duckgres-trino-worker --timeout=300s fi diff --git a/tests/mw-dev/run_sh_test.go b/tests/mw-dev/run_sh_test.go index 5a5698e7..b8e48030 100644 --- a/tests/mw-dev/run_sh_test.go +++ b/tests/mw-dev/run_sh_test.go @@ -126,7 +126,7 @@ func TestTrinoDeployStartsWorkloadsWithoutScaleSubresource(t *testing.T) { } for deployment, replicas := range map[string]int{ "duckgres-trino-coordinator": 1, - "duckgres-trino-worker": 2, + "duckgres-trino-worker": 3, } { want := "patch deployment " + deployment + " --type=merge -p {\"spec\":{\"replicas\":" + strconv.Itoa(replicas) + "}}" if !strings.Contains(calls, want) { @@ -181,18 +181,18 @@ func TestTrinoWorkersMatchDuckgresAggregateCompute(t *testing.T) { resources := worker["resources"].(map[string]any) for _, field := range []string{"requests", "limits"} { values := resources[field].(map[string]any) - if values["cpu"] != "1500m" || values["memory"] != "6Gi" { - t.Errorf("Trino worker %s = %v, want 1500m CPU and 6Gi", field, values) + if values["cpu"] != "1" || values["memory"] != "4Gi" { + t.Errorf("Trino worker %s = %v, want 1 CPU and 4Gi", field, values) } } workerData := workerConfig["data"].(map[string]any) - if jvmConfig := workerData["jvm.config"].(string); !strings.Contains(jvmConfig, "-Xmx4608M") { - t.Errorf("Trino worker JVM config does not increase heap by 50%%:\n%s", jvmConfig) + if jvmConfig := workerData["jvm.config"].(string); !strings.Contains(jvmConfig, "-Xmx3G") { + t.Errorf("Trino worker JVM config does not fit its 4Gi pod:\n%s", jvmConfig) } for name, expectedPerNode := range map[string]string{ "coordinator": "1GB", - "worker": "3GB", + "worker": "2GB", } { manifest := coordinatorConfig if name == "worker" {