From 58182178af793619c4a0dea3ef2d7540ee64073c Mon Sep 17 00:00:00 2001 From: Gustavo Lira e Silva Date: Thu, 3 Sep 2026 16:56:43 -0300 Subject: [PATCH 1/2] fix(ci): wait for the showcase-runtime rollout before running its tests The showcase-runtime phase started Playwright as soon as `helm upgrade` returned, without waiting for the deployment to become ready. The first test in the showcase-runtime-db dependency project scales the deployment to 0, so when the initial rollout is still running that scale-down kills the install-dynamic-plugins init container mid-install. That script holds a lock file on the dynamic-plugins-root PVC and releases it from an atexit handler, which a hard kill skips. The lock survives on the PVC and every pod created afterwards blocks forever on: ======= Waiting for lock release (file: /dynamic-plugins-root/install-dynamic-plugins.lock)... Each Azure DB test then burns its full 10 minute timeout waiting for a pod that can never become ready. Ten attempts consume 100 minutes, the job hits the Prow timeout inside this phase, and showcase-sanity-plugins never runs. The race only opens wide enough to fire when the showcase-runtime namespace also has to extract the catalog index and pull OCI plugins, which happens when CATALOG_INDEX_IMAGE is set. That is empty on the daily nightly, so it passes, but it is always set on the RC and GA verification runs triggered through Gangway with --catalog-index-image. Those runs are the reason the release branch nightlies exist, and they have not reached the sanity plugin check at all. Swap testing::run_tests for testing::check_and_test in both the OCP Helm and OCP Operator runtime phases. It gates on /healthcheck before starting the tests, so the initial plugin install always completes before anything scales the deployment down, and it collects pod logs when tests fail. Also align restartDeployment's internal budgets with the 10 minute timeout its callers set: 5m + 10s + 10m could not fit, so Playwright killed the worker mid-restart, the catch block never logged pod conditions or events, and the retry scaled the deployment down again on top of a rollout that was still starting. 2m + 10s + 7m fits. Verified on an OCP 4.22 cluster by running the full handle_ocp_nightly pipeline with CATALOG_INDEX_IMAGE set, the configuration that reproduces the hang: showcase 35 passed showcase-rbac 17 passed showcase-runtime 19 passed (22.9m, was 100m and never finished) showcase-sanity-plugins 9 passed (had never run) Zero "Waiting for lock release" lines, against 648 in the failing run. Co-Authored-By: Claude Opus 5 (1M context) --- .ci/pipelines/jobs/ocp-nightly.sh | 6 ++++-- .ci/pipelines/jobs/ocp-operator.sh | 5 ++++- e2e-tests/playwright/utils/kube-client.ts | 7 +++++-- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.ci/pipelines/jobs/ocp-nightly.sh b/.ci/pipelines/jobs/ocp-nightly.sh index bb3acccedd..a318d2d1c7 100644 --- a/.ci/pipelines/jobs/ocp-nightly.sh +++ b/.ci/pipelines/jobs/ocp-nightly.sh @@ -75,8 +75,10 @@ run_runtime_config_change_tests() { fi local runtime_url="https://${RELEASE_NAME}-developer-hub-${NAME_SPACE_RUNTIME}.${K8S_CLUSTER_ROUTER_BASE}" - # Run tests - allow failures since schema-mode tests are opt-in - testing::run_tests "${RELEASE_NAME}" "${NAME_SPACE_RUNTIME}" "${PW_PROJECT_SHOWCASE_RUNTIME}" "${runtime_url}" || true + # Gate on /healthcheck first: the initial rollout must finish before a test + # scales the deployment to 0, or install-dynamic-plugins is killed mid-install + # and orphans its lock on the dynamic-plugins-root PVC. + testing::check_and_test "${RELEASE_NAME}" "${NAME_SPACE_RUNTIME}" "${PW_PROJECT_SHOWCASE_RUNTIME}" "${runtime_url}" } run_sanity_plugins_check() { diff --git a/.ci/pipelines/jobs/ocp-operator.sh b/.ci/pipelines/jobs/ocp-operator.sh index cc8d1b24f6..cd87cac051 100644 --- a/.ci/pipelines/jobs/ocp-operator.sh +++ b/.ci/pipelines/jobs/ocp-operator.sh @@ -110,7 +110,10 @@ run_operator_runtime_config_change_tests() { fi fi - testing::run_tests "${RELEASE_NAME}" "${NAME_SPACE_RUNTIME}" "${PW_PROJECT_SHOWCASE_RUNTIME}" "${runtime_url}" || true + # Gate on /healthcheck first: the initial rollout must finish before a test + # scales the deployment to 0, or install-dynamic-plugins is killed mid-install + # and orphans its lock on the dynamic-plugins-root PVC. + testing::check_and_test "${RELEASE_NAME}" "${NAME_SPACE_RUNTIME}" "${PW_PROJECT_SHOWCASE_RUNTIME}" "${runtime_url}" } handle_ocp_operator() { diff --git a/e2e-tests/playwright/utils/kube-client.ts b/e2e-tests/playwright/utils/kube-client.ts index e1d145789b..2a28aa819a 100644 --- a/e2e-tests/playwright/utils/kube-client.ts +++ b/e2e-tests/playwright/utils/kube-client.ts @@ -830,7 +830,7 @@ export class KubeClient { console.log(`Deployment: ${deploymentName}, Namespace: ${namespace}`); await this.logPodConditionsForDeployment(deploymentName, namespace); await this.scaleDeployment(deploymentName, namespace, 0); - await this.waitForDeploymentReady(deploymentName, namespace, 0, 300000); // 5 minutes for scale down + await this.waitForDeploymentReady(deploymentName, namespace, 0, 120000); // 2 minutes for scale down // Wait a bit for pods to be fully terminated console.log("Waiting for pods to be fully terminated..."); @@ -840,7 +840,10 @@ export class KubeClient { console.log(`Scaling up deployment ${deploymentName} to 1 replica.`); await this.scaleDeployment(deploymentName, namespace, 1); - await this.waitForDeploymentReady(deploymentName, namespace, 1, 600000); // 10 minutes for scale up + // 2m + 10s + 7m fits inside the 10 minute timeout callers set for a + // restart, so a slow rollout fails through the catch below instead of + // having the worker killed mid-restart. + await this.waitForDeploymentReady(deploymentName, namespace, 1, 420000); // 7 minutes for scale up console.log( `Restart of deployment ${deploymentName} completed successfully.`, From 42aa09cd0b7acebde4c97872f6d84d37375a2527 Mon Sep 17 00:00:00 2001 From: Gustavo Lira e Silva Date: Fri, 4 Sep 2026 11:11:40 -0300 Subject: [PATCH 2/2] docs(ci): trim the rationale comments to two lines The full reasoning lives in the commit message and the PR body; the inline comments only need to say why the call is shaped this way. --- .ci/pipelines/jobs/ocp-nightly.sh | 5 ++--- .ci/pipelines/jobs/ocp-operator.sh | 5 ++--- e2e-tests/playwright/utils/kube-client.ts | 5 ++--- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/.ci/pipelines/jobs/ocp-nightly.sh b/.ci/pipelines/jobs/ocp-nightly.sh index a318d2d1c7..d451118ace 100644 --- a/.ci/pipelines/jobs/ocp-nightly.sh +++ b/.ci/pipelines/jobs/ocp-nightly.sh @@ -75,9 +75,8 @@ run_runtime_config_change_tests() { fi local runtime_url="https://${RELEASE_NAME}-developer-hub-${NAME_SPACE_RUNTIME}.${K8S_CLUSTER_ROUTER_BASE}" - # Gate on /healthcheck first: the initial rollout must finish before a test - # scales the deployment to 0, or install-dynamic-plugins is killed mid-install - # and orphans its lock on the dynamic-plugins-root PVC. + # Gate on /healthcheck: a test that scales the deployment to 0 mid-rollout + # kills install-dynamic-plugins and orphans its lock on the PVC. testing::check_and_test "${RELEASE_NAME}" "${NAME_SPACE_RUNTIME}" "${PW_PROJECT_SHOWCASE_RUNTIME}" "${runtime_url}" } diff --git a/.ci/pipelines/jobs/ocp-operator.sh b/.ci/pipelines/jobs/ocp-operator.sh index cd87cac051..1187d73914 100644 --- a/.ci/pipelines/jobs/ocp-operator.sh +++ b/.ci/pipelines/jobs/ocp-operator.sh @@ -110,9 +110,8 @@ run_operator_runtime_config_change_tests() { fi fi - # Gate on /healthcheck first: the initial rollout must finish before a test - # scales the deployment to 0, or install-dynamic-plugins is killed mid-install - # and orphans its lock on the dynamic-plugins-root PVC. + # Gate on /healthcheck: a test that scales the deployment to 0 mid-rollout + # kills install-dynamic-plugins and orphans its lock on the PVC. testing::check_and_test "${RELEASE_NAME}" "${NAME_SPACE_RUNTIME}" "${PW_PROJECT_SHOWCASE_RUNTIME}" "${runtime_url}" } diff --git a/e2e-tests/playwright/utils/kube-client.ts b/e2e-tests/playwright/utils/kube-client.ts index 2a28aa819a..700c8867b8 100644 --- a/e2e-tests/playwright/utils/kube-client.ts +++ b/e2e-tests/playwright/utils/kube-client.ts @@ -840,9 +840,8 @@ export class KubeClient { console.log(`Scaling up deployment ${deploymentName} to 1 replica.`); await this.scaleDeployment(deploymentName, namespace, 1); - // 2m + 10s + 7m fits inside the 10 minute timeout callers set for a - // restart, so a slow rollout fails through the catch below instead of - // having the worker killed mid-restart. + // 2m + 10s + 7m fits the 10 minute test timeout, so a slow rollout fails + // through the catch below instead of the worker being killed mid-restart. await this.waitForDeploymentReady(deploymentName, namespace, 1, 420000); // 7 minutes for scale up console.log(