diff --git a/.ci/pipelines/jobs/ocp-nightly.sh b/.ci/pipelines/jobs/ocp-nightly.sh index bb3acccedd..d451118ace 100644 --- a/.ci/pipelines/jobs/ocp-nightly.sh +++ b/.ci/pipelines/jobs/ocp-nightly.sh @@ -75,8 +75,9 @@ run_runtime_config_change_tests() { fi local runtime_url="https://${RELEASE_NAME}-developer-hub-${NAME_SPACE_RUNTIME}.${K8S_CLUSTER_ROUTER_BASE}" - # Run tests - allow failures since schema-mode tests are opt-in - testing::run_tests "${RELEASE_NAME}" "${NAME_SPACE_RUNTIME}" "${PW_PROJECT_SHOWCASE_RUNTIME}" "${runtime_url}" || true + # Gate on /healthcheck: a test that scales the deployment to 0 mid-rollout + # kills install-dynamic-plugins and orphans its lock on the PVC. + testing::check_and_test "${RELEASE_NAME}" "${NAME_SPACE_RUNTIME}" "${PW_PROJECT_SHOWCASE_RUNTIME}" "${runtime_url}" } run_sanity_plugins_check() { diff --git a/.ci/pipelines/jobs/ocp-operator.sh b/.ci/pipelines/jobs/ocp-operator.sh index cc8d1b24f6..1187d73914 100644 --- a/.ci/pipelines/jobs/ocp-operator.sh +++ b/.ci/pipelines/jobs/ocp-operator.sh @@ -110,7 +110,9 @@ run_operator_runtime_config_change_tests() { fi fi - testing::run_tests "${RELEASE_NAME}" "${NAME_SPACE_RUNTIME}" "${PW_PROJECT_SHOWCASE_RUNTIME}" "${runtime_url}" || true + # Gate on /healthcheck: a test that scales the deployment to 0 mid-rollout + # kills install-dynamic-plugins and orphans its lock on the PVC. + testing::check_and_test "${RELEASE_NAME}" "${NAME_SPACE_RUNTIME}" "${PW_PROJECT_SHOWCASE_RUNTIME}" "${runtime_url}" } handle_ocp_operator() { diff --git a/e2e-tests/playwright/utils/kube-client.ts b/e2e-tests/playwright/utils/kube-client.ts index e1d145789b..700c8867b8 100644 --- a/e2e-tests/playwright/utils/kube-client.ts +++ b/e2e-tests/playwright/utils/kube-client.ts @@ -830,7 +830,7 @@ export class KubeClient { console.log(`Deployment: ${deploymentName}, Namespace: ${namespace}`); await this.logPodConditionsForDeployment(deploymentName, namespace); await this.scaleDeployment(deploymentName, namespace, 0); - await this.waitForDeploymentReady(deploymentName, namespace, 0, 300000); // 5 minutes for scale down + await this.waitForDeploymentReady(deploymentName, namespace, 0, 120000); // 2 minutes for scale down // Wait a bit for pods to be fully terminated console.log("Waiting for pods to be fully terminated..."); @@ -840,7 +840,9 @@ export class KubeClient { console.log(`Scaling up deployment ${deploymentName} to 1 replica.`); await this.scaleDeployment(deploymentName, namespace, 1); - await this.waitForDeploymentReady(deploymentName, namespace, 1, 600000); // 10 minutes for scale up + // 2m + 10s + 7m fits the 10 minute test timeout, so a slow rollout fails + // through the catch below instead of the worker being killed mid-restart. + await this.waitForDeploymentReady(deploymentName, namespace, 1, 420000); // 7 minutes for scale up console.log( `Restart of deployment ${deploymentName} completed successfully.`,