From bcf5f1c5e831a013efee69051005fe850c37a3e3 Mon Sep 17 00:00:00 2001 From: Gustavo Lira e Silva Date: Wed, 2 Sep 2026 13:52:02 -0300 Subject: [PATCH 1/5] test: hardcode the 1.10.4 RC image, chart and catalog index Pin every image reference the nightly jobs resolve at runtime so a run always exercises the 1.10.4 RC, regardless of what Gangway or the environment passes: - TAG_NAME=1.10-170 - IMAGE_REGISTRY=quay.io - IMAGE_REPO=rhdh/rhdh-hub-rhel9 - CHART_VERSION=1.10-170-CI - CATALOG_INDEX_IMAGE=quay.io/rhdh/plugin-catalog-index:1.10-102 openshift-ci-tests.sh no longer resolves the chart version from the branch. Cluster configuration is untouched; the cluster still comes from the pool. Not for merge. Verification aid for RHIDP-16019. --- .ci/pipelines/env_variables.sh | 9 +++++---- .ci/pipelines/openshift-ci-tests.sh | 9 ++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.ci/pipelines/env_variables.sh b/.ci/pipelines/env_variables.sh index f417ccd166..b8276bf8de 100755 --- a/.ci/pipelines/env_variables.sh +++ b/.ci/pipelines/env_variables.sh @@ -11,7 +11,7 @@ LOGFILE="test-log" # https://docs.ci.openshift.org/docs/architecture/step-registry/#available-environment-variables # https://docs.prow.k8s.io/docs/jobs/#job-environment-variables JOB_NAME="${JOB_NAME:-unknown-job}" -TAG_NAME="${TAG_NAME:-}" +TAG_NAME="1.10-170" # HARDCODED: 1.10.4 RC verification OPENSHIFT_CI="${OPENSHIFT_CI:-false}" REPO_OWNER="${REPO_OWNER:-redhat-developer}" REPO_NAME="${REPO_NAME:-rhdh}" @@ -50,15 +50,16 @@ HELM_CHART_SANITY_PLUGINS_DIFF_VALUE_FILE_NAME="diff-values_showcase-sanity-plug HELM_CHART_SANITY_PLUGINS_MERGED_VALUE_FILE_NAME="merged-values_showcase-sanity-plugins.yaml" HELM_CHART_URL="oci://quay.io/rhdh/chart" +CHART_VERSION="1.10-170-CI" # HARDCODED: 1.10.4 RC verification K8S_CLUSTER_TOKEN_ENCODED=$(printf "%s" $K8S_CLUSTER_TOKEN | base64 | tr -d '\n') -IMAGE_REGISTRY="${IMAGE_REGISTRY:-quay.io}" -IMAGE_REPO="${IMAGE_REPO:-${QUAY_REPO:-rhdh-community/rhdh}}" +IMAGE_REGISTRY="quay.io" # HARDCODED: 1.10.4 RC verification +IMAGE_REPO="rhdh/rhdh-hub-rhel9" # HARDCODED: 1.10.4 RC verification QUAY_REPO="${IMAGE_REPO}" # Keep QUAY_REPO in sync for backward compatibility # Catalog index image reference. # Override via Gangway for RC (e.g., --catalog-index-image quay.io/rhdh/plugin-catalog-index:1.9-60) or # GA verification (e.g., --catalog-index-image registry.access.redhat.com/rhdh/plugin-catalog-index:1.9.4). -CATALOG_INDEX_IMAGE="${CATALOG_INDEX_IMAGE:-}" +CATALOG_INDEX_IMAGE="quay.io/rhdh/plugin-catalog-index:1.10-102" # HARDCODED: 1.10.4 RC verification if [[ -n "${CATALOG_INDEX_IMAGE}" ]]; then # Derived components for Helm chart (requires separate registry/repository/tag) CATALOG_INDEX_TAG="${CATALOG_INDEX_IMAGE##*:}" diff --git a/.ci/pipelines/openshift-ci-tests.sh b/.ci/pipelines/openshift-ci-tests.sh index 55b3679cbc..049aaff628 100755 --- a/.ci/pipelines/openshift-ci-tests.sh +++ b/.ci/pipelines/openshift-ci-tests.sh @@ -57,11 +57,10 @@ main() { log::info "Log file: ${LOGFILE}" log::info "JOB_NAME : $JOB_NAME" - if [[ -z "${CHART_VERSION:-}" ]]; then - CHART_VERSION=$(helm::get_chart_version) - else - log::info "Using preset CHART_VERSION (pinned or from env): ${CHART_VERSION}" - fi + # HARDCODED: 1.10.4 RC verification. Chart version is pinned in env_variables.sh + # and never resolved from the branch. + CHART_VERSION="1.10-170-CI" + log::info "Using hardcoded CHART_VERSION: ${CHART_VERSION}" export CHART_VERSION case "$JOB_NAME" in From 46dea7f06bb68dba60b340b449d52314992e26fb Mon Sep 17 00:00:00 2001 From: Gustavo Lira e Silva Date: Wed, 2 Sep 2026 13:57:51 -0300 Subject: [PATCH 2/5] test: pin the catalog index tag in the showcase value files [skip-build] The AKS, EKS and GKE deployment scripts build their own helm upgrade command and set only upstream.backstage.image.*. They never set global.catalogIndex.image.*, so on those platforms the value file decides the catalog index and the floating "1.10" tag wins over the RC. Pin it to 1.10-102 in values_showcase.yaml and values_showcase-rbac.yaml. The OCP Helm and Operator paths already pass the same value through helm::get_image_params and operator.sh, so nothing conflicts. This PR changes only CI scripts and value files, so the container image does not need rebuilding. --- .ci/pipelines/value_files/values_showcase-rbac.yaml | 2 +- .ci/pipelines/value_files/values_showcase.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/pipelines/value_files/values_showcase-rbac.yaml b/.ci/pipelines/value_files/values_showcase-rbac.yaml index a16e085dce..dd4be2a984 100644 --- a/.ci/pipelines/value_files/values_showcase-rbac.yaml +++ b/.ci/pipelines/value_files/values_showcase-rbac.yaml @@ -4,7 +4,7 @@ global: image: registry: quay.io repository: rhdh/plugin-catalog-index - tag: "1.10" + tag: "1.10-102" # HARDCODED: 1.10.4 RC verification dynamic: # -- Array of YAML files listing dynamic plugins to include with those listed in the `plugins` field. # Relative paths are resolved from the working directory of the initContainer that will install the plugins (`/opt/app-root/src`). diff --git a/.ci/pipelines/value_files/values_showcase.yaml b/.ci/pipelines/value_files/values_showcase.yaml index 6d86e66fa4..5f17bff534 100644 --- a/.ci/pipelines/value_files/values_showcase.yaml +++ b/.ci/pipelines/value_files/values_showcase.yaml @@ -4,7 +4,7 @@ global: image: registry: quay.io repository: rhdh/plugin-catalog-index - tag: "1.10" + tag: "1.10-102" # HARDCODED: 1.10.4 RC verification dynamic: # -- Array of YAML files listing dynamic plugins to include with those listed in the `plugins` field. # Relative paths are resolved from the working directory of the initContainer that will install the plugins (`/opt/app-root/src`). From 4fa2916bc89a82589bff7ec3d2c3d9f20b37b06d Mon Sep 17 00:00:00 2001 From: Gustavo Lira e Silva Date: Wed, 2 Sep 2026 21:34:08 -0300 Subject: [PATCH 3/5] fix(ci): wait for the showcase-runtime rollout before running its tests The showcase-runtime phase started Playwright as soon as `helm upgrade` returned, without waiting for the deployment to become ready. The first test in the `showcase-runtime-db` dependency project scales the deployment to 0, which killed the `install-dynamic-plugins` init container while it was still installing. That script takes a lock file on the `dynamic-plugins-root` PVC and releases it through an `atexit` handler, which does not run when the container is killed. The lock survived on the PVC, and every pod created afterwards blocked forever on: ======= Waiting for lock release (file: /dynamic-plugins-root/install-dynamic-plugins.lock)... Each Azure DB test then burned its full 10 minute timeout waiting for a pod that could never become ready. Ten attempts consumed 100 minutes and the job hit the Prow timeout inside this phase, so `showcase-sanity-plugins` never ran. Swap `testing::run_tests` for `testing::check_and_test` in both the OCP Helm and OCP Operator runtime phases. It gates on `/healthcheck` before starting the tests, so the initial plugin install always completes before anything scales the deployment down, and it collects pod logs when the tests fail. Also reformat two comment alignments in env_variables.sh that `yarn prettier:check` flags. Co-Authored-By: Claude Opus 5 (1M context) --- .ci/pipelines/env_variables.sh | 4 ++-- .ci/pipelines/jobs/ocp-nightly.sh | 10 ++++++++-- .ci/pipelines/jobs/ocp-operator.sh | 7 ++++++- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/.ci/pipelines/env_variables.sh b/.ci/pipelines/env_variables.sh index b8276bf8de..127c4492bd 100755 --- a/.ci/pipelines/env_variables.sh +++ b/.ci/pipelines/env_variables.sh @@ -52,9 +52,9 @@ HELM_CHART_SANITY_PLUGINS_MERGED_VALUE_FILE_NAME="merged-values_showcase-sanity- HELM_CHART_URL="oci://quay.io/rhdh/chart" CHART_VERSION="1.10-170-CI" # HARDCODED: 1.10.4 RC verification K8S_CLUSTER_TOKEN_ENCODED=$(printf "%s" $K8S_CLUSTER_TOKEN | base64 | tr -d '\n') -IMAGE_REGISTRY="quay.io" # HARDCODED: 1.10.4 RC verification +IMAGE_REGISTRY="quay.io" # HARDCODED: 1.10.4 RC verification IMAGE_REPO="rhdh/rhdh-hub-rhel9" # HARDCODED: 1.10.4 RC verification -QUAY_REPO="${IMAGE_REPO}" # Keep QUAY_REPO in sync for backward compatibility +QUAY_REPO="${IMAGE_REPO}" # Keep QUAY_REPO in sync for backward compatibility # Catalog index image reference. # Override via Gangway for RC (e.g., --catalog-index-image quay.io/rhdh/plugin-catalog-index:1.9-60) or diff --git a/.ci/pipelines/jobs/ocp-nightly.sh b/.ci/pipelines/jobs/ocp-nightly.sh index bb3acccedd..95f97c2a5c 100644 --- a/.ci/pipelines/jobs/ocp-nightly.sh +++ b/.ci/pipelines/jobs/ocp-nightly.sh @@ -75,8 +75,14 @@ run_runtime_config_change_tests() { fi local runtime_url="https://${RELEASE_NAME}-developer-hub-${NAME_SPACE_RUNTIME}.${K8S_CLUSTER_ROUTER_BASE}" - # Run tests - allow failures since schema-mode tests are opt-in - testing::run_tests "${RELEASE_NAME}" "${NAME_SPACE_RUNTIME}" "${PW_PROJECT_SHOWCASE_RUNTIME}" "${runtime_url}" || true + # Wait for the initial rollout before starting the tests. The first + # showcase-runtime test scales the deployment to 0; doing that while + # install-dynamic-plugins is still running kills the init container mid-install + # and leaves install-dynamic-plugins.lock behind on the dynamic-plugins-root + # PVC. Every pod created afterwards blocks forever waiting for that lock, so + # the job burns its entire budget here and never reaches the sanity checks. + # check_and_test gates on /healthcheck and collects pod logs when tests fail. + testing::check_and_test "${RELEASE_NAME}" "${NAME_SPACE_RUNTIME}" "${PW_PROJECT_SHOWCASE_RUNTIME}" "${runtime_url}" } run_sanity_plugins_check() { diff --git a/.ci/pipelines/jobs/ocp-operator.sh b/.ci/pipelines/jobs/ocp-operator.sh index cc8d1b24f6..8f37606a3d 100644 --- a/.ci/pipelines/jobs/ocp-operator.sh +++ b/.ci/pipelines/jobs/ocp-operator.sh @@ -110,7 +110,12 @@ run_operator_runtime_config_change_tests() { fi fi - testing::run_tests "${RELEASE_NAME}" "${NAME_SPACE_RUNTIME}" "${PW_PROJECT_SHOWCASE_RUNTIME}" "${runtime_url}" || true + # Wait for the initial rollout before starting the tests. The first + # showcase-runtime test scales the deployment to 0; doing that while + # install-dynamic-plugins is still running kills the init container mid-install + # and leaves install-dynamic-plugins.lock behind on the dynamic-plugins-root + # PVC. Every pod created afterwards blocks forever waiting for that lock. + testing::check_and_test "${RELEASE_NAME}" "${NAME_SPACE_RUNTIME}" "${PW_PROJECT_SHOWCASE_RUNTIME}" "${runtime_url}" } handle_ocp_operator() { From a7e54d28d40a1d192aebd64cf3b15ff5a1add0fd Mon Sep 17 00:00:00 2001 From: Gustavo Lira e Silva Date: Thu, 3 Sep 2026 15:07:18 -0300 Subject: [PATCH 4/5] fix: recover from an orphaned install-dynamic-plugins lock install-dynamic-plugins.py takes a lock file in dynamic-plugins-root and releases it from an atexit handler. install-dynamic-plugins.sh forwards SIGTERM to the process group (RHDHBUGS-3449), so a graceful shutdown still releases it, but a hard kill - grace period expiry, eviction, OOM, node failure - skips atexit entirely. When dynamic-plugins-root is a PVC the lock file outlives the container, and wait_for_lock_release is an unbounded `while True` with no staleness check, so every later container blocks on it forever. The showcase-runtime phase hit exactly this: the first test scaled the deployment down while the initial install was still running, and every pod after that printed only ======= Waiting for lock release (file: /dynamic-plugins-root/install-dynamic-plugins.lock)... The holder now refreshes the lock's mtime from a daemon thread every 10s, and a waiter that sees no refresh for 60s reports it and breaks the lock. remove_lock tolerates a missing file, which also silences the spurious FileNotFoundError when the process exits before acquiring the lock. Separately, align restartDeployment's internal budgets with the 10 minute timeout its callers set: 5m + 10s + 10m could not fit, so Playwright killed the worker mid-restart, the catch block never logged pod conditions or events, and the retry scaled the deployment down again on top of a rollout that was still starting. 2m + 10s + 7m fits. Co-Authored-By: Claude Opus 5 (1M context) --- e2e-tests/playwright/utils/kube-client.ts | 9 +++- .../install-dynamic-plugins.py | 43 +++++++++++++++++-- 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/e2e-tests/playwright/utils/kube-client.ts b/e2e-tests/playwright/utils/kube-client.ts index e1d145789b..17a4c546f8 100644 --- a/e2e-tests/playwright/utils/kube-client.ts +++ b/e2e-tests/playwright/utils/kube-client.ts @@ -830,7 +830,7 @@ export class KubeClient { console.log(`Deployment: ${deploymentName}, Namespace: ${namespace}`); await this.logPodConditionsForDeployment(deploymentName, namespace); await this.scaleDeployment(deploymentName, namespace, 0); - await this.waitForDeploymentReady(deploymentName, namespace, 0, 300000); // 5 minutes for scale down + await this.waitForDeploymentReady(deploymentName, namespace, 0, 120000); // 2 minutes for scale down // Wait a bit for pods to be fully terminated console.log("Waiting for pods to be fully terminated..."); @@ -840,7 +840,12 @@ export class KubeClient { console.log(`Scaling up deployment ${deploymentName} to 1 replica.`); await this.scaleDeployment(deploymentName, namespace, 1); - await this.waitForDeploymentReady(deploymentName, namespace, 1, 600000); // 10 minutes for scale up + // The budgets above sum to 9m10s so they stay inside the 10 minute test + // timeout callers set for a restart. When they exceeded it, Playwright + // killed the worker mid-restart: the catch below never ran, so the run + // lost its pod conditions and events, and the retry immediately scaled + // the deployment down again on top of a rollout that was still starting. + await this.waitForDeploymentReady(deploymentName, namespace, 1, 420000); // 7 minutes for scale up console.log( `Restart of deployment ${deploymentName} completed successfully.`, diff --git a/scripts/install-dynamic-plugins/install-dynamic-plugins.py b/scripts/install-dynamic-plugins/install-dynamic-plugins.py index cba2a29aa4..c04bc839a0 100755 --- a/scripts/install-dynamic-plugins/install-dynamic-plugins.py +++ b/scripts/install-dynamic-plugins/install-dynamic-plugins.py @@ -28,6 +28,7 @@ import atexit import time import signal +import threading import re """ @@ -1018,26 +1019,62 @@ def verify_package_integrity(plugin: dict, archive: str) -> None: if hash_digest != output.decode('utf-8').strip(): raise InstallException(f'{package}: The hash of the downloaded package {output.decode("utf-8").strip()} does not match the provided integrity hash {hash_digest} provided in the configuration file') +# How often the process holding the lock refreshes its modification time, and +# the age at which a waiter treats the lock as abandoned. install-dynamic-plugins.sh +# forwards SIGTERM (RHDHBUGS-3449) so the lock is released on a graceful shutdown, +# but a hard kill - grace period expiry, eviction, OOM, node failure - skips the +# atexit handler. On a persistent dynamic-plugins-root volume the lock file then +# outlives the container and every later one waits on it forever. +LOCK_HEARTBEAT_SECONDS = 10 +LOCK_STALE_SECONDS = 60 + +# Keep refreshing the lock's mtime so waiters can tell a running install from an +# abandoned lock. Daemon thread: it must not keep the interpreter alive. +def start_lock_heartbeat(lock_file_path): + def heartbeat(): + while True: + time.sleep(LOCK_HEARTBEAT_SECONDS) + try: + os.utime(lock_file_path, None) + except OSError: + return + threading.Thread(target=heartbeat, daemon=True).start() + # Create the lock file, so that other instances of the script will wait for this one to finish def create_lock(lock_file_path): while True: try: with open(lock_file_path, 'x'): print(f"======= Created lock file: {lock_file_path}") + start_lock_heartbeat(lock_file_path) return except FileExistsError: wait_for_lock_release(lock_file_path) # Remove the lock file def remove_lock(lock_file_path): - os.remove(lock_file_path) + try: + os.remove(lock_file_path) + except FileNotFoundError: + # Either this process never acquired the lock, or a waiter already broke it + # as stale. Nothing to clean up. + return print(f"======= Removed lock file: {lock_file_path}") -# Wait for the lock file to be released +# Wait for the lock file to be released, or break it when its holder is gone def wait_for_lock_release(lock_file_path): print(f"======= Waiting for lock release (file: {lock_file_path})...", flush=True) while True: - if not os.path.exists(lock_file_path): + try: + age = time.time() - os.path.getmtime(lock_file_path) + except FileNotFoundError: + break + if age > LOCK_STALE_SECONDS: + print(f"======= Lock file has not been refreshed for {int(age)}s; assuming its holder was killed and removing it", flush=True) + try: + os.remove(lock_file_path) + except FileNotFoundError: + pass break time.sleep(1) print("======= Lock released.") From b5bdb4e80ab2fd469a870f3e0c7d56b81240e3a9 Mon Sep 17 00:00:00 2001 From: Gustavo Lira e Silva Date: Thu, 3 Sep 2026 15:12:33 -0300 Subject: [PATCH 5/5] fix: validate the lock file name before removing it SonarCloud flagged the new os.remove in wait_for_lock_release: the path descends from argv, so a caller could in principle point it anywhere. Route both removals through remove_lock, which now rejects any path whose basename is not install-dynamic-plugins.lock and rebuilds the path from the validated directory. There is a single os.remove left, and it can only ever delete a lock file. Co-Authored-By: Claude Opus 5 (1M context) --- .../install-dynamic-plugins.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/scripts/install-dynamic-plugins/install-dynamic-plugins.py b/scripts/install-dynamic-plugins/install-dynamic-plugins.py index c04bc839a0..2ee19b94dc 100755 --- a/scripts/install-dynamic-plugins/install-dynamic-plugins.py +++ b/scripts/install-dynamic-plugins/install-dynamic-plugins.py @@ -1027,6 +1027,7 @@ def verify_package_integrity(plugin: dict, archive: str) -> None: # outlives the container and every later one waits on it forever. LOCK_HEARTBEAT_SECONDS = 10 LOCK_STALE_SECONDS = 60 +LOCK_FILE_NAME = 'install-dynamic-plugins.lock' # Keep refreshing the lock's mtime so waiters can tell a running install from an # abandoned lock. Daemon thread: it must not keep the interpreter alive. @@ -1051,10 +1052,15 @@ def create_lock(lock_file_path): except FileExistsError: wait_for_lock_release(lock_file_path) -# Remove the lock file +# Remove the lock file. dynamic_plugins_root comes from argv, so validate the +# basename and rebuild the path from its directory before touching the file +# system: this only ever deletes a lock file, whatever it is handed. def remove_lock(lock_file_path): + directory, name = os.path.split(lock_file_path) + if name != LOCK_FILE_NAME: + raise InstallException(f'Refusing to remove {lock_file_path}: not a {LOCK_FILE_NAME} file') try: - os.remove(lock_file_path) + os.remove(os.path.join(directory, LOCK_FILE_NAME)) except FileNotFoundError: # Either this process never acquired the lock, or a waiter already broke it # as stale. Nothing to clean up. @@ -1071,10 +1077,7 @@ def wait_for_lock_release(lock_file_path): break if age > LOCK_STALE_SECONDS: print(f"======= Lock file has not been refreshed for {int(age)}s; assuming its holder was killed and removing it", flush=True) - try: - os.remove(lock_file_path) - except FileNotFoundError: - pass + remove_lock(lock_file_path) break time.sleep(1) print("======= Lock released.") @@ -1419,7 +1422,7 @@ def main(): dynamic_plugins_root = sys.argv[1] - lock_file_path = os.path.join(dynamic_plugins_root, 'install-dynamic-plugins.lock') + lock_file_path = os.path.join(dynamic_plugins_root, LOCK_FILE_NAME) atexit.register(remove_lock, lock_file_path) atexit.register(cleanup_catalog_index_temp_dir, dynamic_plugins_root) signal.signal(signal.SIGTERM, lambda signum, frame: sys.exit(0))