test: hardcode the 1.10.4 RC image, chart and catalog index

.ci/pipelines/env_variables.sh

@@ -11,7 +11,7 @@ LOGFILE="test-log"
 # https://docs.ci.openshift.org/docs/architecture/step-registry/#available-environment-variables
 # https://docs.prow.k8s.io/docs/jobs/#job-environment-variables
 JOB_NAME="${JOB_NAME:-unknown-job}"
-TAG_NAME="${TAG_NAME:-}"
+TAG_NAME="1.10-170" # HARDCODED: 1.10.4 RC verification
 OPENSHIFT_CI="${OPENSHIFT_CI:-false}"
 REPO_OWNER="${REPO_OWNER:-redhat-developer}"
 REPO_NAME="${REPO_NAME:-rhdh}"
@@ -50,15 +50,16 @@ HELM_CHART_SANITY_PLUGINS_DIFF_VALUE_FILE_NAME="diff-values_showcase-sanity-plug
 HELM_CHART_SANITY_PLUGINS_MERGED_VALUE_FILE_NAME="merged-values_showcase-sanity-plugins.yaml"
 
 HELM_CHART_URL="oci://quay.io/rhdh/chart"
+CHART_VERSION="1.10-170-CI" # HARDCODED: 1.10.4 RC verification
 K8S_CLUSTER_TOKEN_ENCODED=$(printf "%s" $K8S_CLUSTER_TOKEN | base64 | tr -d '\n')
-IMAGE_REGISTRY="${IMAGE_REGISTRY:-quay.io}"
-IMAGE_REPO="${IMAGE_REPO:-${QUAY_REPO:-rhdh-community/rhdh}}"
-QUAY_REPO="${IMAGE_REPO}" # Keep QUAY_REPO in sync for backward compatibility
+IMAGE_REGISTRY="quay.io"         # HARDCODED: 1.10.4 RC verification
+IMAGE_REPO="rhdh/rhdh-hub-rhel9" # HARDCODED: 1.10.4 RC verification
+QUAY_REPO="${IMAGE_REPO}"        # Keep QUAY_REPO in sync for backward compatibility
 
 # Catalog index image reference.
 # Override via Gangway for RC (e.g., --catalog-index-image quay.io/rhdh/plugin-catalog-index:1.9-60) or
 # GA verification (e.g., --catalog-index-image registry.access.redhat.com/rhdh/plugin-catalog-index:1.9.4).
-CATALOG_INDEX_IMAGE="${CATALOG_INDEX_IMAGE:-}"
+CATALOG_INDEX_IMAGE="quay.io/rhdh/plugin-catalog-index:1.10-102" # HARDCODED: 1.10.4 RC verification
 if [[ -n "${CATALOG_INDEX_IMAGE}" ]]; then
   # Derived components for Helm chart (requires separate registry/repository/tag)
   CATALOG_INDEX_TAG="${CATALOG_INDEX_IMAGE##*:}"

.ci/pipelines/jobs/ocp-nightly.sh

@@ -75,8 +75,14 @@ run_runtime_config_change_tests() {
   fi
 
   local runtime_url="https://${RELEASE_NAME}-developer-hub-${NAME_SPACE_RUNTIME}.${K8S_CLUSTER_ROUTER_BASE}"
-  # Run tests - allow failures since schema-mode tests are opt-in
-  testing::run_tests "${RELEASE_NAME}" "${NAME_SPACE_RUNTIME}" "${PW_PROJECT_SHOWCASE_RUNTIME}" "${runtime_url}" || true
+  # Wait for the initial rollout before starting the tests. The first
+  # showcase-runtime test scales the deployment to 0; doing that while
+  # install-dynamic-plugins is still running kills the init container mid-install
+  # and leaves install-dynamic-plugins.lock behind on the dynamic-plugins-root
+  # PVC. Every pod created afterwards blocks forever waiting for that lock, so
+  # the job burns its entire budget here and never reaches the sanity checks.
+  # check_and_test gates on /healthcheck and collects pod logs when tests fail.
+  testing::check_and_test "${RELEASE_NAME}" "${NAME_SPACE_RUNTIME}" "${PW_PROJECT_SHOWCASE_RUNTIME}" "${runtime_url}"
 }
 
 run_sanity_plugins_check() {

.ci/pipelines/jobs/ocp-operator.sh

@@ -110,7 +110,12 @@ run_operator_runtime_config_change_tests() {
     fi
   fi
 
-  testing::run_tests "${RELEASE_NAME}" "${NAME_SPACE_RUNTIME}" "${PW_PROJECT_SHOWCASE_RUNTIME}" "${runtime_url}" || true
+  # Wait for the initial rollout before starting the tests. The first
+  # showcase-runtime test scales the deployment to 0; doing that while
+  # install-dynamic-plugins is still running kills the init container mid-install
+  # and leaves install-dynamic-plugins.lock behind on the dynamic-plugins-root
+  # PVC. Every pod created afterwards blocks forever waiting for that lock.
+  testing::check_and_test "${RELEASE_NAME}" "${NAME_SPACE_RUNTIME}" "${PW_PROJECT_SHOWCASE_RUNTIME}" "${runtime_url}"
 }
 
 handle_ocp_operator() {

.ci/pipelines/openshift-ci-tests.sh

@@ -57,11 +57,10 @@ main() {
   log::info "Log file: ${LOGFILE}"
   log::info "JOB_NAME : $JOB_NAME"
 
-  if [[ -z "${CHART_VERSION:-}" ]]; then
-    CHART_VERSION=$(helm::get_chart_version)
-  else
-    log::info "Using preset CHART_VERSION (pinned or from env): ${CHART_VERSION}"
-  fi
+  # HARDCODED: 1.10.4 RC verification. Chart version is pinned in env_variables.sh
+  # and never resolved from the branch.
+  CHART_VERSION="1.10-170-CI"
+  log::info "Using hardcoded CHART_VERSION: ${CHART_VERSION}"
   export CHART_VERSION
 
   case "$JOB_NAME" in

.ci/pipelines/value_files/values_showcase-rbac.yaml

@@ -4,7 +4,7 @@ global:
     image:
       registry: quay.io
       repository: rhdh/plugin-catalog-index
-      tag: "1.10"
+      tag: "1.10-102" # HARDCODED: 1.10.4 RC verification
   dynamic:
     # -- Array of YAML files listing dynamic plugins to include with those listed in the `plugins` field.
     # Relative paths are resolved from the working directory of the initContainer that will install the plugins (`/opt/app-root/src`).

.ci/pipelines/value_files/values_showcase.yaml

@@ -4,7 +4,7 @@ global:
     image:
       registry: quay.io
       repository: rhdh/plugin-catalog-index
-      tag: "1.10"
+      tag: "1.10-102" # HARDCODED: 1.10.4 RC verification
   dynamic:
     # -- Array of YAML files listing dynamic plugins to include with those listed in the `plugins` field.
     # Relative paths are resolved from the working directory of the initContainer that will install the plugins (`/opt/app-root/src`).

e2e-tests/playwright/utils/kube-client.ts

@@ -830,7 +830,7 @@ export class KubeClient {
       console.log(`Deployment: ${deploymentName}, Namespace: ${namespace}`);
       await this.logPodConditionsForDeployment(deploymentName, namespace);
       await this.scaleDeployment(deploymentName, namespace, 0);
-      await this.waitForDeploymentReady(deploymentName, namespace, 0, 300000); // 5 minutes for scale down
+      await this.waitForDeploymentReady(deploymentName, namespace, 0, 120000); // 2 minutes for scale down
 
       // Wait a bit for pods to be fully terminated
       console.log("Waiting for pods to be fully terminated...");
@@ -840,7 +840,12 @@ export class KubeClient {
       console.log(`Scaling up deployment ${deploymentName} to 1 replica.`);
       await this.scaleDeployment(deploymentName, namespace, 1);
 
-      await this.waitForDeploymentReady(deploymentName, namespace, 1, 600000); // 10 minutes for scale up
+      // The budgets above sum to 9m10s so they stay inside the 10 minute test
+      // timeout callers set for a restart. When they exceeded it, Playwright
+      // killed the worker mid-restart: the catch below never ran, so the run
+      // lost its pod conditions and events, and the retry immediately scaled
+      // the deployment down again on top of a rollout that was still starting.
+      await this.waitForDeploymentReady(deploymentName, namespace, 1, 420000); // 7 minutes for scale up
 
       console.log(
         `Restart of deployment ${deploymentName} completed successfully.`,

scripts/install-dynamic-plugins/install-dynamic-plugins.py

@@ -28,6 +28,7 @@
 import atexit
 import time
 import signal
+import threading
 import re
 
 """
@@ -933,7 +934,7 @@
 
     # Create hash file for tracking
     hash_file_path = os.path.join(destination, plugin_path, 'dynamic-plugin-config.hash')
     with open(hash_file_path, 'w') as f:
         f.write(plugin['plugin_hash'])
 
     print(f'\t==> Successfully installed dynamic plugin {package}', flush=True)
@@ -1018,26 +1019,65 @@
     if hash_digest != output.decode('utf-8').strip():
       raise InstallException(f'{package}: The hash of the downloaded package {output.decode("utf-8").strip()} does not match the provided integrity hash {hash_digest} provided in the configuration file')
 
+# How often the process holding the lock refreshes its modification time, and
+# the age at which a waiter treats the lock as abandoned. install-dynamic-plugins.sh
+# forwards SIGTERM (RHDHBUGS-3449) so the lock is released on a graceful shutdown,
+# but a hard kill - grace period expiry, eviction, OOM, node failure - skips the
+# atexit handler. On a persistent dynamic-plugins-root volume the lock file then
+# outlives the container and every later one waits on it forever.
+LOCK_HEARTBEAT_SECONDS = 10
+LOCK_STALE_SECONDS = 60
+LOCK_FILE_NAME = 'install-dynamic-plugins.lock'
+
+# Keep refreshing the lock's mtime so waiters can tell a running install from an
+# abandoned lock. Daemon thread: it must not keep the interpreter alive.
+def start_lock_heartbeat(lock_file_path):
+    def heartbeat():
+      while True:
+        time.sleep(LOCK_HEARTBEAT_SECONDS)
+        try:
+          os.utime(lock_file_path, None)
+        except OSError:
+          return
+    threading.Thread(target=heartbeat, daemon=True).start()
+
 # Create the lock file, so that other instances of the script will wait for this one to finish
 def create_lock(lock_file_path):
     while True:
       try:
         with open(lock_file_path, 'x'):
           print(f"======= Created lock file: {lock_file_path}")
+          start_lock_heartbeat(lock_file_path)
           return
       except FileExistsError:
         wait_for_lock_release(lock_file_path)
 
-# Remove the lock file
+# Remove the lock file. dynamic_plugins_root comes from argv, so validate the
+# basename and rebuild the path from its directory before touching the file
+# system: this only ever deletes a lock file, whatever it is handed.
 def remove_lock(lock_file_path):
-   os.remove(lock_file_path)
+   directory, name = os.path.split(lock_file_path)
+   if name != LOCK_FILE_NAME:
+     raise InstallException(f'Refusing to remove {lock_file_path}: not a {LOCK_FILE_NAME} file')
+   try:
+     os.remove(os.path.join(directory, LOCK_FILE_NAME))
+   except FileNotFoundError:
+     # Either this process never acquired the lock, or a waiter already broke it
+     # as stale. Nothing to clean up.
+     return
    print(f"======= Removed lock file: {lock_file_path}")
 
-# Wait for the lock file to be released
+# Wait for the lock file to be released, or break it when its holder is gone
 def wait_for_lock_release(lock_file_path):
    print(f"======= Waiting for lock release (file: {lock_file_path})...", flush=True)
    while True:
-     if not os.path.exists(lock_file_path):
+     try:
+       age = time.time() - os.path.getmtime(lock_file_path)
+     except FileNotFoundError:
+       break
+     if age > LOCK_STALE_SECONDS:
+       print(f"======= Lock file has not been refreshed for {int(age)}s; assuming its holder was killed and removing it", flush=True)
+       remove_lock(lock_file_path)
        break
      time.sleep(1)
    print("======= Lock released.")
@@ -1081,7 +1121,7 @@
                 if not realpath.startswith(catalog_index_temp_dir):
                     print(f"\t==> WARNING: Skipping link outside archive: {member.name}", flush=True)
                     continue
             tar.extract(member, path=catalog_index_temp_dir, filter='data')
 
 def extract_catalog_index(catalog_index_image: str, catalog_index_mount: str, catalog_entities_parent_dir: str) -> str:
     """Extract the catalog index OCI image and return the path to dynamic-plugins.default.yaml if found."""
@@ -1095,7 +1135,7 @@
     resolved_image = resolve_image_reference(catalog_index_image)
 
     catalog_index_temp_dir = os.path.join(catalog_index_mount, '.catalog-index-temp')
     os.makedirs(catalog_index_temp_dir, exist_ok=True)
 
     with tempfile.TemporaryDirectory() as tmp_dir:
         image_url = resolved_image
@@ -1382,7 +1422,7 @@
 
     dynamic_plugins_root = sys.argv[1]
 
-    lock_file_path = os.path.join(dynamic_plugins_root, 'install-dynamic-plugins.lock')
+    lock_file_path = os.path.join(dynamic_plugins_root, LOCK_FILE_NAME)
     atexit.register(remove_lock, lock_file_path)
     atexit.register(cleanup_catalog_index_temp_dir, dynamic_plugins_root)
     signal.signal(signal.SIGTERM, lambda signum, frame: sys.exit(0))
@@ -1414,7 +1454,7 @@
     # test if file dynamic-plugins.yaml exists
     if not os.path.isfile(dynamic_plugins_file):
         print(f"No {dynamic_plugins_file} file found. Skipping dynamic plugins installation.")
         with open(dynamic_plugins_global_config_file, 'w') as file:
             file.write('')
             file.close()
         exit(0)
@@ -1430,7 +1470,7 @@
 
     if content == '' or content is None:
         print(f"{dynamic_plugins_file} file is empty. Skipping dynamic plugins installation.")
         with open(dynamic_plugins_global_config_file, 'w') as file:
             file.write('')
             file.close()
         exit(0)
@@ -1471,7 +1511,7 @@
             print(f"WARNING: File {include} does not exist, skipping including dynamic packages from {include}", flush=True)
             continue
 
         with open(include, 'r') as file:
             include_content = yaml.safe_load(file)
 
         if not isinstance(include_content, dict):
@@ -1532,7 +1572,7 @@
         if os.path.isdir(dir_path):
             hash_file_path = os.path.join(dir_path, 'dynamic-plugin-config.hash')
             if os.path.isfile(hash_file_path):
                 with open(hash_file_path, 'r') as hash_file:
                     hash_value = hash_file.read().strip()
                     plugin_path_by_hash[hash_value] = dir_name
 
@@ -1544,7 +1584,7 @@
         if plugin_config:
             global_config = maybe_merge_config(plugin_config, global_config)
 
     yaml.safe_dump(global_config, open(dynamic_plugins_global_config_file, 'w'))
 
     # remove plugins that have been removed from the configuration
     for hash_value in plugin_path_by_hash: