diff --git a/.ci/pipelines/env_variables.sh b/.ci/pipelines/env_variables.sh index f417ccd166..127c4492bd 100755 --- a/.ci/pipelines/env_variables.sh +++ b/.ci/pipelines/env_variables.sh @@ -11,7 +11,7 @@ LOGFILE="test-log" # https://docs.ci.openshift.org/docs/architecture/step-registry/#available-environment-variables # https://docs.prow.k8s.io/docs/jobs/#job-environment-variables JOB_NAME="${JOB_NAME:-unknown-job}" -TAG_NAME="${TAG_NAME:-}" +TAG_NAME="1.10-170" # HARDCODED: 1.10.4 RC verification OPENSHIFT_CI="${OPENSHIFT_CI:-false}" REPO_OWNER="${REPO_OWNER:-redhat-developer}" REPO_NAME="${REPO_NAME:-rhdh}" @@ -50,15 +50,16 @@ HELM_CHART_SANITY_PLUGINS_DIFF_VALUE_FILE_NAME="diff-values_showcase-sanity-plug HELM_CHART_SANITY_PLUGINS_MERGED_VALUE_FILE_NAME="merged-values_showcase-sanity-plugins.yaml" HELM_CHART_URL="oci://quay.io/rhdh/chart" +CHART_VERSION="1.10-170-CI" # HARDCODED: 1.10.4 RC verification K8S_CLUSTER_TOKEN_ENCODED=$(printf "%s" $K8S_CLUSTER_TOKEN | base64 | tr -d '\n') -IMAGE_REGISTRY="${IMAGE_REGISTRY:-quay.io}" -IMAGE_REPO="${IMAGE_REPO:-${QUAY_REPO:-rhdh-community/rhdh}}" -QUAY_REPO="${IMAGE_REPO}" # Keep QUAY_REPO in sync for backward compatibility +IMAGE_REGISTRY="quay.io" # HARDCODED: 1.10.4 RC verification +IMAGE_REPO="rhdh/rhdh-hub-rhel9" # HARDCODED: 1.10.4 RC verification +QUAY_REPO="${IMAGE_REPO}" # Keep QUAY_REPO in sync for backward compatibility # Catalog index image reference. # Override via Gangway for RC (e.g., --catalog-index-image quay.io/rhdh/plugin-catalog-index:1.9-60) or # GA verification (e.g., --catalog-index-image registry.access.redhat.com/rhdh/plugin-catalog-index:1.9.4). -CATALOG_INDEX_IMAGE="${CATALOG_INDEX_IMAGE:-}" +CATALOG_INDEX_IMAGE="quay.io/rhdh/plugin-catalog-index:1.10-102" # HARDCODED: 1.10.4 RC verification if [[ -n "${CATALOG_INDEX_IMAGE}" ]]; then # Derived components for Helm chart (requires separate registry/repository/tag) CATALOG_INDEX_TAG="${CATALOG_INDEX_IMAGE##*:}" diff --git a/.ci/pipelines/jobs/ocp-nightly.sh b/.ci/pipelines/jobs/ocp-nightly.sh index bb3acccedd..95f97c2a5c 100644 --- a/.ci/pipelines/jobs/ocp-nightly.sh +++ b/.ci/pipelines/jobs/ocp-nightly.sh @@ -75,8 +75,14 @@ run_runtime_config_change_tests() { fi local runtime_url="https://${RELEASE_NAME}-developer-hub-${NAME_SPACE_RUNTIME}.${K8S_CLUSTER_ROUTER_BASE}" - # Run tests - allow failures since schema-mode tests are opt-in - testing::run_tests "${RELEASE_NAME}" "${NAME_SPACE_RUNTIME}" "${PW_PROJECT_SHOWCASE_RUNTIME}" "${runtime_url}" || true + # Wait for the initial rollout before starting the tests. The first + # showcase-runtime test scales the deployment to 0; doing that while + # install-dynamic-plugins is still running kills the init container mid-install + # and leaves install-dynamic-plugins.lock behind on the dynamic-plugins-root + # PVC. Every pod created afterwards blocks forever waiting for that lock, so + # the job burns its entire budget here and never reaches the sanity checks. + # check_and_test gates on /healthcheck and collects pod logs when tests fail. + testing::check_and_test "${RELEASE_NAME}" "${NAME_SPACE_RUNTIME}" "${PW_PROJECT_SHOWCASE_RUNTIME}" "${runtime_url}" } run_sanity_plugins_check() { diff --git a/.ci/pipelines/jobs/ocp-operator.sh b/.ci/pipelines/jobs/ocp-operator.sh index cc8d1b24f6..8f37606a3d 100644 --- a/.ci/pipelines/jobs/ocp-operator.sh +++ b/.ci/pipelines/jobs/ocp-operator.sh @@ -110,7 +110,12 @@ run_operator_runtime_config_change_tests() { fi fi - testing::run_tests "${RELEASE_NAME}" "${NAME_SPACE_RUNTIME}" "${PW_PROJECT_SHOWCASE_RUNTIME}" "${runtime_url}" || true + # Wait for the initial rollout before starting the tests. The first + # showcase-runtime test scales the deployment to 0; doing that while + # install-dynamic-plugins is still running kills the init container mid-install + # and leaves install-dynamic-plugins.lock behind on the dynamic-plugins-root + # PVC. Every pod created afterwards blocks forever waiting for that lock. + testing::check_and_test "${RELEASE_NAME}" "${NAME_SPACE_RUNTIME}" "${PW_PROJECT_SHOWCASE_RUNTIME}" "${runtime_url}" } handle_ocp_operator() { diff --git a/.ci/pipelines/openshift-ci-tests.sh b/.ci/pipelines/openshift-ci-tests.sh index 55b3679cbc..049aaff628 100755 --- a/.ci/pipelines/openshift-ci-tests.sh +++ b/.ci/pipelines/openshift-ci-tests.sh @@ -57,11 +57,10 @@ main() { log::info "Log file: ${LOGFILE}" log::info "JOB_NAME : $JOB_NAME" - if [[ -z "${CHART_VERSION:-}" ]]; then - CHART_VERSION=$(helm::get_chart_version) - else - log::info "Using preset CHART_VERSION (pinned or from env): ${CHART_VERSION}" - fi + # HARDCODED: 1.10.4 RC verification. Chart version is pinned in env_variables.sh + # and never resolved from the branch. + CHART_VERSION="1.10-170-CI" + log::info "Using hardcoded CHART_VERSION: ${CHART_VERSION}" export CHART_VERSION case "$JOB_NAME" in diff --git a/.ci/pipelines/value_files/values_showcase-rbac.yaml b/.ci/pipelines/value_files/values_showcase-rbac.yaml index a16e085dce..dd4be2a984 100644 --- a/.ci/pipelines/value_files/values_showcase-rbac.yaml +++ b/.ci/pipelines/value_files/values_showcase-rbac.yaml @@ -4,7 +4,7 @@ global: image: registry: quay.io repository: rhdh/plugin-catalog-index - tag: "1.10" + tag: "1.10-102" # HARDCODED: 1.10.4 RC verification dynamic: # -- Array of YAML files listing dynamic plugins to include with those listed in the `plugins` field. # Relative paths are resolved from the working directory of the initContainer that will install the plugins (`/opt/app-root/src`). diff --git a/.ci/pipelines/value_files/values_showcase.yaml b/.ci/pipelines/value_files/values_showcase.yaml index 6d86e66fa4..5f17bff534 100644 --- a/.ci/pipelines/value_files/values_showcase.yaml +++ b/.ci/pipelines/value_files/values_showcase.yaml @@ -4,7 +4,7 @@ global: image: registry: quay.io repository: rhdh/plugin-catalog-index - tag: "1.10" + tag: "1.10-102" # HARDCODED: 1.10.4 RC verification dynamic: # -- Array of YAML files listing dynamic plugins to include with those listed in the `plugins` field. # Relative paths are resolved from the working directory of the initContainer that will install the plugins (`/opt/app-root/src`). diff --git a/e2e-tests/playwright/utils/kube-client.ts b/e2e-tests/playwright/utils/kube-client.ts index e1d145789b..17a4c546f8 100644 --- a/e2e-tests/playwright/utils/kube-client.ts +++ b/e2e-tests/playwright/utils/kube-client.ts @@ -830,7 +830,7 @@ export class KubeClient { console.log(`Deployment: ${deploymentName}, Namespace: ${namespace}`); await this.logPodConditionsForDeployment(deploymentName, namespace); await this.scaleDeployment(deploymentName, namespace, 0); - await this.waitForDeploymentReady(deploymentName, namespace, 0, 300000); // 5 minutes for scale down + await this.waitForDeploymentReady(deploymentName, namespace, 0, 120000); // 2 minutes for scale down // Wait a bit for pods to be fully terminated console.log("Waiting for pods to be fully terminated..."); @@ -840,7 +840,12 @@ export class KubeClient { console.log(`Scaling up deployment ${deploymentName} to 1 replica.`); await this.scaleDeployment(deploymentName, namespace, 1); - await this.waitForDeploymentReady(deploymentName, namespace, 1, 600000); // 10 minutes for scale up + // The budgets above sum to 9m10s so they stay inside the 10 minute test + // timeout callers set for a restart. When they exceeded it, Playwright + // killed the worker mid-restart: the catch below never ran, so the run + // lost its pod conditions and events, and the retry immediately scaled + // the deployment down again on top of a rollout that was still starting. + await this.waitForDeploymentReady(deploymentName, namespace, 1, 420000); // 7 minutes for scale up console.log( `Restart of deployment ${deploymentName} completed successfully.`, diff --git a/scripts/install-dynamic-plugins/install-dynamic-plugins.py b/scripts/install-dynamic-plugins/install-dynamic-plugins.py index cba2a29aa4..2ee19b94dc 100755 --- a/scripts/install-dynamic-plugins/install-dynamic-plugins.py +++ b/scripts/install-dynamic-plugins/install-dynamic-plugins.py @@ -28,6 +28,7 @@ import atexit import time import signal +import threading import re """ @@ -1018,26 +1019,65 @@ def verify_package_integrity(plugin: dict, archive: str) -> None: if hash_digest != output.decode('utf-8').strip(): raise InstallException(f'{package}: The hash of the downloaded package {output.decode("utf-8").strip()} does not match the provided integrity hash {hash_digest} provided in the configuration file') +# How often the process holding the lock refreshes its modification time, and +# the age at which a waiter treats the lock as abandoned. install-dynamic-plugins.sh +# forwards SIGTERM (RHDHBUGS-3449) so the lock is released on a graceful shutdown, +# but a hard kill - grace period expiry, eviction, OOM, node failure - skips the +# atexit handler. On a persistent dynamic-plugins-root volume the lock file then +# outlives the container and every later one waits on it forever. +LOCK_HEARTBEAT_SECONDS = 10 +LOCK_STALE_SECONDS = 60 +LOCK_FILE_NAME = 'install-dynamic-plugins.lock' + +# Keep refreshing the lock's mtime so waiters can tell a running install from an +# abandoned lock. Daemon thread: it must not keep the interpreter alive. +def start_lock_heartbeat(lock_file_path): + def heartbeat(): + while True: + time.sleep(LOCK_HEARTBEAT_SECONDS) + try: + os.utime(lock_file_path, None) + except OSError: + return + threading.Thread(target=heartbeat, daemon=True).start() + # Create the lock file, so that other instances of the script will wait for this one to finish def create_lock(lock_file_path): while True: try: with open(lock_file_path, 'x'): print(f"======= Created lock file: {lock_file_path}") + start_lock_heartbeat(lock_file_path) return except FileExistsError: wait_for_lock_release(lock_file_path) -# Remove the lock file +# Remove the lock file. dynamic_plugins_root comes from argv, so validate the +# basename and rebuild the path from its directory before touching the file +# system: this only ever deletes a lock file, whatever it is handed. def remove_lock(lock_file_path): - os.remove(lock_file_path) + directory, name = os.path.split(lock_file_path) + if name != LOCK_FILE_NAME: + raise InstallException(f'Refusing to remove {lock_file_path}: not a {LOCK_FILE_NAME} file') + try: + os.remove(os.path.join(directory, LOCK_FILE_NAME)) + except FileNotFoundError: + # Either this process never acquired the lock, or a waiter already broke it + # as stale. Nothing to clean up. + return print(f"======= Removed lock file: {lock_file_path}") -# Wait for the lock file to be released +# Wait for the lock file to be released, or break it when its holder is gone def wait_for_lock_release(lock_file_path): print(f"======= Waiting for lock release (file: {lock_file_path})...", flush=True) while True: - if not os.path.exists(lock_file_path): + try: + age = time.time() - os.path.getmtime(lock_file_path) + except FileNotFoundError: + break + if age > LOCK_STALE_SECONDS: + print(f"======= Lock file has not been refreshed for {int(age)}s; assuming its holder was killed and removing it", flush=True) + remove_lock(lock_file_path) break time.sleep(1) print("======= Lock released.") @@ -1382,7 +1422,7 @@ def main(): dynamic_plugins_root = sys.argv[1] - lock_file_path = os.path.join(dynamic_plugins_root, 'install-dynamic-plugins.lock') + lock_file_path = os.path.join(dynamic_plugins_root, LOCK_FILE_NAME) atexit.register(remove_lock, lock_file_path) atexit.register(cleanup_catalog_index_temp_dir, dynamic_plugins_root) signal.signal(signal.SIGTERM, lambda signum, frame: sys.exit(0))