From 7ef71306ecfce2e031b8d9ff217b72283f2981f9 Mon Sep 17 00:00:00 2001 From: oddly Date: Wed, 12 Aug 2026 12:48:45 +0200 Subject: [PATCH] fix(elasticsearch): gate the rolling-upgrade health wait on expected node count MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two 'wait for cluster health' points in elasticsearch-rolling-upgrade.yml (before taking a node down, and after it comes back up) only checked '.status'. Because elasticsearch_api_host defaults to localhost, both waits query the local node — and immediately after a restart the local node briefly reports 'number_of_nodes: 1' while peer discovery is still running. In a 2-node mixed-version window the resulting shape is 'status: yellow, unassigned_shards: 1' for the full retry budget (up to 15 min), even though the cluster is fine and just needs another few seconds to reform. Both until clauses now also require number_of_nodes to have caught up to the expected size ('groups[…] | length', with a single-host default so single-node scenarios still pass). No behavioural change once the cluster is actually reformed — this only removes the transient false positive where we accepted an under-strength cluster reading. Reproduces as the repeat upgrade_multi_node (debian12/rockylinux9, elasticsearch_upgrade_8to9) failure that has been showing up on every non-trivial PR since the wait_status: green default landed. --- .../tasks/elasticsearch-rolling-upgrade.yml | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml index 771fba28..6ca8d2c3 100644 --- a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml +++ b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml @@ -154,11 +154,21 @@ validate_certs: "{{ elasticsearch_validate_api_certs }}" force_basic_auth: true register: elasticsearch_response - until: ((elasticsearch_response.json | default({})).status | default('')) in _elasticsearch_upgrade_health_statuses + # Also gate on the expected node count: elasticsearch_api_host defaults + # to localhost, so a just-restarted node briefly reports + # number_of_nodes: 1 before peer discovery finishes. Without this + # guard, a mixed-version 2-node upgrade can lock at status: yellow + + # unassigned_shards: 1 until the health-retry budget times out — + # even though everything is fine, we're just too early. + until: >- + ((elasticsearch_response.json | default({})).status | default('')) in _elasticsearch_upgrade_health_statuses + and ((elasticsearch_response.json | default({})).number_of_nodes | default(0) | int) + >= _elasticsearch_upgrade_expected_nodes | int retries: "{{ elasticsearch_upgrade_health_retries | int }}" delay: "{{ elasticsearch_upgrade_health_delay | int }}" vars: _elasticsearch_upgrade_health_statuses: "{{ ['green'] if elasticsearch_upgrade_wait_status == 'green' else ['green', 'yellow'] }}" + _elasticsearch_upgrade_expected_nodes: "{{ groups[elasticstack_elasticsearch_group_name] | default([inventory_hostname]) | length }}" no_log: "{{ elasticstack_no_log }}" changed_when: false when: _elasticsearch_cluster_reachable | bool @@ -315,13 +325,19 @@ validate_certs: "{{ elasticsearch_validate_api_certs }}" force_basic_auth: true register: elasticsearch_response - until: ((elasticsearch_response.json | default({})).status | default('')) in _elasticsearch_upgrade_health_statuses + # Same expected-nodes guard as the pre-node-down wait — see comment + # there for the flake it prevents. + until: >- + ((elasticsearch_response.json | default({})).status | default('')) in _elasticsearch_upgrade_health_statuses + and ((elasticsearch_response.json | default({})).number_of_nodes | default(0) | int) + >= _elasticsearch_upgrade_expected_nodes | int retries: 30 delay: 30 vars: # Same gate as the pre-node-down health wait so both sides of the # rolling loop honour the same operator preference. _elasticsearch_upgrade_health_statuses: "{{ ['green'] if elasticsearch_upgrade_wait_status == 'green' else ['green', 'yellow'] }}" + _elasticsearch_upgrade_expected_nodes: "{{ groups[elasticstack_elasticsearch_group_name] | default([inventory_hostname]) | length }}" no_log: "{{ elasticstack_no_log }}" changed_when: false