chore(ci): fail CI when defaults/main.yml drifts from argument_specs.yml

.ansible-lint

@@ -10,7 +10,6 @@ exclude_paths:
 warn_list:
   - experimental
   - key-order[task]
-  - name[casing]
   - name[missing]
   - package-latest
   - schema[meta]
@@ -19,6 +18,12 @@ skip_list:
   - command-instead-of-module
   - galaxy[no-changelog]
   - line-length
+  # The repo uses `role_prefix | Verb` task names (lowercase prefix, uppercase
+  # verb) — matches the commit-message convention. This rule was in warn_list
+  # but ansible-lint still counts warnings against the moderate profile, so
+  # lint_full failed on the pre-existing cert_detect_content_mode tasks any
+  # time the changes filter marked their role for linting.
+  - name[casing]
   - no-handler
   - package-latest
   - role-name

.github/workflows/test_full_stack.yml

@@ -95,9 +95,14 @@ jobs:
       # the shared incus-ci host. The memory-capacity gate in shared/create.yml
       # eventually admits every job, but without a matrix-level cap the full
       # 16-combo PR matrix would swamp the 131 GB host and starve the biggest
-      # scenarios (elasticstack_default, cert_renewal). 6 concurrent slots keep
-      # committed memory around 90-100 GB with head-room to spare.
-      max-parallel: 6
+      # scenarios (elasticstack_default, cert_renewal). 6 concurrent slots was
+      # the previous target, sized for the intended 90-100 GB steady state.
+      # Observed reality after landing several PRs concurrently: with all six
+      # slots filled, the 20 GB + 13.8 GB scenarios still lose the memory
+      # race and time out at the 45-min gate deadline. Halving to 3 keeps
+      # committed memory well under host capacity and lets the heavy jobs
+      # actually acquire without starvation. Wall clock roughly doubles.
+      max-parallel: 3
       matrix:
         # Standardise on rockylinux10 (not 9) for PR runs to match the rest
         # of the workflows. Rocky 10 exercises the EL≥9 branch in

.github/workflows/test_linting.yml

@@ -84,3 +84,11 @@ jobs:
         env:
           ROLENAME: ${{ inputs.rolename }}
         if: ${{ inputs.rolename != '' }}
+
+      # Fails if a role's meta/argument_specs.yml is out of sync with its
+      # defaults/main.yml — catches "I added a var but forgot to update the
+      # spec" and its inverse. See scripts/gen_argspecs.py for regeneration.
+      - name: Check argument_specs drift.
+        run: |
+          python3 scripts/check_argspecs.py
+        if: ${{ inputs.rolename == '' }}

roles/elasticsearch/meta/argument_specs.yml

@@ -340,6 +340,33 @@ argument_specs:
           set to true from a playbook to force renewal regardless of buffer.
         type: bool
         default: false
+      elasticsearch_maintenance_wait_status:
+        description: Minimum cluster health status the node maintenance entry points wait for. Use yellow or green.
+        type: str
+        default: green
+        choices: [green, yellow]
+      elasticsearch_maintenance_health_retries:
+        description: Number of cluster health polling attempts in the node maintenance entry points.
+        type: int
+        default: 60
+      elasticsearch_maintenance_health_delay:
+        description: Delay in seconds between cluster health polling attempts in the node maintenance entry points.
+        type: int
+        default: 30
+      elasticsearch_maintenance_wait_health:
+        description: Wait for cluster health at the end of node maintenance. Disable for a defensive state reset at the start of a run.
+        type: bool
+        default: true
+      elasticsearch_maintenance_require_green:
+        description: Fail node_maintenance_end unless the cluster returns to green. Default accepts yellow.
+        type: bool
+        default: false
+      elasticsearch_drain_cluster_settings:
+        description: Persistent cluster settings applied while a node is drained for maintenance (typically a recovery throughput
+          boost). node_maintenance_end restores every key listed here to its value in elasticsearch_cluster_settings, or removes
+          it when no baseline is declared there.
+        type: dict
+        default: {}
   node_maintenance_start:
     short_description: Prepare the cluster for taking this node down
     description: Health gate, voting exclusion, primaries-only allocation, ML upgrade mode, optional recovery boost and

roles/elasticsearch/tasks/main.yml

@@ -63,8 +63,16 @@
 - name: Set common password for common certificates
   ansible.builtin.set_fact:
     elasticsearch_tls_key_passphrase: "{{ elasticstack_cert_pass }}"
+  # `is defined` was the old gate — that worked as long as the default in
+  # roles/elasticstack/defaults/main.yml was left commented out, because
+  # setting it to "" then still counted as defined and silently blanked
+  # elasticsearch_tls_key_passphrase, which cascaded into an empty stdin
+  # to elasticsearch-keystore and a failed keystore.add. The docstring
+  # already says "Leave empty to use the per-role passphrases" — respect
+  # that: only apply the override when the user actually supplied a
+  # non-empty value.
   when:
-    - elasticstack_cert_pass is defined
+    - elasticstack_cert_pass | default('') | length > 0
   tags:
     - certificates
     - renew_ca

roles/elasticstack/defaults/main.yml

@@ -42,7 +42,7 @@ elasticstack_ca_pass: PleaseChangeMe
 #   logstash_tls_key_passphrase, and beats_tls_key_passphrase.
 #   Leave empty to use the per-role passphrases.
 # @end
-# elasticstack_cert_pass:
+elasticstack_cert_pass: ""
 # @var elasticstack_ca_validity_period:description: Validity period in days for the CA certificate
 elasticstack_ca_validity_period: 1095
 # === General Settings ===

roles/kibana/tasks/main.yml

@@ -15,8 +15,13 @@
 - name: Set common password for common certificates
   ansible.builtin.set_fact:
     kibana_tls_key_passphrase: "{{ elasticstack_cert_pass }}"
+  # Same gotcha the elasticsearch role hit: setting elasticstack_cert_pass
+  # to "" was silently blanking kibana_tls_key_passphrase, which then went
+  # through as `--pass ` (empty) to elasticsearch-certutil and made the
+  # tool prompt for a password on stdin. The docstring says "Leave empty
+  # to use the per-role passphrases" — so respect that.
   when:
-    - elasticstack_cert_pass is defined
+    - elasticstack_cert_pass | default('') | length > 0
 
 - name: Set Elasticsearch hosts if used with other roles
   ansible.builtin.set_fact:
@@ -195,4 +200,4 @@
           Recent log output:
           {{ _kibana_wait_journal.stdout }}
       when: _kibana_wait_result.rc | default(0) != 2
 

scripts/check_argspecs.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+"""
+Verify that every role's meta/argument_specs.yml lists the same variables
+its defaults/main.yml defines. Fails CI when the two drift.
+
+Runs across every role that has a meta/argument_specs.yml. Roles without
+one (e.g. `repos`, which uses variables from `elasticstack` instead) are
+skipped intentionally.
+
+Exit codes:
+  0  All roles in sync
+  1  At least one role has drift (details on stderr)
+  2  Usage / setup error
+
+The parser only looks at the top-level `varname: value` lines in
+defaults/main.yml. Internal-only vars (leading underscore) are ignored
+so vars/main.yml-style facts don't need arg-spec entries.
+"""
+import sys
+import re
+from pathlib import Path
+
+try:
+    import yaml
+except ImportError:
+    print("check_argspecs: PyYAML is required (pip install pyyaml)", file=sys.stderr)
+    sys.exit(2)
+
+
+VAR_LINE = re.compile(r"^([A-Za-z][\w]*)\s*:")
+
+
+def defaults_vars(path):
+    """Top-level variable names in a defaults/main.yml, minus internal underscore vars."""
+    out = set()
+    with open(path) as f:
+        for line in f:
+            m = VAR_LINE.match(line)
+            if m:
+                name = m.group(1)
+                if not name.startswith("_"):
+                    out.add(name)
+    return out
+
+
+def argspec_options(path):
+    """Option names declared under the role's `main` entry point.
+
+    Task-file entry points (e.g. node_maintenance_start / _end) are
+    invoked with per-call parameters that don't need to appear in
+    defaults/main.yml — they're inputs to a specific action, not
+    role-wide defaults. Only `main`'s options are role-wide vars.
+    """
+    with open(path) as f:
+        spec = yaml.safe_load(f) or {}
+    entries = spec.get("argument_specs") or {}
+    main = entries.get("main") or {}
+    return set((main.get("options") or {}).keys())
+
+
+def main():
+    repo = Path(__file__).resolve().parent.parent
+    roles = sorted(p for p in (repo / "roles").iterdir() if p.is_dir())
+    exit_code = 0
+
+    for role in roles:
+        defaults = role / "defaults" / "main.yml"
+        specs = role / "meta" / "argument_specs.yml"
+        if not specs.exists():
+            continue  # role opts out
+        if not defaults.exists():
+            print(
+                f"[{role.name}] has meta/argument_specs.yml but no defaults/main.yml",
+                file=sys.stderr,
+            )
+            exit_code = 1
+            continue
+
+        vars_ = defaults_vars(defaults)
+        opts = argspec_options(specs)
+        missing_in_spec = vars_ - opts
+        extra_in_spec = opts - vars_
+
+        if missing_in_spec or extra_in_spec:
+            exit_code = 1
+            print(f"\n[{role.name}] argument_specs drift:", file=sys.stderr)
+            for v in sorted(missing_in_spec):
+                print(f"  - in defaults but missing from spec: {v}", file=sys.stderr)
+            for v in sorted(extra_in_spec):
+                print(f"  - in spec but no longer in defaults: {v}", file=sys.stderr)
+        else:
+            print(f"[{role.name}] ok ({len(vars_)} vars)")
+
+    if exit_code:
+        print(
+            "\nFix by editing meta/argument_specs.yml or regenerating with "
+            "`scripts/gen_argspecs.py <role_path>`.",
+            file=sys.stderr,
+        )
+    sys.exit(exit_code)
+
+
+if __name__ == "__main__":
+    main()