From a6722a96168ee8e0a4b4151b4b0ccdd18bc52849 Mon Sep 17 00:00:00 2001 From: Jayson Grace Date: Mon, 3 Aug 2026 12:59:47 -0700 Subject: [PATCH] fix: harden Windows update flow and parameterize warpgate templates **Added:** - Connection recovery steps after reboot and after updates using `wait_for_connection`, which re-attempts downed connections to ride out a wedged SSM channel that `until`/`rescue` cannot handle - `settings_updates/tasks/main.yml` - Explicit assertion that fails with update attempt count and error message if Windows updates did not complete - `settings_updates/tasks/main.yml` - Documentation of the new recovery, wait, and assert steps - `settings_updates/README.md` **Changed:** - Increased reboot resilience by raising `reboot_timeout` to 1800 and `post_reboot_delay` to 60, and tolerating errors on reboot and update tasks so failures fall through to the recovery and assert logic - `settings_updates/tasks/main.yml` - Parameterized the base AMI SSM image ARN with `${aws_region}` instead of a hardcoded `us-west-1` across all warpgate templates - Reduced AMI `volume_size` from 100 to 50 across all warpgate templates --- ansible/roles/settings_updates/README.md | 3 ++ ansible/roles/settings_updates/tasks/main.yml | 28 +++++++++++++++++-- .../goad-dc-base-2016/warpgate.yaml | 4 +-- warpgate-templates/goad-dc-base/warpgate.yaml | 4 +-- .../goad-mssql-base-2016/warpgate.yaml | 4 +-- .../goad-mssql-base/warpgate.yaml | 4 +-- 6 files changed, 37 insertions(+), 10 deletions(-) diff --git a/ansible/roles/settings_updates/README.md b/ansible/roles/settings_updates/README.md index 522d8af4..4a361854 100644 --- a/ansible/roles/settings_updates/README.md +++ b/ansible/roles/settings_updates/README.md @@ -19,8 +19,11 @@ Install Windows updates on managed hosts - **Prevent forced user registry unload (fixes WUA 0x800703FA)** (ansible.windows.win_regedit) - **Reset Windows Update components** (ansible.windows.win_shell) - **Reboot to clear pending registry operations** (ansible.windows.win_reboot) +- **Ensure host recovered after reboot** (ansible.builtin.wait_for_connection) - **Enable update service** (ansible.windows.win_service) - **Install all updates and reboot as many times as needed** (ansible.windows.win_updates) +- **Ensure host recovered after updates** (ansible.builtin.wait_for_connection) +- **Fail if Windows updates did not complete** (ansible.builtin.assert) ## Example Playbook diff --git a/ansible/roles/settings_updates/tasks/main.yml b/ansible/roles/settings_updates/tasks/main.yml index c6a9b678..9f912e94 100644 --- a/ansible/roles/settings_updates/tasks/main.yml +++ b/ansible/roles/settings_updates/tasks/main.yml @@ -28,8 +28,16 @@ - name: Reboot to clear pending registry operations ansible.windows.win_reboot: - reboot_timeout: 600 - post_reboot_delay: 30 + reboot_timeout: 1800 + post_reboot_delay: 60 + ignore_errors: true # tolerate a wedged SSM poll; a genuinely dead box is caught below + + - name: Ensure host recovered after reboot + # wait_for_connection re-attempts a downed connection; until/rescue do not + # retry on 'unreachable', so they cannot ride out a wedged SSM channel. + ansible.builtin.wait_for_connection: + timeout: 600 + sleep: 20 - name: Enable update service ansible.windows.win_service: @@ -46,3 +54,19 @@ retries: 5 delay: 120 until: update_result is not failed + ignore_errors: true # let a wedged SSM poll fall through to recovery + assert below + + - name: Ensure host recovered after updates + # win_updates performs its own internal reboots; a wedged SSM channel during + # one of them can exhaust the retries above. Re-attempt the connection before + # deciding the box is actually dead. + ansible.builtin.wait_for_connection: + timeout: 600 + sleep: 20 + + - name: Fail if Windows updates did not complete + ansible.builtin.assert: + that: update_result is not failed + fail_msg: >- + Windows updates did not complete after {{ update_result.attempts | default('?') }} + attempt(s): {{ update_result.msg | default('unknown error') }} diff --git a/warpgate-templates/goad-dc-base-2016/warpgate.yaml b/warpgate-templates/goad-dc-base-2016/warpgate.yaml index 99dd070e..eec7acb8 100644 --- a/warpgate-templates/goad-dc-base-2016/warpgate.yaml +++ b/warpgate-templates/goad-dc-base-2016/warpgate.yaml @@ -18,7 +18,7 @@ name: goad-dc-base-2016 version: latest base: - image: "arn:aws:ssm:us-west-1::parameter/aws/service/ami-windows-latest/Windows_Server-2016-English-Full-Base" + image: "arn:aws:ssm:${aws_region}::parameter/aws/service/ami-windows-latest/Windows_Server-2016-English-Full-Base" variables: aws_region: us-west-1 @@ -41,7 +41,7 @@ targets: region: "${aws_region}" instance_type: "${instance_type}" ami_name: "goad-dc-base-2016-{{timestamp}}" - volume_size: 100 + volume_size: 50 ami_tags: Name: goad-dc-base-2016 Lab: GOAD diff --git a/warpgate-templates/goad-dc-base/warpgate.yaml b/warpgate-templates/goad-dc-base/warpgate.yaml index b8f3316d..44819237 100644 --- a/warpgate-templates/goad-dc-base/warpgate.yaml +++ b/warpgate-templates/goad-dc-base/warpgate.yaml @@ -17,7 +17,7 @@ name: goad-dc-base version: latest base: - image: "arn:aws:ssm:us-west-1::parameter/aws/service/ami-windows-latest/Windows_Server-2019-English-Full-Base" + image: "arn:aws:ssm:${aws_region}::parameter/aws/service/ami-windows-latest/Windows_Server-2019-English-Full-Base" variables: aws_region: us-west-1 @@ -40,7 +40,7 @@ targets: region: "${aws_region}" instance_type: "${instance_type}" ami_name: "goad-dc-base-{{timestamp}}" - volume_size: 100 + volume_size: 50 ami_tags: Name: goad-dc-base Lab: GOAD diff --git a/warpgate-templates/goad-mssql-base-2016/warpgate.yaml b/warpgate-templates/goad-mssql-base-2016/warpgate.yaml index 9af0b2b2..60e11052 100644 --- a/warpgate-templates/goad-mssql-base-2016/warpgate.yaml +++ b/warpgate-templates/goad-mssql-base-2016/warpgate.yaml @@ -18,7 +18,7 @@ name: goad-mssql-base-2016 version: latest base: - image: "arn:aws:ssm:us-west-1::parameter/aws/service/ami-windows-latest/Windows_Server-2016-English-Full-Base" + image: "arn:aws:ssm:${aws_region}::parameter/aws/service/ami-windows-latest/Windows_Server-2016-English-Full-Base" variables: aws_region: us-west-1 @@ -48,7 +48,7 @@ targets: region: "${aws_region}" instance_type: "${instance_type}" ami_name: "goad-mssql-base-2016-{{timestamp}}" - volume_size: 100 + volume_size: 50 ami_tags: Name: goad-mssql-base-2016 Lab: GOAD diff --git a/warpgate-templates/goad-mssql-base/warpgate.yaml b/warpgate-templates/goad-mssql-base/warpgate.yaml index 8050124a..57d78c7a 100644 --- a/warpgate-templates/goad-mssql-base/warpgate.yaml +++ b/warpgate-templates/goad-mssql-base/warpgate.yaml @@ -17,7 +17,7 @@ name: goad-mssql-base version: latest base: - image: "arn:aws:ssm:us-west-1::parameter/aws/service/ami-windows-latest/Windows_Server-2019-English-Full-Base" + image: "arn:aws:ssm:${aws_region}::parameter/aws/service/ami-windows-latest/Windows_Server-2019-English-Full-Base" variables: aws_region: us-west-1 @@ -47,7 +47,7 @@ targets: region: "${aws_region}" instance_type: "${instance_type}" ami_name: "goad-mssql-base-{{timestamp}}" - volume_size: 100 + volume_size: 50 ami_tags: Name: goad-mssql-base Lab: GOAD