fix: harden Windows reboots and AD group creation against aws_ssm reconnect timeouts

ansible/roles/ad/README.md

@@ -16,8 +16,11 @@ Configure Active Directory domain administrator membership and settings
 ### groups.yml
 
 - **Create Universal Groups** (microsoft.ad.group) - Conditional
+- **Wait for Universal group creation to complete** (ansible.builtin.async_status) - Conditional
 - **Create Global Groups** (microsoft.ad.group) - Conditional
+- **Wait for Global group creation to complete** (ansible.builtin.async_status) - Conditional
 - **Create DomainLocal Groups** (microsoft.ad.group) - Conditional
+- **Wait for DomainLocal group creation to complete** (ansible.builtin.async_status) - Conditional
 
 ### main.yml
 

ansible/roles/ad/tasks/groups.yml

@@ -12,6 +12,26 @@
   loop_control:
     label: "Creating universal group: {{ item.key }}"
   when: ad_groups['universal'] is defined
+  register: universal_group_result
+  retries: 3
+  delay: 10
+  until: universal_group_result is not failed
+  async: 120
+  poll: 0
+  vars:
+    ansible_win_async_startup_timeout: 30
+
+- name: Wait for Universal group creation to complete
+  ansible.builtin.async_status:
+    jid: "{{ item.ansible_job_id }}"
+  register: universal_group_status
+  until: universal_group_status.finished
+  retries: 30
+  delay: 5
+  loop: "{{ universal_group_result.results }}"
+  loop_control:
+    label: "Waiting for universal group: {{ item.item.key }}"
+  when: universal_group_result is not skipped and item.ansible_job_id is defined
 
 - name: Create Global Groups
   microsoft.ad.group:
@@ -24,6 +44,26 @@
   loop_control:
     label: "Creating global group: {{ item.key }}"
   when: ad_groups['global'] is defined
+  register: global_group_result
+  retries: 3
+  delay: 10
+  until: global_group_result is not failed
+  async: 120
+  poll: 0
+  vars:
+    ansible_win_async_startup_timeout: 30
+
+- name: Wait for Global group creation to complete
+  ansible.builtin.async_status:
+    jid: "{{ item.ansible_job_id }}"
+  register: global_group_status
+  until: global_group_status.finished
+  retries: 30
+  delay: 5
+  loop: "{{ global_group_result.results }}"
+  loop_control:
+    label: "Waiting for global group: {{ item.item.key }}"
+  when: global_group_result is not skipped and item.ansible_job_id is defined
 
 - name: Create DomainLocal Groups
   microsoft.ad.group:
@@ -36,3 +76,23 @@
   loop_control:
     label: "Creating domainlocal group: {{ item.key }}"
   when: ad_groups['domainlocal'] is defined
+  register: domainlocal_group_result
+  retries: 3
+  delay: 10
+  until: domainlocal_group_result is not failed
+  async: 120
+  poll: 0
+  vars:
+    ansible_win_async_startup_timeout: 30
+
+- name: Wait for DomainLocal group creation to complete
+  ansible.builtin.async_status:
+    jid: "{{ item.ansible_job_id }}"
+  register: domainlocal_group_status
+  until: domainlocal_group_status.finished
+  retries: 30
+  delay: 5
+  loop: "{{ domainlocal_group_result.results }}"
+  loop_control:
+    label: "Waiting for domainlocal group: {{ item.item.key }}"
+  when: domainlocal_group_result is not skipped and item.ansible_job_id is defined

ansible/roles/groups_domains/README.md

@@ -15,7 +15,9 @@ Create and configure Active Directory groups across domains
 
 ### main.yml
 
-- **Reboot and wait for the AD system to restart** (ansible.windows.win_reboot)
+- **Record pre-reboot boot time baseline** (ansible.windows.win_powershell)
+- **Reboot and wait for the AD system to restart** (block)
+- **Trigger reboot via win_reboot** (ansible.windows.win_reboot)
 - **Synchronize all domains with proper credentials** (ansible.windows.win_powershell)
 - **Add cross-domain users/groups using PowerShell Direct** (ansible.windows.win_powershell) - Conditional
 

ansible/roles/groups_domains/tasks/main.yml

@@ -1,8 +1,28 @@
 ---
+- name: Record pre-reboot boot time baseline
+  ansible.windows.win_powershell:
+    script: |
+      (Get-CimInstance -ClassName Win32_OperatingSystem -Property LastBootUpTime).LastBootUpTime.ToFileTime()
+  register: reboot_baseline
+
 - name: Reboot and wait for the AD system to restart
-  ansible.windows.win_reboot:
-    test_command: "Get-ADUser -Identity Administrator -Properties *"
-    post_reboot_delay: 100
+  block:
+    - name: Trigger reboot via win_reboot
+      ansible.windows.win_reboot:
+        test_command: "Get-ADUser -Identity Administrator -Properties *"
+        reboot_timeout: 300
+        post_reboot_delay: 60
+  rescue:
+    - name: Verify reboot completed (aws_ssm reconnect timeout is expected)
+      ansible.windows.win_powershell:
+        script: |
+          $current = (Get-CimInstance -ClassName Win32_OperatingSystem -Property LastBootUpTime).LastBootUpTime.ToFileTime()
+          if ($current -le {{ reboot_baseline.output[0] }}) { throw "host has not rebooted yet" }
+          Get-ADUser -Identity Administrator -Properties * | Out-Null
+      register: reboot_verify
+      until: reboot_verify is succeeded
+      retries: 30
+      delay: 10
 
 - name: Synchronize all domains with proper credentials
   ansible.windows.win_powershell:

ansible/roles/iis/README.md

@@ -21,7 +21,9 @@ Install and configure Internet Information Services web server
 - **Add SYSTEM allow rights to machine keys** (ansible.windows.win_acl)
 - **Create IIS directories** (ansible.windows.win_file)
 - **Deploy default website index** (ansible.windows.win_copy)
-- **Reboot if required** (ansible.windows.win_reboot) - Conditional
+- **Record pre-reboot boot time baseline (IIS install)** (ansible.windows.win_powershell) - Conditional
+- **Reboot if required** (block) - Conditional
+- **Trigger reboot via win_reboot** (ansible.windows.win_reboot)
 
 ## Example Playbook
 

ansible/roles/iis/tasks/main.yml

@@ -76,6 +76,27 @@
     src: files/index.html
     dest: "C:\\inetpub\\wwwroot\\index.html"
 
+- name: Record pre-reboot boot time baseline (IIS install)
+  ansible.windows.win_powershell:
+    script: |
+      (Get-CimInstance -ClassName Win32_OperatingSystem -Property LastBootUpTime).LastBootUpTime.ToFileTime()
+  register: reboot_baseline
+  when: win_feature.reboot_required | default(false)
+
 - name: Reboot if required
-  ansible.windows.win_reboot:
+  block:
+    - name: Trigger reboot via win_reboot
+      ansible.windows.win_reboot:
+        reboot_timeout: 300
+        post_reboot_delay: 60
+  rescue:
+    - name: Verify reboot completed (aws_ssm reconnect timeout is expected)
+      ansible.windows.win_powershell:
+        script: |
+          $current = (Get-CimInstance -ClassName Win32_OperatingSystem -Property LastBootUpTime).LastBootUpTime.ToFileTime()
+          if ($current -le {{ reboot_baseline.output[0] }}) { throw "host has not rebooted yet" }
+      register: reboot_verify
+      until: reboot_verify is succeeded
+      retries: 30
+      delay: 10
   when: win_feature.reboot_required | default(false)

ansible/roles/mssql/README.md

@@ -43,7 +43,9 @@ Install and configure Microsoft SQL Server Express
 ### install.yml
 
 - **Check if reboot is pending before install** (ansible.windows.win_shell)
-- **Reboot before install if pending (long timeout in case of update)** (ansible.windows.win_reboot) - Conditional
+- **Record pre-reboot boot time baseline (pre-MSSQL install)** (ansible.windows.win_powershell) - Conditional
+- **Reboot before install if pending** (block) - Conditional
+- **Trigger reboot via win_reboot** (ansible.windows.win_reboot)
 - **Create SQL Server installation directories** (ansible.windows.win_file)
 - **Create and load user profile** (ansible.windows.win_shell)
 - **Create SQL Server configuration file** (ansible.windows.win_template)
@@ -58,7 +60,9 @@ Install and configure Microsoft SQL Server Express
 - **Install SQL Server** (ansible.windows.win_command) - Conditional
 - **Add or update registry for ip port (2022)** (ansible.windows.win_regedit) - Conditional
 - **Add or update registry for ip port (2019)** (ansible.windows.win_regedit) - Conditional
-- **Reboot if registry was changed** (ansible.windows.win_reboot) - Conditional
+- **Record pre-reboot boot time baseline (post-registry change)** (ansible.windows.win_powershell) - Conditional
+- **Reboot if registry was changed** (block) - Conditional
+- **Trigger reboot via win_reboot** (ansible.windows.win_reboot)
 - **Firewall ¦ Allow MSSQL through Firewall** (ansible.windows.win_dsc)
 - **Firewall ¦ Allow MSSQL discover through Firewall** (ansible.windows.win_dsc)
 - **Be sure service is started** (ansible.windows.win_service)

ansible/roles/mssql/tasks/install.yml

@@ -34,9 +34,29 @@
   failed_when: false
   changed_when: false
 
-- name: Reboot before install if pending (long timeout in case of update)
-  ansible.windows.win_reboot:
-    reboot_timeout: 1200
+- name: Record pre-reboot boot time baseline (pre-MSSQL install)
+  ansible.windows.win_powershell:
+    script: |
+      (Get-CimInstance -ClassName Win32_OperatingSystem -Property LastBootUpTime).LastBootUpTime.ToFileTime()
+  register: reboot_baseline
+  when: reboot_check.rc == 1
+
+- name: Reboot before install if pending
+  block:
+    - name: Trigger reboot via win_reboot
+      ansible.windows.win_reboot:
+        reboot_timeout: 300
+        post_reboot_delay: 60
+  rescue:
+    - name: Verify reboot completed (aws_ssm reconnect timeout is expected)
+      ansible.windows.win_powershell:
+        script: |
+          $current = (Get-CimInstance -ClassName Win32_OperatingSystem -Property LastBootUpTime).LastBootUpTime.ToFileTime()
+          if ($current -le {{ reboot_baseline.output[0] }}) { throw "host has not rebooted yet" }
+      register: reboot_verify
+      until: reboot_verify is succeeded
+      retries: 60
+      delay: 10
   when: reboot_check.rc == 1
 
 - name: Create SQL Server installation directories
@@ -156,8 +176,29 @@
   register: win_reg
   when: sql_version == "MSSQL_2019"
 
+- name: Record pre-reboot boot time baseline (post-registry change)
+  ansible.windows.win_powershell:
+    script: |
+      (Get-CimInstance -ClassName Win32_OperatingSystem -Property LastBootUpTime).LastBootUpTime.ToFileTime()
+  register: reboot_baseline
+  when: win_reg.changed
+
 - name: Reboot if registry was changed
-  ansible.windows.win_reboot:
+  block:
+    - name: Trigger reboot via win_reboot
+      ansible.windows.win_reboot:
+        reboot_timeout: 300
+        post_reboot_delay: 60
+  rescue:
+    - name: Verify reboot completed (aws_ssm reconnect timeout is expected)
+      ansible.windows.win_powershell:
+        script: |
+          $current = (Get-CimInstance -ClassName Win32_OperatingSystem -Property LastBootUpTime).LastBootUpTime.ToFileTime()
+          if ($current -le {{ reboot_baseline.output[0] }}) { throw "host has not rebooted yet" }
+      register: reboot_verify
+      until: reboot_verify is succeeded
+      retries: 30
+      delay: 10
   when: win_reg.changed
 
 - name: Firewall | Allow MSSQL through Firewall

ansible/roles/trusts/README.md

@@ -17,7 +17,9 @@ Configure Active Directory domain trust relationships
 
 - **Prepare to trust flush and renew dns** (ansible.windows.win_shell)
 - **Configure forest trust to remote domain** (ansible.windows.win_powershell)
-- **Reboot and wait for the AD system to restart** (ansible.windows.win_reboot) - Conditional
+- **Record pre-reboot boot time baseline** (ansible.windows.win_powershell) - Conditional
+- **Reboot and wait for the AD system to restart** (block) - Conditional
+- **Trigger reboot via win_reboot** (ansible.windows.win_reboot)
 
 ## Example Playbook
 

ansible/roles/trusts/tasks/main.yml

@@ -50,7 +50,29 @@
       }
   register: trust_result
 
+- name: Record pre-reboot boot time baseline
+  ansible.windows.win_powershell:
+    script: |
+      (Get-CimInstance -ClassName Win32_OperatingSystem -Property LastBootUpTime).LastBootUpTime.ToFileTime()
+  register: reboot_baseline
+  when: trust_result.changed
+
 - name: Reboot and wait for the AD system to restart
-  ansible.windows.win_reboot:
-    test_command: "Get-ADUser -Identity {{ admin_user }} -Properties *"
+  block:
+    - name: Trigger reboot via win_reboot
+      ansible.windows.win_reboot:
+        test_command: "Get-ADUser -Identity {{ admin_user }} -Properties *"
+        reboot_timeout: 300
+        post_reboot_delay: 60
+  rescue:
+    - name: Verify reboot completed (aws_ssm reconnect timeout is expected)
+      ansible.windows.win_powershell:
+        script: |
+          $current = (Get-CimInstance -ClassName Win32_OperatingSystem -Property LastBootUpTime).LastBootUpTime.ToFileTime()
+          if ($current -le {{ reboot_baseline.output[0] }}) { throw "host has not rebooted yet" }
+          Get-ADUser -Identity {{ admin_user }} -Properties * | Out-Null
+      register: reboot_verify
+      until: reboot_verify is succeeded
+      retries: 30
+      delay: 10
   when: trust_result.changed

ansible/roles/webdav/README.md

@@ -16,7 +16,9 @@ Install and configure WebDAV client on Windows hosts
 ### main.yml
 
 - **Ensure WebDAV client feature is installed** (ansible.windows.win_feature)
-- **Reboot after installing WebDAV client feature** (ansible.windows.win_reboot) - Conditional
+- **Record pre-reboot boot time baseline (WebDAV install)** (ansible.windows.win_powershell) - Conditional
+- **Reboot after installing WebDAV client feature** (block) - Conditional
+- **Trigger reboot via win_reboot** (ansible.windows.win_reboot)
 
 ## Example Playbook
 

ansible/roles/webdav/tasks/main.yml

@@ -5,6 +5,27 @@
     state: present
   register: webdav_feature
 
+- name: Record pre-reboot boot time baseline (WebDAV install)
+  ansible.windows.win_powershell:
+    script: |
+      (Get-CimInstance -ClassName Win32_OperatingSystem -Property LastBootUpTime).LastBootUpTime.ToFileTime()
+  register: reboot_baseline
+  when: webdav_feature.reboot_required
+
 - name: Reboot after installing WebDAV client feature
-  ansible.windows.win_reboot:
+  block:
+    - name: Trigger reboot via win_reboot
+      ansible.windows.win_reboot:
+        reboot_timeout: 300
+        post_reboot_delay: 60
+  rescue:
+    - name: Verify reboot completed (aws_ssm reconnect timeout is expected)
+      ansible.windows.win_powershell:
+        script: |
+          $current = (Get-CimInstance -ClassName Win32_OperatingSystem -Property LastBootUpTime).LastBootUpTime.ToFileTime()
+          if ($current -le {{ reboot_baseline.output[0] }}) { throw "host has not rebooted yet" }
+      register: reboot_verify
+      until: reboot_verify is succeeded
+      retries: 30
+      delay: 10
   when: webdav_feature.reboot_required

cli/internal/ansible/logparser.go

@@ -13,21 +13,22 @@ var (
 )
 
 // CheckAnsibleSuccess analyzes Ansible output to determine if the run succeeded.
-// It reports whether no failures or unreachable hosts were detected in the
-// PLAY RECAP and no unignored fatal errors appear in the output.
+// When a PLAY RECAP is present it is treated as authoritative: ansible's own
+// failed/unreachable counters already discount fatals that were rescued or
+// explicitly ignored. If no recap exists (the play aborted before printing one)
+// the output is scanned for unignored fatal errors as a fallback.
 func CheckAnsibleSuccess(output string) bool {
 	if idx := strings.Index(output, "PLAY RECAP"); idx >= 0 {
 		recap := output[idx:]
 		if failedRe.MatchString(recap) || unreachableRe.MatchString(recap) {
 			return false
 		}
+		return !strings.Contains(output, "to retry, use:")
 	}
 
-	// Secondary: check for fatal errors not followed by "...ignoring"
 	lines := strings.Split(output, "\n")
 	for i, line := range lines {
 		if strings.HasPrefix(line, "fatal:") {
-			// Check next 20 lines for "...ignoring" (multi-line YAML output can be long)
 			end := i + 21
 			if end > len(lines) {
 				end = len(lines)

cli/internal/ansible/logparser_test.go

@@ -45,6 +45,17 @@ fatal: [DC01]: FAILED! => {"msg": "critical error"}
 NO MORE HOSTS LEFT *************************************************************`,
 			want: false,
 		},
+		{
+			name: "fatal rescued by block/rescue with clean recap",
+			output: `TASK [Reboot]
+fatal: [DC01]: FAILED! => {"msg": "Timed out waiting for last boot time check (timeout=600)"}
+
+TASK [Verify reboot completed]
+ok: [DC01]
+PLAY RECAP *********************************************************************
+DC01                       : ok=12   changed=1    unreachable=0    failed=0    skipped=0    rescued=1    ignored=0`,
+			want: true,
+		},
 		{
 			name: "retry indicator present",
 			output: `PLAY RECAP *********************************************************************